Skip to content

Commit 7b582b7

Browse files
zihaomufengyuentau
zihaomu
and
fengyuentau
authored
Merge pull request opencv#21036 from fengyuentau:timvx_backend_support
dnn: TIM-VX NPU backend support * Add TimVX NPU backend for DNN module. * use official branch from tim-vx repo; fix detecting viv sdk Co-authored-by: fytao <[email protected]>
1 parent 9390c56 commit 7b582b7

37 files changed

+2982
-30
lines changed

‎3rdparty/libtim-vx/tim-vx.cmake

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
set(TIMVX_COMMIT_HASH "1d9c7ab941b3d8d9c4d28d80058402725731e3d6")
2+
set(OCV_TIMVX_DIR "${OpenCV_BINARY_DIR}/3rdparty/libtim-vx")
3+
set(OCV_TIMVX_SOURCE_PATH "${OCV_TIMVX_DIR}/TIM-VX-${TIMVX_COMMIT_HASH}")
4+
5+
# Download TIM-VX source code
6+
if(EXISTS "${OCV_TIMVX_SOURCE_PATH}")
7+
message(STATUS "TIM-VX: Use cache of TIM-VX source code at ${OCV_TIMVX_SOURCE_PATH}")
8+
set(TIMVX_FOUND ON)
9+
else()
10+
set(OCV_TIMVX_FILENAME "${TIMVX_COMMIT_HASH}.zip")
11+
set(OCV_TIMVX_URL "https://github.com/VeriSilicon/TIM-VX/archive/")
12+
set(timvx_zip_md5sum 92619cc4498014ac7a09834d5e33ebd5)
13+
14+
ocv_download(FILENAME ${OCV_TIMVX_FILENAME}
15+
HASH ${timvx_zip_md5sum}
16+
URL "${OCV_TIMVX_URL}"
17+
DESTINATION_DIR "${OCV_TIMVX_DIR}"
18+
ID "TIM-VX"
19+
STATUS res
20+
UNPACK RELATIVE_URL)
21+
if(res)
22+
set(TIMVX_FOUND ON)
23+
message(STATUS "TIM-VX: Source code downloaded at ${OCV_TIMVX_SOURCE_PATH}.")
24+
else()
25+
set(TIMVX_FOUND OFF)
26+
message(STATUS "TIM-VX: Failed to download source code from github. Turning off TIMVX_FOUND")
27+
return()
28+
endif()
29+
endif()
30+
31+
# set VIVANTE SDK especially for x86_64 which comes along with TIM-VX source code
32+
if(CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64)
33+
set(VIVANTE_SDK_DIR "${OCV_TIMVX_SOURCE_PATH}/prebuilt-sdk/x86_64_linux")
34+
message(STATUS "TIM-VX: Build from source using prebuilt x86_64 VIVANTE SDK.")
35+
endif()
36+
37+
# Verify if requested VIVANTE SDK libraries are all found
38+
find_vivante_sdk_libs(missing ${VIVANTE_SDK_DIR})
39+
if(missing)
40+
message(STATUS "TIM-VX: Failed to find ${missing} in ${VIVANTE_SDK_DIR}/lib. Turning off TIMVX_VIV_FOUND")
41+
set(TIMVX_VIV_FOUND OFF)
42+
else()
43+
message(STATUS "TIM-VX: dependent VIVANTE SDK libraries are found at ${VIVANTE_SDK_DIR}/lib.")
44+
set(TIMVX_VIV_FOUND ON)
45+
endif()
46+
47+
if(TIMVX_VIV_FOUND)
48+
# vars used by TIM-VX CMake scripts
49+
set(EXTERNAL_VIV_SDK "${VIVANTE_SDK_DIR}" CACHE INTERNAL "" FORCE)
50+
set(VIV_SDK_DRIVER_PREFIX "lib" CACHE INTERNAL "" FORCE)
51+
endif()
52+
53+
if(TIMVX_FOUND AND TIMVX_VIV_FOUND)
54+
set(BUILD_TIMVX ON)
55+
else()
56+
return()
57+
endif()
58+
59+
if(BUILD_TIMVX)
60+
set(HAVE_TIMVX 1)
61+
62+
ocv_warnings_disable(CMAKE_C_FLAGS -Wunused-parameter -Wstrict-prototypes -Wundef -Wsign-compare -Wmissing-prototypes -Wmissing-declarations -Wstrict-aliasing -Wunused-but-set-variable -Wmaybe-uninitialized -Wshadow -Wsuggest-override -Wswitch)
63+
ocv_warnings_disable(CMAKE_CXX_FLAGS -Wunused-parameter -Wstrict-prototypes -Wundef -Wsign-compare -Wunused-but-set-variable -Wshadow -Wsuggest-override -Wmissing-declarations -Wswitch)
64+
65+
set(TIMVX_INC_DIR "${OCV_TIMVX_SOURCE_PATH}/include" CACHE INTERNAL "TIM-VX include directory")
66+
if(EXISTS "${OCV_TIMVX_SOURCE_PATH}/CMakeLists.txt")
67+
add_subdirectory("${OCV_TIMVX_SOURCE_PATH}" "${OCV_TIMVX_DIR}/build")
68+
else()
69+
message(WARNING "TIM-VX: Missing 'CMakeLists.txt' in the source code: ${OCV_TIMVX_SOURCE_PATH}")
70+
endif()
71+
ocv_install_target(tim-vx EXPORT OpenCVModules ARCHIVE DESTINATION ${OPENCV_3P_LIB_INSTALL_PATH} COMPONENT dev)
72+
set(TIMVX_LIB "tim-vx")
73+
endif()

‎CMakeLists.txt

+16
Original file line numberDiff line numberDiff line change
@@ -453,6 +453,9 @@ OCV_OPTION(WITH_TENGINE "Include Arm Inference Tengine support" OFF
453453
OCV_OPTION(WITH_ONNX "Include Microsoft ONNX Runtime support" OFF
454454
VISIBLE_IF TRUE
455455
VERIFY HAVE_ONNX)
456+
OCV_OPTION(WITH_TIMVX "Include Tim-VX support" OFF
457+
VISIBLE_IF TRUE
458+
VERIFY HAVE_TIMVX)
456459

457460
# OpenCV build components
458461
# ===================================================
@@ -733,6 +736,9 @@ include(cmake/OpenCVFindProtobuf.cmake)
733736
if(WITH_TENGINE)
734737
include(cmake/OpenCVFindTengine.cmake)
735738
endif()
739+
if(WITH_TIMVX)
740+
include(cmake/OpenCVFindTIMVX.cmake)
741+
endif()
736742

737743
# ----------------------------------------------------------------------------
738744
# Detect other 3rd-party libraries/tools
@@ -1645,6 +1651,16 @@ if(WITH_WEBNN OR HAVE_WEBNN)
16451651
endif()
16461652
endif()
16471653

1654+
if(WITH_TIMVX)
1655+
status("")
1656+
status(" Tim-VX:" HAVE_TIMVX THEN "YES" ELSE "NO")
1657+
if(HAVE_TIMVX)
1658+
status(" Include path" TIMVX_INCLUDE_DIR THEN "${TIMVX_INCLUDE_DIR}" ELSE "NO")
1659+
status(" Link libraries:" TIMVX_LIBRARY THEN "${TIMVX_LIBRARY}" ELSE "NO")
1660+
status(" VIVANTE SDK path" VIVANTE_SDK_DIR THEN "${VIVANTE_SDK_DIR}" ELSE "NO")
1661+
endif()
1662+
endif()
1663+
16481664
if(WITH_OPENCL OR HAVE_OPENCL)
16491665
ocv_build_features_string(opencl_features
16501666
IF HAVE_OPENCL_SVM THEN "SVM"

‎cmake/OpenCVFindTIMVX.cmake

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
set(TIMVX_INSTALL_DIR "" CACHE PATH "Path to libtim-vx installation")
2+
set(VIVANTE_SDK_DIR "" CACHE PATH "Path to VIVANTE SDK needed by TIM-VX.")
3+
set(VIVANTE_SDK_LIB_CANDIDATES "OpenVX;VSC;GAL;ArchModelSw;NNArchPerf" CACHE STRING "VIVANTE SDK library candidates")
4+
5+
# Ensure VIVANTE SDK library candidates are present in given search path
6+
function(find_vivante_sdk_libs _viv_notfound _viv_search_path)
7+
foreach(one ${VIVANTE_SDK_LIB_CANDIDATES})
8+
#NO_DEFAULT_PATH is used to ensure VIVANTE SDK libs are from one only source
9+
find_library(VIV_${one}_LIB ${one} PATHS "${_viv_search_path}/lib" NO_DEFAULT_PATH)
10+
if(NOT VIV_${one}_LIB)
11+
list(APPEND _viv_notfound_list ${one})
12+
endif()
13+
endforeach()
14+
set(${_viv_notfound} ${_viv_notfound_list} PARENT_SCOPE)
15+
endfunction()
16+
# Default value for VIVANTE_SDK_DIR: /usr
17+
if(NOT VIVANTE_SDK_DIR)
18+
set(VIVANTE_SDK_DIR "/usr")
19+
endif()
20+
# Environment variable VIVANTE_SDK_DIR overrides the one in this script
21+
if(DEFINED ENV{VIVANTE_SDK_DIR})
22+
set(VIVANTE_SDK_DIR $ENV{VIVANTE_SDK_DIR})
23+
message(STATUS "TIM-VX: Load VIVANTE_SDK_DIR from system environment: ${VIVANTE_SDK_DIR}")
24+
endif()
25+
26+
27+
# Compile with pre-installed TIM-VX; Or compile together with TIM-VX from source
28+
if(TIMVX_INSTALL_DIR AND NOT BUILD_TIMVX)
29+
message(STATUS "TIM-VX: Use binaries at ${TIMVX_INSTALL_DIR}")
30+
set(BUILD_TIMVX OFF)
31+
32+
set(TIMVX_INC_DIR "${TIMVX_INSTALL_DIR}/include" CACHE INTERNAL "TIM-VX include directory")
33+
find_library(TIMVX_LIB "tim-vx" PATHS "${TIMVX_INSTALL_DIR}/lib")
34+
if(TIMVX_LIB)
35+
set(TIMVX_FOUND ON)
36+
else()
37+
set(TIMVX_FOUND OFF)
38+
endif()
39+
40+
# Verify if requested VIVANTE SDK libraries are all found
41+
find_vivante_sdk_libs(missing ${VIVANTE_SDK_DIR})
42+
if(missing)
43+
message(STATUS "TIM-VX: Failed to find ${missing} in ${VIVANTE_SDK_DIR}/lib. Turning off TIMVX_VIV_FOUND")
44+
set(TIMVX_VIV_FOUND OFF)
45+
else()
46+
message(STATUS "TIM-VX: dependent VIVANTE SDK libraries are found at ${VIVANTE_SDK_DIR}/lib.")
47+
set(TIMVX_VIV_FOUND ON)
48+
endif()
49+
else()
50+
message(STATUS "TIM-VX: Build from source")
51+
include("${OpenCV_SOURCE_DIR}/3rdparty/libtim-vx/tim-vx.cmake")
52+
endif()
53+
54+
if(TIMVX_FOUND AND TIMVX_VIV_FOUND)
55+
set(HAVE_TIMVX 1)
56+
57+
message(STATUS "TIM-VX: Found TIM-VX includes: ${TIMVX_INC_DIR}")
58+
message(STATUS "TIM-VX: Found TIM-VX library: ${TIMVX_LIB}")
59+
set(TIMVX_LIBRARY ${TIMVX_LIB})
60+
set(TIMVX_INCLUDE_DIR ${TIMVX_INC_DIR})
61+
62+
message(STATUS "TIM-VX: Found VIVANTE SDK libraries: ${VIVANTE_SDK_DIR}/lib")
63+
link_directories(${VIVANTE_SDK_DIR}/lib)
64+
endif()
65+
66+
MARK_AS_ADVANCED(
67+
TIMVX_INC_DIR
68+
TIMVX_LIB
69+
)

‎modules/dnn/CMakeLists.txt

+9
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ if(WITH_WEBNN AND HAVE_WEBNN)
2323
add_definitions(-DHAVE_WEBNN=1)
2424
endif()
2525

26+
if(HAVE_TIMVX)
27+
add_definitions(-DHAVE_TIMVX=1)
28+
endif()
29+
2630
ocv_option(OPENCV_DNN_CUDA "Build with CUDA support"
2731
HAVE_CUDA
2832
AND HAVE_CUBLAS
@@ -146,6 +150,11 @@ if(HAVE_TENGINE)
146150
list(APPEND libs -Wl,--whole-archive ${TENGINE_LIBRARIES} -Wl,--no-whole-archive)
147151
endif()
148152

153+
if(HAVE_TIMVX)
154+
list(APPEND include_dirs ${TIMVX_INCLUDE_DIR})
155+
list(APPEND libs -Wl,--whole-archive ${TIMVX_LIBRARY} -Wl,--no-whole-archive)
156+
endif()
157+
149158
set(webnn_srcs "")
150159
if(NOT EMSCRIPTEN)
151160
if(HAVE_WEBNN)

‎modules/dnn/include/opencv2/dnn/all_layers.hpp

+4-2
Original file line numberDiff line numberDiff line change
@@ -262,7 +262,7 @@ CV__DNN_INLINE_NS_BEGIN
262262
{
263263
public:
264264
int input_zp, output_zp;
265-
float output_sc;
265+
float input_sc, output_sc;
266266
static Ptr<BaseConvolutionLayer> create(const LayerParams& params);
267267
};
268268

@@ -322,6 +322,7 @@ CV__DNN_INLINE_NS_BEGIN
322322
{
323323
public:
324324
int input_zp, output_zp;
325+
float input_sc, output_sc;
325326
static Ptr<PoolingLayerInt8> create(const LayerParams& params);
326327
};
327328

@@ -365,7 +366,8 @@ CV__DNN_INLINE_NS_BEGIN
365366
class CV_EXPORTS InnerProductLayerInt8 : public InnerProductLayer
366367
{
367368
public:
368-
int output_zp;
369+
int input_zp, output_zp;
370+
float input_sc, output_sc;
369371
static Ptr<InnerProductLayerInt8> create(const LayerParams& params);
370372
};
371373

‎modules/dnn/include/opencv2/dnn/dnn.hpp

+16-1
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ CV__DNN_INLINE_NS_BEGIN
7575
DNN_BACKEND_VKCOM,
7676
DNN_BACKEND_CUDA,
7777
DNN_BACKEND_WEBNN,
78+
DNN_BACKEND_TIMVX,
7879
#ifdef __OPENCV_BUILD
7980
DNN_BACKEND_INFERENCE_ENGINE_NGRAPH = 1000000, // internal - use DNN_BACKEND_INFERENCE_ENGINE + setInferenceEngineBackendType()
8081
DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019, // internal - use DNN_BACKEND_INFERENCE_ENGINE + setInferenceEngineBackendType()
@@ -95,7 +96,8 @@ CV__DNN_INLINE_NS_BEGIN
9596
DNN_TARGET_FPGA, //!< FPGA device with CPU fallbacks using Inference Engine's Heterogeneous plugin.
9697
DNN_TARGET_CUDA,
9798
DNN_TARGET_CUDA_FP16,
98-
DNN_TARGET_HDDL
99+
DNN_TARGET_HDDL,
100+
DNN_TARGET_NPU,
99101
};
100102

101103
CV_EXPORTS std::vector< std::pair<Backend, Target> > getAvailableBackends();
@@ -321,6 +323,19 @@ CV__DNN_INLINE_NS_BEGIN
321323
const std::vector<Ptr<BackendWrapper>>& outputs
322324
);
323325

326+
/**
327+
* @brief Returns a TimVX backend node
328+
*
329+
* @param timVxInfo void pointer to CSLContext object
330+
* @param inputsWrapper layer inputs
331+
* @param outputsWrapper layer outputs
332+
* @param isLast if the node is the last one of the TimVX Graph.
333+
*/
334+
virtual Ptr<BackendNode> initTimVX(void* timVxInfo,
335+
const std::vector<Ptr<BackendWrapper> > &inputsWrapper,
336+
const std::vector<Ptr<BackendWrapper> > &outputsWrapper,
337+
bool isLast);
338+
324339
/**
325340
* @brief Automatic Halide scheduling based on layer hyper-parameters.
326341
* @param[in] node Backend node with Halide functions.

0 commit comments

Comments
 (0)