53 support building without tflite support

CMakeLists.txt

@@ -19,10 +19,7 @@ include(cmake/variables.cmake)
 
 # ---- Declare library ----
 
-add_library(
-    edgerunner_edgerunner source/edgerunner.cpp source/tflite/model.cpp
-                          source/tflite/tensor.cpp
-)
+add_library(edgerunner_edgerunner source/edgerunner.cpp)
 add_library(edgerunner::edgerunner ALIAS edgerunner_edgerunner)
 
 include(GenerateExportHeader)
@@ -70,8 +67,18 @@ target_link_libraries(edgerunner_edgerunner PRIVATE fmt::fmt)
 find_package(span-lite REQUIRED)
 target_link_libraries(edgerunner_edgerunner PUBLIC nonstd::span-lite)
 
-find_package(tensorflowlite REQUIRED)
-target_link_libraries(edgerunner_edgerunner PRIVATE tensorflow::tensorflowlite)
+if(edgerunner_ENABLE_TFLITE)
+    find_package(tensorflowlite REQUIRED)
+    target_link_libraries(
+        edgerunner_edgerunner PRIVATE tensorflow::tensorflowlite
+    )
+
+    target_sources(
+        edgerunner_edgerunner PRIVATE source/tflite/model.cpp
+                                      source/tflite/tensor.cpp
+    )
+    target_compile_definitions(edgerunner_edgerunner PUBLIC EDGERUNNER_TFLITE)
+endif()
 
 if(edgerunner_ENABLE_GPU)
     target_compile_definitions(edgerunner_edgerunner PUBLIC EDGERUNNER_GPU)
@@ -89,8 +96,10 @@ if(edgerunner_ENABLE_NPU)
     target_link_libraries(edgerunner_edgerunner PRIVATE qnn::headers)
 
     if(ANDROID)
-        target_link_libraries(edgerunner_edgerunner PRIVATE qnn::tflite)
         target_compile_definitions(edgerunner_edgerunner PUBLIC EDGERUNNER_QNN)
+        if(edgerunner_ENABLE_TFLITE)
+            target_link_libraries(edgerunner_edgerunner PRIVATE qnn::tflite)
+        endif()
     endif()
 endif()
 

cmake/variables.cmake

@@ -1,9 +1,9 @@
 # ---- Developer mode ----
 
 # Developer mode enables targets and code paths in the CMake scripts that are
-# only relevant for the developer(s) of edgerunner
-# Targets necessary to build the project must be provided unconditionally, so
-# consumers can trivially build and package the project
+# only relevant for the developer(s) of edgerunner Targets necessary to build
+# the project must be provided unconditionally, so consumers can trivially build
+# and package the project
 if(PROJECT_IS_TOP_LEVEL)
     option(edgerunner_DEVELOPER_MODE "Enable developer mode" OFF)
     option(BUILD_SHARED_LIBS "Build shared libs." OFF)
@@ -12,28 +12,29 @@ endif()
 # ---- Suppress C4251 on Windows ----
 
 # Please see include/edgerunner/edgerunner.hpp for more details
-set(pragma_suppress_c4251 "
+set(pragma_suppress_c4251
+    "
 /* This needs to suppress only for MSVC */
 #if defined(_MSC_VER) && !defined(__ICL)
 #  define EDGERUNNER_SUPPRESS_C4251 _Pragma(\"warning(suppress:4251)\")
 #else
 #  define EDGERUNNER_SUPPRESS_C4251
 #endif
-")
+"
+)
 
 # ---- Warning guard ----
 
 # target_include_directories with the SYSTEM modifier will request the compiler
-# to omit warnings from the provided paths, if the compiler supports that
-# This is to provide a user experience similar to find_package when
-# add_subdirectory or FetchContent is used to consume this project
+# to omit warnings from the provided paths, if the compiler supports that This
+# is to provide a user experience similar to find_package when add_subdirectory
+# or FetchContent is used to consume this project
 set(warning_guard "")
 if(NOT PROJECT_IS_TOP_LEVEL)
-    option(
-      edgerunner_INCLUDES_WITH_SYSTEM
-      "Use SYSTEM modifier for edgerunner's includes, disabling warnings"
-      ON
-  )
+    option(edgerunner_INCLUDES_WITH_SYSTEM
+           "Use SYSTEM modifier for edgerunner's includes, disabling warnings"
+           ON
+    )
     mark_as_advanced(edgerunner_INCLUDES_WITH_SYSTEM)
     if(edgerunner_INCLUDES_WITH_SYSTEM)
         set(warning_guard SYSTEM)
@@ -42,3 +43,4 @@ endif()
 
 option(edgerunner_ENABLE_GPU "Enable GPU support" OFF)
 option(edgerunner_ENABLE_NPU "Enable NPU support" OFF)
+option(edgerunner_ENABLE_TFLITE "Enable TFLite support" OFF)

conanfile.py

@@ -21,6 +21,7 @@ class EdgerunnerRecipe(ConanFile):
         "fPIC": [True, False],
         "with_gpu": [True, False],
         "with_npu": [True, False],
+        "with_tflite": [True, False],
         "examples": [True, False],
     }
 
@@ -29,6 +30,7 @@ class EdgerunnerRecipe(ConanFile):
         "fPIC": True,
         "with_gpu": False,
         "with_npu": False,
+        "with_tflite": True,
         "examples": False,
     }
 
@@ -54,7 +56,8 @@ def layout(self):
     def requirements(self):
         self.requires("fmt/10.2.1")
         self.requires("span-lite/0.11.0", transitive_headers=True)
-        self.requires("tensorflow-lite/2.12.0")
+        if self.options.with_tflite:
+            self.requires("tensorflow-lite/2.12.0")
 
         if self.options.examples:
             self.requires("opencv/4.9.0")
@@ -66,7 +69,11 @@ def build_requirements(self):
         self.test_requires("catch2/3.6.0")
 
     def configure(self):
-        self.options["tensorflow-lite"].with_gpu = self.options.with_gpu
+        if self.options.with_tflite:
+            self.options["tensorflow-lite"].with_gpu = self.options.with_gpu
+
+        if self.options.with_npu:
+            self.options["qnn"].with_tflite = self.options.with_tflite
 
         if self.options.examples:
             self.options["opencv"].with_quirc = False
@@ -89,6 +96,7 @@ def generate(self):
         toolchain.variables["BUILD_EXAMPLES"] = self.options.examples
         toolchain.variables["edgerunner_ENABLE_GPU"] = self.options.with_gpu
         toolchain.variables["edgerunner_ENABLE_NPU"] = self.options.with_npu
+        toolchain.variables["edgerunner_ENABLE_TFLITE"] = self.options.with_tflite
 
         toolchain.generate()
 
@@ -117,5 +125,8 @@ def package_info(self):
         if self.options.with_npu:
             defines.append("EDGERUNNER_QNN")
 
+        if self.options.with_tflite:
+            defines.append("EDGERUNNER_TFLITE")
+
         self.cpp_info.defines = defines
         self.cpp_info.libs = ["edgerunner"]

include/edgerunner/qnn/graph.hpp

@@ -45,17 +45,16 @@ using GraphConfigInfoT = struct GraphConfigInfo {
     const QnnGraph_Config_t** graphConfigs;
 };
 
-using ComposeGraphsFnHandleTypeT =
-    GraphErrorT (*)(Qnn_BackendHandle_t,
-                    QnnInterface_ImplementationV2_16_t,
-                    Qnn_ContextHandle_t,
-                    const GraphConfigInfoT**,
-                    const uint32_t,
-                    GraphInfoT***,
-                    uint32_t*,
-                    bool,
-                    QnnLog_Callback_t,
-                    QnnLog_Level_t);
+using ComposeGraphsFnHandleTypeT = GraphErrorT (*)(Qnn_BackendHandle_t,
+                                                   QNN_INTERFACE_VER_TYPE,
+                                                   Qnn_ContextHandle_t,
+                                                   const GraphConfigInfoT**,
+                                                   const uint32_t,
+                                                   GraphInfoT***,
+                                                   uint32_t*,
+                                                   bool,
+                                                   QnnLog_Callback_t,
+                                                   QnnLog_Level_t);
 
 using FreeGraphInfoFnHandleTypeT = GraphErrorT (*)(GraphInfoT***, uint32_t);
 
@@ -74,9 +73,7 @@ class GraphsInfo {
 
     auto accessGraphs() -> auto& { return m_graphsInfo; }
 
-    auto setGraph() {
-        m_graphInfo = std::unique_ptr<GraphInfoT>(m_graphsInfo[0] /* NOLINT */);
-    }
+    auto setGraph() { m_graphInfo = m_graphsInfo[0] /* NOLINT */; }
 
     auto getGraphsCountPtr() -> uint32_t* { return &m_graphsCount; }
 
@@ -114,12 +111,11 @@ class GraphsInfo {
         FreeGraphInfoFnHandleTypeT freeGraphInfoFnHandle) -> STATUS;
 
     auto composeGraphs(Qnn_BackendHandle_t& qnnBackendHandle,
-                       QnnInterface_ImplementationV2_16_t& qnnInterface,
+                       QNN_INTERFACE_VER_TYPE& qnnInterface,
                        Qnn_ContextHandle_t& qnnContext) -> STATUS;
 
-    auto retrieveGraphFromContext(
-        QnnInterface_ImplementationV2_16_t& qnnInterface,
-        Qnn_ContextHandle_t& qnnContext) -> STATUS;
+    auto retrieveGraphFromContext(QNN_INTERFACE_VER_TYPE& qnnInterface,
+                                  Qnn_ContextHandle_t& qnnContext) -> STATUS;
 
     auto copyGraphsInfoV1(const QnnSystemContext_GraphInfoV1_t* graphInfoSrc,
                           GraphInfoT* graphInfoDst) -> bool;
@@ -134,11 +130,11 @@ class GraphsInfo {
     std::vector<GraphInfoT> m_graphs;
     std::vector<GraphInfoT*> m_graphPtrs;
 
+    GraphInfoT* m_graphInfo {};
+
     GraphInfoT** m_graphsInfo {};
     uint32_t m_graphsCount {};
 
-    std::unique_ptr<GraphInfoT> m_graphInfo;
-
     ComposeGraphsFnHandleTypeT m_composeGraphsFnHandle {};
     FreeGraphInfoFnHandleTypeT m_freeGraphInfoFnHandle {};
 

source/edgerunner.cpp

@@ -8,7 +8,10 @@
 #include <nonstd/span.hpp>
 
 #include "edgerunner/model.hpp"
-#include "edgerunner/tflite/model.hpp"
+
+#ifdef EDGERUNNER_TFLITE
+#    include "edgerunner/tflite/model.hpp"
+#endif
 
 #ifdef EDGERUNNER_QNN
 #    include "edgerunner/qnn/model.hpp"
@@ -22,9 +25,11 @@ auto createModel(const std::filesystem::path& modelPath)
 
     std::unique_ptr<Model> model;
 
+#ifdef EDGERUNNER_TFLITE
     if (modelExtension == "tflite") {
         model = std::make_unique<tflite::ModelImpl>(modelPath);
     }
+#endif
 
 #ifdef EDGERUNNER_QNN
     if (modelExtension == "so" || modelExtension == "bin") {
@@ -44,9 +49,11 @@ auto createModel(const nonstd::span<uint8_t>& modelBuffer,
                  const std::string& modelExtension) -> std::unique_ptr<Model> {
     std::unique_ptr<Model> model;
 
+#ifdef EDGERUNNER_TFLITE
     if (modelExtension == "tflite") {
         model = std::make_unique<tflite::ModelImpl>(modelBuffer);
     }
+#endif
 
 #ifdef EDGERUNNER_QNN
     if (modelExtension == "so" || modelExtension == "bin") {

source/qnn/graph.cpp

@@ -138,7 +138,7 @@ auto GraphsInfo::setFreeGraphInfoFnHandle(
 }
 
 auto GraphsInfo::composeGraphs(Qnn_BackendHandle_t& qnnBackendHandle,
-                               QnnInterface_ImplementationV2_16_t& qnnInterface,
+                               QNN_INTERFACE_VER_TYPE& qnnInterface,
                                Qnn_ContextHandle_t& qnnContext) -> STATUS {
     const auto status = m_composeGraphsFnHandle(qnnBackendHandle,
                                                 qnnInterface,
@@ -160,9 +160,9 @@ auto GraphsInfo::composeGraphs(Qnn_BackendHandle_t& qnnBackendHandle,
     return STATUS::SUCCESS;
 }
 
-auto GraphsInfo::retrieveGraphFromContext(
-    QnnInterface_ImplementationV2_16_t& qnnInterface,
-    Qnn_ContextHandle_t& qnnContext) -> STATUS {
+auto GraphsInfo::retrieveGraphFromContext(QNN_INTERFACE_VER_TYPE& qnnInterface,
+                                          Qnn_ContextHandle_t& qnnContext)
+    -> STATUS {
     for (size_t graphIdx = 0; graphIdx < m_graphsCount; ++graphIdx) {
         if (nullptr == qnnInterface.graphRetrieve) {
             return STATUS::FAIL;

test/CMakeLists.txt

@@ -36,20 +36,24 @@ endif()
 
 # ---- Tests ----
 
-set(TEST_SOURCES
-    source/tflite_test.cpp source/tflite_from_buffer_test.cpp
-    source/tflite_quantized_test.cpp source/tflite_bad_model_test.cpp
-    source/tflite_delegate_test.cpp
-)
+set(TEST_SOURCES source/bad_model_test.cpp)
 
-if(edgerunner_ENABLE_GPU)
-    list(APPEND TEST_SOURCES source/tflite_gpu_test.cpp)
+if(edgerunner_ENABLE_TFLITE)
+    list(APPEND TEST_SOURCES source/tflite_test.cpp
+         source/tflite_from_buffer_test.cpp source/tflite_delegate_test.cpp
+         source/tflite_quantized_test.cpp
+    )
+    if(edgerunner_ENABLE_GPU)
+        list(APPEND TEST_SOURCES source/tflite_gpu_test.cpp)
+    endif()
+    if(edgerunner_ENABLE_NPU)
+        list(APPEND TEST_SOURCES source/tflite_npu_test.cpp)
+    endif()
 endif()
 
 if(edgerunner_ENABLE_NPU)
-    list(APPEND TEST_SOURCES source/tflite_npu_test.cpp
-         source/qnn_shared_library_npu_test.cpp
-         source/qnn_context_binary_npu_test.cpp
+    list(APPEND TEST_SOURCES source/qnn_shared_library_npu_test.cpp
+         source/qnn_context_binary_npu_test.cpp source/qnn_quantized_test.cpp
     )
 endif()
 

test/source/tflite_bad_model_test.cpp → test/source/bad_model_test.cpp

@@ -11,7 +11,7 @@
 #include "edgerunner/edgerunner.hpp"
 #include "utils.hpp"
 
-TEST_CASE("Tflite bad model", "[tflite][misuse]") {
+TEST_CASE("Bad model", "[model][misuse]") {
     const std::string badPath = "test.bin";
     auto badPathModel = edge::createModel(badPath);
     REQUIRE(badPathModel == nullptr);

test/source/qnn_quantized_test.cpp

@@ -0,0 +1,66 @@
+#include <cstddef>
+#include <cstdint>
+#include <string>
+#include <vector>
+
+#include <catch2/catch_test_macros.hpp>
+
+#include "edgerunner/edgerunner.hpp"
+#include "edgerunner/model.hpp"
+#include "edgerunner/tensor.hpp"
+#include "utils.hpp"
+
+TEST_CASE("QNN runtime quantized (NPU)", "[qnn][npu][quantized]") {
+    const std::string modelPath = "models/qnn/mobilenet_v3_large_quantized.so";
+
+    auto model = edge::createModel(modelPath);
+    REQUIRE(model != nullptr);
+    REQUIRE(std::string {"mobilenet_v3_large_quantized"} == model->name());
+
+    model->applyDelegate(edge::DELEGATE::CPU);
+    REQUIRE(model->getDelegate() == edge::DELEGATE::CPU);
+
+    REQUIRE(model->getPrecision() == edge::TensorType::UINT8);
+
+    const auto inputs = model->getInputs();
+    const auto numInputs = model->getNumInputs();
+    REQUIRE(numInputs == 1);
+    REQUIRE(numInputs == inputs.size());
+
+    const auto outputs = model->getOutputs();
+    const auto numOutputs = model->getNumOutputs();
+    REQUIRE(numOutputs == 1);
+    REQUIRE(numOutputs == outputs.size());
+
+    auto input = model->getInput(0);
+    REQUIRE(input->getName() == "image_tensor");
+    REQUIRE(input->getDimensions() == std::vector<size_t> {1, 224, 224, 3});
+    REQUIRE(input->getType() == edge::TensorType::UINT8);
+    REQUIRE(input.get() == inputs[0].get());
+
+    auto inputData = input->getTensorAs<uint8_t>();
+    REQUIRE(inputData.size() == input->getSize());
+
+    auto badInput = model->getInput(1);
+    REQUIRE(badInput == nullptr);
+
+    auto output = model->getOutput(0);
+    REQUIRE(output->getName() == "_668");
+    REQUIRE(output->getDimensions() == std::vector<size_t> {1, 1000});
+    REQUIRE(output->getType() == edge::TensorType::UINT8);
+    REQUIRE(output.get() == outputs[0].get());
+
+    auto outputBuffer = output->getTensorAs<uint8_t>();
+    REQUIRE(outputBuffer.size() == output->getSize());
+
+    auto badOutput = model->getOutput(1);
+    REQUIRE(badOutput == nullptr);
+
+    const auto executionStatus = model->execute();
+    REQUIRE(executionStatus == edge::STATUS::SUCCESS);
+
+    /* verify output buffer is persistent across execution */
+    const auto newOutputBuffer = model->getOutput(0)->getTensorAs<uint8_t>();
+    REQUIRE(outputBuffer.data() == newOutputBuffer.data());
+    REQUIRE(outputBuffer.size() == newOutputBuffer.size());
+}