add open vocabulary yolo world

examples/ax650/CMakeLists.txt

@@ -39,6 +39,7 @@ axera_example(ax_yolox ax_yolox_steps.cc)
 axera_example(ax_yolov10 ax_yolov10s_steps.cc)
 axera_example(ax_yolo_nas ax_yolo_nas_steps.cc)
 axera_example(ax_yolo_world ax_yolo_world_steps.cc)
+axera_example(ax_yolo_world_open_vocabulary ax_yolo_world_open_vocabulary_steps.cc)
 
 axera_example(ax_ppyoloe ax_ppyoloe_steps.cc)
 axera_example(ax_ppyoloe_obj365 ax_ppyoloe_obj365_steps.cc)

examples/ax650/ax_yolo_world_open_vocabulary_steps.cc

@@ -0,0 +1,266 @@
+/*
+* AXERA is pleased to support the open source community by making ax-samples available.
+*
+* Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved.
+*
+* Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
+* in compliance with the License. You may obtain a copy of the License at
+*
+* https://opensource.org/licenses/BSD-3-Clause
+*
+* Unless required by applicable law or agreed to in writing, software distributed
+* under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+* CONDITIONS OF ANY KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations under the License.
+*/
+
+/*
+* Author: ZHEQIUSHUI
+*/
+
+#include <cstdio>
+#include <cstring>
+#include <numeric>
+
+#include <opencv2/opencv.hpp>
+#include "base/common.hpp"
+#include "base/detection.hpp"
+#include "middleware/io.hpp"
+
+#include "utilities/args.hpp"
+#include "utilities/cmdline.hpp"
+#include "utilities/file.hpp"
+#include "utilities/timer.hpp"
+
+#include <ax_sys_api.h>
+#include <ax_engine_api.h>
+
+const int DEFAULT_IMG_H = 640;
+const int DEFAULT_IMG_W = 640;
+
+const char* CLASS_NAMES[] = {
+    "object"};
+
+int NUM_CLASS = 1;
+
+const int DEFAULT_LOOP_COUNT = 1;
+
+const float PROB_THRESHOLD = 0.25f;
+const float NMS_THRESHOLD = 0.45f;
+namespace ax
+{
+    void post_process(AX_ENGINE_IO_INFO_T* io_info, AX_ENGINE_IO_T* io_data, const cv::Mat& mat, int input_w, int input_h, const std::vector<float>& time_costs)
+    {
+        std::vector<detection::Object> proposals;
+        std::vector<detection::Object> objects;
+        timer timer_postprocess;
+        for (int i = 0; i < 3; ++i)
+        {
+            auto feat_ptr = (float*)io_data->pOutputs[i].pVirAddr;
+            int32_t stride = (1 << i) * 8;
+            detection::generate_proposals_yolov8_native(stride, feat_ptr, PROB_THRESHOLD, proposals, input_w, input_h, NUM_CLASS);
+        }
+
+        detection::get_out_bbox(proposals, objects, NMS_THRESHOLD, input_h, input_w, mat.rows, mat.cols);
+        fprintf(stdout, "post process cost time:%.2f ms \n", timer_postprocess.cost());
+        fprintf(stdout, "--------------------------------------\n");
+        auto total_time = std::accumulate(time_costs.begin(), time_costs.end(), 0.f);
+        auto min_max_time = std::minmax_element(time_costs.begin(), time_costs.end());
+        fprintf(stdout,
+                "Repeat %d times, avg time %.2f ms, max_time %.2f ms, min_time %.2f ms\n",
+                (int)time_costs.size(),
+                total_time / (float)time_costs.size(),
+                *min_max_time.second,
+                *min_max_time.first);
+        fprintf(stdout, "--------------------------------------\n");
+        fprintf(stdout, "detection num: %zu\n", objects.size());
+
+        detection::draw_objects(mat, objects, CLASS_NAMES, "yolo_world_out");
+    }
+
+    bool run_model(const std::string& model, const std::vector<uint8_t>& data, const std::vector<uchar>& text_feature, const int& repeat, cv::Mat& mat, int input_h, int input_w)
+    {
+        // 1. init engine
+#ifdef AXERA_TARGET_CHIP_AX620E
+        auto ret = AX_ENGINE_Init();
+#else
+        AX_ENGINE_NPU_ATTR_T npu_attr;
+        memset(&npu_attr, 0, sizeof(npu_attr));
+        npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_DISABLE;
+        auto ret = AX_ENGINE_Init(&npu_attr);
+#endif
+        if (0 != ret)
+        {
+            return ret;
+        }
+
+        // 2. load model
+        std::vector<char> model_buffer;
+        if (!utilities::read_file(model, model_buffer))
+        {
+            fprintf(stderr, "Read Run-Joint model(%s) file failed.\n", model.c_str());
+            return false;
+        }
+
+        // 3. create handle
+        AX_ENGINE_HANDLE handle;
+        ret = AX_ENGINE_CreateHandle(&handle, model_buffer.data(), model_buffer.size());
+        SAMPLE_AX_ENGINE_DEAL_HANDLE
+        fprintf(stdout, "Engine creating handle is done.\n");
+
+        // 4. create context
+        ret = AX_ENGINE_CreateContext(handle);
+        SAMPLE_AX_ENGINE_DEAL_HANDLE
+        fprintf(stdout, "Engine creating context is done.\n");
+
+        // 5. set io
+        AX_ENGINE_IO_INFO_T* io_info;
+        ret = AX_ENGINE_GetIOInfo(handle, &io_info);
+        SAMPLE_AX_ENGINE_DEAL_HANDLE
+        fprintf(stdout, "Engine get io info is done. \n");
+        middleware::print_io_info(io_info);
+
+        // 6. alloc io
+        AX_ENGINE_IO_T io_data;
+        ret = middleware::prepare_io(io_info, &io_data, std::make_pair(AX_ENGINE_ABST_DEFAULT, AX_ENGINE_ABST_CACHED));
+        SAMPLE_AX_ENGINE_DEAL_HANDLE
+        fprintf(stdout, "Engine alloc io is done. \n");
+
+        // 7. insert input
+        memcpy(io_data.pInputs[0].pVirAddr, data.data(), data.size());
+        memcpy(io_data.pInputs[1].pVirAddr, text_feature.data(), text_feature.size());
+        SAMPLE_AX_ENGINE_DEAL_HANDLE_IO
+        fprintf(stdout, "Engine push input is done. \n");
+        fprintf(stdout, "--------------------------------------\n");
+
+        // 8. warn up
+        for (int i = 0; i < 5; ++i)
+        {
+            AX_ENGINE_RunSync(handle, &io_data);
+        }
+
+        // 9. run model
+        std::vector<float> time_costs(repeat, 0);
+        for (int i = 0; i < repeat; ++i)
+        {
+            timer tick;
+            ret = AX_ENGINE_RunSync(handle, &io_data);
+            time_costs[i] = tick.cost();
+            SAMPLE_AX_ENGINE_DEAL_HANDLE_IO
+        }
+
+        // 10. get result
+        post_process(io_info, &io_data, mat, input_w, input_h, time_costs);
+        fprintf(stdout, "--------------------------------------\n");
+
+        middleware::free_io(&io_data);
+        return AX_ENGINE_DestroyHandle(handle);
+    }
+} // namespace ax
+
+int main(int argc, char* argv[])
+{
+    cmdline::parser cmd;
+    cmd.add<std::string>("model", 'm', "joint file(a.k.a. joint model)", true, "");
+    cmd.add<std::string>("image", 'i', "image file", true, "");
+    cmd.add<std::string>("text_feature", 't', "text feature file", true, "");
+    cmd.add<std::string>("size", 'g', "input_h, input_w", false, std::to_string(DEFAULT_IMG_H) + "," + std::to_string(DEFAULT_IMG_W));
+
+    cmd.add<int>("repeat", 'r', "repeat count", false, DEFAULT_LOOP_COUNT);
+    cmd.parse_check(argc, argv);
+
+    // 0. get app args, can be removed from user's app
+    auto model_file = cmd.get<std::string>("model");
+    auto image_file = cmd.get<std::string>("image");
+    auto text_feature_file = cmd.get<std::string>("text_feature");
+
+    auto model_file_flag = utilities::file_exist(model_file);
+    auto image_file_flag = utilities::file_exist(image_file);
+    auto text_feature_file_flag = utilities::file_exist(text_feature_file);
+
+    if (!model_file_flag | !image_file_flag | !text_feature_file_flag)
+    {
+        auto show_error = [](const std::string& kind, const std::string& value) {
+            fprintf(stderr, "Input file %s(%s) is not exist, please check it.\n", kind.c_str(), value.c_str());
+        };
+
+        if (!model_file_flag) { show_error("model", model_file); }
+        if (!image_file_flag) { show_error("image", image_file); }
+        if (!text_feature_file_flag) { show_error("text_feature", text_feature_file); }
+
+        return -1;
+    }
+
+    auto input_size_string = cmd.get<std::string>("size");
+
+    std::array<int, 2> input_size = {DEFAULT_IMG_H, DEFAULT_IMG_W};
+
+    auto input_size_flag = utilities::parse_string(input_size_string, input_size);
+
+    if (!input_size_flag)
+    {
+        auto show_error = [](const std::string& kind, const std::string& value) {
+            fprintf(stderr, "Input %s(%s) is not allowed, please check it.\n", kind.c_str(), value.c_str());
+        };
+
+        show_error("size", input_size_string);
+
+        return -1;
+    }
+
+    auto repeat = cmd.get<int>("repeat");
+
+    // 1. print args
+    fprintf(stdout, "--------------------------------------\n");
+    fprintf(stdout, "model file : %s\n", model_file.c_str());
+    fprintf(stdout, "image file : %s\n", image_file.c_str());
+    fprintf(stdout, "img_h, img_w : %d %d\n", input_size[0], input_size[1]);
+    fprintf(stdout, "--------------------------------------\n");
+
+    // 2. read image & resize & transpose
+    std::vector<uint8_t> image(input_size[0] * input_size[1] * 3, 0);
+    cv::Mat mat = cv::imread(image_file);
+    if (mat.empty())
+    {
+        fprintf(stderr, "Read image failed.\n");
+        return -1;
+    }
+    common::get_input_data_letterbox(mat, image, input_size[0], input_size[1], true);
+
+    std::vector<uchar> text_feature;
+    common::read_file(text_feature_file.c_str(), text_feature);
+
+    std::string filename; //filename of text_feature_file
+    if (text_feature_file.find_last_of("/") != std::string::npos)
+    {
+        filename = text_feature_file.substr(text_feature_file.find_last_of("/") + 1);
+    }
+    else
+    {
+        filename = text_feature_file;
+    }
+
+    if (filename.find_last_of(".") != std::string::npos)
+    {
+        filename = filename.substr(0, filename.find_last_of("."));
+    }
+
+    CLASS_NAMES[0] = filename.c_str();
+
+    // 3. sys_init
+    AX_SYS_Init();
+
+    // 4. -  engine model  -  can only use AX_ENGINE** inside
+    {
+        // AX_ENGINE_NPUReset(); // todo ??
+        ax::run_model(model_file, image, text_feature, repeat, mat, input_size[0], input_size[1]);
+
+        // 4.3 engine de init
+        AX_ENGINE_Deinit();
+        // AX_ENGINE_NPUReset();
+    }
+    // 4. -  engine model  -
+
+    AX_SYS_Deinit();
+    return 0;
+}