Support video understanding

Makefile

@@ -1465,13 +1465,16 @@ llama-llava-cli: examples/llava/llava-cli.cpp \
 	$(OBJ_ALL)
 	$(CXX) $(CXXFLAGS) $< $(filter-out %.h $<,$^) -o $@ $(LDFLAGS) -Wno-cast-qual
 
+FFMPEG_CFLAGS := $(shell pkg-config --cflags libavformat libavcodec libavutil)
+FFMPEG_LIBS := $(shell pkg-config --libs libavformat libavcodec libavutil) -lswscale
+
 llama-minicpmv-cli: examples/llava/minicpmv-cli.cpp \
 	examples/llava/llava.cpp \
 	examples/llava/llava.h \
 	examples/llava/clip.cpp \
 	examples/llava/clip.h \
 	$(OBJ_ALL)
-	$(CXX) $(CXXFLAGS) $< $(filter-out %.h $<,$^) -o $@ $(LDFLAGS) -Wno-cast-qual
+	$(CXX) $(CXXFLAGS) $(FFMPEG_CFLAGS) $< $(filter-out %.h $<,$^) -o $@ $(LDFLAGS) $(FFMPEG_LIBS) -Wno-cast-qual
 
 ifeq ($(UNAME_S),Darwin)
 swift: examples/batched.swift

common/common.cpp

@@ -816,6 +816,19 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
         params.image.emplace_back(argv[i]);
         return true;
     }
+    if (arg == "--video") {
+        if (++i >= argc) {
+            invalid_param = true;
+            return true;
+        }
+        params.video = argv[i];
+        return true;
+    }
+    if (arg == "--frame-num") {
+        CHECK_ARG
+        params.frame_num = std::stoi(argv[i]);
+        return true;
+    }
     if (arg == "-i" || arg == "--interactive") {
         params.interactive = true;
         return true;
@@ -1639,6 +1652,8 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param
     options.push_back({ "multi-modality" });
     options.push_back({ "*",           "       --mmproj FILE",          "path to a multimodal projector file for LLaVA. see examples/llava/README.md" });
     options.push_back({ "*",           "       --image FILE",           "path to an image file. use with multimodal models. Specify multiple times for batching" });
+    options.push_back({ "*",           "       --video FILE",           "path to an video file. use with multimodal models. Specify multiple times for batching" });
+    options.push_back({ "*",           "       --frame-num N",          "number of max video frame (default: 16)" });
 
     options.push_back({ "backend" });
     options.push_back({ "*",           "       --rpc SERVERS",          "comma separated list of RPC servers" });

common/common.h

@@ -193,6 +193,8 @@ struct gpt_params {
     // multimodal models (see examples/llava)
     std::string mmproj = "";        // path to multimodal projector
     std::vector<std::string> image; // path to image file(s)
+    std::string video = ""; 
+    int frame_num = 16;
 
     // embedding
     bool embedding         = false; // get only sentence embedding

examples/llava/clip.cpp

@@ -3,6 +3,7 @@
 // I'll gradually clean and extend it
 // Note: Even when using identical normalized image inputs (see normalize_image_u8_to_f32()) we have a significant difference in resulting embeddings compared to pytorch
 #include "clip.h"
+#include "common.h"
 #include "log.h"
 #include "ggml.h"
 #include "ggml-alloc.h"
@@ -538,6 +539,7 @@ struct clip_ctx {
     bool has_llava_projector = false;
     bool has_minicpmv_projector = false;
     int minicpmv_version = 2;
+    int max_slice_nums = 9;
 
     struct clip_vision_model vision_model;
     projector_type proj_type = PROJECTOR_TYPE_MLP;
@@ -1623,7 +1625,7 @@ static void normalize_image_u8_to_f32(const clip_image_u8* src, clip_image_f32*
     }
 }
 
-inline float clip(float x, float lower, float upper) {
+inline int clip(int x, int lower, int upper) {
     return std::max(lower, std::min(x, upper));
 }
 
@@ -1827,10 +1829,6 @@ static std::pair<int, int> uhd_get_refine_size(std::pair<int, int> original_size
     return refine_size;
 }
 
-inline int clip(int x, int lower, int upper) {
-    return std::max(lower, std::min(x, upper));
-}
-
 static std::pair<int, int> uhd_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) {
     std::vector<int> candidate_split_grids_nums;
     for (int i : {multiple - 1, multiple, multiple + 1}) {
@@ -1932,7 +1930,7 @@ static std::vector<std::vector<clip_image_u8 *>> uhd_slice_image(const clip_imag
 }
 
 int clip_uhd_num_image_embeds_col(struct clip_ctx * ctx_clip) {
-    const int max_slice_nums=9;
+    const int max_slice_nums=ctx_clip->max_slice_nums;
     const int scale_resolution=448;
     const int original_width = ctx_clip->load_image_size->width;
     const int original_height = ctx_clip->load_image_size->height;
@@ -1948,7 +1946,7 @@ int clip_uhd_num_image_embeds_col(struct clip_ctx * ctx_clip) {
 bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, clip_image_f32_batch * res_imgs) {
 
     if(clip_is_minicpmv(ctx)){
-        int max_slice_nums = 9;
+        int max_slice_nums = ctx->max_slice_nums;
         std::vector<std::vector<clip_image_u8 *>> imgs = uhd_slice_image(img, max_slice_nums);
         res_imgs->size = 0;
         for (size_t i = 0; i < imgs.size(); ++i){
@@ -2626,3 +2624,7 @@ int clip_is_minicpmv(const struct clip_ctx * ctx) {
     }
     return 0;
 }
+
+void clip_uhd_max_slice_nums(struct clip_ctx * ctx, int max_slice_nums) {
+    ctx->max_slice_nums = max_slice_nums;
+}

examples/llava/clip.h

@@ -86,6 +86,7 @@ CLIP_API bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, cons
 CLIP_API bool clip_model_quantize(const char * fname_inp, const char * fname_out, int itype);
 
 CLIP_API int clip_is_minicpmv(const struct clip_ctx * ctx);
+CLIP_API void clip_uhd_max_slice_nums(struct clip_ctx * ctx, int max_slice_nums);
 
 #ifdef __cplusplus
 }