Repository: li-plus/chatglm.cpp (language: python)

Issue:
请问是否支持了GLM-4V？
请问是否支持了GLM-4V？
支持glm4v多模态模型
你好，我们试用下来GLM4V-9B模型效果不错，请问本项目能支持该模型的部署吗？


Discussion / hints:
同问，后续会有支持计划吗 。: )
目前还不支持，可以有
给力，太好了，希望能尽快支持！
> 目前还不支持，可以有

慢慢來，之前肝得辛苦了 :-)
感谢你们努力💪
大神加油，glm-4-9b-chat Loading checkpoint 80%停止了
@li-plus 您好，4V的支持在进行中了吗？



Fix patch:
diff --git a/.gitmodules b/.gitmodules
index 7ac75854..9033490c 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -14,3 +14,6 @@
 [submodule "third_party/abseil-cpp"]
 	path = third_party/abseil-cpp
 	url = https://github.com/abseil/abseil-cpp.git
+[submodule "third_party/stb"]
+	path = third_party/stb
+	url = https://github.com/nothings/stb.git
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3a9fe5ed..18b547bb 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -58,6 +58,8 @@ add_subdirectory(third_party/abseil-cpp)
 
 add_subdirectory(third_party/re2)
 
+include_directories(third_party/stb)
+
 if (GGML_METAL)
     add_compile_definitions(GGML_USE_METAL)
     configure_file(third_party/ggml/src/ggml-metal.metal ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/ggml-metal.metal COPYONLY)
@@ -135,7 +137,7 @@ add_custom_target(check-all
     COMMAND cmake --build build -j
     COMMAND ./build/bin/chatglm_test
     COMMAND python3 setup.py develop
-    COMMAND python3 -m pytest tests/test_chatglm_cpp.py
+    COMMAND python3 -m pytest --forked tests/test_chatglm_cpp.py
     WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
 )
 
diff --git a/MANIFEST.in b/MANIFEST.in
index feae2cbd..968f3a64 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -20,3 +20,7 @@ graft third_party/re2
 
 # absl
 graft third_party/abseil-cpp
+
+# stb
+include third_party/stb/stb_image.h
+include third_party/stb/stb_image_resize2.h
diff --git a/README.md b/README.md
index d65c70fa..80e2f28d 100644
--- a/README.md
+++ b/README.md
@@ -6,7 +6,7 @@
 ![Python](https://img.shields.io/pypi/pyversions/chatglm-cpp)
 [![License: MIT](https://img.shields.io/badge/license-MIT-blue)](LICENSE)
 
-C++ implementation of [ChatGLM-6B](https://github.com/THUDM/ChatGLM-6B), [ChatGLM2-6B](https://github.com/THUDM/ChatGLM2-6B), [ChatGLM3](https://github.com/THUDM/ChatGLM3) and [GLM-4](https://github.com/THUDM/GLM-4) for real-time chatting on your MacBook.
+C++ implementation of [ChatGLM-6B](https://github.com/THUDM/ChatGLM-6B), [ChatGLM2-6B](https://github.com/THUDM/ChatGLM2-6B), [ChatGLM3](https://github.com/THUDM/ChatGLM3) and [GLM-4](https://github.com/THUDM/GLM-4)(V) for real-time chatting on your MacBook.
 
 ![demo](docs/demo.gif)
 
@@ -22,7 +22,7 @@ Highlights:
 Support Matrix:
 * Hardwares: x86/arm CPU, NVIDIA GPU, Apple Silicon GPU
 * Platforms: Linux, MacOS, Windows
-* Models: [ChatGLM-6B](https://github.com/THUDM/ChatGLM-6B), [ChatGLM2-6B](https://github.com/THUDM/ChatGLM2-6B), [ChatGLM3](https://github.com/THUDM/ChatGLM3), [GLM-4](https://github.com/THUDM/GLM-4), [CodeGeeX2](https://github.com/THUDM/CodeGeeX2)
+* Models: [ChatGLM-6B](https://github.com/THUDM/ChatGLM-6B), [ChatGLM2-6B](https://github.com/THUDM/ChatGLM2-6B), [ChatGLM3](https://github.com/THUDM/ChatGLM3), [GLM-4](https://github.com/THUDM/GLM-4)(V), [CodeGeeX2](https://github.com/THUDM/CodeGeeX2)
 
 ## Getting Started
 
@@ -53,9 +53,9 @@ python3 chatglm_cpp/convert.py -i THUDM/chatglm-6b -t q4_0 -o models/chatglm-ggm
 
 The original model (`-i <model_name_or_path>`) can be a Hugging Face model name or a local path to your pre-downloaded model. Currently supported models are:
 * ChatGLM-6B: `THUDM/chatglm-6b`, `THUDM/chatglm-6b-int8`, `THUDM/chatglm-6b-int4`
-* ChatGLM2-6B: `THUDM/chatglm2-6b`, `THUDM/chatglm2-6b-int4`
-* ChatGLM3-6B: `THUDM/chatglm3-6b`
-* ChatGLM4-9B: `THUDM/glm-4-9b-chat`
+* ChatGLM2-6B: `THUDM/chatglm2-6b`, `THUDM/chatglm2-6b-int4`, `THUDM/chatglm2-6b-32k`, `THUDM/chatglm2-6b-32k-int4`
+* ChatGLM3-6B: `THUDM/chatglm3-6b`, `THUDM/chatglm3-6b-32k`, `THUDM/chatglm3-6b-128k`, `THUDM/chatglm3-6b-base`
+* ChatGLM4(V)-9B: `THUDM/glm-4-9b-chat`, `THUDM/glm-4-9b-chat-1m`, `THUDM/glm-4-9b`, `THUDM/glm-4v-9b`
 * CodeGeeX2: `THUDM/codegeex2-6b`, `THUDM/codegeex2-6b-int4`
 
 You are free to try any of the below quantization types by specifying `-t <type>`:
@@ -188,6 +188,22 @@ python3 chatglm_cpp/convert.py -i THUDM/glm-4-9b-chat -t q4_0 -o models/chatglm4
 
 </details>
 
+<details open>
+<summary>ChatGLM4V-9B</summary>
+
+[![03-Confusing-Pictures](examples/03-Confusing-Pictures.jpg)](https://www.barnorama.com/wp-content/uploads/2016/12/03-Confusing-Pictures.jpg)
+
+You may use `-vt <vision_type>` to set quantization type for the vision encoder. It is recommended to run GLM4V on GPU since vision encoding runs too slow on CPU even with 4-bit quantization.
+```sh
+python3 chatglm_cpp/convert.py -i THUDM/glm-4v-9b -t q4_0 -vt q4_0 -o models/chatglm4v-ggml.bin
+./build/bin/main -m models/chatglm4v-ggml.bin --image examples/03-Confusing-Pictures.jpg -p "这张图片有什么不寻常之处" --temp 0
+# 这张图片中不寻常的是，一个男人站在一辆黄色SUV的后备箱上，正在使用一个铁板熨烫衣物。
+# 通常情况下，熨衣是在室内进行的，使用的是家用电熨斗，而不是在户外使用汽车后备箱作为工作台。
+# 此外，他似乎是在一个繁忙的城市街道上，周围有行驶的车辆和建筑物，这增加了场景的荒谬性。
+```
+
+</details>
+
 <details>
 <summary>CodeGeeX2</summary>
 
@@ -361,6 +377,15 @@ python3 cli_demo.py -m ../models/chatglm4-ggml.bin -p 你好 --temp 0.8 --top_p
 ```
 </details>
 
+<details open>
+<summary>ChatGLM4V-9B</summary>
+
+Chat mode:
+```sh
+python3 cli_demo.py -m ../models/chatglm4v-ggml.bin --image 03-Confusing-Pictures.jpg -p "这张图片有什么不寻常之处" --temp 0
+```
+</details>
+
 <details>
 <summary>CodeGeeX2</summary>
 
@@ -450,12 +475,22 @@ Use the OpenAI client to chat with your model:
 
 For stream response, check out the example client script:
 ```sh
-OPENAI_BASE_URL=http://127.0.0.1:8000/v1 python3 examples/openai_client.py --stream --prompt 你好
+python3 examples/openai_client.py --base_url http://127.0.0.1:8000/v1 --stream --prompt 你好
 ```
 
 Tool calling is also supported:
 ```sh
-OPENAI_BASE_URL=http://127.0.0.1:8000/v1 python3 examples/openai_client.py --tool_call --prompt 上海天气怎么样
+python3 examples/openai_client.py --base_url http://127.0.0.1:8000/v1 --tool_call --prompt 上海天气怎么样
+```
+
+Request GLM4V with image inputs:
+```sh
+# request with local image file
+python3 examples/openai_client.py --base_url http://127.0.0.1:8000/v1 --prompt "描述这张图片" \
+    --image examples/03-Confusing-Pictures.jpg --temp 0
+# request with image url
+python3 examples/openai_client.py --base_url http://127.0.0.1:8000/v1 --prompt "描述这张图片" \
+    --image https://www.barnorama.com/wp-content/uploads/2016/12/03-Confusing-Pictures.jpg --temp 0
 ```
 
 With this API server as backend, ChatGLM.cpp models can be seamlessly integrated into any frontend that uses OpenAI-style API, including [mckaywrigley/chatbot-ui](https://github.com/mckaywrigley/chatbot-ui), [fuergaosi233/wechat-chatgpt](https://github.com/fuergaosi233/wechat-chatgpt), [Yidadaa/ChatGPT-Next-Web](https://github.com/Yidadaa/ChatGPT-Next-Web), and more.
diff --git a/chatglm.cpp b/chatglm.cpp
index a7fc3369..c9f943bb 100644
--- a/chatglm.cpp
+++ b/chatglm.cpp
@@ -17,6 +17,11 @@
 #include <sys/stat.h>
 #include <thread>
 
+#define STB_IMAGE_IMPLEMENTATION
+#define STB_IMAGE_RESIZE2_IMPLEMENTATION
+#include <stb_image.h>
+#include <stb_image_resize2.h>
+
 #ifdef __has_include
 #if __has_include(<unistd.h>)
 #include <unistd.h>
@@ -390,11 +395,27 @@ StateDict ModelLoader::read_state_dict() {
     return sd;
 }
 
-ModelContext::ModelContext(ggml_type dtype)
-    : dtype(dtype), compute_meta(ggml_tensor_overhead() * GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead()),
+Image Image::open(const std::string &path) {
+    int width, height, channels;
+    uint8_t *buffer = stbi_load(path.c_str(), &width, &height, &channels, 3);
+    CHATGLM_CHECK(channels == 3);
+    Image image(width, height, channels, buffer);
+    stbi_image_free(buffer);
+    return image;
+}
+
+Image Image::resize(size_t new_width, size_t new_height) const {
+    Image output(new_width, new_height, channels);
+    stbir_resize_uint8_srgb(pixels.data(), width, height, 0, output.pixels.data(), new_width, new_height, 0, STBIR_RGB);
+    return output;
+}
+
+ModelContext::ModelContext()
+    : compute_meta(ggml_tensor_overhead() * (GGML_DEFAULT_GRAPH_SIZE * 4) + ggml_graph_overhead()),
       ctx_w(make_unique_ggml_context(ggml_tensor_overhead() * GGML_DEFAULT_GRAPH_SIZE, nullptr, true)),
       ctx_kv(make_unique_ggml_context(ggml_tensor_overhead() * GGML_DEFAULT_GRAPH_SIZE, nullptr, true)),
-      ctx_b(make_unique_ggml_context(compute_meta.size(), compute_meta.data(), true)), gf(ggml_new_graph(ctx_b.get())) {
+      ctx_b(make_unique_ggml_context(compute_meta.size(), compute_meta.data(), true)),
+      gf(ggml_new_graph_custom(ctx_b.get(), GGML_DEFAULT_GRAPH_SIZE * 4, false)) {
 
 #if defined(GGML_USE_CUDA)
     backend = unique_ggml_backend_t(ggml_backend_cuda_init(0));
@@ -428,8 +449,7 @@ ggml_tensor *Linear::forward(ModelContext *mctx, ggml_tensor *input) const {
 ggml_tensor *LayerNorm::forward(ModelContext *mctx, ggml_tensor *input) const {
     // input: [seqlen, normalized_shape]
     ggml_context *ctx = mctx->ctx_b.get();
-    auto ggml_norm_fn = inplace ? ggml_norm_inplace : ggml_norm;
-    ggml_tensor *output = ggml_norm_fn(ctx, input, eps);
+    ggml_tensor *output = ggml_norm(ctx, input, eps);
     output = ggml_mul_inplace(ctx, output, weight);
     output = ggml_add_inplace(ctx, output, bias);
     return output;
@@ -437,8 +457,7 @@ ggml_tensor *LayerNorm::forward(ModelContext *mctx, ggml_tensor *input) const {
 
 ggml_tensor *RMSNorm::forward(ModelContext *mctx, ggml_tensor *input) const {
     ggml_context *ctx = mctx->ctx_b.get();
-    auto ggml_rms_norm_fn = inplace ? ggml_rms_norm_inplace : ggml_rms_norm;
-    ggml_tensor *output = ggml_rms_norm_fn(ctx, input, eps);
+    ggml_tensor *output = ggml_rms_norm(ctx, input, eps);
     output = ggml_mul_inplace(ctx, output, weight);
     return output;
 }
@@ -506,18 +525,12 @@ static void set_default_num_threads(ggml_backend_t backend, int num_tokens) {
 }
 
 std::string to_string(ModelType model_type) {
-    switch (model_type) {
-    case ModelType::CHATGLM:
-        return "ChatGLM";
-    case ModelType::CHATGLM2:
-        return "ChatGLM2";
-    case ModelType::CHATGLM3:
-        return "ChatGLM3";
-    case ModelType::CHATGLM4:
-        return "ChatGLM4";
-    default:
-        CHATGLM_THROW << "unknown model type " << (int)model_type;
-    }
+    static const std::unordered_map<ModelType, std::string> m{{ModelType::CHATGLM, "ChatGLM"},
+                                                              {ModelType::CHATGLM2, "ChatGLM2"},
+                                                              {ModelType::CHATGLM3, "ChatGLM3"},
+                                                              {ModelType::CHATGLM4, "ChatGLM4"},
+                                                              {ModelType::CHATGLM4V, "ChatGLM4V"}};
+    return m.at(model_type);
 }
 
 static ggml_tensor *apply_rotary_emb_basic(ModelContext *mctx, ggml_tensor *layer, ggml_tensor *position_ids,
@@ -664,23 +677,28 @@ ggml_tensor *BasicAttention::forward(ModelContext *mctx, ggml_tensor *hidden_sta
     key_layer = ggml_permute(ctx, key_layer, 0, 2, 1, 3);     // [#kvh, s, d]
     value_layer = ggml_permute(ctx, value_layer, 1, 2, 0, 3); // [#kvh, d, s]
 
-    // store key & value to cache
-    ggml_tensor *k_cache_view =
-        ggml_view_3d(ctx, k_cache, head_size, qlen, num_key_value_heads, k_cache->nb[1], k_cache->nb[2],
-                     (num_virtual_tokens + n_past) * head_size * ggml_element_size(k_cache)); // [#kvh, s, d]
-    ggml_build_forward_expand(mctx->gf, ggml_cpy(ctx, key_layer, k_cache_view));
-    ggml_tensor *v_cache_view =
-        ggml_view_3d(ctx, v_cache, qlen, head_size, num_key_value_heads, v_cache->nb[1], v_cache->nb[2],
-                     (num_virtual_tokens + n_past) * ggml_element_size(v_cache)); // [#kvh, d, s]
-    ggml_build_forward_expand(mctx->gf, ggml_cpy(ctx, value_layer, v_cache_view));
-
-    // concat key & value with past kv
-    key_layer = ggml_view_3d(ctx, k_cache, head_size, num_virtual_tokens + n_past + qlen, num_key_value_heads,
-                             k_cache->nb[1], k_cache->nb[2],
-                             0); // [#kvh, kvs, d]
-    value_layer = ggml_view_3d(ctx, v_cache, num_virtual_tokens + n_past + qlen, head_size, num_key_value_heads,
-                               v_cache->nb[1], v_cache->nb[2],
-                               0); // [#kvh, d, kvs]
+    if (k_cache && v_cache) {
+        // store key & value to cache
+        ggml_tensor *k_cache_view =
+            ggml_view_3d(ctx, k_cache, head_size, qlen, num_key_value_heads, k_cache->nb[1], k_cache->nb[2],
+                         (num_virtual_tokens + n_past) * k_cache->nb[1]); // [#kvh, s, d]
+        ggml_tensor *v_cache_view =
+            ggml_view_3d(ctx, v_cache, qlen, head_size, num_key_value_heads, v_cache->nb[1], v_cache->nb[2],
+                         (num_virtual_tokens + n_past) * v_cache->nb[0]); // [#kvh, d, s]
+        ggml_build_forward_expand(mctx->gf, ggml_cpy(ctx, key_layer, k_cache_view));
+        ggml_build_forward_expand(mctx->gf, ggml_cpy(ctx, value_layer, v_cache_view));
+
+        // concat key & value with past kv
+        key_layer = ggml_view_3d(ctx, k_cache, head_size, num_virtual_tokens + n_past + qlen, num_key_value_heads,
+                                 k_cache->nb[1], k_cache->nb[2],
+                                 0); // [#kvh, kvs, d]
+        value_layer = ggml_view_3d(ctx, v_cache, num_virtual_tokens + n_past + qlen, head_size, num_key_value_heads,
+                                   v_cache->nb[1], v_cache->nb[2],
+                                   0); // [#kvh, d, kvs]
+    } else {
+        key_layer = ggml_cont(ctx, key_layer);
+        value_layer = ggml_cont(ctx, value_layer);
+    }
 
     // attention
     query_layer = ggml_scale_inplace(ctx, query_layer, 1.f / std::sqrt(head_size));
@@ -693,7 +711,9 @@ ggml_tensor *BasicAttention::forward(ModelContext *mctx, ggml_tensor *hidden_sta
                                           num_attention_heads); // [#h, s, kvs]
         }
 
-        if (attn_mask_type == AttentionMaskType::CAUSAL) {
+        if (attn_mask_type == AttentionMaskType::BIDIRECTIONAL) {
+            // pass
+        } else if (attn_mask_type == AttentionMaskType::CAUSAL) {
             attn_scores = ggml_diag_mask_inf_inplace(ctx, attn_scores, num_virtual_tokens + n_past);
         } else {
             attn_scores = ggml_add_inplace(ctx, attn_scores, attention_mask);
@@ -721,28 +741,40 @@ ggml_tensor *BasicAttention::forward(ModelContext *mctx, ggml_tensor *hidden_sta
 }
 
 BaseModelForCausalLM::BaseModelForCausalLM(ModelConfig config)
-    : config(config), mctx_(std::make_unique<ModelContext>(config.dtype)) {}
+    : config(config), mctx_(std::make_unique<ModelContext>()) {}
 
-ggml_tensor *BaseModelForCausalLM::forward_graph_compute(const std::vector<int> &input_ids, int n_past, int n_ctx,
+ggml_tensor *BaseModelForCausalLM::forward_graph_compute(const std::vector<int> &input_ids,
+                                                         const std::optional<Image> &image, int n_past, int n_ctx,
                                                          bool is_decoding) {
     mctx_->ctx_b = make_unique_ggml_context(mctx_->compute_meta.size(), mctx_->compute_meta.data(), true);
-    mctx_->gf = ggml_new_graph(mctx_->ctx_b.get());
+    mctx_->gf = ggml_new_graph_custom(mctx_->ctx_b.get(), GGML_DEFAULT_GRAPH_SIZE * 4, false);
 
-    const int qlen = input_ids.size() - n_past;
+    const int qlen = (n_past == 0) ? input_ids.size() : 1;
 
     ggml_tensor *curr_input_ids = ggml_new_tensor_1d(mctx_->ctx_b.get(), GGML_TYPE_I32, qlen);
     ggml_set_name(curr_input_ids, "input_ids");
     ggml_set_input(curr_input_ids);
 
-    ggml_tensor *lm_logits = forward(mctx_.get(), curr_input_ids, n_past, n_ctx, is_decoding);
+    ggml_tensor *curr_image = nullptr;
+    if (n_past == 0 && image) {
+        curr_image = ggml_new_tensor_3d(mctx_->ctx_b.get(), GGML_TYPE_F32, config.vision.image_size,
+                                        config.vision.image_size, config.vision.in_channels);
+        ggml_set_name(curr_image, "image");
+        ggml_set_input(curr_image);
+    }
+
+    ggml_tensor *lm_logits = forward(mctx_.get(), curr_input_ids, curr_image, input_ids, n_past, is_decoding);
     ggml_set_output(lm_logits);
 
     ggml_build_forward_expand(mctx_->gf, lm_logits);
     CHATGLM_CHECK(ggml_gallocr_alloc_graph(mctx_->allocr.get(), mctx_->gf));
 
-    ggml_backend_tensor_set(curr_input_ids, input_ids.data() + n_past, 0, qlen * sizeof(int));
-
-    set_graph_inputs(qlen, n_past, n_ctx);
+    // TODO: move into set_graph_inputs
+    curr_input_ids = ggml_graph_get_tensor(mctx_->gf, "input_ids");
+    if (curr_input_ids) {
+        ggml_backend_tensor_set(curr_input_ids, input_ids.data() + input_ids.size() - qlen, 0, qlen * sizeof(int));
+    }
+    set_graph_inputs(input_ids, image, n_past, n_ctx);
 
     set_default_num_threads(mctx_->backend.get(), qlen);
     CHATGLM_CHECK(ggml_backend_graph_compute(mctx_->backend.get(), mctx_->gf) == GGML_STATUS_SUCCESS);
@@ -754,9 +786,9 @@ ggml_tensor *BaseModelForCausalLM::forward_graph_compute(const std::vector<int>
     return lm_logits;
 }
 
-int BaseModelForCausalLM::generate_next_token(const std::vector<int> &input_ids, const GenerationConfig &gen_config,
-                                              int n_past, int n_ctx) {
-    ggml_tensor *lm_logits = forward_graph_compute(input_ids, n_past, n_ctx, true);
+int BaseModelForCausalLM::generate_next_token(const std::vector<int> &input_ids, const std::optional<Image> &image,
+                                              const GenerationConfig &gen_config, int n_past, int n_ctx) {
+    ggml_tensor *lm_logits = forward_graph_compute(input_ids, image, n_past, n_ctx, true);
     CHATGLM_CHECK(ggml_n_dims(lm_logits) == 1);
 
     int vocab_size = lm_logits->ne[0];
@@ -883,8 +915,8 @@ void BaseModelForCausalLM::sampling_softmax_inplace(TokenIdScore *first, TokenId
     }
 }
 
-std::vector<int> BaseModelForCausalLM::generate(const std::vector<int> &input_ids, const GenerationConfig &gen_config,
-                                                BaseStreamer *streamer) {
+std::vector<int> BaseModelForCausalLM::generate(const std::vector<int> &input_ids, const std::optional<Image> &image,
+                                                const GenerationConfig &gen_config, BaseStreamer *streamer) {
     CHATGLM_CHECK(gen_config.max_length <= config.max_length)
         << "Requested max_length (" << gen_config.max_length << ") exceeds pre-configured model max_length ("
         << config.max_length << ")";
@@ -901,9 +933,9 @@ std::vector<int> BaseModelForCausalLM::generate(const std::vector<int> &input_id
     const int max_new_tokens = (gen_config.max_new_tokens > 0) ? gen_config.max_new_tokens : gen_config.max_length;
 
     while ((int)output_ids.size() < std::min(gen_config.max_length, n_ctx + max_new_tokens)) {
-        int next_token_id = generate_next_token(output_ids, gen_config, n_past, n_ctx);
+        int next_token_id = generate_next_token(output_ids, image, gen_config, n_past, n_ctx);
 
-        n_past = output_ids.size();
+        n_past = count_tokens(output_ids, image);
         output_ids.emplace_back(next_token_id);
 
         if (streamer) {
@@ -1086,25 +1118,8 @@ ggml_tensor *GLMBlock::forward(ModelContext *mctx, ggml_tensor *hidden_states, g
     return output;
 }
 
-ChatGLMForCausalLM::ChatGLMForCausalLM(const ModelConfig &config) : BasicModelForCausalLM(config) {}
-
 void ChatGLMForCausalLM::load_state_dict(const StateDict &sd) {
-    void *sd_buf_base = ggml_backend_buffer_get_base(sd.buf.get());
-    const size_t sd_buf_size = ggml_backend_buffer_get_size(sd.buf.get());
-    if (ggml_backend_is_cpu(mctx_->backend.get())) {
-        mctx_->buf_w = unique_ggml_backend_buffer_t(ggml_backend_cpu_buffer_from_ptr(sd_buf_base, sd_buf_size));
-    }
-#ifdef GGML_USE_METAL
-    else if (ggml_backend_is_metal(mctx_->backend.get())) {
-        const size_t max_size = ggml_get_max_tensor_size(mctx_->ctx_w.get());
-        mctx_->buf_w =
-            unique_ggml_backend_buffer_t(ggml_backend_metal_buffer_from_ptr(sd_buf_base, sd_buf_size, max_size));
-    }
-#endif
-    else {
-        mctx_->buf_w =
-            unique_ggml_backend_buffer_t(ggml_backend_alloc_ctx_tensors(mctx_->ctx_w.get(), mctx_->backend.get()));
-    }
+    alloc_weight_context(sd.buf.get());
 
     StateDict self_sd = state_dict();
     for (auto &item : self_sd.kv) {
@@ -1120,13 +1135,11 @@ void ChatGLMForCausalLM::load_state_dict(const StateDict &sd) {
     }
 }
 
-void ChatGLMForCausalLM::set_graph_inputs(int qlen, int n_past, int n_ctx) const {
-    set_graph_inputs(mctx_->gf, qlen, n_past, n_ctx);
-}
-
-void ChatGLMForCausalLM::set_graph_inputs(ggml_cgraph *gf, int qlen, int n_past, int n_ctx) {
+void ChatGLMModel::set_graph_inputs(ggml_cgraph *gf, const std::vector<int> &input_ids,
+                                    const std::optional<Image> &image, int n_past, int n_ctx) const {
     // attention_mask: [s, kvs] auto broadcast to [#h, s, kvs]
     // semantic: attn_scores[:, :-1, -1] = -inf
+    const int qlen = input_ids.size() - n_past;
     if (n_past == 0) {
         ggml_tensor *attention_mask = ggml_graph_get_tensor(gf, "attention_mask");
         const int kvlen = attention_mask->ne[0];
@@ -1245,25 +1258,8 @@ bool ChatGLM2Tokenizer::is_special_id(int id) const {
            id == eop_token_id;
 }
 
-ChatGLM2ForCausalLM::ChatGLM2ForCausalLM(const ModelConfig &config) : BasicModelForCausalLM(config) {}
-
 void ChatGLM2ForCausalLM::load_state_dict(const StateDict &sd) {
-    void *sd_buf_base = ggml_backend_buffer_get_base(sd.buf.get());
-    const size_t sd_buf_size = ggml_backend_buffer_get_size(sd.buf.get());
-    if (ggml_backend_is_cpu(mctx_->backend.get())) {
-        mctx_->buf_w = unique_ggml_backend_buffer_t(ggml_backend_cpu_buffer_from_ptr(sd_buf_base, sd_buf_size));
-    }
-#ifdef GGML_USE_METAL
-    else if (ggml_backend_is_metal(mctx_->backend.get())) {
-        const size_t max_size = ggml_get_max_tensor_size(mctx_->ctx_w.get());
-        mctx_->buf_w =
-            unique_ggml_backend_buffer_t(ggml_backend_metal_buffer_from_ptr(sd_buf_base, sd_buf_size, max_size));
-    }
-#endif
-    else {
-        mctx_->buf_w =
-            unique_ggml_backend_buffer_t(ggml_backend_alloc_ctx_tensors(mctx_->ctx_w.get(), mctx_->backend.get()));
-    }
+    alloc_weight_context(sd.buf.get());
 
     if (config.num_virtual_tokens > 0) {
         ggml_tensor *past_key_values = sd.kv.at("past_key_values");
@@ -1271,7 +1267,11 @@ void ChatGLM2ForCausalLM::load_state_dict(const StateDict &sd) {
     }
 
     auto self_sd = state_dict();
-    for (auto it = sd.kv.begin(); it != sd.kv.end(); it++) {
+    load_state_dict(mctx_.get(), self_sd, sd);
+}
+
+void ChatGLM2ForCausalLM::load_state_dict(ModelContext *mctx, StateDict &dst, const StateDict &src) {
+    for (auto it = src.kv.begin(); it != src.kv.end(); it++) {
         const std::string &name = it->first;
         ggml_tensor *ckpt_weight = it->second;
 
@@ -1283,16 +1283,16 @@ void ChatGLM2ForCausalLM::load_state_dict(const StateDict &sd) {
         if (pos != std::string::npos) {
             // split dense_h_to_4h to gate & up
             std::string gate_name = name.substr(0, pos) + "mlp.gate_proj.weight";
-            ggml_tensor *gate_proj = self_sd.kv.at(gate_name);
+            ggml_tensor *gate_proj = dst.kv.at(gate_name);
 
             std::string up_name = name.substr(0, pos) + "mlp.up_proj.weight";
-            ggml_tensor *up_proj = self_sd.kv.at(up_name);
+            ggml_tensor *up_proj = dst.kv.at(up_name);
 
-            CHATGLM_CHECK(ggml_nbytes(ckpt_weight) == ggml_nbytes(gate_proj) + ggml_nbytes(up_proj));
+            CHATGLM_CHECK(ggml_nbytes(ckpt_weight) == ggml_nbytes(gate_proj) + ggml_nbytes(up_proj)) << name;
 
-            if (ggml_backend_is_cpu(mctx_->backend.get()) || ggml_cpu_has_metal()) {
-                ggml_backend_tensor_alloc(mctx_->buf_w.get(), gate_proj, ckpt_weight->data);
-                ggml_backend_tensor_alloc(mctx_->buf_w.get(), up_proj,
+            if (ggml_backend_is_cpu(mctx->backend.get()) || ggml_cpu_has_metal()) {
+                ggml_backend_tensor_alloc(mctx->buf_w.get(), gate_proj, ckpt_weight->data);
+                ggml_backend_tensor_alloc(mctx->buf_w.get(), up_proj,
                                           (char *)ckpt_weight->data + ggml_nbytes(gate_proj));
             } else {
                 ggml_backend_tensor_set(gate_proj, ckpt_weight->data, 0, ggml_nbytes(gate_proj));
@@ -1301,10 +1301,10 @@ void ChatGLM2ForCausalLM::load_state_dict(const StateDict &sd) {
             }
         } else {
             // normal weight
-            ggml_tensor *self_weight = self_sd.kv.at(name);
-            CHATGLM_CHECK(ggml_nbytes(self_weight) == ggml_nbytes(ckpt_weight));
-            if (ggml_backend_is_cpu(mctx_->backend.get()) || ggml_cpu_has_metal()) {
-                ggml_backend_tensor_alloc(mctx_->buf_w.get(), self_weight, ckpt_weight->data);
+            ggml_tensor *self_weight = dst.kv.at(name);
+            CHATGLM_CHECK(ggml_nbytes(self_weight) == ggml_nbytes(ckpt_weight)) << name;
+            if (ggml_backend_is_cpu(mctx->backend.get()) || ggml_cpu_has_metal()) {
+                ggml_backend_tensor_alloc(mctx->buf_w.get(), self_weight, ckpt_weight->data);
             } else {
                 ggml_backend_tensor_set(self_weight, ckpt_weight->data, 0, ggml_nbytes(self_weight));
             }
@@ -1312,12 +1312,10 @@ void ChatGLM2ForCausalLM::load_state_dict(const StateDict &sd) {
     }
 }
 
-void ChatGLM2ForCausalLM::set_graph_inputs(int qlen, int n_past, int n_ctx) const {
-    set_graph_inputs(mctx_->gf, qlen, n_past, n_ctx);
-}
-
-void ChatGLM2ForCausalLM::set_graph_inputs(ggml_cgraph *gf, int qlen, int n_past, int n_ctx) {
+void ChatGLM2Model::set_graph_inputs(ggml_cgraph *gf, const std::vector<int> &input_ids,
+                                     const std::optional<Image> &image, int n_past, int n_ctx) const {
     ggml_tensor *position_ids = ggml_graph_get_tensor(gf, "position_ids");
+    const int qlen = input_ids.size() - n_past;
     CHATGLM_CHECK(ggml_n_dims(position_ids) == 1 && position_ids->ne[0] == qlen)
         << "invalid position ids size " << position_ids->ne[0];
 
@@ -1470,7 +1468,7 @@ ChatMessage ChatGLM3Tokenizer::decode_message(const std::vector<int> &ids) const
             std::string code_output = output.substr(ci_pos + ci_delim.size());
             code_output = remove_special_tokens(code_output);
             trim(code_output);
-            message = ChatMessage(ChatMessage::ROLE_ASSISTANT, std::move(chat_output),
+            message = ChatMessage(ChatMessage::ROLE_ASSISTANT, std::move(chat_output), std::nullopt,
                                   {ToolCallMessage(CodeMessage(std::move(code_output)))});
         } else {
             // tool call
@@ -1498,7 +1496,7 @@ ChatMessage ChatGLM3Tokenizer::decode_message(const std::vector<int> &ids) const
                 tool_args = sm[1];
             }
 
-            message = ChatMessage(ChatMessage::ROLE_ASSISTANT, std::move(output),
+            message = ChatMessage(ChatMessage::ROLE_ASSISTANT, std::move(output), std::nullopt,
                                   {ToolCallMessage(FunctionMessage(std::move(tool_name), std::move(tool_args)))});
         }
     } else {
@@ -1691,11 +1689,14 @@ ChatGLM4Tokenizer::ChatGLM4Tokenizer(const std::string &vocab_text) {
         special_tokens_encoder.emplace(token, vocab_size++);
     }
     // common special token ids
+    eos_token_id = special_tokens_encoder.at("
");
     gmask_token_id = special_tokens_encoder.at("[gMASK]");
     sop_token_id = special_tokens_encoder.at("<sop>");
     user_token_id = special_tokens_encoder.at("
");
     assistant_token_id = special_tokens_encoder.at("
");
     observation_token_id = special_tokens_encoder.at("
");
+    boi_token_id = special_tokens_encoder.at("
");
+    eoi_token_id = special_tokens_encoder.at("
");
 
     const std::string pattern =
         R"((?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|(\s+)(?:\s)|\s+)";
@@ -1733,6 +1734,9 @@ std::vector<int> ChatGLM4Tokenizer::apply_chat_template(const std::vector<ChatMe
     for (const auto &msg : messages) {
         input_ids.emplace_back(core_bpe.special_tokens_encoder.at("<|" + msg.role + "|>"));
         input_ids.insert(input_ids.end(), newline_ids.begin(), newline_ids.end());
+        if (msg.image.has_value()) {
+            input_ids.insert(input_ids.end(), {boi_token_id, eos_token_id, eoi_token_id});
+        }
         std::vector<int> content_ids = core_bpe.encode_ordinary(msg.content);
         input_ids.insert(input_ids.end(), content_ids.begin(), content_ids.end());
     }
@@ -1749,6 +1753,282 @@ void ChatGLM4Tokenizer::truncate(std::vector<int> &ids, int max_length) {
     }
 }
 
+// ===== GLM4V-9B =====
+
+ggml_tensor *Conv2d::forward(ModelContext *mctx, ggml_tensor *input) const {
+    // input: [b, c, h, w]
+    ggml_context *ctx = mctx->ctx_b.get();
+    ggml_tensor *output = ggml_conv_2d(ctx, weight, input, stride, stride, 0, 0, 1, 1); // [b, oc, oh, ow]
+    output = ggml_add_inplace(ctx, output, bias);
+    return output;
+}
+
+ggml_tensor *PatchEmbedding::forward(ModelContext *mctx, ggml_tensor *input) const {
+    // input: [c, h, w]
+    ggml_context *ctx = mctx->ctx_b.get();
+    input = proj.forward(mctx, input);                                              // [oc, oh, ow]
+    input = ggml_reshape_2d(ctx, input, input->ne[0] * input->ne[1], input->ne[2]); // [oc, oh * ow]
+    input = ggml_permute(ctx, input, 1, 0, 2, 3);                                   // [oh * ow, oc] view as [s, h]
+
+    ggml_tensor *embeddings = ggml_new_tensor_2d(ctx, input->type, hidden_size(), num_positions());
+    ggml_set_name(embeddings, "embeddings");
+    ggml_set_input(embeddings);
+
+    // concat (cls, x)
+    ggml_tensor *cls_embedding_view =
+        ggml_cpy(ctx, cls_embedding, ggml_view_1d(ctx, embeddings, cls_embedding->ne[0], 0));
+    ggml_tensor *img_embedding_view = ggml_cpy(
+        ctx, input, ggml_view_2d(ctx, embeddings, input->ne[0], input->ne[1], embeddings->nb[1], embeddings->nb[1]));
+    ggml_build_forward_expand(mctx->gf, cls_embedding_view);
+    ggml_build_forward_expand(mctx->gf, img_embedding_view);
+
+    embeddings = ggml_add_inplace(ctx, embeddings, position_embedding.weight); // [s, h]
+    return embeddings;
+}
+
+ggml_tensor *EVA2CLIPBlock::forward(ModelContext *mctx, ggml_tensor *hidden_states, ggml_tensor *attention_mask,
+                                    ggml_tensor *position_ids, int n_past) const {
+    ggml_context *ctx = mctx->ctx_b.get();
+
+    ggml_tensor *residual = hidden_states;
+    hidden_states = attention.forward(mctx, hidden_states, attention_mask, position_ids, n_past);
+    hidden_states = input_layernorm.forward(mctx, hidden_states);
+    hidden_states = ggml_add_inplace(ctx, hidden_states, residual);
+
+    residual = hidden_states;
+    hidden_states = mlp.forward(mctx, hidden_states);
+    hidden_states = post_attention_layernorm.forward(mctx, hidden_states);
+    hidden_states = ggml_add_inplace(ctx, hidden_states, residual);
+
+    return hidden_states;
+}
+
+EVA2CLIPTransformer::EVA2CLIPTransformer(ModelContext *mctx, const VisionModelConfig &config) {
+    layers.reserve(config.num_hidden_layers);
+    for (int layer_id = 0; layer_id < config.num_hidden_layers; layer_id++) {
+        layers.emplace_back(mctx, config.dtype, config.hidden_size, config.num_attention_heads,
+                            config.num_attention_heads, config.intermediate_size, config.num_positions, config.norm_eps,
+                            config.hidden_act, true, true, false, RopeType::DISABLED, -1,
+                            AttentionMaskType::BIDIRECTIONAL, 0, false);
+    }
+}
+
+ggml_tensor *EVA2CLIPTransformer::forward(ModelContext *mctx, ggml_tensor *hidden_states) const {
+    for (const auto &layer : layers) {
+        hidden_states = layer.forward(mctx, hidden_states, nullptr, nullptr, 0);
+    }
+    return hidden_states;
+}
+
+ggml_tensor *EVA2CLIPModel::forward(ModelContext *mctx, ggml_tensor *input) const {
+    ggml_context *ctx = mctx->ctx_b.get();
+
+    ggml_tensor *hidden_states = patch_embedding.forward(mctx, input);
+    hidden_states = transformer.forward(mctx, hidden_states); // [s, hd]
+
+    const int grid_size = std::round(std::sqrt(hidden_states->ne[1] - 1));
+    hidden_states = ggml_view_3d(ctx, hidden_states, hidden_states->ne[0], grid_size, grid_size, hidden_states->nb[1],
+                                 grid_size * hidden_states->nb[1], hidden_states->nb[1]); // [g, g, hd]
+    // TODO: must use this cont?
+    hidden_states = ggml_cont(ctx, ggml_permute(ctx, hidden_states, 2, 0, 1, 3)); // [hd, g, g]
+    hidden_states = conv.forward(mctx, hidden_states);                            // [hd, g/2, g/2]
+    hidden_states = ggml_reshape_2d(ctx, hidden_states, hidden_states->ne[0] * hidden_states->ne[1],
+                                    hidden_states->ne[2]); // [hd, s]
+    // TODO: this cont?
+    hidden_states = ggml_cont(ctx, ggml_permute(ctx, hidden_states, 1, 0, 2, 3)); // [s, hd]
+
+    hidden_states = linear_proj.forward(mctx, hidden_states);
+    hidden_states = norm1.forward(mctx, hidden_states);
+    hidden_states = ggml_gelu_inplace(ctx, hidden_states);
+    hidden_states = glu.forward(mctx, hidden_states);
+
+    ggml_tensor *output = ggml_new_tensor_2d(ctx, hidden_states->type, hidden_states->ne[0], hidden_states->ne[1] + 2);
+
+    // concat (boi, x, eoi)
+    ggml_tensor *boi_view = ggml_cpy(ctx, boi, ggml_view_1d(ctx, output, hidden_states->ne[0], 0));
+    ggml_tensor *hidden_states_view =
+        ggml_cpy(ctx, hidden_states,
+                 ggml_view_2d(ctx, output, hidden_states->ne[0], hidden_states->ne[1], output->nb[1], output->nb[1]));
+    ggml_tensor *eoi_view =
+        ggml_cpy(ctx, eoi, ggml_view_1d(ctx, output, hidden_states->ne[0], (hidden_states->ne[1] + 1) * output->nb[1]));
+    ggml_build_forward_expand(mctx->gf, boi_view);
+    ggml_build_forward_expand(mctx->gf, hidden_states_view);
+    ggml_build_forward_expand(mctx->gf, eoi_view);
+
+    output = ggml_scale_inplace(ctx, output, 1.f / scaling_factor);
+
+    return output;
+}
+
+ggml_tensor *ChatGLM4VModel::forward_embeddings(ModelContext *mctx, ggml_tensor *input_ids, ggml_tensor *images,
+                                                const std::vector<int> &input_ids_vec, int n_past) const {
+    ggml_context *ctx = mctx->ctx_b.get();
+
+    ggml_tensor *text_emb = word_embeddings.forward(mctx, input_ids);
+
+    if (!(n_past == 0 && images)) {
+        return text_emb;
+    }
+
+    CHATGLM_CHECK(images->ne[0] == images->ne[1] && images->ne[2] == 3);
+
+    const int vision_begin =
+        std::find(input_ids_vec.begin(), input_ids_vec.end(), config.boi_token_id) - input_ids_vec.begin();
+    const int vision_end =
+        std::find(input_ids_vec.begin(), input_ids_vec.end(), config.eoi_token_id) - input_ids_vec.begin() + 1;
+    CHATGLM_CHECK(vision_begin < (int)input_ids_vec.size() && vision_end <= (int)input_ids_vec.size() &&
+                  vision_begin + 3 == vision_end);
+
+    ggml_tensor *vision_emb = vision.forward(mctx, images);
+
+    const int text_tokens = input_ids->ne[0];
+    const int total_tokens = text_tokens - 1 + num_vision_tokens();
+
+    ggml_tensor *embeddings = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, config.hidden_size, total_tokens);
+    // before boi
+    ggml_tensor *text_emb0_view =
+        ggml_cpy(ctx, ggml_view_2d(ctx, text_emb, config.hidden_size, vision_begin, text_emb->nb[1], 0),
+                 ggml_view_2d(ctx, embeddings, config.hidden_size, vision_begin, embeddings->nb[1], 0));
+    // vision tokens
+    ggml_tensor *vision_emb_view = ggml_cpy(ctx, vision_emb,
+                                            ggml_view_2d(ctx, embeddings, config.hidden_size, vision_emb->ne[1],
+                                                         embeddings->nb[1], vision_begin * embeddings->nb[1]));
+    // after eoi
+    ggml_tensor *text_emb1_view =
+        ggml_cpy(ctx,
+                 ggml_view_2d(ctx, text_emb, config.hidden_size, text_tokens - vision_end, text_emb->nb[1],
+                              vision_end * text_emb->nb[1]),
+                 ggml_view_2d(ctx, embeddings, config.hidden_size, text_tokens - vision_end, embeddings->nb[1],
+                              (total_tokens - (text_tokens - vision_end)) * embeddings->nb[1]));
+
+    ggml_build_forward_expand(mctx->gf, text_emb0_view);
+    ggml_build_forward_expand(mctx->gf, vision_emb_view);
+    ggml_build_forward_expand(mctx->gf, text_emb1_view);
+
+    return embeddings;
+}
+
+void ChatGLM4VModel::set_graph_inputs(ggml_cgraph *gf, const std::vector<int> &input_ids,
+                                      const std::optional<Image> &image, int n_past, int n_ctx) const {
+    // set position_ids
+    ggml_tensor *position_ids = ggml_graph_get_tensor(gf, "position_ids");
+    if (position_ids) {
+        std::vector<int> position_ids_buf(position_ids->ne[0]);
+        const auto vision_idx =
+            std::find(input_ids.begin(), input_ids.end(), config.boi_token_id) - input_ids.begin() + 1;
+        if (n_past == 0 && vision_idx < (int)input_ids.size()) {
+            std::iota(position_ids_buf.begin(), position_ids_buf.begin() + vision_idx, 0);
+            std::fill(position_ids_buf.begin() + vision_idx,
+                      position_ids_buf.begin() + vision_idx + num_vision_tokens(), vision_idx);
+            std::iota(position_ids_buf.begin() + vision_idx + num_vision_tokens(), position_ids_buf.end(),
+                      vision_idx + 1);
+        } else {
+            const int n_past_text = (n_past > 0) ? input_ids.size() - 1 : 0;
+            std::iota(position_ids_buf.begin(), position_ids_buf.end(), n_past_text);
+        }
+        ggml_backend_tensor_set(position_ids, position_ids_buf.data(), 0, position_ids_buf.size() * sizeof(int));
+    }
+
+    // set image
+    ggml_tensor *image_tensor = ggml_graph_get_tensor(gf, "image");
+    if (image_tensor) {
+        // resize
+        Image src_resized = image->resize(config.vision.image_size, config.vision.image_size);
+        // to float, normalize, channel first
+        std::vector<float> pixels_f32(src_resized.pixels.size());
+        const size_t num_pixels = src_resized.height * src_resized.width;
+        for (size_t i = 0; i < num_pixels; i++) {
+            pixels_f32[0 * num_pixels + i] =
+                (src_resized.pixels[i * src_resized.channels + 0] / 255.f - 0.48145466f) / 0.26862954f;
+            pixels_f32[1 * num_pixels + i] =
+                (src_resized.pixels[i * src_resized.channels + 1] / 255.f - 0.45782750f) / 0.26130258f;
+            pixels_f32[2 * num_pixels + i] =
+                (src_resized.pixels[i * src_resized.channels + 2] / 255.f - 0.40821073f) / 0.27577711f;
+        }
+        // copy to tensor
+        ggml_backend_tensor_set(image_tensor, pixels_f32.data(), 0, ggml_nbytes(image_tensor));
+    }
+}
+
+int ChatGLM4VForCausalLM::count_tokens(const std::vector<int> &input_ids, const std::optional<Image> &image) const {
+    int token_cnt = input_ids.size();
+    if (image) {
+        token_cnt += transformer.num_vision_tokens() - 1;
+    }
+    return token_cnt;
+}
+
+void ChatGLM4VForCausalLM::load_state_dict(const StateDict &sd) {
+    alloc_weight_context(sd.buf.get());
+
+    auto self_sd = state_dict();
+    ChatGLM2ForCausalLM::load_state_dict(mctx_.get(), self_sd, sd);
+}
+
+StateDict ChatGLM4VForCausalLM::state_dict() const {
+    StateDict sd;
+
+    // vision
+    sd.kv.emplace("transformer.vision.patch_embedding.cls_embedding", transformer.vision.patch_embedding.cls_embedding);
+    sd.kv.emplace("transformer.vision.patch_embedding.proj.weight", transformer.vision.patch_embedding.proj.weight);
+    sd.kv.emplace("transformer.vision.patch_embedding.proj.bias", transformer.vision.patch_embedding.proj.bias);
+    sd.kv.emplace("transformer.vision.patch_embedding.position_embedding.weight",
+                  transformer.vision.patch_embedding.position_embedding.weight);
+
+    for (size_t i = 0; i < transformer.vision.transformer.layers.size(); i++) {
+        const std::string prefix = "transformer.vision.transformer.layers." + std::to_string(i) + '.';
+        sd.kv.emplace(prefix + "input_layernorm.weight",
+                      transformer.vision.transformer.layers[i].input_layernorm.weight);
+        sd.kv.emplace(prefix + "input_layernorm.bias", transformer.vision.transformer.layers[i].input_layernorm.bias);
+        sd.kv.emplace(prefix + "attention.query_key_value.weight",
+                      transformer.vision.transformer.layers[i].attention.query_key_value.weight);
+        sd.kv.emplace(prefix + "attention.query_key_value.bias",
+                      transformer.vision.transformer.layers[i].attention.query_key_value.bias);
+        sd.kv.emplace(prefix + "attention.dense.weight",
+                      transformer.vision.transformer.layers[i].attention.dense.weight);
+        sd.kv.emplace(prefix + "attention.dense.bias", transformer.vision.transformer.layers[i].attention.dense.bias);
+        sd.kv.emplace(prefix + "mlp.fc1.weight", transformer.vision.transformer.layers[i].mlp.dense_h_to_4h.weight);
+        sd.kv.emplace(prefix + "mlp.fc1.bias", transformer.vision.transformer.layers[i].mlp.dense_h_to_4h.bias);
+        sd.kv.emplace(prefix + "mlp.fc2.weight", transformer.vision.transformer.layers[i].mlp.dense_4h_to_h.weight);
+        sd.kv.emplace(prefix + "mlp.fc2.bias", transformer.vision.transformer.layers[i].mlp.dense_4h_to_h.bias);
+        sd.kv.emplace(prefix + "post_attention_layernorm.weight",
+                      transformer.vision.transformer.layers[i].post_attention_layernorm.weight);
+        sd.kv.emplace(prefix + "post_attention_layernorm.bias",
+                      transformer.vision.transformer.layers[i].post_attention_layernorm.bias);
+    }
+    sd.kv.emplace("transformer.vision.conv.weight", transformer.vision.conv.weight);
+    sd.kv.emplace("transformer.vision.conv.bias", transformer.vision.conv.bias);
+    sd.kv.emplace("transformer.vision.linear_proj.linear_proj.weight", transformer.vision.linear_proj.weight);
+    sd.kv.emplace("transformer.vision.linear_proj.norm1.weight", transformer.vision.norm1.weight);
+    sd.kv.emplace("transformer.vision.linear_proj.norm1.bias", transformer.vision.norm1.bias);
+    sd.kv.emplace("transformer.vision.linear_proj.gate_proj.weight", transformer.vision.glu.gate_proj.weight);
+    sd.kv.emplace("transformer.vision.linear_proj.dense_h_to_4h.weight", transformer.vision.glu.up_proj.weight);
+    sd.kv.emplace("transformer.vision.linear_proj.dense_4h_to_h.weight", transformer.vision.glu.down_proj.weight);
+    sd.kv.emplace("transformer.vision.boi", transformer.vision.boi);
+    sd.kv.emplace("transformer.vision.eoi", transformer.vision.eoi);
+
+    // text
+    sd.kv.emplace("transformer.embedding.word_embeddings.weight", transformer.word_embeddings.weight);
+    for (int i = 0; i < config.num_hidden_layers; i++) {
+        const std::string layer_prefix = "transformer.encoder.layers." + std::to_string(i) + '.';
+        sd.kv.emplace(layer_prefix + "input_layernorm.weight", transformer.layers[i].input_layernorm.weight);
+        sd.kv.emplace(layer_prefix + "self_attention.query_key_value.weight",
+                      transformer.layers[i].attention.query_key_value.weight);
+        sd.kv.emplace(layer_prefix + "self_attention.query_key_value.bias",
+                      transformer.layers[i].attention.query_key_value.bias);
+        sd.kv.emplace(layer_prefix + "self_attention.dense.weight", transformer.layers[i].attention.dense.weight);
+        sd.kv.emplace(layer_prefix + "post_attention_layernorm.weight",
+                      transformer.layers[i].post_attention_layernorm.weight);
+        sd.kv.emplace(layer_prefix + "mlp.gate_proj.weight", transformer.layers[i].mlp.gate_proj.weight);
+        sd.kv.emplace(layer_prefix + "mlp.up_proj.weight", transformer.layers[i].mlp.up_proj.weight);
+        // for compatibility
+        sd.kv.emplace(layer_prefix + "mlp.dense_4h_to_h.weight", transformer.layers[i].mlp.down_proj.weight);
+    }
+    sd.kv.emplace("transformer.encoder.final_layernorm.weight", transformer.final_layernorm.weight);
+    sd.kv.emplace("transformer.output_layer.weight", lm_head.weight);
+    return sd;
+}
+
 // ===== pipeline =====
 
 Pipeline::Pipeline(const std::string &path, int max_length) {
@@ -1776,9 +2056,9 @@ Pipeline::Pipeline(const std::string &path, int max_length) {
         // load config
         ModelConfig config;
         if (version == 1) {
-            config = ModelConfig(model_type, loader.read_basic<ConfigRecordV1>(), 1e-5f, 10000.f, 0);
+            config = ModelConfig(model_type, loader.read_basic<ModelConfigRecordV1>(), 1e-5f, 10000.f, 0);
         } else if (version == 2) {
-            config = ModelConfig(model_type, loader.read_basic<ConfigRecordV2>());
+            config = ModelConfig(model_type, loader.read_basic<ModelConfigRecordV2>());
         } else {
             CHATGLM_THROW << "only support version 1 or 2 for now but got " << version;
         }
@@ -1798,9 +2078,9 @@ Pipeline::Pipeline(const std::string &path, int max_length) {
         // load config
         ModelConfig config;
         if (version == 1) {
-            config = ModelConfig(model_type, loader.read_basic<ConfigRecordV1GQA>(), 1e-5f, 10000.f, 0);
+            config = ModelConfig(model_type, loader.read_basic<ModelConfigRecordV1GQA>(), 1e-5f, 10000.f, 0);
         } else if (version == 2) {
-            config = ModelConfig(model_type, loader.read_basic<ConfigRecordV2>());
+            config = ModelConfig(model_type, loader.read_basic<ModelConfigRecordV2>());
         } else {
             CHATGLM_THROW << "only support version 1 or 2 for now but got " << version;
         }
@@ -1825,21 +2105,31 @@ Pipeline::Pipeline(const std::string &path, int max_length) {
         // load model
         StateDict sd = loader.read_state_dict();
         model->load_state_dict(sd);
-    } else if (model_type == ModelType::CHATGLM4) {
+    } else if (model_type == ModelType::CHATGLM4 || model_type == ModelType::CHATGLM4V) {
         // load config
         CHATGLM_CHECK(version == 2) << "only support version 2 for now but got " << version;
-        ModelConfig config(model_type, loader.read_basic<ConfigRecordV2>());
+        ModelConfig config(model_type, loader.read_basic<ModelConfigRecordV2>());
         _update_config_max_length(config, max_length);
 
+        if (model_type == ModelType::CHATGLM4V) {
+            config.vision = VisionModelConfig(loader.read_basic<VisionModelConfigRecord>());
+        }
+
         // load tokenizer
         int vocab_text_size = loader.read_basic<int>();
         std::string vocab_text = loader.read_string(vocab_text_size);
         auto chatglm4_tokenizer = std::make_unique<ChatGLM4Tokenizer>(vocab_text);
         config.extra_eos_token_ids = {chatglm4_tokenizer->observation_token_id, chatglm4_tokenizer->user_token_id};
+        config.boi_token_id = chatglm4_tokenizer->boi_token_id;
+        config.eoi_token_id = chatglm4_tokenizer->eoi_token_id;
         tokenizer = std::move(chatglm4_tokenizer);
 
         // load model
-        model = std::make_unique<ChatGLM4ForCausalLM>(config);
+        if (model_type == ModelType::CHATGLM4V) {
+            model = std::make_unique<ChatGLM4VForCausalLM>(config);
+        } else {
+            model = std::make_unique<ChatGLM4ForCausalLM>(config);
+        }
         StateDict sd = loader.read_state_dict();
         model->load_state_dict(sd);
     } else {
@@ -1847,9 +2137,9 @@ Pipeline::Pipeline(const std::string &path, int max_length) {
     }
 }
 
-std::vector<int> Pipeline::generate(const std::vector<int> &input_ids, const GenerationConfig &gen_config,
-                                    BaseStreamer *streamer) const {
-    std::vector<int> output_ids = model->generate(input_ids, gen_config, streamer);
+std::vector<int> Pipeline::generate(const std::vector<int> &input_ids, const std::optional<Image> &image,
+                                    const GenerationConfig &gen_config, BaseStreamer *streamer) const {
+    std::vector<int> output_ids = model->generate(input_ids, image, gen_config, streamer);
     std::vector<int> new_output_ids(output_ids.begin() + input_ids.size(), output_ids.end());
     return new_output_ids;
 }
@@ -1857,15 +2147,18 @@ std::vector<int> Pipeline::generate(const std::vector<int> &input_ids, const Gen
 std::string Pipeline::generate(const std::string &prompt, const GenerationConfig &gen_config,
                                BaseStreamer *streamer) const {
     std::vector<int> input_ids = tokenizer->encode(prompt, gen_config.max_context_length);
-    std::vector<int> new_output_ids = generate(input_ids, gen_config, streamer);
+    std::vector<int> new_output_ids = generate(input_ids, std::nullopt, gen_config, streamer);
     std::string output = tokenizer->decode(new_output_ids);
     return output;
 }
 
 ChatMessage Pipeline::chat(const std::vector<ChatMessage> &messages, const GenerationConfig &gen_config,
                            BaseStreamer *streamer) const {
+    auto it =
+        std::find_if(messages.begin(), messages.end(), [](const ChatMessage &msg) { return msg.image.has_value(); });
+    const std::optional<Image> image = (it != messages.end()) ? it->image : std::nullopt;
     std::vector<int> input_ids = tokenizer->apply_chat_template(messages, gen_config.max_context_length);
-    std::vector<int> new_output_ids = generate(input_ids, gen_config, streamer);
+    std::vector<int> new_output_ids = generate(input_ids, image, gen_config, streamer);
     ChatMessage output = tokenizer->decode_message(new_output_ids);
     return output;
 }
diff --git a/chatglm.h b/chatglm.h
index f42be536..bc910247 100644
--- a/chatglm.h
+++ b/chatglm.h
@@ -41,12 +41,13 @@ enum class ModelType {
     CHATGLM2 = 2,
     CHATGLM3 = 3,
     CHATGLM4 = 4,
+    CHATGLM4V = 1004,
 };
 
 std::string to_string(ModelType model_type);
 
 // For compatibility
-struct ConfigRecordV1 {
+struct ModelConfigRecordV1 {
     // common attributes
     ggml_type dtype;
     int vocab_size;
@@ -64,12 +65,25 @@ struct ConfigRecordV1 {
 };
 
 // For compatibility
-struct ConfigRecordV1GQA : public ConfigRecordV1 {
+struct ModelConfigRecordV1GQA {
+    // ModelConfigRecordV1
+    ggml_type dtype;
+    int vocab_size;
+    int hidden_size;
+    int num_attention_heads;
+    int num_hidden_layers;
+    int intermediate_size;
+    int max_length;
+    int bos_token_id;
+    int eos_token_id;
+    int pad_token_id;
+    int sep_token_id;
+    // GQA
     int num_key_value_heads;
 };
 
 // TODO: use json to serialize config
-struct ConfigRecordV2 {
+struct ModelConfigRecordV2 {
     ggml_type dtype;
     int vocab_size;
     int hidden_size;
@@ -99,10 +113,64 @@ enum class RopeType {
 };
 
 enum class AttentionMaskType {
+    BIDIRECTIONAL,
     CAUSAL,
     CHATGLM,
 };
 
+struct VisionModelConfigRecord {
+    ggml_type dtype;
+    int hidden_size;
+    int image_size;
+    int in_channels;
+    int intermediate_size;
+    float norm_eps;
+    int num_attention_heads;
+    int num_hidden_layers;
+    int num_positions;
+    int patch_size;
+    float scaling_factor;
+};
+
+struct VisionModelConfig {
+    ggml_type dtype;
+    ActivationType hidden_act;
+    int hidden_size;
+    int image_size;
+    int in_channels;
+    int intermediate_size;
+    float norm_eps;
+    int num_attention_heads;
+    int num_hidden_layers;
+    int num_positions;
+    int patch_size;
+    float scaling_factor;
+
+    VisionModelConfig() = default;
+
+    VisionModelConfig(ggml_type dtype, ActivationType hidden_act, int hidden_size, int image_size, int in_channels,
+                      int intermediate_size, float norm_eps, int num_attention_heads, int num_hidden_layers,
+                      int num_positions, int patch_size, float scaling_factor)
+        : dtype(dtype), hidden_act(hidden_act), hidden_size(hidden_size), image_size(image_size),
+          in_channels(in_channels), intermediate_size(intermediate_size), norm_eps(norm_eps),
+          num_attention_heads(num_attention_heads), num_hidden_layers(num_hidden_layers), num_positions(num_positions),
+          patch_size(patch_size), scaling_factor(scaling_factor) {}
+
+    VisionModelConfig(const VisionModelConfigRecord &rec)
+        : VisionModelConfig(rec.dtype, ActivationType::GELU, rec.hidden_size, rec.image_size, rec.in_channels,
+                            rec.intermediate_size, rec.norm_eps, rec.num_attention_heads, rec.num_hidden_layers,
+                            rec.num_positions, rec.patch_size, rec.scaling_factor) {}
+
+    friend std::ostream &operator<<(std::ostream &os, const VisionModelConfig &self) {
+        return os << "VisionModelConfig(dtype=" << self.dtype << ", hidden_act=" << (int)self.hidden_act
+                  << ", hidden_size=" << self.hidden_size << ", image_size=" << self.image_size
+                  << ", in_channels=" << self.in_channels << ", intermediate_size=" << self.intermediate_size
+                  << ", norm_eps=" << self.norm_eps << ", num_attention_heads=" << self.num_attention_heads
+                  << ", num_hidden_layers=" << self.num_hidden_layers << ", num_positions="
+                  << ", patch_size=" << self.patch_size << ", scaling_factor=" << self.scaling_factor << ")";
+    }
+};
+
 // Should save kv record of ModelConfig in the future
 class ModelConfig {
   public:
@@ -111,13 +179,15 @@ class ModelConfig {
     ModelConfig(ModelType model_type, ggml_type dtype, int vocab_size, int hidden_size, int num_attention_heads,
                 int num_key_value_heads, int num_hidden_layers, int intermediate_size, float norm_eps, float rope_theta,
                 int num_virtual_tokens, int max_length, int bos_token_id, int eos_token_id, int pad_token_id,
-                int sep_token_id, std::vector<int> extra_eos_token_ids)
+                int sep_token_id, int boi_token_id, int eoi_token_id, std::vector<int> extra_eos_token_ids,
+                const VisionModelConfig &vision)
         : model_type(model_type), dtype(dtype), vocab_size(vocab_size), hidden_size(hidden_size),
           num_attention_heads(num_attention_heads), num_key_value_heads(num_key_value_heads),
           num_hidden_layers(num_hidden_layers), intermediate_size(intermediate_size), norm_eps(norm_eps),
           rope_theta(rope_theta), num_virtual_tokens(num_virtual_tokens), max_length(max_length),
           bos_token_id(bos_token_id), eos_token_id(eos_token_id), pad_token_id(pad_token_id),
-          sep_token_id(sep_token_id), extra_eos_token_ids(std::move(extra_eos_token_ids)) {
+          sep_token_id(sep_token_id), boi_token_id(boi_token_id), eoi_token_id(eoi_token_id),
+          extra_eos_token_ids(std::move(extra_eos_token_ids)), vision(vision) {
         if (model_type == ModelType::CHATGLM) {
             hidden_act = ActivationType::GELU;
             use_qkv_bias = true;
@@ -137,25 +207,25 @@ class ModelConfig {
         }
     }
 
-    ModelConfig(ModelType model_type, const ConfigRecordV1 &rec, float norm_eps, float rope_theta,
+    ModelConfig(ModelType model_type, const ModelConfigRecordV1 &rec, float norm_eps, float rope_theta,
                 int num_virtual_tokens)
         : ModelConfig(model_type, rec.dtype, rec.vocab_size, rec.hidden_size, rec.num_attention_heads,
                       rec.num_attention_heads, rec.num_hidden_layers, rec.intermediate_size, norm_eps, rope_theta,
                       num_virtual_tokens, rec.max_length, rec.bos_token_id, rec.eos_token_id, rec.pad_token_id,
-                      rec.sep_token_id, {}) {}
+                      rec.sep_token_id, -1, -1, {}, {}) {}
 
-    ModelConfig(ModelType model_type, const ConfigRecordV1GQA &rec, float norm_eps, float rope_theta,
+    ModelConfig(ModelType model_type, const ModelConfigRecordV1GQA &rec, float norm_eps, float rope_theta,
                 int num_virtual_tokens)
         : ModelConfig(model_type, rec.dtype, rec.vocab_size, rec.hidden_size, rec.num_attention_heads,
                       rec.num_key_value_heads, rec.num_hidden_layers, rec.intermediate_size, norm_eps, rope_theta,
                       num_virtual_tokens, rec.max_length, rec.bos_token_id, rec.eos_token_id, rec.pad_token_id,
-                      rec.sep_token_id, {}) {}
+                      rec.sep_token_id, -1, -1, {}, {}) {}
 
-    ModelConfig(ModelType model_type, const ConfigRecordV2 &rec)
+    ModelConfig(ModelType model_type, const ModelConfigRecordV2 &rec)
         : ModelConfig(model_type, rec.dtype, rec.vocab_size, rec.hidden_size, rec.num_attention_heads,
                       rec.num_key_value_heads, rec.num_hidden_layers, rec.intermediate_size, rec.norm_eps,
                       rec.rope_theta, rec.num_virtual_tokens, rec.max_length, -1, rec.eos_token_id, rec.pad_token_id,
-                      -1, {}) {}
+                      -1, -1, -1, {}, {}) {}
 
     std::string model_type_name() const { return to_string(model_type); }
 
@@ -176,7 +246,7 @@ class ModelConfig {
         for (size_t i = 0; i < self.extra_eos_token_ids.size(); i++) {
             os << (i > 0 ? ", " : "") << self.extra_eos_token_ids[i];
         }
-        return os << "})";
+        return os << "}, vision=" << self.vision << ")";
     }
 
   public:
@@ -203,7 +273,10 @@ class ModelConfig {
     int eos_token_id;
     int pad_token_id;
     int sep_token_id;
+    int boi_token_id;
+    int eoi_token_id;
     std::vector<int> extra_eos_token_ids;
+    VisionModelConfig vision;
 };
 
 struct FunctionMessage {
@@ -248,9 +321,53 @@ struct ToolCallMessage {
     }
 };
 
+struct Image {
+    size_t width = 0;
+    size_t height = 0;
+    size_t channels = 0;
+    std::vector<uint8_t> pixels;
+
+    Image() = default;
+
+    Image(size_t width, size_t height, size_t channels)
+        : width(width), height(height), channels(channels), pixels(width * height * channels) {}
+
+    Image(size_t width, size_t height, size_t channels, uint8_t *data)
+        : width(width), height(height), channels(channels), pixels(data, data + width * height * channels) {}
+
+    Image(const Image &other) = default;
+
+    Image(Image &&other) { *this = std::move(other); }
+
+    Image &operator=(const Image &other) = default;
+
+    Image &operator=(Image &&other) {
+        width = other.width;
+        height = other.height;
+        channels = other.channels;
+        pixels = std::move(other.pixels);
+        other.clear();
+        return *this;
+    }
+
+    static Image open(const std::string &path);
+
+    Image resize(size_t new_width, size_t new_height) const;
+
+    void clear() {
+        width = height = channels = 0;
+        pixels.clear();
+    }
+
+    friend std::ostream &operator<<(std::ostream &os, const Image &self) {
+        return os << "Image(mode=RGB, size=" << self.width << "x" << self.height << ")";
+    }
+};
+
 struct ChatMessage {
     std::string role;
     std::string content;
+    std::optional<Image> image;
     std::vector<ToolCallMessage> tool_calls;
 
     static const std::string ROLE_USER;
@@ -259,12 +376,18 @@ struct ChatMessage {
     static const std::string ROLE_OBSERVATION;
 
     ChatMessage() = default;
-    ChatMessage(std::string role, std::string content, std::vector<ToolCallMessage> tool_calls = {})
-        : role(std::move(role)), content(std::move(content)), tool_calls(std::move(tool_calls)) {}
+
+    ChatMessage(std::string role, std::string content, std::optional<Image> image = std::nullopt,
+                std::vector<ToolCallMessage> tool_calls = {})
+        : role(std::move(role)), content(std::move(content)), image(std::move(image)),
+          tool_calls(std::move(tool_calls)) {}
 
     friend std::ostream &operator<<(std::ostream &os, const ChatMessage &self) {
-        os << "ChatMessage(role=" << std::quoted(self.role) << ", content=" << std::quoted(self.content)
-           << ", tool_calls=[";
+        os << "ChatMessage(role=" << std::quoted(self.role) << ", content=" << std::quoted(self.content);
+        if (self.image.has_value()) {
+            os << ", image=" << *self.image;
+        }
+        os << ", tool_calls=[";
         for (size_t i = 0; i < self.tool_calls.size(); i++) {
             os << (i > 0 ? ", " : "") << self.tool_calls[i];
         }
@@ -329,8 +452,6 @@ struct no_init {
 };
 
 struct ModelContext {
-    ggml_type dtype;
-
     std::vector<no_init<char>> compute_meta;
 
     unique_ggml_context_t ctx_w;  // weight
@@ -344,26 +465,32 @@ struct ModelContext {
     unique_ggml_backend_buffer_t buf_w;
     unique_ggml_backend_buffer_t buf_kv;
 
-    ModelContext(ggml_type dtype);
+    ModelContext();
 };
 
 class Embedding {
   public:
-    Embedding() : weight(nullptr) {}
-    Embedding(ModelContext *mctx, int num_embeddings, int embedding_dim)
-        : weight(ggml_new_tensor_2d(mctx->ctx_w.get(), mctx->dtype, embedding_dim, num_embeddings)) {}
+    Embedding() = default;
+
+    Embedding(ModelContext *mctx, ggml_type dtype, int num_embeddings, int embedding_dim)
+        : weight(ggml_new_tensor_2d(mctx->ctx_w.get(), dtype, embedding_dim, num_embeddings)) {}
+
+    int num_embeddings() const { return weight->ne[1]; }
+
+    int embedding_dim() const { return weight->ne[0]; }
 
     ggml_tensor *forward(ModelContext *mctx, ggml_tensor *input) const;
 
   public:
-    ggml_tensor *weight;
+    ggml_tensor *weight = nullptr;
 };
 
 class Linear {
   public:
-    Linear() : weight(nullptr), bias(nullptr) {}
-    Linear(ModelContext *mctx, int in_features, int out_features, bool use_bias = true)
-        : weight(ggml_new_tensor_2d(mctx->ctx_w.get(), mctx->dtype, in_features, out_features)),
+    Linear() = default;
+
+    Linear(ModelContext *mctx, ggml_type dtype, int in_features, int out_features, bool use_bias = true)
+        : weight(ggml_new_tensor_2d(mctx->ctx_w.get(), dtype, in_features, out_features)),
           bias(use_bias ? ggml_new_tensor_1d(mctx->ctx_w.get(), GGML_TYPE_F32, out_features) : nullptr) {}
 
     int in_features() const { return weight->ne[0]; }
@@ -372,46 +499,47 @@ class Linear {
     ggml_tensor *forward(ModelContext *mctx, ggml_tensor *input) const;
 
   public:
-    ggml_tensor *weight; // [out_features, in_features]
-    ggml_tensor *bias;   // [out_features]
+    ggml_tensor *weight = nullptr; // [out_features, in_features]
+    ggml_tensor *bias = nullptr;   // [out_features]
 };
 
 class LayerNorm {
   public:
     LayerNorm() = default;
-    LayerNorm(ModelContext *mctx, int normalized_shape, bool inplace = true, float eps = 1e-5f)
+
+    LayerNorm(ModelContext *mctx, int normalized_shape, float eps = 1e-5f)
         : weight(ggml_new_tensor_1d(mctx->ctx_w.get(), GGML_TYPE_F32, normalized_shape)),
-          bias(ggml_new_tensor_1d(mctx->ctx_w.get(), GGML_TYPE_F32, normalized_shape)), inplace(inplace), eps(eps) {}
+          bias(ggml_new_tensor_1d(mctx->ctx_w.get(), GGML_TYPE_F32, normalized_shape)), eps(eps) {}
 
     ggml_tensor *forward(ModelContext *mctx, ggml_tensor *input) const;
 
   public:
-    ggml_tensor *weight; // [normalized_shape]
-    ggml_tensor *bias;   // [normalized_shape]
-    bool inplace;
-    float eps;
+    ggml_tensor *weight = nullptr; // [normalized_shape]
+    ggml_tensor *bias = nullptr;   // [normalized_shape]
+    float eps = 0.f;
 };
 
 class RMSNorm {
   public:
     RMSNorm() = default;
-    RMSNorm(ModelContext *mctx, int normalized_shape, bool inplace = true, float eps = 1e-5f)
-        : weight(ggml_new_tensor_1d(mctx->ctx_w.get(), GGML_TYPE_F32, normalized_shape)), inplace(inplace), eps(eps) {}
+
+    RMSNorm(ModelContext *mctx, int normalized_shape, float eps = 1e-5f)
+        : weight(ggml_new_tensor_1d(mctx->ctx_w.get(), GGML_TYPE_F32, normalized_shape)), eps(eps) {}
 
     ggml_tensor *forward(ModelContext *mctx, ggml_tensor *input) const;
 
   public:
-    ggml_tensor *weight; // [normalized_shape]
-    bool inplace;
-    float eps;
+    ggml_tensor *weight = nullptr; // [normalized_shape]
+    float eps = 0.f;
 };
 
 class BasicMLP {
   public:
     BasicMLP() = default;
-    BasicMLP(ModelContext *mctx, int hidden_size, int intermediate_size, ActivationType hidden_act)
-        : dense_h_to_4h(mctx, hidden_size, intermediate_size), dense_4h_to_h(mctx, intermediate_size, hidden_size),
-          hidden_act(hidden_act) {}
+
+    BasicMLP(ModelContext *mctx, ggml_type dtype, int hidden_size, int intermediate_size, ActivationType hidden_act)
+        : dense_h_to_4h(mctx, dtype, hidden_size, intermediate_size),
+          dense_4h_to_h(mctx, dtype, intermediate_size, hidden_size), hidden_act(hidden_act) {}
 
     ggml_tensor *forward(ModelContext *mctx, ggml_tensor *hidden_states) const;
 
@@ -424,9 +552,11 @@ class BasicMLP {
 class BasicGLU {
   public:
     BasicGLU() = default;
-    BasicGLU(ModelContext *mctx, int hidden_size, int intermediate_size, ActivationType hidden_act)
-        : gate_proj(mctx, hidden_size, intermediate_size, false), up_proj(mctx, hidden_size, intermediate_size, false),
-          down_proj(mctx, intermediate_size, hidden_size, false), hidden_act(hidden_act) {}
+
+    BasicGLU(ModelContext *mctx, ggml_type dtype, int hidden_size, int intermediate_size, ActivationType hidden_act)
+        : gate_proj(mctx, dtype, hidden_size, intermediate_size, false),
+          up_proj(mctx, dtype, hidden_size, intermediate_size, false),
+          down_proj(mctx, dtype, intermediate_size, hidden_size, false), hidden_act(hidden_act) {}
 
     ggml_tensor *forward(ModelContext *mctx, ggml_tensor *hidden_states) const;
 
@@ -440,19 +570,23 @@ class BasicGLU {
 class BasicAttention {
   public:
     BasicAttention() = default;
-    BasicAttention(ModelContext *mctx, int hidden_size, int num_attention_heads, int num_key_value_heads,
-                   int max_length, bool use_qkv_bias, bool use_dense_bias, bool interleaved_qkv, RopeType rope_type,
-                   float rope_theta, AttentionMaskType attn_mask_type, int num_virtual_tokens)
+
+    BasicAttention(ModelContext *mctx, ggml_type dtype, int hidden_size, int num_attention_heads,
+                   int num_key_value_heads, int max_length, bool use_qkv_bias, bool use_dense_bias,
+                   bool interleaved_qkv, RopeType rope_type, float rope_theta, AttentionMaskType attn_mask_type,
+                   int num_virtual_tokens, bool use_cache)
         : num_attention_heads(num_attention_heads), num_key_value_heads(num_key_value_heads),
           interleaved_qkv(interleaved_qkv), rope_type(rope_type), rope_theta(rope_theta),
           attn_mask_type(attn_mask_type), num_virtual_tokens(num_virtual_tokens),
-          query_key_value(mctx, hidden_size,
+          query_key_value(mctx, dtype, hidden_size,
                           hidden_size + 2 * (hidden_size / num_attention_heads) * num_key_value_heads, use_qkv_bias),
-          dense(mctx, hidden_size, hidden_size, use_dense_bias),
-          k_cache(ggml_new_tensor_3d(mctx->ctx_kv.get(), GGML_TYPE_F16, hidden_size / num_attention_heads,
-                                     max_length + num_virtual_tokens, num_key_value_heads)),
-          v_cache(ggml_new_tensor_3d(mctx->ctx_kv.get(), GGML_TYPE_F16, max_length + num_virtual_tokens,
-                                     hidden_size / num_attention_heads, num_key_value_heads)) {}
+          dense(mctx, dtype, hidden_size, hidden_size, use_dense_bias),
+          k_cache(use_cache ? ggml_new_tensor_3d(mctx->ctx_kv.get(), GGML_TYPE_F32, hidden_size / num_attention_heads,
+                                                 max_length + num_virtual_tokens, num_key_value_heads)
+                            : nullptr),
+          v_cache(use_cache ? ggml_new_tensor_3d(mctx->ctx_kv.get(), GGML_TYPE_F32, max_length + num_virtual_tokens,
+                                                 hidden_size / num_attention_heads, num_key_value_heads)
+                            : nullptr) {}
 
     ggml_tensor *forward(ModelContext *mctx, ggml_tensor *hidden_states, ggml_tensor *attention_mask,
                          ggml_tensor *position_ids, int n_past) const;
@@ -467,23 +601,24 @@ class BasicAttention {
     int num_virtual_tokens;
     Linear query_key_value;
     Linear dense;
-    ggml_tensor *k_cache; // [#kvh, s, d]
-    ggml_tensor *v_cache; // [#kvh, d, s]
+    ggml_tensor *k_cache = nullptr; // [#kvh, s, d]
+    ggml_tensor *v_cache = nullptr; // [#kvh, d, s]
 };
 
 template <typename Norm, typename MLP>
 class BasicBlock {
   public:
     BasicBlock() = default;
-    BasicBlock(ModelContext *mctx, int hidden_size, int num_attention_heads, int num_key_value_heads,
+    BasicBlock(ModelContext *mctx, ggml_type dtype, int hidden_size, int num_attention_heads, int num_key_value_heads,
                int intermediate_size, int max_length, float norm_eps, ActivationType hidden_act, bool use_qkv_bias,
                bool use_dense_bias, bool interleaved_qkv, RopeType rope_type, float rope_theta,
-               AttentionMaskType attn_mask_type, int num_virtual_tokens)
-        : input_layernorm(mctx, hidden_size, false, norm_eps),
-          attention(mctx, hidden_size, num_attention_heads, num_key_value_heads, max_length, use_qkv_bias,
-                    use_dense_bias, interleaved_qkv, rope_type, rope_theta, attn_mask_type, num_virtual_tokens),
-          post_attention_layernorm(mctx, hidden_size, false, norm_eps),
-          mlp(mctx, hidden_size, intermediate_size, hidden_act) {}
+               AttentionMaskType attn_mask_type, int num_virtual_tokens, bool use_cache)
+        : input_layernorm(mctx, hidden_size, norm_eps),
+          attention(mctx, dtype, hidden_size, num_attention_heads, num_key_value_heads, max_length, use_qkv_bias,
+                    use_dense_bias, interleaved_qkv, rope_type, rope_theta, attn_mask_type, num_virtual_tokens,
+                    use_cache),
+          post_attention_layernorm(mctx, hidden_size, norm_eps),
+          mlp(mctx, dtype, hidden_size, intermediate_size, hidden_act) {}
 
     ggml_tensor *forward(ModelContext *mctx, ggml_tensor *hidden_states, ggml_tensor *attention_mask,
                          ggml_tensor *position_ids, int n_past) const {
@@ -547,13 +682,16 @@ class BasicModel {
         : word_embeddings(word_embeddings), layers(std::move(layers)), final_layernorm(final_layernorm) {}
 
     BasicModel(ModelContext *mctx, const ModelConfig &config)
-        : word_embeddings(mctx, config.vocab_size, config.hidden_size), layers(build_layers(mctx, config)),
-          final_layernorm(mctx, config.hidden_size) {}
+        : word_embeddings(mctx, config.dtype, config.vocab_size, config.hidden_size),
+          layers(build_layers(mctx, config)), final_layernorm(mctx, config.hidden_size) {}
 
-    ggml_tensor *forward(ModelContext *mctx, ggml_tensor *input_ids, int n_past) const {
+    ggml_tensor *forward(ModelContext *mctx, ggml_tensor *input_ids, ggml_tensor *images,
+                         const std::vector<int> &input_ids_vec, int n_past) const {
         ggml_context *ctx = mctx->ctx_b.get();
 
-        const int qlen = input_ids->ne[0];
+        ggml_tensor *hidden_states = forward_embeddings(mctx, input_ids, images, input_ids_vec, n_past);
+
+        const int qlen = hidden_states->ne[1];
         const int kvlen = layers.front().attention.num_virtual_tokens + n_past + qlen;
 
         ggml_tensor *position_ids = pos_ids_alloc_(ctx, qlen);
@@ -568,7 +706,6 @@ class BasicModel {
             ggml_set_input(attention_mask);
         }
 
-        ggml_tensor *hidden_states = word_embeddings.forward(mctx, input_ids);
         for (const auto &layer : layers) {
             hidden_states = layer.forward(mctx, hidden_states, attention_mask, position_ids, n_past);
         }
@@ -577,9 +714,15 @@ class BasicModel {
         return hidden_states;
     }
 
+    virtual ggml_tensor *forward_embeddings(ModelContext *mctx, ggml_tensor *input_ids, ggml_tensor *images,
+                                            const std::vector<int> &input_ids_vec, int n_past) const {
+        CHATGLM_CHECK(images == nullptr) << "unimplemented";
+        return word_embeddings.forward(mctx, input_ids);
+    }
+
     void load_prefix_cache(const ModelConfig &config, ggml_tensor *past_key_values) {
         // past_key_values: [l * 2, #h, v, d]
-        ModelContext mctx(config.dtype);
+        ModelContext mctx;
 
         ggml_tensor *backend_past_key_values = ggml_new_tensor(mctx.ctx_kv.get(), past_key_values->type,
                                                                ggml_n_dims(past_key_values), past_key_values->ne);
@@ -619,10 +762,11 @@ class BasicModel {
         std::vector<Block> layers;
         layers.reserve(config.num_hidden_layers);
         for (int layer_id = 0; layer_id < config.num_hidden_layers; layer_id++) {
-            layers.emplace_back(mctx, config.hidden_size, config.num_attention_heads, config.num_key_value_heads,
-                                config.intermediate_size, config.max_length, config.norm_eps, config.hidden_act,
-                                config.use_qkv_bias, config.use_dense_bias, config.interleaved_qkv, config.rope_type,
-                                config.rope_theta, config.attn_mask_type, config.num_virtual_tokens);
+            layers.emplace_back(mctx, config.dtype, config.hidden_size, config.num_attention_heads,
+                                config.num_key_value_heads, config.intermediate_size, config.max_length,
+                                config.norm_eps, config.hidden_act, config.use_qkv_bias, config.use_dense_bias,
+                                config.interleaved_qkv, config.rope_type, config.rope_theta, config.attn_mask_type,
+                                config.num_virtual_tokens, true);
         }
         mctx->buf_kv =
             unique_ggml_backend_buffer_t(ggml_backend_alloc_ctx_tensors(mctx->ctx_kv.get(), mctx->backend.get()));
@@ -785,18 +929,22 @@ class BaseModelForCausalLM {
 
     virtual void load_state_dict(const StateDict &sd) = 0;
 
-    virtual ggml_tensor *forward(ModelContext *mctx, ggml_tensor *input_ids, int n_past, int n_ctx,
-                                 bool is_decoding) const = 0;
+    virtual ggml_tensor *forward(ModelContext *mctx, ggml_tensor *input_ids, ggml_tensor *images,
+                                 const std::vector<int> &input_ids_vec, int n_past, bool is_decoding) const = 0;
 
-    virtual void set_graph_inputs(int qlen, int n_past, int n_ctx) const = 0;
+    virtual void set_graph_inputs(const std::vector<int> &input_ids, const std::optional<Image> &image, int n_past,
+                                  int n_ctx) const = 0;
 
-    ggml_tensor *forward_graph_compute(const std::vector<int> &input_ids, int n_past, int n_ctx, bool is_decoding);
+    virtual int count_tokens(const std::vector<int> &input_ids, const std::optional<Image> &image) const = 0;
 
-    std::vector<int> generate(const std::vector<int> &input_ids, const GenerationConfig &gen_config,
-                              BaseStreamer *streamer = nullptr);
+    ggml_tensor *forward_graph_compute(const std::vector<int> &input_ids, const std::optional<Image> &image, int n_past,
+                                       int n_ctx, bool is_decoding);
 
-    int generate_next_token(const std::vector<int> &input_ids, const GenerationConfig &gen_config, int n_past,
-                            int n_ctx);
+    std::vector<int> generate(const std::vector<int> &input_ids, const std::optional<Image> &image,
+                              const GenerationConfig &gen_config, BaseStreamer *streamer = nullptr);
+
+    int generate_next_token(const std::vector<int> &input_ids, const std::optional<Image> &image,
+                            const GenerationConfig &gen_config, int n_past, int n_ctx);
 
     // logits processor
     static void sampling_repetition_penalty(float *first, float *last, const std::vector<int> &input_ids,
@@ -820,28 +968,57 @@ class BasicModelForCausalLM : public BaseModelForCausalLM {
   protected:
     BasicModelForCausalLM(const ModelConfig &config)
         : BaseModelForCausalLM(config), transformer(mctx_.get(), config),
-          lm_head(mctx_.get(), config.hidden_size, config.vocab_size, false) {
+          lm_head(mctx_.get(), config.dtype, config.hidden_size, config.vocab_size, false) {
         if (config.tie_word_embeddings) {
             lm_head.weight = transformer.word_embeddings.weight;
         }
     }
 
   public:
-    ggml_tensor *forward(ModelContext *mctx, ggml_tensor *input_ids, int n_past, int n_ctx,
-                         bool is_decoding) const override {
-        ggml_tensor *transformer_outputs = transformer.forward(mctx, input_ids, n_past);
+    ggml_tensor *forward(ModelContext *mctx, ggml_tensor *input_ids, ggml_tensor *images,
+                         const std::vector<int> &input_ids_vec, int n_past, bool is_decoding) const override {
+        ggml_tensor *transformer_outputs = transformer.forward(mctx, input_ids, images, input_ids_vec, n_past);
         // NOTE: only compute next token logits for decoding
-        if (is_decoding && input_ids->ne[0] > 1) {
-            transformer_outputs =
-                ggml_view_1d(mctx->ctx_b.get(), transformer_outputs, config.hidden_size,
-                             (input_ids->ne[0] - 1) * config.hidden_size * ggml_element_size(transformer_outputs));
+        if (is_decoding && transformer_outputs->ne[1] > 1) {
+            transformer_outputs = ggml_view_1d(mctx->ctx_b.get(), transformer_outputs, transformer_outputs->ne[0],
+                                               (transformer_outputs->ne[1] - 1) * transformer_outputs->nb[1]);
         }
         ggml_tensor *lm_logits = lm_head.forward(mctx, transformer_outputs);
         return lm_logits;
     }
 
+    void set_graph_inputs(const std::vector<int> &input_ids, const std::optional<Image> &image, int n_past,
+                          int n_ctx) const override {
+        transformer.set_graph_inputs(mctx_->gf, input_ids, image, n_past, n_ctx);
+    }
+
+    int count_tokens(const std::vector<int> &input_ids, const std::optional<Image> &image) const override {
+        CHATGLM_CHECK(!image) << "unimplemented";
+        return input_ids.size();
+    }
+
     void load_prefix_cache(ggml_tensor *past_key_values) { transformer.load_prefix_cache(config, past_key_values); }
 
+  protected:
+    void alloc_weight_context(const ggml_backend_buffer_t sd_buf) const {
+        void *sd_buf_base = ggml_backend_buffer_get_base(sd_buf);
+        const size_t sd_buf_size = ggml_backend_buffer_get_size(sd_buf);
+        if (ggml_backend_is_cpu(mctx_->backend.get())) {
+            mctx_->buf_w = unique_ggml_backend_buffer_t(ggml_backend_cpu_buffer_from_ptr(sd_buf_base, sd_buf_size));
+        }
+#ifdef GGML_USE_METAL
+        else if (ggml_backend_is_metal(mctx_->backend.get())) {
+            const size_t max_size = ggml_get_max_tensor_size(mctx_->ctx_w.get());
+            mctx_->buf_w =
+                unique_ggml_backend_buffer_t(ggml_backend_metal_buffer_from_ptr(sd_buf_base, sd_buf_size, max_size));
+        }
+#endif
+        else {
+            mctx_->buf_w =
+                unique_ggml_backend_buffer_t(ggml_backend_alloc_ctx_tensors(mctx_->ctx_w.get(), mctx_->backend.get()));
+        }
+    }
+
   public:
     Model transformer;
     Linear lm_head;
@@ -879,16 +1056,17 @@ class ChatGLMTokenizer : public BaseTokenizer {
 class GLMBlock : public BasicBlock<LayerNorm, BasicMLP> {
   public:
     GLMBlock() = default;
-    GLMBlock(ModelContext *mctx, int hidden_size, int num_attention_heads, int num_key_value_heads,
+
+    GLMBlock(ModelContext *mctx, ggml_type dtype, int hidden_size, int num_attention_heads, int num_key_value_heads,
              int intermediate_size, int max_length, float norm_eps, ActivationType hidden_act, bool use_qkv_bias,
              bool use_dense_bias, bool interleaved_qkv, RopeType rope_type, float rope_theta,
-             AttentionMaskType attn_mask_type, int num_virtual_tokens)
-        : BasicBlock(LayerNorm(mctx, hidden_size, false, norm_eps),
-                     BasicAttention(mctx, hidden_size, num_attention_heads, num_attention_heads, max_length,
+             AttentionMaskType attn_mask_type, int num_virtual_tokens, bool use_cache)
+        : BasicBlock(LayerNorm(mctx, hidden_size, norm_eps),
+                     BasicAttention(mctx, dtype, hidden_size, num_attention_heads, num_attention_heads, max_length,
                                     use_qkv_bias, use_dense_bias, interleaved_qkv, rope_type, rope_theta,
-                                    attn_mask_type, num_virtual_tokens),
-                     LayerNorm(mctx, hidden_size, false, norm_eps),
-                     BasicMLP(mctx, hidden_size, intermediate_size, hidden_act)),
+                                    attn_mask_type, num_virtual_tokens, use_cache),
+                     LayerNorm(mctx, hidden_size, norm_eps),
+                     BasicMLP(mctx, dtype, hidden_size, intermediate_size, hidden_act)),
           alpha(std::sqrt(2.f * 28)) {}
 
     ggml_tensor *forward(ModelContext *mctx, ggml_tensor *hidden_states, ggml_tensor *attention_mask,
@@ -898,18 +1076,22 @@ class GLMBlock : public BasicBlock<LayerNorm, BasicMLP> {
     float alpha;
 };
 
-using ChatGLMModel = BasicModel<GLMBlock, LayerNorm, BasicAttentionMaskAllocator, GLMPositionIdsAllocator>;
+class ChatGLMModel : public BasicModel<GLMBlock, LayerNorm, BasicAttentionMaskAllocator, GLMPositionIdsAllocator> {
+  public:
+    ChatGLMModel() = default;
+
+    ChatGLMModel(ModelContext *mctx, const ModelConfig &config) : BasicModel(mctx, config) {}
+
+    void set_graph_inputs(ggml_cgraph *gf, const std::vector<int> &input_ids, const std::optional<Image> &image,
+                          int n_past, int n_ctx) const;
+};
 
 class ChatGLMForCausalLM : public BasicModelForCausalLM<ChatGLMModel> {
   public:
-    ChatGLMForCausalLM(const ModelConfig &config);
+    ChatGLMForCausalLM(const ModelConfig &config) : BasicModelForCausalLM(config) {}
 
     void load_state_dict(const StateDict &sd) override;
 
-    void set_graph_inputs(int qlen, int n_past, int n_ctx) const override;
-
-    static void set_graph_inputs(ggml_cgraph *gf, int qlen, int n_past, int n_ctx);
-
   private:
     StateDict state_dict() const;
 };
@@ -942,17 +1124,23 @@ class ChatGLM2Tokenizer : public BaseTokenizer {
 
 using GLM2Block = BasicBlock<RMSNorm, BasicGLU>;
 
-using ChatGLM2Model = BasicModel<GLM2Block, RMSNorm, NoopAttentionMaskAllocator, BasicPositionIdsAllocator>;
+class ChatGLM2Model : public BasicModel<GLM2Block, RMSNorm, NoopAttentionMaskAllocator, BasicPositionIdsAllocator> {
+  public:
+    ChatGLM2Model() = default;
+
+    ChatGLM2Model(ModelContext *mctx, const ModelConfig &config) : BasicModel(mctx, config) {}
+
+    void set_graph_inputs(ggml_cgraph *gf, const std::vector<int> &input_ids, const std::optional<Image> &image,
+                          int n_past, int n_ctx) const;
+};
 
 class ChatGLM2ForCausalLM : public BasicModelForCausalLM<ChatGLM2Model> {
   public:
-    ChatGLM2ForCausalLM(const ModelConfig &config);
+    ChatGLM2ForCausalLM(const ModelConfig &config) : BasicModelForCausalLM(config) {}
 
     void load_state_dict(const StateDict &sd) override;
 
-    void set_graph_inputs(int qlen, int n_past, int n_ctx) const override;
-
-    static void set_graph_inputs(ggml_cgraph *gf, int qlen, int n_past, int n_ctx);
+    static void load_state_dict(ModelContext *mctx, StateDict &dst, const StateDict &src);
 
   private:
     StateDict state_dict() const;
@@ -1052,7 +1240,7 @@ class ChatGLM4Tokenizer : public BaseTokenizer {
 
   public:
     TiktokenCoreBPE core_bpe;
-    // int eos_token_id;
+    int eos_token_id;
     // int mask_token_id;
     int gmask_token_id;
     // int smask_token_id;
@@ -1062,20 +1250,159 @@ class ChatGLM4Tokenizer : public BaseTokenizer {
     int user_token_id;
     int assistant_token_id;
     int observation_token_id;
+    int boi_token_id;
+    int eoi_token_id;
 };
 
 using ChatGLM4Model = ChatGLM2Model;
 
 using ChatGLM4ForCausalLM = ChatGLM2ForCausalLM;
 
+// ===== GLM4V-9B =====
+
+class Conv2d {
+  public:
+    Conv2d() : weight(nullptr), bias(nullptr), stride(0) {}
+
+    Conv2d(ModelContext *mctx, int in_channels, int out_channels, int kernel_size, int stride)
+        : weight(ggml_new_tensor_4d(mctx->ctx_w.get(), GGML_TYPE_F16, kernel_size, kernel_size, in_channels,
+                                    out_channels)),
+          bias(ggml_new_tensor_3d(mctx->ctx_w.get(), GGML_TYPE_F32, 1, 1, out_channels)), stride(stride) {}
+
+    int in_channels() const { return weight->ne[2]; }
+
+    int out_channels() const { return weight->ne[3]; }
+
+    ggml_tensor *forward(ModelContext *mctx, ggml_tensor *input) const;
+
+  public:
+    ggml_tensor *weight;
+    ggml_tensor *bias;
+    int stride;
+};
+
+class PatchEmbedding {
+  public:
+    PatchEmbedding() = default;
+
+    PatchEmbedding(ModelContext *mctx, int in_channels, int hidden_size, int patch_size, int num_positions)
+        : proj(mctx, in_channels, hidden_size, patch_size, patch_size),
+          position_embedding(mctx, GGML_TYPE_F32, num_positions, hidden_size),
+          cls_embedding(ggml_new_tensor_1d(mctx->ctx_w.get(), GGML_TYPE_F16, hidden_size)) {}
+
+    int hidden_size() const { return proj.out_channels(); }
+
+    int num_positions() const { return position_embedding.num_embeddings(); }
+
+    ggml_tensor *forward(ModelContext *mctx, ggml_tensor *input) const;
+
+  public:
+    Conv2d proj;
+    Embedding position_embedding;
+    ggml_tensor *cls_embedding = nullptr; // [H]
+};
+
+class EVA2CLIPBlock : public BasicBlock<LayerNorm, BasicMLP> {
+  public:
+    EVA2CLIPBlock() = default;
+
+    EVA2CLIPBlock(ModelContext *mctx, ggml_type dtype, int hidden_size, int num_attention_heads,
+                  int num_key_value_heads, int intermediate_size, int max_length, float norm_eps,
+                  ActivationType hidden_act, bool use_qkv_bias, bool use_dense_bias, bool interleaved_qkv,
+                  RopeType rope_type, float rope_theta, AttentionMaskType attn_mask_type, int num_virtual_tokens,
+                  bool use_cache)
+        : BasicBlock(mctx, dtype, hidden_size, num_attention_heads, num_key_value_heads, intermediate_size, max_length,
+                     norm_eps, hidden_act, use_qkv_bias, use_dense_bias, interleaved_qkv, rope_type, rope_theta,
+                     attn_mask_type, num_virtual_tokens, use_cache) {}
+
+    ggml_tensor *forward(ModelContext *mctx, ggml_tensor *hidden_states, ggml_tensor *attention_mask,
+                         ggml_tensor *position_ids, int n_past) const;
+};
+
+class EVA2CLIPTransformer {
+  public:
+    EVA2CLIPTransformer() = default;
+
+    EVA2CLIPTransformer(ModelContext *mctx, const VisionModelConfig &config);
+
+    ggml_tensor *forward(ModelContext *mctx, ggml_tensor *hidden_states) const;
+
+  public:
+    std::vector<EVA2CLIPBlock> layers;
+};
+
+class EVA2CLIPModel {
+  public:
+    EVA2CLIPModel() = default;
+
+    EVA2CLIPModel(ModelContext *mctx, const ModelConfig &config)
+        : patch_embedding(mctx, config.vision.in_channels, config.vision.hidden_size, config.vision.patch_size,
+                          config.vision.num_positions),
+          transformer(mctx, config.vision), conv(mctx, config.vision.hidden_size, config.hidden_size, 2, 2),
+          linear_proj(mctx, config.vision.dtype, config.hidden_size, config.hidden_size, false),
+          norm1(mctx, config.hidden_size),
+          glu(mctx, config.vision.dtype, config.hidden_size, config.intermediate_size, ActivationType::SILU),
+          boi(ggml_new_tensor_1d(mctx->ctx_w.get(), GGML_TYPE_F16, config.hidden_size)),
+          eoi(ggml_new_tensor_1d(mctx->ctx_w.get(), GGML_TYPE_F16, config.hidden_size)),
+          scaling_factor(config.vision.scaling_factor) {}
+
+    ggml_tensor *forward(ModelContext *mctx, ggml_tensor *input) const;
+
+  public:
+    PatchEmbedding patch_embedding;
+    EVA2CLIPTransformer transformer;
+    Conv2d conv;
+    Linear linear_proj;
+    LayerNorm norm1;
+    BasicGLU glu;
+    ggml_tensor *boi = nullptr;
+    ggml_tensor *eoi = nullptr;
+    float scaling_factor = 0.f;
+};
+
+class ChatGLM4VModel : public ChatGLM4Model {
+  public:
+    ChatGLM4VModel() = default;
+
+    ChatGLM4VModel(ModelContext *mctx, const ModelConfig &config)
+        : ChatGLM4Model(mctx, config), config(config), vision(mctx, config) {}
+
+    ggml_tensor *forward_embeddings(ModelContext *mctx, ggml_tensor *input_ids, ggml_tensor *images,
+                                    const std::vector<int> &input_ids_vec, int n_past) const override;
+
+    int num_vision_tokens() const {
+        auto square = [](int x) { return x * x; };
+        return square(config.vision.image_size / config.vision.patch_size / 2);
+    }
+
+    void set_graph_inputs(ggml_cgraph *gf, const std::vector<int> &input_ids, const std::optional<Image> &image,
+                          int n_past, int n_ctx) const;
+
+  public:
+    ModelConfig config;
+    EVA2CLIPModel vision;
+};
+
+class ChatGLM4VForCausalLM : public BasicModelForCausalLM<ChatGLM4VModel> {
+  public:
+    ChatGLM4VForCausalLM(const ModelConfig &config) : BasicModelForCausalLM(config) {}
+
+    int count_tokens(const std::vector<int> &input_ids, const std::optional<Image> &image) const override;
+
+    void load_state_dict(const StateDict &sd) override;
+
+  private:
+    StateDict state_dict() const;
+};
+
 // ===== pipeline =====
 
 class Pipeline {
   public:
     Pipeline(const std::string &path, int max_length = -1);
 
-    std::vector<int> generate(const std::vector<int> &input_ids, const GenerationConfig &gen_config,
-                              BaseStreamer *streamer = nullptr) const;
+    std::vector<int> generate(const std::vector<int> &input_ids, const std::optional<Image> &image,
+                              const GenerationConfig &gen_config, BaseStreamer *streamer = nullptr) const;
 
     std::string generate(const std::string &prompt, const GenerationConfig &gen_config,
                          BaseStreamer *streamer = nullptr) const;
diff --git a/chatglm_cpp/_C.pyi b/chatglm_cpp/_C.pyi
index cae0f2f1..18191bda 100644
--- a/chatglm_cpp/_C.pyi
+++ b/chatglm_cpp/_C.pyi
@@ -3,9 +3,12 @@ ChatGLM.cpp python binding
 """
 from __future__ import annotations
 import typing
-__all__ = ['BaseModelForCausalLM', 'BaseTokenizer', 'ChatGLM2ForCausalLM', 'ChatGLM2Tokenizer', 'ChatGLM3Tokenizer', 'ChatGLM4Tokenizer', 'ChatGLMForCausalLM', 'ChatGLMTokenizer', 'ChatMessage', 'CodeMessage', 'FunctionMessage', 'GenerationConfig', 'ModelConfig', 'ModelType', 'Pipeline', 'ToolCallMessage']
+import typing_extensions
+__all__ = ['BaseModelForCausalLM', 'BaseTokenizer', 'ChatGLM2ForCausalLM', 'ChatGLM2Tokenizer', 'ChatGLM3Tokenizer', 'ChatGLM4Tokenizer', 'ChatGLMForCausalLM', 'ChatGLMTokenizer', 'ChatMessage', 'CodeMessage', 'FunctionMessage', 'GenerationConfig', 'Image', 'ModelConfig', 'ModelType', 'Pipeline', 'ToolCallMessage', 'VisionModelConfig']
 class BaseModelForCausalLM:
-    def generate_next_token(self, input_ids: list[int], gen_config: GenerationConfig, n_past: int, n_ctx: int) -> int:
+    def count_tokens(self, input_ids: list[int], image: Image | None) -> int:
+        ...
+    def generate_next_token(self, input_ids: list[int], image: Image | None, gen_config: GenerationConfig, n_past: int, n_ctx: int) -> int:
         ...
     @property
     def config(self) -> ModelConfig:
@@ -37,9 +40,10 @@ class ChatMessage:
     ROLE_SYSTEM: typing.ClassVar[str] = 'system'
     ROLE_USER: typing.ClassVar[str] = 'user'
     content: str
+    image: Image | None
     role: str
     tool_calls: list[ToolCallMessage]
-    def __init__(self, role: str, content: str, tool_calls: list[ToolCallMessage] = []) -> None:
+    def __init__(self, role: str, content: str, image: Image | None = None, tool_calls: list[ToolCallMessage] = []) -> None:
         ...
     def __repr__(self) -> str:
         ...
@@ -69,6 +73,25 @@ class GenerationConfig:
     top_p: float
     def __init__(self, max_length: int = 2048, max_new_tokens: int = -1, max_context_length: int = 512, do_sample: bool = True, top_k: int = 0, top_p: float = 0.7, temperature: float = 0.95, repetition_penalty: float = 1.0) -> None:
         ...
+class Image:
+    def __init__(self, arg0: typing_extensions.Buffer) -> None:
+        ...
+    def __repr__(self) -> str:
+        ...
+    def __str__(self) -> str:
+        ...
+    @property
+    def channels(self) -> int:
+        ...
+    @property
+    def height(self) -> int:
+        ...
+    @property
+    def pixels(self) -> list[int]:
+        ...
+    @property
+    def width(self) -> int:
+        ...
 class ModelConfig:
     @property
     def bos_token_id(self) -> int:
@@ -113,6 +136,9 @@ class ModelConfig:
     def sep_token_id(self) -> int:
         ...
     @property
+    def vision(self) -> VisionModelConfig:
+        ...
+    @property
     def vocab_size(self) -> int:
         ...
 class ModelType:
@@ -175,3 +201,34 @@ class ToolCallMessage:
         ...
     def __str__(self) -> str:
         ...
+class VisionModelConfig:
+    @property
+    def hidden_size(self) -> int:
+        ...
+    @property
+    def image_size(self) -> int:
+        ...
+    @property
+    def in_channels(self) -> int:
+        ...
+    @property
+    def intermediate_size(self) -> int:
+        ...
+    @property
+    def norm_eps(self) -> float:
+        ...
+    @property
+    def num_attention_heads(self) -> int:
+        ...
+    @property
+    def num_hidden_layers(self) -> int:
+        ...
+    @property
+    def num_positions(self) -> int:
+        ...
+    @property
+    def patch_size(self) -> int:
+        ...
+    @property
+    def scaling_factor(self) -> float:
+        ...
diff --git a/chatglm_cpp/__init__.py b/chatglm_cpp/__init__.py
index 48da0b5a..209bf241 100644
--- a/chatglm_cpp/__init__.py
+++ b/chatglm_cpp/__init__.py
@@ -4,7 +4,7 @@
 from typing import Any, Dict, Iterator, List, Optional, Union
 
 import chatglm_cpp._C as _C
-from chatglm_cpp._C import ChatMessage
+from chatglm_cpp._C import ChatMessage, Image
 
 __version__ = "0.4.1"
 
@@ -72,9 +72,16 @@ def chat(
             temperature=temperature,
             repetition_penalty=repetition_penalty,
         )
+
+        image = None
+        for msg in messages:
+            if msg.image is not None:
+                image = msg.image
+                break
+
         if stream:
-            return self._stream_chat(input_ids=input_ids, gen_config=gen_config)
-        return self._sync_chat(input_ids=input_ids, gen_config=gen_config)
+            return self._stream_chat(input_ids=input_ids, image=image, gen_config=gen_config)
+        return self._sync_chat(input_ids=input_ids, image=image, gen_config=gen_config)
 
     def generate(
         self,
@@ -105,26 +112,30 @@ def generate(
             return self._stream_generate(input_ids=input_ids, gen_config=gen_config)
         return self._sync_generate(input_ids=input_ids, gen_config=gen_config)
 
-    def _stream_generate_ids(self, input_ids: List[int], gen_config: _C.GenerationConfig) -> Iterator[int]:
+    def _stream_generate_ids(
+        self, input_ids: List[int], image: Optional[Image], gen_config: _C.GenerationConfig
+    ) -> Iterator[int]:
         input_ids = input_ids.copy()
         n_past = 0
         n_ctx = len(input_ids)
         max_new_tokens = gen_config.max_new_tokens if gen_config.max_new_tokens > 0 else gen_config.max_length
 
         while len(input_ids) < min(gen_config.max_length, n_ctx + max_new_tokens):
-            next_token_id = self.model.generate_next_token(input_ids, gen_config, n_past, n_ctx)
+            next_token_id = self.model.generate_next_token(input_ids, image, gen_config, n_past, n_ctx)
             yield next_token_id
-            n_past = len(input_ids)
+            n_past = self.model.count_tokens(input_ids, image)
             input_ids.append(next_token_id)
 
             if next_token_id in [self.model.config.eos_token_id, *self.model.config.extra_eos_token_ids]:
                 break
 
-    def _stream_chat(self, input_ids: List[int], gen_config: _C.GenerationConfig) -> Iterator[DeltaMessage]:
+    def _stream_chat(
+        self, input_ids: List[int], image: Optional[Image], gen_config: _C.GenerationConfig
+    ) -> Iterator[DeltaMessage]:
         token_cache = []
         print_len = 0
         print_token_len = 0
-        for next_token_id in self._stream_generate_ids(input_ids=input_ids, gen_config=gen_config):
+        for next_token_id in self._stream_generate_ids(input_ids=input_ids, image=image, gen_config=gen_config):
             token_cache.append(next_token_id)
 
             try:
@@ -157,15 +168,17 @@ def _stream_generate(self, input_ids: List[int], gen_config: _C.GenerationConfig
         for msg in self._stream_chat(input_ids=input_ids, gen_config=gen_config):
             yield msg.content
 
-    def _sync_generate_ids(self, input_ids: List[int], gen_config: _C.GenerationConfig) -> List[int]:
-        return list(self._stream_generate_ids(input_ids=input_ids, gen_config=gen_config))
+    def _sync_generate_ids(
+        self, input_ids: List[int], image: Optional[Image], gen_config: _C.GenerationConfig
+    ) -> List[int]:
+        return list(self._stream_generate_ids(input_ids=input_ids, image=image, gen_config=gen_config))
 
-    def _sync_generate(self, input_ids: List[int], gen_config: _C.GenerationConfig) -> str:
-        output_ids = self._sync_generate_ids(input_ids=input_ids, gen_config=gen_config)
+    def _sync_generate(self, input_ids: List[int], image: Optional[Image], gen_config: _C.GenerationConfig) -> str:
+        output_ids = self._sync_generate_ids(input_ids=input_ids, image=image, gen_config=gen_config)
         return self.tokenizer.decode(output_ids)
 
-    def _sync_chat(self, input_ids: List[int], gen_config: _C.GenerationConfig) -> ChatMessage:
-        output_ids = self._sync_generate_ids(input_ids=input_ids, gen_config=gen_config)
+    def _sync_chat(self, input_ids: List[int], image: Optional[Image], gen_config: _C.GenerationConfig) -> ChatMessage:
+        output_ids = self._sync_generate_ids(input_ids=input_ids, image=image, gen_config=gen_config)
         return self.tokenizer.decode_message(output_ids)
 
     def merge_streaming_messages(self, chunks: List[DeltaMessage]) -> ChatMessage:
diff --git a/chatglm_cpp/convert.py b/chatglm_cpp/convert.py
index f24a8133..ccc25691 100644
--- a/chatglm_cpp/convert.py
+++ b/chatglm_cpp/convert.py
@@ -8,7 +8,7 @@
 import sys
 from enum import Enum
 from pathlib import Path
-from typing import BinaryIO, Optional
+from typing import BinaryIO, NamedTuple, Optional
 
 import torch
 import torch.nn.functional as F
@@ -49,6 +49,12 @@ class ModelType(Enum):
     CHATGLM2 = 2
     CHATGLM3 = 3
     CHATGLM4 = 4
+    CHATGLM4V = 1004
+
+
+class WeightMeta(NamedTuple):
+    name: str
+    dtype: ModelType
 
 
 def quantize_q8_0(tensor: torch.Tensor) -> torch.Tensor:
@@ -162,51 +168,36 @@ def dump_tensor(f, name: str, tensor: torch.Tensor, ggml_type: GGMLType):
     tensor.numpy().tofile(f)
 
 
-def dump_state_dict(f, weight_names, state_dict, quantization_bit, ggml_type):
+def dump_state_dict(f, weight_meta, state_dict, quantization_bit):
     tensor_info = []
-    for name in tqdm(weight_names, desc="Processing model states"):
-        tensor = state_dict[name]
-        if name == "past_key_values":
-            tensor_ggml_type = GGMLType.F16
-        elif tensor.ndim == 2:
-            # 2d weight: should quantize it if needed
-
-            # step 1: de-quantize it back to float32
-            if tensor.dtype == torch.int8:
-                assert quantization_bit in [4, 8]
-                scale = state_dict[f"{name}_scale"].float()  # channel-wise scale
-
-                if quantization_bit == 4:
-                    # convert int4 weight to int8
-                    low_bits = ((tensor << 4) & 0xF0) >> 4
-                    high_bits = (tensor & 0xF0) >> 4
-                    tensor = torch.stack((high_bits, low_bits), dim=-1).view(tensor.shape[0], -1)
-                tensor = tensor * scale[:, None]
-            else:
-                tensor = tensor.float()
-
-            # step 2: quantize it into ggml format
-            tensor_ggml_type = ggml_type
-        else:
-            # 1d weight: convert it to float32
-            assert tensor.ndim == 1
-            tensor = tensor.float()
-            tensor_ggml_type = GGMLType.F32
-
-        dump_tensor(f, name, tensor, tensor_ggml_type)
-        tensor_info.append((name, tuple(tensor.shape), tensor_ggml_type.name))
+    for meta in tqdm(weight_meta, desc="Processing model states"):
+        tensor = state_dict[meta.name]
+        if tensor.ndim == 2 and tensor.dtype == torch.int8:
+            # de-quantize gemm weight back to float32
+            assert quantization_bit in [4, 8]
+            scale = state_dict[f"{meta.name}_scale"].float()  # channel-wise scale
+
+            if quantization_bit == 4:
+                # convert int4 weight to int8
+                low_bits = ((tensor << 4) & 0xF0) >> 4
+                high_bits = (tensor & 0xF0) >> 4
+                tensor = torch.stack((high_bits, low_bits), dim=-1).view(tensor.shape[0], -1)
+            tensor = tensor * scale[:, None]
+
+        dump_tensor(f, meta.name, tensor, meta.dtype)
+        tensor_info.append((meta.name, tuple(tensor.shape), meta.dtype.name))
 
     print(tabulate(tensor_info, headers=["name", "shape", "dtype"], tablefmt="psql"))
 
 
 class BaseConverter:
     @classmethod
-    def convert(cls, f, model, tokenizer, ggml_type):
+    def convert(cls, f, model, tokenizer, ggml_type, vision_type=None):
         f.write(b"ggml")  # magic
         f.write(struct.pack("i", cls.MODEL_TYPE.value))  # model type
-        cls.dump_config(f, model.config, ggml_type)
+        cls.dump_config(f, model.config, ggml_type, vision_type)
         cls.dump_tokenizer(f, tokenizer)
-        cls.dump_model(f, model, ggml_type)
+        cls.dump_model(f, model, ggml_type, vision_type)
 
 
 def get_prefix_cache(prefix_encoder, pre_seq_len, num_layers, num_key_value_heads, head_size):
@@ -226,7 +217,7 @@ class ChatGLMConverter(BaseConverter):
     MODEL_TYPE = ModelType.CHATGLM
 
     @staticmethod
-    def dump_config(f, config, ggml_type):
+    def dump_config(f, config, ggml_type, vision_type):
         assert config.position_encoding_2d, "unimplemented: position_encoding_2d should be True"
         assert (
             config.inner_hidden_size == 4 * config.hidden_size
@@ -257,39 +248,39 @@ def dump_tokenizer(f, tokenizer):
         f.write(serialized_model_proto)
 
     @staticmethod
-    def dump_model(f, model, ggml_type):
+    def dump_model(f, model, ggml_type, vision_type):
         assert torch.allclose(
             model.state_dict()["transformer.word_embeddings.weight"], model.state_dict()["lm_head.weight"]
         ), "unimplemented: lm_head weight must be tied to input embedding"
 
-        weight_names = ["transformer.word_embeddings.weight"]
+        weight_meta = [WeightMeta("transformer.word_embeddings.weight", ggml_type)]
         for i in range(model.config.num_layers):
-            weight_names += [
-                f"transformer.layers.{i}.input_layernorm.weight",
-                f"transformer.layers.{i}.input_layernorm.bias",
-                f"transformer.layers.{i}.attention.query_key_value.weight",
-                f"transformer.layers.{i}.attention.query_key_value.bias",
-                f"transformer.layers.{i}.attention.dense.weight",
-                f"transformer.layers.{i}.attention.dense.bias",
-                f"transformer.layers.{i}.post_attention_layernorm.weight",
-                f"transformer.layers.{i}.post_attention_layernorm.bias",
-                f"transformer.layers.{i}.mlp.dense_h_to_4h.weight",
-                f"transformer.layers.{i}.mlp.dense_h_to_4h.bias",
-                f"transformer.layers.{i}.mlp.dense_4h_to_h.weight",
-                f"transformer.layers.{i}.mlp.dense_4h_to_h.bias",
+            weight_meta += [
+                WeightMeta(f"transformer.layers.{i}.input_layernorm.weight", GGMLType.F32),
+                WeightMeta(f"transformer.layers.{i}.input_layernorm.bias", GGMLType.F32),
+                WeightMeta(f"transformer.layers.{i}.attention.query_key_value.weight", ggml_type),
+                WeightMeta(f"transformer.layers.{i}.attention.query_key_value.bias", GGMLType.F32),
+                WeightMeta(f"transformer.layers.{i}.attention.dense.weight", ggml_type),
+                WeightMeta(f"transformer.layers.{i}.attention.dense.bias", GGMLType.F32),
+                WeightMeta(f"transformer.layers.{i}.post_attention_layernorm.weight", GGMLType.F32),
+                WeightMeta(f"transformer.layers.{i}.post_attention_layernorm.bias", GGMLType.F32),
+                WeightMeta(f"transformer.layers.{i}.mlp.dense_h_to_4h.weight", ggml_type),
+                WeightMeta(f"transformer.layers.{i}.mlp.dense_h_to_4h.bias", GGMLType.F32),
+                WeightMeta(f"transformer.layers.{i}.mlp.dense_4h_to_h.weight", ggml_type),
+                WeightMeta(f"transformer.layers.{i}.mlp.dense_4h_to_h.bias", GGMLType.F32),
             ]
-        weight_names += [
-            "transformer.final_layernorm.weight",
-            "transformer.final_layernorm.bias",
+        weight_meta += [
+            WeightMeta("transformer.final_layernorm.weight", GGMLType.F32),
+            WeightMeta("transformer.final_layernorm.bias", GGMLType.F32),
         ]
-        dump_state_dict(f, weight_names, model.state_dict(), model.config.quantization_bit, ggml_type)
+        dump_state_dict(f, weight_meta, model.state_dict(), model.config.quantization_bit)
 
 
 class ChatGLM2Converter(BaseConverter):
     MODEL_TYPE = ModelType.CHATGLM2
 
     @classmethod
-    def dump_config(cls, f, config, ggml_type):
+    def dump_config(cls, f, config, ggml_type, vision_type):
         assert config.add_bias_linear is False, "unimplemented: add_bias_linear must be false"
         assert config.add_qkv_bias is True, "unimplemented: add_qkv_bias must be true"
         assert (
@@ -329,12 +320,12 @@ def dump_tokenizer(f, tokenizer):
         f.write(serialized_model_proto)
 
     @staticmethod
-    def dump_model(f, model, ggml_type):
+    def dump_model(f, model, ggml_type, vision_type):
         config = model.config
 
         state_dict = model.state_dict()
 
-        weight_names = []
+        weight_meta = []
         if getattr(config, "pre_seq_len", None) is not None and config.pre_seq_len > 0:
             past_key_values = get_prefix_cache(
                 model.transformer.prefix_encoder,
@@ -344,29 +335,28 @@ def dump_model(f, model, ggml_type):
                 config.kv_channels,
             )
             state_dict["past_key_values"] = past_key_values
-            weight_names.append("past_key_values")
+            weight_meta.append(WeightMeta("past_key_values", GGMLType.F16))
 
-        weight_names.append("transformer.embedding.word_embeddings.weight")
+        weight_meta.append(WeightMeta("transformer.embedding.word_embeddings.weight", ggml_type))
         for i in range(config.num_layers):
-            weight_names += [
-                f"transformer.encoder.layers.{i}.input_layernorm.weight",
-                f"transformer.encoder.layers.{i}.self_attention.query_key_value.weight",
-                f"transformer.encoder.layers.{i}.self_attention.query_key_value.bias",
-                f"transformer.encoder.layers.{i}.self_attention.dense.weight",
-                f"transformer.encoder.layers.{i}.post_attention_layernorm.weight",
-                f"transformer.encoder.layers.{i}.mlp.dense_h_to_4h.weight",
-                f"transformer.encoder.layers.{i}.mlp.dense_4h_to_h.weight",
+            weight_meta += [
+                WeightMeta(f"transformer.encoder.layers.{i}.input_layernorm.weight", GGMLType.F32),
+                WeightMeta(f"transformer.encoder.layers.{i}.self_attention.query_key_value.weight", ggml_type),
+                WeightMeta(f"transformer.encoder.layers.{i}.self_attention.query_key_value.bias", GGMLType.F32),
+                WeightMeta(f"transformer.encoder.layers.{i}.self_attention.dense.weight", ggml_type),
+                WeightMeta(f"transformer.encoder.layers.{i}.post_attention_layernorm.weight", GGMLType.F32),
+                WeightMeta(f"transformer.encoder.layers.{i}.mlp.dense_h_to_4h.weight", ggml_type),
+                WeightMeta(f"transformer.encoder.layers.{i}.mlp.dense_4h_to_h.weight", ggml_type),
             ]
-        weight_names += [
-            "transformer.encoder.final_layernorm.weight",
-            "transformer.output_layer.weight",
+        weight_meta += [
+            WeightMeta("transformer.encoder.final_layernorm.weight", GGMLType.F32),
+            WeightMeta("transformer.output_layer.weight", ggml_type),
         ]
         dump_state_dict(
             f,
-            weight_names=weight_names,
+            weight_meta=weight_meta,
             state_dict=state_dict,
             quantization_bit=getattr(config, "quantization_bit", None),
-            ggml_type=ggml_type,
         )
 
 
@@ -384,8 +374,103 @@ def dump_tokenizer(f, tokenizer):
         f.write(vocab_text)
 
 
-def convert(f: BinaryIO, model_name_or_path: str, lora_model_name_or_path: Optional[str] = None, dtype: str = "q4_0"):
+class ChatGLM4VConverter(ChatGLM4Converter):
+    MODEL_TYPE = ModelType.CHATGLM4V
+
+    @classmethod
+    def dump_config(cls, f, config, ggml_type, vision_type):
+        ChatGLM4Converter.dump_config(f, config, ggml_type, vision_type)
+
+        config_values = [
+            vision_type.value,
+            config.vision_config["hidden_size"],
+            config.vision_config["image_size"],
+            config.vision_config["in_channels"],
+            config.vision_config["intermediate_size"],
+            config.vision_config["layer_norm_eps"],
+            config.vision_config["num_heads"],
+            config.vision_config["num_hidden_layers"],
+            config.vision_config["num_positions"],
+            config.vision_config["patch_size"],
+            config.vision_config["scaling_factor"],
+        ]
+
+        f.write(struct.pack("iiiiifiiiif", *config_values))
+
+    @staticmethod
+    def dump_model(f, model, ggml_type, vision_type):
+        config = model.config
+
+        state_dict = model.state_dict()
+
+        # vision
+        weight_meta = [
+            WeightMeta("transformer.vision.patch_embedding.cls_embedding", GGMLType.F16),
+            WeightMeta("transformer.vision.patch_embedding.proj.weight", GGMLType.F16),
+            WeightMeta("transformer.vision.patch_embedding.proj.bias", GGMLType.F32),
+            WeightMeta("transformer.vision.patch_embedding.position_embedding.weight", GGMLType.F32),
+        ]
+        for i in range(config.vision_config["num_hidden_layers"]):
+            weight_meta += [
+                WeightMeta(f"transformer.vision.transformer.layers.{i}.input_layernorm.weight", GGMLType.F32),
+                WeightMeta(f"transformer.vision.transformer.layers.{i}.input_layernorm.bias", GGMLType.F32),
+                WeightMeta(f"transformer.vision.transformer.layers.{i}.attention.query_key_value.weight", vision_type),
+                WeightMeta(f"transformer.vision.transformer.layers.{i}.attention.query_key_value.bias", GGMLType.F32),
+                WeightMeta(f"transformer.vision.transformer.layers.{i}.attention.dense.weight", vision_type),
+                WeightMeta(f"transformer.vision.transformer.layers.{i}.attention.dense.bias", GGMLType.F32),
+                WeightMeta(f"transformer.vision.transformer.layers.{i}.mlp.fc1.weight", vision_type),
+                WeightMeta(f"transformer.vision.transformer.layers.{i}.mlp.fc1.bias", GGMLType.F32),
+                WeightMeta(f"transformer.vision.transformer.layers.{i}.mlp.fc2.weight", vision_type),
+                WeightMeta(f"transformer.vision.transformer.layers.{i}.mlp.fc2.bias", GGMLType.F32),
+                WeightMeta(f"transformer.vision.transformer.layers.{i}.post_attention_layernorm.weight", GGMLType.F32),
+                WeightMeta(f"transformer.vision.transformer.layers.{i}.post_attention_layernorm.bias", GGMLType.F32),
+            ]
+        weight_meta += [
+            WeightMeta("transformer.vision.conv.weight", GGMLType.F16),
+            WeightMeta("transformer.vision.conv.bias", GGMLType.F32),
+            WeightMeta("transformer.vision.linear_proj.linear_proj.weight", vision_type),
+            WeightMeta("transformer.vision.linear_proj.norm1.weight", GGMLType.F32),
+            WeightMeta("transformer.vision.linear_proj.norm1.bias", GGMLType.F32),
+            WeightMeta("transformer.vision.linear_proj.gate_proj.weight", vision_type),
+            WeightMeta("transformer.vision.linear_proj.dense_h_to_4h.weight", vision_type),
+            WeightMeta("transformer.vision.linear_proj.dense_4h_to_h.weight", vision_type),
+            WeightMeta("transformer.vision.boi", GGMLType.F16),
+            WeightMeta("transformer.vision.eoi", GGMLType.F16),
+        ]
+
+        # text
+        weight_meta.append(WeightMeta("transformer.embedding.word_embeddings.weight", ggml_type))
+        for i in range(config.num_layers):
+            weight_meta += [
+                WeightMeta(f"transformer.encoder.layers.{i}.input_layernorm.weight", GGMLType.F32),
+                WeightMeta(f"transformer.encoder.layers.{i}.self_attention.query_key_value.weight", ggml_type),
+                WeightMeta(f"transformer.encoder.layers.{i}.self_attention.query_key_value.bias", GGMLType.F32),
+                WeightMeta(f"transformer.encoder.layers.{i}.self_attention.dense.weight", ggml_type),
+                WeightMeta(f"transformer.encoder.layers.{i}.post_attention_layernorm.weight", GGMLType.F32),
+                WeightMeta(f"transformer.encoder.layers.{i}.mlp.dense_h_to_4h.weight", ggml_type),
+                WeightMeta(f"transformer.encoder.layers.{i}.mlp.dense_4h_to_h.weight", ggml_type),
+            ]
+        weight_meta += [
+            WeightMeta("transformer.encoder.final_layernorm.weight", GGMLType.F32),
+            WeightMeta("transformer.output_layer.weight", ggml_type),
+        ]
+        dump_state_dict(
+            f,
+            weight_meta=weight_meta,
+            state_dict=state_dict,
+            quantization_bit=getattr(config, "quantization_bit", None),
+        )
+
+
+def convert(
+    f: BinaryIO,
+    model_name_or_path: str,
+    lora_model_name_or_path: Optional[str] = None,
+    dtype: str = "q4_0",
+    vision_dtype: str = "f16",
+):
     ggml_type = GGMLType[dtype.upper()]
+    vision_type = GGMLType[vision_dtype.upper()]
 
     tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=True)
 
@@ -415,7 +500,10 @@ def convert(f: BinaryIO, model_name_or_path: str, lora_model_name_or_path: Optio
             if tiktoken is not None and isinstance(tokenizer.tokenizer, tiktoken.Encoding):
                 # TODO: store all eos token ids
                 model.config.eos_token_id = tokenizer.eos_token_id
-                ChatGLM4Converter.convert(f, model, tokenizer, ggml_type)
+                if getattr(model.config, "vision_config", None) is not None:
+                    ChatGLM4VConverter.convert(f, model, tokenizer, ggml_type, vision_type)
+                else:
+                    ChatGLM4Converter.convert(f, model, tokenizer, ggml_type)
             elif "
" in tokenizer.tokenizer.special_tokens:
                 ChatGLM3Converter.convert(f, model, tokenizer, ggml_type)
             else:
@@ -453,10 +541,20 @@ def main():
         choices=["f32", "f16", "q8_0", "q4_0", "q4_1", "q5_0", "q5_1"],
         help="GGML model quantization type",
     )
+    parser.add_argument(
+        "-vt",
+        "--vision_type",
+        default="f16",
+        type=str,
+        choices=["f32", "f16", "q8_0", "q4_0", "q4_1", "q5_0", "q5_1"],
+        help="Vision model quantization type",
+    )
     args = parser.parse_args()
 
     with open(args.save_path, "wb") as f:
-        convert(f, args.model_name_or_path, args.lora_model_name_or_path, dtype=args.type)
+        convert(
+            f, args.model_name_or_path, args.lora_model_name_or_path, dtype=args.type, vision_dtype=args.vision_type
+        )
 
     print(f"GGML model saved to {args.save_path}")
 
diff --git a/chatglm_cpp/openai_api.py b/chatglm_cpp/openai_api.py
index 6d37844d..9fad4263 100644
--- a/chatglm_cpp/openai_api.py
+++ b/chatglm_cpp/openai_api.py
@@ -1,4 +1,6 @@
 import asyncio
+import base64
+import io
 import json
 import logging
 import time
@@ -30,9 +32,24 @@ class ToolCall(BaseModel):
     type: Literal["function"]
 
 
+class ContentText(BaseModel):
+    type: Literal["text"] = "text"
+    text: str
+
+
+class ContentImageUrlData(BaseModel):
+    url: str
+    detail: str = "high"
+
+
+class ContentImageUrl(BaseModel):
+    type: Literal["image_url"] = "image_url"
+    image_url: ContentImageUrlData
+
+
 class ChatMessage(BaseModel):
     role: Literal["system", "user", "assistant"]
-    content: str
+    content: Union[str, List[Union[ContentText, ContentImageUrl]]]
     tool_calls: Optional[List[ToolCall]] = None
 
 
@@ -186,7 +203,32 @@ def tool_call(**kwargs):
     if not body.messages:
         raise HTTPException(status.HTTP_400_BAD_REQUEST, "empty messages")
 
-    messages = [chatglm_cpp.ChatMessage(role=msg.role, content=msg.content) for msg in body.messages]
+    messages = []
+    for msg in body.messages:
+        if isinstance(msg.content, str):
+            messages.append(chatglm_cpp.ChatMessage(role=msg.role, content=msg.content))
+        else:
+            if not (len(msg.content) == 2 and msg.content[0].type == "text" and msg.content[1].type == "image_url"):
+                raise HTTPException(
+                    status.HTTP_400_BAD_REQUEST,
+                    "multimodal content must have a text item followed by an image_url item",
+                )
+
+            import numpy as np
+            from PIL import Image
+
+            text = msg.content[0].text
+            image_url = msg.content[1].image_url.url
+            if image_url.startswith("data:"):
+                image_bytes = base64.b64decode(image_url.split(",")[1])
+            else:
+                import requests
+
+                image_bytes = requests.get(image_url).content
+            image = chatglm_cpp.Image(np.asarray(Image.open(io.BytesIO(image_bytes))))
+
+            messages.append(chatglm_cpp.ChatMessage(role=msg.role, content=text, image=image))
+
     if body.tools:
         system_content = (
             "Answer the following questions as best as you can. You have access to the following tools:\n"
diff --git a/chatglm_pybind.cpp b/chatglm_pybind.cpp
index 5143e04a..1cf08c1c 100644
--- a/chatglm_pybind.cpp
+++ b/chatglm_pybind.cpp
@@ -30,14 +30,19 @@ class PyBaseModelForCausalLM : public BaseModelForCausalLM {
         PYBIND11_OVERLOAD_PURE(void, PyBaseModelForCausalLM, load_state_dict, sd);
     }
 
-    ggml_tensor *forward(ModelContext *ctx, ggml_tensor *input_ids, int n_past, int n_ctx,
-                         bool is_decoding) const override {
-        PYBIND11_OVERLOAD_PURE(ggml_tensor *, PyBaseModelForCausalLM, forward, ctx, input_ids, n_past, n_ctx,
-                               is_decoding)
+    ggml_tensor *forward(ModelContext *mctx, ggml_tensor *input_ids, ggml_tensor *images,
+                         const std::vector<int> &input_ids_vec, int n_past, bool is_decoding) const override {
+        PYBIND11_OVERLOAD_PURE(ggml_tensor *, PyBaseModelForCausalLM, forward, mctx, input_ids, images, input_ids_vec,
+                               n_past, is_decoding);
     }
 
-    void set_graph_inputs(int qlen, int n_past, int n_ctx) const override {
-        PYBIND11_OVERLOAD_PURE(void, PyBaseModelForCausalLM, set_graph_inputs, qlen, n_past, n_ctx);
+    void set_graph_inputs(const std::vector<int> &input_ids, const std::optional<Image> &image, int n_past,
+                          int n_ctx) const override {
+        PYBIND11_OVERLOAD_PURE(void, PyBaseModelForCausalLM, set_graph_inputs, input_ids, image, n_past, n_ctx);
+    }
+
+    int count_tokens(const std::vector<int> &input_ids, const std::optional<Image> &image) const override {
+        PYBIND11_OVERLOAD_PURE(int, PyBaseModelForCausalLM, count_tokens, input_ids, image);
     }
 };
 
@@ -57,6 +62,20 @@ PYBIND11_MODULE(_C, m) {
         .value("CHATGLM3", ModelType::CHATGLM3)
         .value("CHATGLM4", ModelType::CHATGLM4);
 
+    py::class_<VisionModelConfig>(m, "VisionModelConfig")
+        // .def_readonly("dtype", &VisionModelConfig::dtype)
+        // .def_readonly("hidden_act", &VisionModelConfig::hidden_act)
+        .def_readonly("hidden_size", &VisionModelConfig::hidden_size)
+        .def_readonly("image_size", &VisionModelConfig::image_size)
+        .def_readonly("in_channels", &VisionModelConfig::in_channels)
+        .def_readonly("intermediate_size", &VisionModelConfig::intermediate_size)
+        .def_readonly("norm_eps", &VisionModelConfig::norm_eps)
+        .def_readonly("num_attention_heads", &VisionModelConfig::num_attention_heads)
+        .def_readonly("num_hidden_layers", &VisionModelConfig::num_hidden_layers)
+        .def_readonly("num_positions", &VisionModelConfig::num_positions)
+        .def_readonly("patch_size", &VisionModelConfig::patch_size)
+        .def_readonly("scaling_factor", &VisionModelConfig::scaling_factor);
+
     py::class_<ModelConfig>(m, "ModelConfig")
         .def_readonly("model_type", &ModelConfig::model_type)
         // .def_readonly("dtype", &ModelConfig::dtype)
@@ -73,6 +92,7 @@ PYBIND11_MODULE(_C, m) {
         .def_readonly("pad_token_id", &ModelConfig::pad_token_id)
         .def_readonly("sep_token_id", &ModelConfig::sep_token_id)
         .def_readonly("extra_eos_token_ids", &ModelConfig::extra_eos_token_ids)
+        .def_readonly("vision", &ModelConfig::vision)
         .def_property_readonly("model_type_name", &ModelConfig::model_type_name);
 
     py::class_<GenerationConfig>(m, "GenerationConfig")
@@ -106,9 +126,37 @@ PYBIND11_MODULE(_C, m) {
         .def_readwrite("function", &ToolCallMessage::function)
         .def_readwrite("code", &ToolCallMessage::code);
 
+    py::class_<Image>(m, "Image", py::buffer_protocol())
+        .def(py::init([](py::buffer b) {
+            py::buffer_info info = b.request();
+
+            CHATGLM_CHECK(info.format == py::format_descriptor<uint8_t>::format())
+                << "Incompatible format: expect a byte array!";
+            CHATGLM_CHECK(info.ndim == 3 && info.shape[2] == 3) << "Only support RGB image for now";
+
+            for (int i = 1; i < info.ndim; i++) {
+                CHATGLM_CHECK(info.strides[i] * info.shape[i] == info.strides[i - 1])
+                    << "Only support contiguous array";
+            }
+
+            return Image(info.shape[1], info.shape[0], info.shape[2], (uint8_t *)info.ptr);
+        }))
+        .def_buffer([](Image &self) {
+            return py::buffer_info(
+                self.pixels.data(), sizeof(uint8_t), py::format_descriptor<uint8_t>::format(), 3,
+                {self.height, self.width, self.channels},
+                {self.width * self.channels * sizeof(uint8_t), self.channels * sizeof(uint8_t), sizeof(uint8_t)});
+        })
+        .def("__repr__", &to_string<Image>)
+        .def("__str__", &to_string<Image>)
+        .def_readonly("width", &Image::width)
+        .def_readonly("height", &Image::height)
+        .def_readonly("channels", &Image::channels)
+        .def_readonly("pixels", &Image::pixels);
+
     py::class_<ChatMessage>(m, "ChatMessage")
-        .def(py::init<std::string, std::string, std::vector<ToolCallMessage>>(), "role"_a, "content"_a,
-             "tool_calls"_a = std::vector<ToolCallMessage>{})
+        .def(py::init<std::string, std::string, std::optional<Image>, std::vector<ToolCallMessage>>(), "role"_a,
+             "content"_a, "image"_a = std::nullopt, "tool_calls"_a = std::vector<ToolCallMessage>{})
         .def("__repr__", &to_string<ChatMessage>)
         .def("__str__", &to_string<ChatMessage>)
         .def_readonly_static("ROLE_SYSTEM", &ChatMessage::ROLE_SYSTEM)
@@ -117,6 +165,7 @@ PYBIND11_MODULE(_C, m) {
         .def_readonly_static("ROLE_OBSERVATION", &ChatMessage::ROLE_OBSERVATION)
         .def_readwrite("role", &ChatMessage::role)
         .def_readwrite("content", &ChatMessage::content)
+        .def_readwrite("image", &ChatMessage::image)
         .def_readwrite("tool_calls", &ChatMessage::tool_calls);
 
     py::class_<BaseTokenizer, PyBaseTokenizer>(m, "BaseTokenizer")
@@ -126,8 +175,9 @@ PYBIND11_MODULE(_C, m) {
         .def("decode_message", &BaseTokenizer::decode_message, "ids"_a);
 
     py::class_<BaseModelForCausalLM, PyBaseModelForCausalLM>(m, "BaseModelForCausalLM")
-        .def("generate_next_token", &BaseModelForCausalLM::generate_next_token, "input_ids"_a, "gen_config"_a,
-             "n_past"_a, "n_ctx"_a)
+        .def("generate_next_token", &BaseModelForCausalLM::generate_next_token, "input_ids"_a, "image"_a,
+             "gen_config"_a, "n_past"_a, "n_ctx"_a)
+        .def("count_tokens", &BaseModelForCausalLM::count_tokens, "input_ids"_a, "image"_a)
         .def_readonly("config", &BaseModelForCausalLM::config);
 
     // ===== ChatGLM =====
diff --git a/examples/03-Confusing-Pictures.jpg b/examples/03-Confusing-Pictures.jpg
new file mode 100644
index 00000000..f3bb120c
Binary files /dev/null and b/examples/03-Confusing-Pictures.jpg differ
diff --git a/examples/cli_demo.py b/examples/cli_demo.py
index 6cd3b9a0..429b7f82 100644
--- a/examples/cli_demo.py
+++ b/examples/cli_demo.py
@@ -37,6 +37,7 @@ def main() -> None:
         type=Path,
         help="path to the plain text file that stores the system message",
     )
+    parser.add_argument("--image", default=None, type=str, help="path to the input image for visual language models")
     parser.add_argument("-i", "--interactive", action="store_true", help="run in interactive mode")
     parser.add_argument(
         "-l", "--max_length", default=2048, type=int, help="max total length including prompt and output"
@@ -86,9 +87,16 @@ def main() -> None:
 
     messages = system_messages.copy()
 
+    image = None
+    if args.image is not None:
+        import numpy as np
+        from PIL import Image
+
+        image = chatglm_cpp.Image(np.asarray(Image.open(args.image).convert("RGB")))
+
     if not args.interactive:
         if args.mode == "chat":
-            messages.append(chatglm_cpp.ChatMessage(role="user", content=prompt))
+            messages.append(chatglm_cpp.ChatMessage(role="user", content=prompt, image=image))
             for chunk in pipeline.chat(messages, **generation_kwargs):
                 print(chunk.content, sep="", end="", flush=True)
         else:
@@ -107,6 +115,7 @@ def main() -> None:
     if system:
         print(f"{'System':{prompt_width}} > {system}")
 
+    prompt_image = image
     while True:
         if messages and messages[-1].tool_calls:
             (tool_call,) = messages[-1].tool_calls
@@ -135,10 +144,12 @@ def main() -> None:
         if prompt == "stop":
             break
         if prompt == "clear":
-            messages = system_messages
+            messages = system_messages.copy()
+            prompt_image = image
             continue
 
-        messages.append(chatglm_cpp.ChatMessage(role=role, content=prompt))
+        messages.append(chatglm_cpp.ChatMessage(role=role, content=prompt, image=prompt_image))
+        prompt_image = None
         print(f"{pipeline.model.config.model_type_name} > ", sep="", end="")
         chunks = []
         for chunk in pipeline.chat(messages, **generation_kwargs):
diff --git a/examples/openai_client.py b/examples/openai_client.py
index 99cadc1d..96550ec6 100644
--- a/examples/openai_client.py
+++ b/examples/openai_client.py
@@ -1,14 +1,21 @@
 import argparse
+import base64
+from pathlib import Path
 
 from openai import OpenAI
 
 parser = argparse.ArgumentParser()
-parser.add_argument("--stream", action="store_true")
-parser.add_argument("--prompt", default="你好", type=str)
-parser.add_argument("--tool_call", action="store_true")
+parser.add_argument("--api_key", default="Bearer chatglm-cpp-example", type=str, help="API key of OpenAI api server")
+parser.add_argument("--base_url", default=None, type=str, help="base url of OpenAI api server")
+parser.add_argument("--stream", action="store_true", help="enable stream generation")
+parser.add_argument("-p", "--prompt", default="你好", type=str, help="prompt to start generation with")
+parser.add_argument("--tool_call", action="store_true", help="enable function call")
+parser.add_argument("--image", default=None, type=str, help="path to the input image for visual language models")
+parser.add_argument("--temp", default=0.95, type=float, help="temperature")
+parser.add_argument("--top_p", default=0.7, type=float, help="top-p sampling")
 args = parser.parse_args()
 
-client = OpenAI()
+client = OpenAI(api_key=args.api_key, base_url=args.base_url)
 
 tools = None
 if args.tool_call:
@@ -33,14 +40,24 @@
         }
     ]
 
-messages = [{"role": "user", "content": args.prompt}]
+if args.image is not None:
+    image_url = args.image
+    if not image_url.startswith(("http://", "https://")):
+        base64_image = base64.b64encode(Path(image_url).read_bytes()).decode()
+        image_url = f"data:image/jpeg;base64,{base64_image}"
+    user_content = [{"type": "text", "text": args.prompt}, {"type": "image_url", "image_url": {"url": image_url}}]
+else:
+    user_content = args.prompt
+
+messages = [{"role": "user", "content": user_content}]
+response = client.chat.completions.create(
+    model="default-model", messages=messages, stream=args.stream, temperature=args.temp, top_p=args.top_p, tools=tools
+)
 if args.stream:
-    response = client.chat.completions.create(model="default-model", messages=messages, stream=True, tools=tools)
     for chunk in response:
         content = chunk.choices[0].delta.content
         if content is not None:
             print(content, end="", flush=True)
     print()
 else:
-    response = client.chat.completions.create(model="default-model", messages=messages, tools=tools)
     print(response.choices[0].message.content)
diff --git a/main.cpp b/main.cpp
index 037f8340..bff31e5a 100644
--- a/main.cpp
+++ b/main.cpp
@@ -27,6 +27,7 @@ struct Args {
     bool sync = false;
     std::string prompt = "你好";
     std::string system = "";
+    std::string image_path = "";
     int max_length = 2048;
     int max_new_tokens = -1;
     int max_context_length = 512;
@@ -50,6 +51,7 @@ static void usage(const std::string &prog) {
   --pp, --prompt_path   path to the plain text file that stores the prompt
   -s, --system SYSTEM   system message to set the behavior of the assistant
   --sp, --system_path   path to the plain text file that stores the system message
+  --image               path to the input image for visual language models
   -i, --interactive     run in interactive mode
   -l, --max_length N    max total length including prompt and output (default: 2048)
   --max_new_tokens N    max number of tokens to generate, ignoring the number of prompt tokens
@@ -94,6 +96,8 @@ static Args parse_args(const std::vector<std::string> &argv) {
             args.system = argv.at(++i);
         } else if (arg == "--sp" || arg == "--system_path") {
             args.system = read_text(argv.at(++i));
+        } else if (arg == "--image") {
+            args.image_path = argv.at(++i);
         } else if (arg == "-i" || arg == "--interactive") {
             args.interactive = true;
         } else if (arg == "-l" || arg == "--max_length") {
@@ -219,6 +223,16 @@ static void chat(Args &args) {
         args.interactive = false;
     }
 
+    if (!args.image_path.empty() && pipeline.model->config.model_type != chatglm::ModelType::CHATGLM4V) {
+        std::cerr << "image is specified for model without visual ability, falling back to language mode\n";
+        args.image_path.clear();
+    }
+
+    std::optional<chatglm::Image> image;
+    if (!args.image_path.empty()) {
+        image = chatglm::Image::open(args.image_path);
+    }
+
     std::vector<chatglm::ChatMessage> system_messages;
     if (!args.system.empty()) {
         system_messages.emplace_back(chatglm::ChatMessage::ROLE_SYSTEM, args.system);
@@ -242,7 +256,8 @@ static void chat(Args &args) {
             std::cout << std::setw(model_name.size()) << std::left << "System"
                       << " > " << args.system << std::endl;
         }
-        while (1) {
+        auto prompt_image = image;
+        while (true) {
             std::string role;
             if (!messages.empty() && !messages.back().tool_calls.empty()) {
                 const auto &tool_call = messages.back().tool_calls.front();
@@ -273,9 +288,11 @@ static void chat(Args &args) {
             }
             if (prompt == "clear") {
                 messages = system_messages;
+                prompt_image = image;
                 continue;
             }
-            messages.emplace_back(std::move(role), std::move(prompt));
+            messages.emplace_back(std::move(role), std::move(prompt), std::move(prompt_image));
+            prompt_image.reset();
             std::cout << model_name << " > ";
             chatglm::ChatMessage output = pipeline.chat(messages, gen_config, streamer.get());
             if (args.sync) {
@@ -291,7 +308,7 @@ static void chat(Args &args) {
     } else {
         if (args.mode == INFERENCE_MODE_CHAT) {
             std::vector<chatglm::ChatMessage> messages = system_messages;
-            messages.emplace_back(chatglm::ChatMessage::ROLE_USER, args.prompt);
+            messages.emplace_back(chatglm::ChatMessage::ROLE_USER, args.prompt, std::move(image));
             chatglm::ChatMessage output = pipeline.chat(messages, gen_config, streamer.get());
             if (args.sync) {
                 print_message(output);
diff --git a/third_party/stb b/third_party/stb
new file mode 160000
index 00000000..013ac3be
--- /dev/null
+++ b/third_party/stb
@@ -0,0 +1,1 @@
+Subproject commit 013ac3beddff3dbffafd5177e7972067cd2b5083


Test patch:
diff --git a/chatglm_test.cpp b/chatglm_test.cpp
index 96c66fdf..ce9e2784 100644
--- a/chatglm_test.cpp
+++ b/chatglm_test.cpp
@@ -52,7 +52,7 @@ static inline void expect_all_close(ggml_tensor *a, ggml_tensor *b, float atol =
 
 static inline void read_backend_tensor_data(std::istream &is, ggml_tensor *tensor) {
     std::vector<no_init<char>> buf(ggml_nbytes(tensor));
-    is.read((char *)buf.data(), buf.size());
+    CHATGLM_CHECK(is.read((char *)buf.data(), buf.size()));
     ggml_backend_tensor_set(tensor, buf.data(), 0, buf.size());
 }
 
@@ -261,7 +261,7 @@ class ChatGLMTest : public ::testing::Test {
   protected:
     std::unique_ptr<ModelContext> mctx_;
 
-    void SetUp() override { mctx_ = std::make_unique<ModelContext>(GGML_TYPE_F32); }
+    void SetUp() override { mctx_ = std::make_unique<ModelContext>(); }
 
     float perf_graph_compute() {
         auto fn = [this] {
@@ -274,17 +274,6 @@ class ChatGLMTest : public ::testing::Test {
         }
     }
 
-    template <typename Model>
-    static void set_graph_inputs(ggml_cgraph *gf, int qlen, int n_past, int n_ctx) {
-        static_assert(std::is_same_v<Model, ChatGLMModel> || std::is_same_v<Model, ChatGLM2Model>,
-                      "invalid model type");
-        if (std::is_same_v<Model, ChatGLMModel>) {
-            ChatGLMForCausalLM::set_graph_inputs(gf, qlen, n_past, n_ctx);
-        } else {
-            ChatGLM2ForCausalLM::set_graph_inputs(gf, qlen, n_past, n_ctx);
-        }
-    }
-
     template <typename Model>
     void test_model(Model &model, const ModelConfig &config, const fs::path &data_path, int seq_len,
                     const std::vector<ggml_tensor *> &all_weights) {
@@ -326,35 +315,48 @@ class ChatGLMTest : public ::testing::Test {
         }
         ASSERT_TRUE(ifs.peek() == EOF);
 
-        // self attention
+        auto input_ids_to_vec = [](ggml_tensor *input_ids) {
+            std::vector<int> input_ids_vec(ggml_nelements(input_ids));
+            ggml_backend_tensor_get(input_ids, input_ids_vec.data(), 0, ggml_nbytes(input_ids));
+            return input_ids_vec;
+        };
+
+        std::vector<int> input_ids;
+
+        // prefill
         {
+            std::vector<int> x1_vec = input_ids_to_vec(x1);
+            input_ids.insert(input_ids.end(), x1_vec.begin(), x1_vec.end());
             ggml_graph_clear(mctx_->gf);
-            ggml_tensor *out_y1 = model.forward(mctx_.get(), x1, 0);
+            ggml_tensor *out_y1 = model.forward(mctx_.get(), x1, nullptr, input_ids, 0);
             ggml_build_forward_expand(mctx_->gf, out_y1);
             CHATGLM_CHECK(ggml_gallocr_alloc_graph(mctx_->allocr.get(), mctx_->gf));
-            set_graph_inputs<Model>(mctx_->gf, seq_len, 0, seq_len);
+            model.set_graph_inputs(mctx_->gf, input_ids, std::nullopt, 0, seq_len);
             CHATGLM_CHECK(ggml_backend_graph_compute(mctx_->backend.get(), mctx_->gf) == GGML_STATUS_SUCCESS);
 
             expect_all_close(ref_y1, out_y1, 5e-4);
         }
-
-        // cross attention
+        // decode
         {
+            std::vector<int> x2_vec = input_ids_to_vec(x2);
+            input_ids.insert(input_ids.end(), x2_vec.begin(), x2_vec.end());
             ggml_graph_clear(mctx_->gf);
-            ggml_tensor *out_y2 = model.forward(mctx_.get(), x2, seq_len);
+            ggml_tensor *out_y2 = model.forward(mctx_.get(), x2, nullptr, input_ids, seq_len);
             ggml_build_forward_expand(mctx_->gf, out_y2);
             CHATGLM_CHECK(ggml_gallocr_alloc_graph(mctx_->allocr.get(), mctx_->gf));
-            set_graph_inputs<Model>(mctx_->gf, 1, seq_len, seq_len);
+            model.set_graph_inputs(mctx_->gf, input_ids, std::nullopt, seq_len, seq_len);
             CHATGLM_CHECK(ggml_backend_graph_compute(mctx_->backend.get(), mctx_->gf) == GGML_STATUS_SUCCESS);
 
             expect_all_close(ref_y2, out_y2, 5e-4);
         }
         {
+            std::vector<int> x3_vec = input_ids_to_vec(x3);
+            input_ids.insert(input_ids.end(), x3_vec.begin(), x3_vec.end());
             ggml_graph_clear(mctx_->gf);
-            ggml_tensor *out_y3 = model.forward(mctx_.get(), x3, seq_len + 1);
+            ggml_tensor *out_y3 = model.forward(mctx_.get(), x3, nullptr, input_ids, seq_len + 1);
             ggml_build_forward_expand(mctx_->gf, out_y3);
             CHATGLM_CHECK(ggml_gallocr_alloc_graph(mctx_->allocr.get(), mctx_->gf));
-            set_graph_inputs<Model>(mctx_->gf, 1, seq_len + 1, seq_len);
+            model.set_graph_inputs(mctx_->gf, input_ids, std::nullopt, seq_len + 1, seq_len);
             CHATGLM_CHECK(ggml_backend_graph_compute(mctx_->backend.get(), mctx_->gf) == GGML_STATUS_SUCCESS);
 
             expect_all_close(ref_y3, out_y3, 5e-4);
@@ -370,7 +372,7 @@ TEST_F(ChatGLMTest, Embedding) {
                    -2.1788, 0.4033,  0.8380,  -0.7193, -0.4033, -0.5966, 0.1820};
 
     ggml_tensor *x = ggml_new_tensor_1d(mctx_->ctx_b.get(), GGML_TYPE_I32, 5);
-    Embedding model(mctx_.get(), 4, 3);
+    Embedding model(mctx_.get(), GGML_TYPE_F32, 4, 3);
     ggml_tensor *ref = ggml_new_tensor_2d(mctx_->ctx_b.get(), GGML_TYPE_F32, 3, 5);
 
     auto buf_b = unique_ggml_backend_buffer_t(ggml_backend_alloc_ctx_tensors(mctx_->ctx_b.get(), mctx_->backend.get()));
@@ -432,8 +434,7 @@ TEST_F(ChatGLMTest, Linear) {
     };
 
     for (const auto &config : test_configs) {
-        mctx_->dtype = config.dtype;
-        Linear model(mctx_.get(), 64, 32);
+        Linear model(mctx_.get(), config.dtype, 64, 32);
         auto buf_w =
             unique_ggml_backend_buffer_t(ggml_backend_alloc_ctx_tensors(mctx_->ctx_w.get(), mctx_->backend.get()));
 
@@ -471,9 +472,9 @@ TEST_F(ChatGLMTest, BenchmarkLinear) {
     std::vector<ggml_type> dtypes{GGML_TYPE_F32,  GGML_TYPE_F16,  GGML_TYPE_Q8_0, GGML_TYPE_Q5_1,
                                   GGML_TYPE_Q5_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0};
     for (ggml_type dtype : dtypes) {
-        mctx_ = std::make_unique<ModelContext>(dtype);
+        mctx_ = std::make_unique<ModelContext>();
 
-        Linear m(mctx_.get(), K, N);
+        Linear m(mctx_.get(), dtype, K, N);
         ggml_tensor *x = ggml_new_tensor_2d(mctx_->ctx_b.get(), GGML_TYPE_F32, K, M);
 
         ggml_tensor *y = m.forward(mctx_.get(), x);
@@ -485,8 +486,7 @@ TEST_F(ChatGLMTest, BenchmarkLinear) {
             randn_(tensor);
         }
 
-        std::cout << "[Benchmark] Linear " << ggml_type_name(mctx_->dtype) << " time: " << perf_graph_compute()
-                  << " ms\n";
+        std::cout << "[Benchmark] Linear " << ggml_type_name(dtype) << " time: " << perf_graph_compute() << " ms\n";
     }
 }
 
@@ -537,7 +537,7 @@ TEST_F(ChatGLMTest, BenchmarkLayerNorm) {
     ggml_tensor *y = m.forward(mctx_.get(), x);
     ggml_build_forward_expand(mctx_->gf, y);
     CHATGLM_CHECK(ggml_gallocr_alloc_graph(mctx_->allocr.get(), mctx_->gf));
-    std::cout << "[Benchmark] LayerNorm " << ggml_type_name(mctx_->dtype) << " time: " << perf_graph_compute()
+    std::cout << "[Benchmark] LayerNorm " << ggml_type_name(GGML_TYPE_F32) << " time: " << perf_graph_compute()
               << " ms\n";
 }
 
@@ -588,19 +588,19 @@ TEST_F(ChatGLMTest, BenchmarkRMSNorm) {
     ggml_tensor *y = m.forward(mctx_.get(), x);
     ggml_build_forward_expand(mctx_->gf, y);
     CHATGLM_CHECK(ggml_gallocr_alloc_graph(mctx_->allocr.get(), mctx_->gf));
-    std::cout << "[Benchmark] RMSNorm " << ggml_type_name(mctx_->dtype) << " time: " << perf_graph_compute() << " ms\n";
+    std::cout << "[Benchmark] RMSNorm " << ggml_type_name(GGML_TYPE_F32) << " time: " << perf_graph_compute()
+              << " ms\n";
 }
 
 TEST_F(ChatGLMTest, GLMModel) {
     fs::path data_path = fs::path(__FILE__).parent_path() / "tests/data/glm_model.data";
 
-    ModelConfig config(
-        ModelType::CHATGLM, GGML_TYPE_F32, /*vocab_size=*/5, /*hidden_size=*/32, /*num_attention_heads=*/8,
-        /*num_key_value_heads=*/8, /*num_hidden_layers=*/1, /*intermediate_size=*/128, /*norm_eps=*/1e-5f,
-        /*rope_theta=*/10000.f,
-        /*num_virtual_tokens=*/0,
-        /*max_length=*/8, /*bos_token_id=*/-1, /*eos_token_id=*/-1, /*pad_token_id=*/-1, /*sep_token_id=*/-1,
-        /*extra_eos_token_ids=*/{});
+    ModelConfig config(ModelType::CHATGLM, GGML_TYPE_F32, /*vocab_size=*/5, /*hidden_size=*/32,
+                       /*num_attention_heads=*/8, /*num_key_value_heads=*/8, /*num_hidden_layers=*/1,
+                       /*intermediate_size=*/128, /*norm_eps=*/1e-5f, /*rope_theta=*/10000.f, /*num_virtual_tokens=*/0,
+                       /*max_length=*/8, /*bos_token_id=*/-1, /*eos_token_id=*/-1, /*pad_token_id=*/-1,
+                       /*sep_token_id=*/-1, /*boi_token_id=*/-1, /*eoi_token_id=*/-1, /*extra_eos_token_ids=*/{},
+                       /*vision=*/{});
 
     constexpr int seq_len = 3;
 
@@ -628,13 +628,12 @@ TEST_F(ChatGLMTest, GLMModel) {
 TEST_F(ChatGLMTest, GLMPTuningV2Model) {
     fs::path data_path = fs::path(__FILE__).parent_path() / "tests/data/glm_ptuning_v2_model.data";
 
-    ModelConfig config(
-        ModelType::CHATGLM, GGML_TYPE_F32, /*vocab_size=*/5, /*hidden_size=*/32, /*num_attention_heads=*/8,
-        /*num_key_value_heads=*/8, /*num_hidden_layers=*/1, /*intermediate_size=*/128, /*norm_eps=*/1e-5f,
-        /*rope_theta=*/10000.f,
-        /*num_virtual_tokens=*/5,
-        /*max_length=*/8, /*bos_token_id=*/-1, /*eos_token_id=*/-1, /*pad_token_id=*/-1, /*sep_token_id=*/-1,
-        /*extra_eos_token_ids=*/{});
+    ModelConfig config(ModelType::CHATGLM, GGML_TYPE_F32, /*vocab_size=*/5, /*hidden_size=*/32,
+                       /*num_attention_heads=*/8, /*num_key_value_heads=*/8, /*num_hidden_layers=*/1,
+                       /*intermediate_size=*/128, /*norm_eps=*/1e-5f, /*rope_theta=*/10000.f, /*num_virtual_tokens=*/5,
+                       /*max_length=*/8, /*bos_token_id=*/-1, /*eos_token_id=*/-1, /*pad_token_id=*/-1,
+                       /*sep_token_id=*/-1, /*boi_token_id=*/-1, /*eoi_token_id=*/-1, /*extra_eos_token_ids=*/{},
+                       /*vision=*/{});
 
     constexpr int seq_len = 3;
 
@@ -662,13 +661,12 @@ TEST_F(ChatGLMTest, GLMPTuningV2Model) {
 TEST_F(ChatGLMTest, GLM2Model) {
     fs::path data_path = fs::path(__FILE__).parent_path() / "tests/data/glm2_model.data";
 
-    ModelConfig config(
-        ModelType::CHATGLM2, GGML_TYPE_F32, /*vocab_size=*/5, /*hidden_size=*/32, /*num_attention_heads=*/8,
-        /*num_key_value_heads=*/2, /*num_hidden_layers=*/1, /*intermediate_size=*/48, /*norm_eps=*/1e-5f,
-        /*rope_theta=*/10000.f,
-        /*num_virtual_tokens=*/0,
-        /*max_length=*/8, /*bos_token_id=*/-1, /*eos_token_id=*/-1, /*pad_token_id=*/-1, /*sep_token_id=*/-1,
-        /*extra_eos_token_ids=*/{});
+    ModelConfig config(ModelType::CHATGLM2, GGML_TYPE_F32, /*vocab_size=*/5, /*hidden_size=*/32,
+                       /*num_attention_heads=*/8, /*num_key_value_heads=*/2, /*num_hidden_layers=*/1,
+                       /*intermediate_size=*/48, /*norm_eps=*/1e-5f, /*rope_theta=*/10000.f, /*num_virtual_tokens=*/0,
+                       /*max_length=*/8, /*bos_token_id=*/-1, /*eos_token_id=*/-1, /*pad_token_id=*/-1,
+                       /*sep_token_id=*/-1, /*boi_token_id=*/-1, /*eoi_token_id=*/-1, /*extra_eos_token_ids=*/{},
+                       /*vision=*/{});
 
     constexpr int seq_len = 3;
 
@@ -691,13 +689,12 @@ TEST_F(ChatGLMTest, GLM2Model) {
 TEST_F(ChatGLMTest, GLM3Model) {
     fs::path data_path = fs::path(__FILE__).parent_path() / "tests/data/glm3_model.data";
 
-    ModelConfig config(
-        ModelType::CHATGLM3, GGML_TYPE_F32, /*vocab_size=*/5, /*hidden_size=*/32, /*num_attention_heads=*/8,
-        /*num_key_value_heads=*/2, /*num_hidden_layers=*/1, /*intermediate_size=*/48, /*norm_eps=*/1e-5f,
-        /*rope_theta=*/10000.f,
-        /*num_virtual_tokens=*/0,
-        /*max_length=*/8, /*bos_token_id=*/-1, /*eos_token_id=*/-1, /*pad_token_id=*/-1, /*sep_token_id=*/-1,
-        /*extra_eos_token_ids=*/{});
+    ModelConfig config(ModelType::CHATGLM3, GGML_TYPE_F32, /*vocab_size=*/5, /*hidden_size=*/32,
+                       /*num_attention_heads=*/8, /*num_key_value_heads=*/2, /*num_hidden_layers=*/1,
+                       /*intermediate_size=*/48, /*norm_eps=*/1e-5f, /*rope_theta=*/10000.f, /*num_virtual_tokens=*/0,
+                       /*max_length=*/8, /*bos_token_id=*/-1, /*eos_token_id=*/-1, /*pad_token_id=*/-1,
+                       /*sep_token_id=*/-1, /*boi_token_id=*/-1, /*eoi_token_id=*/-1, /*extra_eos_token_ids=*/{},
+                       /*vision=*/{});
 
     constexpr int seq_len = 3;
 
@@ -720,13 +717,12 @@ TEST_F(ChatGLMTest, GLM3Model) {
 TEST_F(ChatGLMTest, GLM3PTuningV2Model) {
     fs::path data_path = fs::path(__FILE__).parent_path() / "tests/data/glm3_ptuning_v2_model.data";
 
-    ModelConfig config(
-        ModelType::CHATGLM3, GGML_TYPE_F32, /*vocab_size=*/5, /*hidden_size=*/32, /*num_attention_heads=*/8,
-        /*num_key_value_heads=*/2, /*num_hidden_layers=*/1, /*intermediate_size=*/48, /*norm_eps=*/1e-5f,
-        /*rope_theta=*/10000.f,
-        /*num_virtual_tokens=*/5,
-        /*max_length=*/8, /*bos_token_id=*/-1, /*eos_token_id=*/-1, /*pad_token_id=*/-1, /*sep_token_id=*/-1,
-        /*extra_eos_token_ids=*/{});
+    ModelConfig config(ModelType::CHATGLM3, GGML_TYPE_F32, /*vocab_size=*/5, /*hidden_size=*/32,
+                       /*num_attention_heads=*/8, /*num_key_value_heads=*/2, /*num_hidden_layers=*/1,
+                       /*intermediate_size=*/48, /*norm_eps=*/1e-5f, /*rope_theta=*/10000.f, /*num_virtual_tokens=*/5,
+                       /*max_length=*/8, /*bos_token_id=*/-1, /*eos_token_id=*/-1, /*pad_token_id=*/-1,
+                       /*sep_token_id=*/-1, /*boi_token_id=*/-1, /*eoi_token_id=*/-1, /*extra_eos_token_ids=*/{},
+                       /*vision=*/{});
 
     constexpr int seq_len = 3;
 
@@ -749,13 +745,12 @@ TEST_F(ChatGLMTest, GLM3PTuningV2Model) {
 TEST_F(ChatGLMTest, GLM4Model) {
     fs::path data_path = fs::path(__FILE__).parent_path() / "tests/data/glm4_model.data";
 
-    ModelConfig config(
-        ModelType::CHATGLM4, GGML_TYPE_F32, /*vocab_size=*/5, /*hidden_size=*/32, /*num_attention_heads=*/8,
-        /*num_key_value_heads=*/2, /*num_hidden_layers=*/1, /*intermediate_size=*/48, /*norm_eps=*/1e-5f,
-        /*rope_theta=*/10000.f,
-        /*num_virtual_tokens=*/0,
-        /*max_length=*/8, /*bos_token_id=*/-1, /*eos_token_id=*/-1, /*pad_token_id=*/-1, /*sep_token_id=*/-1,
-        /*extra_eos_token_ids=*/{});
+    ModelConfig config(ModelType::CHATGLM4, GGML_TYPE_F32, /*vocab_size=*/5, /*hidden_size=*/32,
+                       /*num_attention_heads=*/8, /*num_key_value_heads=*/2, /*num_hidden_layers=*/1,
+                       /*intermediate_size=*/48, /*norm_eps=*/1e-5f, /*rope_theta=*/10000.f, /*num_virtual_tokens=*/0,
+                       /*max_length=*/8, /*bos_token_id=*/-1, /*eos_token_id=*/-1, /*pad_token_id=*/-1,
+                       /*sep_token_id=*/-1, /*boi_token_id=*/-1, /*eoi_token_id=*/-1, /*extra_eos_token_ids=*/{},
+                       /*vision=*/{});
 
     constexpr int seq_len = 3;
 
@@ -775,6 +770,175 @@ TEST_F(ChatGLMTest, GLM4Model) {
     test_model(model, config, data_path, seq_len, all_weights);
 }
 
+TEST_F(ChatGLMTest, GLM4VModelText) {
+    fs::path data_path = fs::path(__FILE__).parent_path() / "tests/data/glm4v_model_text.data";
+
+    VisionModelConfig vision(GGML_TYPE_F32, ActivationType::GELU, 32, 28, 3, 56, 1e-6, 2, 1, 17, 7, 8);
+
+    ModelConfig config(ModelType::CHATGLM4V, GGML_TYPE_F32, /*vocab_size=*/8, /*hidden_size=*/32,
+                       /*num_attention_heads=*/8, /*num_key_value_heads=*/2, /*num_hidden_layers=*/1,
+                       /*intermediate_size=*/48, /*norm_eps=*/0.00000015625f, /*rope_theta=*/10000.f,
+                       /*num_virtual_tokens=*/0,
+                       /*max_length=*/16, /*bos_token_id=*/-1, /*eos_token_id=*/4, /*pad_token_id=*/-1,
+                       /*sep_token_id=*/-1, /*boi_token_id=*/2, /*eoi_token_id=*/4, /*extra_eos_token_ids=*/{},
+                       /*vision=*/vision);
+
+    constexpr int seq_len = 3;
+
+    ChatGLM4VModel model(mctx_.get(), config);
+
+    std::vector<ggml_tensor *> all_weights{model.word_embeddings.weight,
+                                           model.layers[0].input_layernorm.weight,
+                                           model.layers[0].attention.query_key_value.weight,
+                                           model.layers[0].attention.query_key_value.bias,
+                                           model.layers[0].attention.dense.weight,
+                                           model.layers[0].post_attention_layernorm.weight,
+                                           model.layers[0].mlp.gate_proj.weight,
+                                           model.layers[0].mlp.up_proj.weight,
+                                           model.layers[0].mlp.down_proj.weight,
+                                           model.final_layernorm.weight};
+
+    test_model(model, config, data_path, seq_len, all_weights);
+}
+
+TEST_F(ChatGLMTest, GLM4VModel) {
+    fs::path data_path = fs::path(__FILE__).parent_path() / "tests/data/glm4v_model.data";
+
+    VisionModelConfig vision(GGML_TYPE_F32, ActivationType::GELU, 32, 28, 3, 56, 1e-6, 2, 1, 17, 7, 8);
+
+    ModelConfig config(ModelType::CHATGLM4V, GGML_TYPE_F32, /*vocab_size=*/8, /*hidden_size=*/32,
+                       /*num_attention_heads=*/8, /*num_key_value_heads=*/2, /*num_hidden_layers=*/1,
+                       /*intermediate_size=*/48, /*norm_eps=*/0.00000015625f, /*rope_theta=*/10000.f,
+                       /*num_virtual_tokens=*/0,
+                       /*max_length=*/16, /*bos_token_id=*/-1, /*eos_token_id=*/4, /*pad_token_id=*/-1,
+                       /*sep_token_id=*/-1, /*boi_token_id=*/2, /*eoi_token_id=*/4, /*extra_eos_token_ids=*/{},
+                       /*vision=*/vision);
+
+    std::ifstream fin(data_path, std::ios::binary);
+    ASSERT_TRUE(fin) << "cannot open file " << data_path;
+
+    ChatGLM4VModel model(mctx_.get(), config);
+
+    ggml_tensor *images = ggml_new_tensor_3d(mctx_->ctx_b.get(), GGML_TYPE_F32, config.vision.image_size,
+                                             config.vision.image_size, config.vision.in_channels);
+
+    const int vision_tokens = model.num_vision_tokens();
+
+    const int seq_len = 6;
+    ggml_tensor *x1 = ggml_new_tensor_1d(mctx_->ctx_b.get(), GGML_TYPE_I32, seq_len);
+    ggml_tensor *ref_y1 =
+        ggml_new_tensor_2d(mctx_->ctx_b.get(), GGML_TYPE_F32, config.hidden_size, seq_len - 1 + vision_tokens);
+    ggml_tensor *x2 = ggml_new_tensor_1d(mctx_->ctx_b.get(), GGML_TYPE_I32, 1);
+    ggml_tensor *ref_y2 = ggml_new_tensor_1d(mctx_->ctx_b.get(), GGML_TYPE_F32, config.hidden_size);
+    ggml_tensor *x3 = ggml_new_tensor_1d(mctx_->ctx_b.get(), GGML_TYPE_I32, 1);
+    ggml_tensor *ref_y3 = ggml_new_tensor_1d(mctx_->ctx_b.get(), GGML_TYPE_F32, config.hidden_size);
+
+    auto buf_b = unique_ggml_backend_buffer_t(ggml_backend_alloc_ctx_tensors(mctx_->ctx_b.get(), mctx_->backend.get()));
+    auto buf_w = unique_ggml_backend_buffer_t(ggml_backend_alloc_ctx_tensors(mctx_->ctx_w.get(), mctx_->backend.get()));
+
+    std::vector<ggml_tensor *> all_tensors{
+        // vision
+        model.vision.patch_embedding.cls_embedding,
+        model.vision.patch_embedding.proj.weight,
+        model.vision.patch_embedding.proj.bias,
+        model.vision.patch_embedding.position_embedding.weight,
+        model.vision.transformer.layers[0].input_layernorm.weight,
+        model.vision.transformer.layers[0].input_layernorm.bias,
+        model.vision.transformer.layers[0].attention.query_key_value.weight,
+        model.vision.transformer.layers[0].attention.query_key_value.bias,
+        model.vision.transformer.layers[0].attention.dense.weight,
+        model.vision.transformer.layers[0].attention.dense.bias,
+        model.vision.transformer.layers[0].mlp.dense_h_to_4h.weight,
+        model.vision.transformer.layers[0].mlp.dense_h_to_4h.bias,
+        model.vision.transformer.layers[0].mlp.dense_4h_to_h.weight,
+        model.vision.transformer.layers[0].mlp.dense_4h_to_h.bias,
+        model.vision.transformer.layers[0].post_attention_layernorm.weight,
+        model.vision.transformer.layers[0].post_attention_layernorm.bias,
+        model.vision.conv.weight,
+        model.vision.conv.bias,
+        model.vision.linear_proj.weight,
+        model.vision.norm1.weight,
+        model.vision.norm1.bias,
+        model.vision.glu.gate_proj.weight,
+        model.vision.glu.up_proj.weight,
+        model.vision.glu.down_proj.weight,
+        model.vision.boi,
+        model.vision.eoi,
+        // text
+        model.word_embeddings.weight,
+        model.layers[0].input_layernorm.weight,
+        model.layers[0].attention.query_key_value.weight,
+        model.layers[0].attention.query_key_value.bias,
+        model.layers[0].attention.dense.weight,
+        model.layers[0].post_attention_layernorm.weight,
+        model.layers[0].mlp.gate_proj.weight,
+        model.layers[0].mlp.up_proj.weight,
+        model.layers[0].mlp.down_proj.weight,
+        model.final_layernorm.weight,
+        // inputs & outputs
+        images,
+        x1,
+        ref_y1,
+        x2,
+        ref_y2,
+        x3,
+        ref_y3,
+    };
+
+    for (ggml_tensor *tensor : all_tensors) {
+        read_backend_tensor_data(fin, tensor);
+    }
+    ASSERT_TRUE(fin.peek() == EOF);
+
+    auto input_ids_to_vec = [](ggml_tensor *input_ids) {
+        std::vector<int> input_ids_vec(ggml_nelements(input_ids));
+        ggml_backend_tensor_get(input_ids, input_ids_vec.data(), 0, ggml_nbytes(input_ids));
+        return input_ids_vec;
+    };
+
+    std::vector<int> input_ids;
+
+    // prefill
+    {
+        std::vector<int> x1_vec = input_ids_to_vec(x1);
+        input_ids.insert(input_ids.end(), x1_vec.begin(), x1_vec.end());
+        ggml_graph_clear(mctx_->gf);
+        ggml_tensor *out_y1 = model.forward(mctx_.get(), x1, images, input_ids, 0);
+        ggml_build_forward_expand(mctx_->gf, out_y1);
+        CHATGLM_CHECK(ggml_gallocr_alloc_graph(mctx_->allocr.get(), mctx_->gf));
+        model.set_graph_inputs(mctx_->gf, input_ids, std::nullopt, 0, ref_y1->ne[1]);
+        CHATGLM_CHECK(ggml_backend_graph_compute(mctx_->backend.get(), mctx_->gf) == GGML_STATUS_SUCCESS);
+
+        expect_all_close(ref_y1, out_y1, 1e-2);
+    }
+    // decode
+    {
+        std::vector<int> x2_vec = input_ids_to_vec(x2);
+        input_ids.insert(input_ids.end(), x2_vec.begin(), x2_vec.end());
+        ggml_graph_clear(mctx_->gf);
+        ggml_tensor *out_y2 =
+            model.forward(mctx_.get(), x2, images, input_ids, seq_len - 1 + model.num_vision_tokens());
+        ggml_build_forward_expand(mctx_->gf, out_y2);
+        CHATGLM_CHECK(ggml_gallocr_alloc_graph(mctx_->allocr.get(), mctx_->gf));
+        model.set_graph_inputs(mctx_->gf, input_ids, std::nullopt, seq_len, seq_len);
+        CHATGLM_CHECK(ggml_backend_graph_compute(mctx_->backend.get(), mctx_->gf) == GGML_STATUS_SUCCESS);
+
+        expect_all_close(ref_y2, out_y2, 1e-2);
+    }
+    {
+        std::vector<int> x3_vec = input_ids_to_vec(x3);
+        input_ids.insert(input_ids.end(), x3_vec.begin(), x3_vec.end());
+        ggml_graph_clear(mctx_->gf);
+        ggml_tensor *out_y3 = model.forward(mctx_.get(), x3, images, input_ids, seq_len + model.num_vision_tokens());
+        ggml_build_forward_expand(mctx_->gf, out_y3);
+        CHATGLM_CHECK(ggml_gallocr_alloc_graph(mctx_->allocr.get(), mctx_->gf));
+        model.set_graph_inputs(mctx_->gf, input_ids, std::nullopt, seq_len + 1, seq_len);
+        CHATGLM_CHECK(ggml_backend_graph_compute(mctx_->backend.get(), mctx_->gf) == GGML_STATUS_SUCCESS);
+
+        expect_all_close(ref_y3, out_y3, 1e-2);
+    }
+}
+
 TEST_F(ChatGLMTest, quantize) {
     const float src_data[]{
         -1.1258e+00, -1.1524e+00, -2.5058e-01, -4.3388e-01, 8.4871e-01,  6.9201e-01,  -3.1601e-01, -2.1152e+00,
@@ -996,21 +1160,6 @@ TEST(Pipeline, ChatGLM) {
             "ChatGLM-6B，很高兴见到你，欢迎问我任何问题。\n[Round 1]\n问：晚上睡不着应该怎么办\n答：");
     }
 
-    // memory test
-    {
-        GenerationConfig gen_config;
-        gen_config.max_length = 2048;
-        gen_config.max_context_length = gen_config.max_length - 1;
-        gen_config.do_sample = false;
-
-        std::ostringstream oss;
-        for (int i = 0; i < gen_config.max_context_length; i++) {
-            oss << "你好";
-        }
-        std::vector<ChatMessage> messages{{ChatMessage::ROLE_USER, oss.str()}};
-        pipeline.chat(messages, gen_config);
-    }
-
     // chat
     {
         check_chat_format(pipeline);
@@ -1063,21 +1212,6 @@ TEST(Pipeline, ChatGLM2) {
             "ChatGLM2-6B，很高兴见到你，欢迎问我任何问题。\n\n[Round 2]\n\n问：晚上睡不着应该怎么办\n\n答：");
     }
 
-    // memory test
-    {
-        GenerationConfig gen_config;
-        gen_config.max_length = 2048;
-        gen_config.max_context_length = gen_config.max_length - 1;
-        gen_config.do_sample = false;
-
-        std::ostringstream oss;
-        for (int i = 0; i < gen_config.max_context_length; i++) {
-            oss << "你好";
-        }
-        std::vector<ChatMessage> messages{{ChatMessage::ROLE_USER, oss.str()}};
-        pipeline.chat(messages, gen_config);
-    }
-
     // chat
     {
         check_chat_format(pipeline);
@@ -1163,21 +1297,6 @@ TEST(Pipeline, ChatGLM3) {
         EXPECT_EQ(input_ids, target_ids);
     }
 
-    // memory test
-    {
-        GenerationConfig gen_config;
-        gen_config.max_length = 2048;
-        gen_config.max_context_length = gen_config.max_length - 1;
-        gen_config.do_sample = false;
-
-        std::ostringstream oss;
-        for (int i = 0; i < gen_config.max_context_length; i++) {
-            oss << "你好";
-        }
-        std::vector<ChatMessage> messages{{ChatMessage::ROLE_USER, oss.str()}};
-        pipeline.chat(messages, gen_config);
-    }
-
     // chat
     {
         // check_chat_format(pipeline);
@@ -1218,31 +1337,41 @@ TEST(Pipeline, ChatGLM3) {
         gen_config.do_sample = false;
         std::vector<ChatMessage> messages{
             {ChatMessage::ROLE_SYSTEM, system_ci},
-            {ChatMessage::ROLE_USER, "列出100以内的所有质数"},
+            {ChatMessage::ROLE_USER, "找出100以内的所有质数"},
         };
         {
             ChatMessage output = pipeline.chat(messages, gen_config);
             EXPECT_EQ(output.role, ChatMessage::ROLE_ASSISTANT);
-            EXPECT_EQ(output.content, R"(好的，我会为您列出100以内的所有质数。
-
-(Note: 质数是指只能被1和它本身整除的正整数。))");
-            EXPECT_EQ(output.tool_calls.front().code.input, R"(```python
-def is_prime(n):
-    """Check if a number is prime."""
-    if n <= 1:
-        return False
-    if n <= 3:
-        return True
-    if n % 2 == 0 or n % 3 == 0:
-        return False
-    i = 5
-    while i * i <= n:
-        if n % i == 0 or n % (i + 2) == 0:
-            return False
-        i += 6
-    return True
-
-primes_upto_100 = [i for i in range(2, 101) if is_prime(i)]
+            EXPECT_EQ(output.content,
+                      R"(质数是只能被1和它本身整除的正整数。我们可以通过简单的算法来找出100以内的所有质数。
+
+这里我们将使用一个简单的线性筛法来找出100以内的所有质数。
+
+线性筛法的基本思想是：
+1. 创建一个列表，其中包含1到n的所有整数。
+2. 从列表中删除所有可以被2整除的数。
+3. 然后从剩余的数中删除所有可以被3整除的数。
+4. 重复上述步骤，直到列表中的数少于100为止。
+
+让我们开始计算。)");
+            EXPECT_EQ(output.tool_calls.at(0).code.input, R"(```python
+def sieve_of_eratosthenes(n):
+    # Create a boolean array "prime[0..n]" and initialize all entries as true.
+    # A value in prime[i] will finally be false if i is Not a prime, else true bool val.
+    prime = [True for _ in range(n+1)]
+    p = 2
+    while p**2 <= n:
+        # If prime[p] is not changed, then it is a prime
+        if prime[p]:
+            # Update all multiples of p
+            for i in range(p**2, n+1, p):
+                prime[i] = False
+        p += 1
+
+    # Return the list of prime numbers
+    return [p for p in range(2, n+1) if prime[p]]
+
+primes_upto_100 = sieve_of_eratosthenes(100)
 primes_upto_100
 ```)");
             messages.emplace_back(std::move(output));
@@ -1253,9 +1382,9 @@ primes_upto_100
         {
             ChatMessage output = pipeline.chat(messages, gen_config);
             EXPECT_EQ(output.role, ChatMessage::ROLE_ASSISTANT);
-            EXPECT_EQ(output.content, R"(100以内的所有质数为：
-
-$$2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97$$)");
+            EXPECT_EQ(
+                output.content,
+                R"(100以内的所有质数是：2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97。)");
         }
     }
 }
@@ -1378,21 +1507,6 @@ if __name__ == '__main__':
     //     EXPECT_EQ(input_ids, target_ids);
     // }
 
-    // memory test
-    {
-        GenerationConfig gen_config;
-        gen_config.max_length = 2048;
-        gen_config.max_context_length = gen_config.max_length - 1;
-        gen_config.do_sample = false;
-
-        std::ostringstream oss;
-        for (int i = 0; i < gen_config.max_context_length; i++) {
-            oss << "你好 ";
-        }
-        std::vector<ChatMessage> messages{{ChatMessage::ROLE_USER, oss.str()}};
-        pipeline.chat(messages, gen_config);
-    }
-
     // chat
     {
         // check_chat_format(pipeline);
@@ -1400,7 +1514,7 @@ if __name__ == '__main__':
         gen_config.do_sample = false;
         std::vector<ChatMessage> messages{{ChatMessage::ROLE_USER, "你好"}};
         ChatMessage output = pipeline.chat(messages, gen_config);
-        EXPECT_EQ(output.content, "你好👋！我是人工智能助手，很高兴见到你，有什么可以帮助你的吗？");
+        EXPECT_EQ(output.content, "你好👋！很高兴能帮助你，有什么问题或者需要帮助的地方吗？");
     }
 }
 
@@ -1445,6 +1559,49 @@ print(bubble_sort([5, 4, 3, 2, 1])))";
     }
 }
 
+TEST(Pipeline, ChatGLM4V) {
+    fs::path model_path = fs::path(__FILE__).parent_path() / "models/chatglm4v-ggml.bin";
+    if (!fs::exists(model_path)) {
+        GTEST_SKIP() << "Skipping ChatGLM4V e2e test (ggml model not found)";
+    }
+    Pipeline pipeline(model_path.string());
+    ASSERT_TRUE(dynamic_cast<ChatGLM4Tokenizer *>(pipeline.tokenizer.get()));
+    ASSERT_TRUE(dynamic_cast<ChatGLM4VForCausalLM *>(pipeline.model.get()));
+
+    // tokenizer
+    {
+        fs::path image_path = fs::path(__FILE__).parent_path() / "examples/03-Confusing-Pictures.jpg";
+        Image image = Image::open(image_path.string());
+        std::vector<ChatMessage> messages{{ChatMessage::ROLE_USER, "描述这张图片", image}};
+        std::vector<int> target_ids{151331, 151333, 151336, 198,    151339, 151329,
+                                    151340, 100395, 108627, 100736, 151337};
+        std::vector<int> input_ids = pipeline.tokenizer->apply_chat_template(messages, 2048);
+        EXPECT_EQ(input_ids, target_ids);
+    }
+    // chat
+    {
+        // check_chat_format(pipeline);
+        GenerationConfig gen_config;
+        gen_config.do_sample = false;
+        std::vector<ChatMessage> messages{{ChatMessage::ROLE_USER, "你好"}};
+        ChatMessage output = pipeline.chat(messages, gen_config);
+        EXPECT_EQ(output.content, "你好👋！很高兴见到你，欢迎问我任何问题。");
+    }
+    // chat with image
+    {
+        GenerationConfig gen_config;
+        gen_config.do_sample = false;
+        fs::path image_path = fs::path(__FILE__).parent_path() / "examples/03-Confusing-Pictures.jpg";
+        Image image = Image::open(image_path.string());
+        std::vector<ChatMessage> messages{{ChatMessage::ROLE_USER, "描述这张图片", std::move(image)}};
+        ChatMessage output = pipeline.chat(messages, gen_config);
+        EXPECT_EQ(output.content,
+                  "这张图片的幽默之处在于场景的荒谬性。看到一个人在出租车后面熨衣服是非常不寻常的，因为这不是出租车通常"
+                  "被用来做的事情。熨衣板和衣服悬挂在车尾，男子站在上面，似乎在专心熨烫衣物。出租车在行驶中，而男子却稳"
+                  "稳地站在上面熨衣服，这样的场景给人一种不稳定的感觉，增加了喜剧效果。");
+    }
+}
+
 static void run_benchmark(const fs::path &model_path) {
     if (!fs::exists(model_path)) {
         GTEST_SKIP() << "Skipping benchmark test (model " << model_path << " not found)";
diff --git a/tests/data/baichuan13b_model.data b/tests/data/baichuan13b_model.data
deleted file mode 100644
index 380b9a3a..00000000
Binary files a/tests/data/baichuan13b_model.data and /dev/null differ
diff --git a/tests/data/baichuan7b_model.data b/tests/data/baichuan7b_model.data
deleted file mode 100644
index d5d05e91..00000000
Binary files a/tests/data/baichuan7b_model.data and /dev/null differ
diff --git a/tests/data/glm4v_model.data b/tests/data/glm4v_model.data
new file mode 100644
index 00000000..df2133ef
Binary files /dev/null and b/tests/data/glm4v_model.data differ
diff --git a/tests/data/glm4v_model_text.data b/tests/data/glm4v_model_text.data
new file mode 100644
index 00000000..7990ca0b
Binary files /dev/null and b/tests/data/glm4v_model_text.data differ
diff --git a/tests/data/internlm_model.data b/tests/data/internlm_model.data
deleted file mode 100644
index 821b36e4..00000000
Binary files a/tests/data/internlm_model.data and /dev/null differ
diff --git a/tests/perplexity.cpp b/tests/perplexity.cpp
index 22ddff16..775c64a8 100644
--- a/tests/perplexity.cpp
+++ b/tests/perplexity.cpp
@@ -113,7 +113,7 @@ static void perplexity(Args &args) {
         size_t target_len = std::min(end - prev_end, size_t(args.max_length - 1));
         std::vector<int> input_ids(corpus_ids.begin() + begin, corpus_ids.begin() + end);
 
-        ggml_tensor *lm_logits = pipeline.model->forward_graph_compute(input_ids, 0, 0, false);
+        ggml_tensor *lm_logits = pipeline.model->forward_graph_compute(input_ids, std::nullopt, 0, 0, false);
 
         const auto clk_fwd = std::chrono::system_clock::now();
 
diff --git a/tests/test_chatglm_cpp.py b/tests/test_chatglm_cpp.py
index a490740d..f0b90e27 100644
--- a/tests/test_chatglm_cpp.py
+++ b/tests/test_chatglm_cpp.py
@@ -1,3 +1,4 @@
+import base64
 from pathlib import Path
 
 import chatglm_cpp
@@ -9,6 +10,7 @@
 CHATGLM2_MODEL_PATH = PROJECT_ROOT / "models/chatglm2-ggml.bin"
 CHATGLM3_MODEL_PATH = PROJECT_ROOT / "models/chatglm3-ggml.bin"
 CHATGLM4_MODEL_PATH = PROJECT_ROOT / "models/chatglm4-ggml.bin"
+CHATGLM4V_MODEL_PATH = PROJECT_ROOT / "models/chatglm4v-ggml.bin"
 CODEGEEX2_MODEL_PATH = PROJECT_ROOT / "models/codegeex2-ggml.bin"
 
 
@@ -16,8 +18,8 @@ def test_chatglm_version():
     print(chatglm_cpp.__version__)
 
 
-def check_pipeline(model_path, prompt, target, gen_kwargs={}):
-    messages = [chatglm_cpp.ChatMessage(role="user", content=prompt)]
+def check_pipeline(model_path, prompt, target, gen_kwargs={}, image=None):
+    messages = [chatglm_cpp.ChatMessage(role="user", content=prompt, image=image)]
 
     pipeline = chatglm_cpp.Pipeline(model_path)
     output = pipeline.chat(messages, do_sample=False, **gen_kwargs).content
@@ -76,7 +78,29 @@ def test_chatglm4_pipeline():
     check_pipeline(
         model_path=CHATGLM4_MODEL_PATH,
         prompt="你好",
-        target="你好👋！我是人工智能助手，很高兴见到你，有什么可以帮助你的吗？",
+        target="你好👋！很高兴能帮助你，有什么问题或者需要帮助的地方吗？",
+    )
+
+
+@pytest.mark.skipif(not CHATGLM4V_MODEL_PATH.exists(), reason="model file not found")
+def test_chatglm4v_pipeline():
+    import numpy as np
+    from PIL import Image
+
+    check_pipeline(
+        model_path=CHATGLM4V_MODEL_PATH,
+        prompt="你好",
+        target="你好👋！很高兴见到你，欢迎问我任何问题。",
+    )
+
+    image = chatglm_cpp.Image(
+        np.asarray(Image.open(PROJECT_ROOT / "examples/03-Confusing-Pictures.jpg").convert("RGB"))
+    )
+    check_pipeline(
+        model_path=CHATGLM4V_MODEL_PATH,
+        prompt="这张图片有什么不寻常之处",
+        image=image,
+        target="这张图片中不寻常的是，一个男人站在一辆黄色SUV的后备箱上，正在使用一个铁板熨烫衣物。通常情况下，熨衣是在室内进行的，使用的是家用电熨斗，而不是在户外使用汽车后备箱作为工作台。此外，他似乎是在一个繁忙的城市街道上，周围有行驶的车辆和建筑物，这增加了场景的异想天开性。",
     )
 
 
@@ -117,7 +141,7 @@ def test_langchain_api():
     client = TestClient(app)
     response = client.post("/", json={"prompt": "你好", "temperature": 0})
     assert response.status_code == 200
-    assert response.json()["response"] == "你好👋！我是人工智能助手，很高兴见到你，有什么可以帮助你的吗？"
+    assert response.json()["response"] == "你好👋！很高兴能帮助你，有什么问题或者需要帮助的地方吗？"
 
 
 @pytest.mark.skipif(not CHATGLM4_MODEL_PATH.exists(), reason="model file not found")
@@ -137,4 +161,67 @@ def test_openai_api():
     assert response.status_code == 200
     response_message = response.json()["choices"][0]["message"]
     assert response_message["role"] == "assistant"
-    assert response_message["content"] == "你好👋！我是人工智能助手，很高兴见到你，有什么可以帮助你的吗？"
+    assert response_message["content"] == "你好👋！很高兴能帮助你，有什么问题或者需要帮助的地方吗？"
+
+
+@pytest.mark.skipif(not CHATGLM4V_MODEL_PATH.exists(), reason="model file not found")
+def test_openai_api_vision():
+    import os
+    from unittest.mock import patch
+
+    from fastapi.testclient import TestClient
+
+    with patch.dict(os.environ, {"MODEL": str(CHATGLM4V_MODEL_PATH)}):
+        from chatglm_cpp.openai_api import app
+
+    client = TestClient(app)
+
+    # request with image url
+    payload = {
+        "messages": [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "这张图片有什么不寻常之处"},
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": "https://www.barnorama.com/wp-content/uploads/2016/12/03-Confusing-Pictures.jpg"
+                        },
+                    },
+                ],
+            }
+        ],
+        "temperature": 0,
+    }
+    response = client.post("/v1/chat/completions", json=payload)
+    assert response.status_code == 200
+    response_message = response.json()["choices"][0]["message"]
+    assert response_message["role"] == "assistant"
+    assert (
+        response_message["content"]
+        == "这张图片中不寻常的是，一个男人站在一辆黄色SUV的后备箱上，正在使用一个铁板熨烫衣物。通常情况下，熨衣是在室内进行的，使用的是家用电熨斗，而不是在户外使用汽车后备箱作为工作台。此外，他似乎是在一个繁忙的城市街道上，周围有行驶的车辆和建筑物，这增加了场景的异想天开性。"
+    )
+
+    # request with base64 image
+    base64_image = base64.b64encode(Path(PROJECT_ROOT / "examples/03-Confusing-Pictures.jpg").read_bytes()).decode()
+    payload = {
+        "messages": [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "这张图片有什么不寻常之处"},
+                    {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}},
+                ],
+            }
+        ],
+        "temperature": 0,
+    }
+    response = client.post("/v1/chat/completions", json=payload)
+    assert response.status_code == 200
+    response_message = response.json()["choices"][0]["message"]
+    assert response_message["role"] == "assistant"
+    assert (
+        response_message["content"]
+        == "这张图片中不寻常的是，一个男人站在一辆黄色SUV的后备箱上，正在使用一个铁板熨烫衣物。通常情况下，熨衣是在室内进行的，使用的是家用电熨斗，而不是在户外使用汽车后备箱作为工作台。此外，他似乎是在一个繁忙的城市街道上，周围有行驶的车辆和建筑物，这增加了场景的异想天开性。"
+    )
diff --git a/tests/test_convert.py b/tests/test_convert.py
index 25955999..b74a495e 100644
--- a/tests/test_convert.py
+++ b/tests/test_convert.py
@@ -1,4 +1,6 @@
+import re
 import sys
+from contextlib import contextmanager
 from pathlib import Path
 
 import torch
@@ -175,8 +177,19 @@ def test_quantize_q5_1():
     assert (q_tensor == ggml_q_tensor).all()
 
 
+@contextmanager
+def relative_to_absolute_import(path):
+    code = Path(path).read_text()
+    fix_code = re.sub(r"(from )\.(.+ import)", r"\1\2", code)
+    Path(path).write_text(fix_code)
+    try:
+        yield
+    finally:
+        Path(path).write_text(code)
+
+
 CHATGLM2_MODEL_PATH = Path(
-    "~/.cache/huggingface/hub/models--THUDM--chatglm2-6b/snapshots/b1502f4f75c71499a3d566b14463edd62620ce9f"
+    "~/.cache/huggingface/hub/models--THUDM--chatglm2-6b/snapshots/7fabe56db91e085c9c027f56f1c654d137bdba40"
 ).expanduser()
 
 
@@ -277,7 +290,8 @@ def _forward_steps(model, seq_len):
     ).expanduser()
 
     sys.path.append(str(CHATGLM_MODEL_PATH))
-    from modeling_chatglm import ChatGLMModel
+    with relative_to_absolute_import(CHATGLM_MODEL_PATH / "modeling_chatglm.py"):
+        from modeling_chatglm import ChatGLMModel
     from transformers import AutoConfig
 
     config = AutoConfig.from_pretrained(CHATGLM_MODEL_PATH, trust_remote_code=True)
@@ -366,33 +380,13 @@ def _forward_steps(model, seq_len):
         y3.data.numpy().tofile(f)
 
 
-def make_data_glm2_model():
-    sys.path.append(str(CHATGLM2_MODEL_PATH))
-    from modeling_chatglm import ChatGLMModel
-    from transformers import AutoConfig
-
-    config = AutoConfig.from_pretrained(CHATGLM2_MODEL_PATH, trust_remote_code=True)
-    config.hidden_size = 32
-    config.num_attention_heads = 8
-    config.num_layers = 1
-    config.padded_vocab_size = 5
-    config.multi_query_group_num = 2
-    config.ffn_hidden_size = 48
-    config.kv_channels = config.hidden_size // config.num_attention_heads
-    config.torch_dtype = torch.float32
-
-    m = ChatGLMModel(config).float().eval()
-    for param in m.parameters():
-        param.data.uniform_(-0.5, 0.5)
-
-    seq_len = 3
-
+def _forward_steps(model, seq_len, **kwargs):
     # self attention
     x1 = torch.arange(seq_len, dtype=torch.int64)[None, :]
     position_ids = torch.arange(seq_len, dtype=torch.int64)[None, :]
     attn_mask = torch.ones(1, seq_len, dtype=torch.int64)
     with torch.no_grad():
-        out = m(x1, position_ids=position_ids, attention_mask=attn_mask, use_cache=True)
+        out = model(x1, position_ids=position_ids, attention_mask=attn_mask, use_cache=True, **kwargs)
         y1 = out.last_hidden_state
         kv_cache = out.past_key_values
 
@@ -401,7 +395,9 @@ def make_data_glm2_model():
     position_ids = torch.tensor([[seq_len]], dtype=torch.int64)
     attn_mask = torch.ones(1, seq_len + 1, dtype=torch.int64)
     with torch.no_grad():
-        out = m(x2, position_ids=position_ids, attention_mask=attn_mask, past_key_values=kv_cache, use_cache=True)
+        out = model(
+            x2, position_ids=position_ids, attention_mask=attn_mask, past_key_values=kv_cache, use_cache=True, **kwargs
+        )
         y2 = out.last_hidden_state
         kv_cache = out.past_key_values
 
@@ -410,10 +406,37 @@ def make_data_glm2_model():
     position_ids = torch.tensor([[seq_len + 1]], dtype=torch.int64)
     attn_mask = torch.ones(1, seq_len + 2, dtype=torch.int64)
     with torch.no_grad():
-        out = m(x3, position_ids=position_ids, attention_mask=attn_mask, past_key_values=kv_cache, use_cache=True)
+        out = model(
+            x3, position_ids=position_ids, attention_mask=attn_mask, past_key_values=kv_cache, use_cache=True, **kwargs
+        )
         y3 = out.last_hidden_state
         kv_cache = out.past_key_values
 
+    return x1, y1, x2, y2, x3, y3
+
+
+def make_data_glm2_model():
+    sys.path.append(str(CHATGLM2_MODEL_PATH))
+    with relative_to_absolute_import(CHATGLM2_MODEL_PATH / "modeling_chatglm.py"):
+        from modeling_chatglm import ChatGLMModel
+    from transformers import AutoConfig
+
+    config = AutoConfig.from_pretrained(CHATGLM2_MODEL_PATH, trust_remote_code=True)
+    config.hidden_size = 32
+    config.num_attention_heads = 8
+    config.num_layers = 1
+    config.padded_vocab_size = 5
+    config.multi_query_group_num = 2
+    config.ffn_hidden_size = 48
+    config.kv_channels = config.hidden_size // config.num_attention_heads
+    config.torch_dtype = torch.float32
+
+    m = ChatGLMModel(config).float().eval()
+    for param in m.parameters():
+        param.data.uniform_(-0.5, 0.5)
+
+    x1, y1, x2, y2, x3, y3 = _forward_steps(m, seq_len=3)
+
     print(m)
 
     with open(HERE / "data/glm2_model.data", "wb") as f:
@@ -437,46 +460,13 @@ def make_data_glm2_model():
 
 
 def make_data_glm3_model():
-    def _forward_steps(model, seq_len):
-        # self attention
-        x1 = torch.arange(seq_len, dtype=torch.int64)[None, :]
-        position_ids = torch.arange(seq_len, dtype=torch.int64)[None, :]
-        attn_mask = torch.ones(1, seq_len, dtype=torch.int64)
-        with torch.no_grad():
-            out = model(x1, position_ids=position_ids, attention_mask=attn_mask, use_cache=True)
-            y1 = out.last_hidden_state
-            kv_cache = out.past_key_values
-
-        # cross attention
-        x2 = torch.tensor([[seq_len]], dtype=torch.int64)
-        position_ids = torch.tensor([[seq_len]], dtype=torch.int64)
-        attn_mask = torch.ones(1, seq_len + 1, dtype=torch.int64)
-        with torch.no_grad():
-            out = model(
-                x2, position_ids=position_ids, attention_mask=attn_mask, past_key_values=kv_cache, use_cache=True
-            )
-            y2 = out.last_hidden_state
-            kv_cache = out.past_key_values
-
-        # cross attention
-        x3 = torch.tensor([[seq_len + 1]], dtype=torch.int64)
-        position_ids = torch.tensor([[seq_len + 1]], dtype=torch.int64)
-        attn_mask = torch.ones(1, seq_len + 2, dtype=torch.int64)
-        with torch.no_grad():
-            out = model(
-                x3, position_ids=position_ids, attention_mask=attn_mask, past_key_values=kv_cache, use_cache=True
-            )
-            y3 = out.last_hidden_state
-            kv_cache = out.past_key_values
-
-        return x1, y1, x2, y2, x3, y3
-
     CHATGLM3_MODEL_PATH = Path(
-        "~/.cache/huggingface/hub/models--THUDM--chatglm3-6b/snapshots/a5ba5501eb873d40d48bd0983bd2a8dd006bb838"
+        "~/.cache/huggingface/hub/models--THUDM--chatglm3-6b/snapshots/6f3b58ec10f088978ae174398f9d20b6dfc71552"
     ).expanduser()
 
     sys.path.append(str(CHATGLM3_MODEL_PATH))
-    from modeling_chatglm import ChatGLMModel
+    with relative_to_absolute_import(CHATGLM3_MODEL_PATH / "modeling_chatglm.py"):
+        from modeling_chatglm import ChatGLMModel
     from transformers import AutoConfig
 
     config = AutoConfig.from_pretrained(CHATGLM3_MODEL_PATH, trust_remote_code=True)
@@ -552,46 +542,13 @@ def _forward_steps(model, seq_len):
 
 
 def make_data_glm4_model():
-    def _forward_steps(model, seq_len):
-        # self attention
-        x1 = torch.arange(seq_len, dtype=torch.int64)[None, :]
-        position_ids = torch.arange(seq_len, dtype=torch.int64)[None, :]
-        attn_mask = torch.ones(1, seq_len, dtype=torch.int64)
-        with torch.no_grad():
-            out = model(x1, position_ids=position_ids, attention_mask=attn_mask, use_cache=True)
-            y1 = out.last_hidden_state
-            kv_cache = out.past_key_values
-
-        # cross attention
-        x2 = torch.tensor([[seq_len]], dtype=torch.int64)
-        position_ids = torch.tensor([[seq_len]], dtype=torch.int64)
-        attn_mask = torch.ones(1, seq_len + 1, dtype=torch.int64)
-        with torch.no_grad():
-            out = model(
-                x2, position_ids=position_ids, attention_mask=attn_mask, past_key_values=kv_cache, use_cache=True
-            )
-            y2 = out.last_hidden_state
-            kv_cache = out.past_key_values
-
-        # cross attention
-        x3 = torch.tensor([[seq_len + 1]], dtype=torch.int64)
-        position_ids = torch.tensor([[seq_len + 1]], dtype=torch.int64)
-        attn_mask = torch.ones(1, seq_len + 2, dtype=torch.int64)
-        with torch.no_grad():
-            out = model(
-                x3, position_ids=position_ids, attention_mask=attn_mask, past_key_values=kv_cache, use_cache=True
-            )
-            y3 = out.last_hidden_state
-            kv_cache = out.past_key_values
-
-        return x1, y1, x2, y2, x3, y3
-
     CHATGLM4_MODEL_PATH = Path(
-        "~/.cache/huggingface/hub/models--THUDM--glm-4-9b-chat/snapshots/6f060dcb542d48b6bc743628634924b271545428"
+        "~/.cache/huggingface/hub/models--THUDM--glm-4-9b-chat/snapshots/04419001bc63e05e70991ade6da1f91c4aeec278"
     ).expanduser()
 
     sys.path.append(str(CHATGLM4_MODEL_PATH))
-    from modeling_chatglm import ChatGLMModel
+    with relative_to_absolute_import(CHATGLM4_MODEL_PATH / "modeling_chatglm.py"):
+        from modeling_chatglm import ChatGLMModel
     from transformers import AutoConfig
 
     config = AutoConfig.from_pretrained(CHATGLM4_MODEL_PATH, trust_remote_code=True)
@@ -633,59 +590,124 @@ def _forward_steps(model, seq_len):
         y3.numpy().tofile(f)
 
 
-def make_glm4_pipeline_data():
-    from transformers import AutoModelForCausalLM, AutoTokenizer
-
-    tokenizer = AutoTokenizer.from_pretrained("THUDM/glm-4-9b-chat", trust_remote_code=True)
-
-    # tiktoken
-    chktxt = "\n \n\n \n\n\n \t \t\t \t\n  \n   \n    \n     \n🚀 (normal) 😶\u200d🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български ''''''```````\"\"\"\"......!!!!!!?????? I've been 'told he's there, 'RE you sure? 'M not sure I'll make it, 'D you like some tea? We'Ve a'lL"
-    print("tiktoken:", tokenizer.tokenizer.encode(chktxt, disallowed_special=()))
+def make_data_glm4v_model():
+    CHATGLM4V_MODEL_PATH = Path(
+        "~/.cache/huggingface/hub/models--THUDM--glm-4v-9b/snapshots/6c2e4732db8443f64a48d5af04b74425a7d169c4"
+    ).expanduser()
 
-    # tokenizer
-    inputs = tokenizer("你好")
-    print(f"encode: {inputs=}")
+    sys.path.append(str(CHATGLM4V_MODEL_PATH))
+    with relative_to_absolute_import(CHATGLM4V_MODEL_PATH / "modeling_chatglm.py"):
+        from modeling_chatglm import ChatGLMModel
+    from transformers import AutoConfig
 
-    conversation = [{"role": "user", "content": "你好"}]
-    inputs = tokenizer.apply_chat_template(
-        conversation,
-        add_generation_prompt=True,
-        tokenize=True,
-        return_tensors="pt",
-        return_dict=True,
+    config = AutoConfig.from_pretrained(CHATGLM4V_MODEL_PATH, trust_remote_code=True)
+    config.hidden_size = 32
+    config.num_attention_heads = 8
+    config.num_layers = 1
+    config.padded_vocab_size = 8
+    config.multi_query_group_num = 2
+    config.ffn_hidden_size = 48
+    config.kv_channels = config.hidden_size // config.num_attention_heads
+    config.torch_dtype = torch.float32
+    config.vision_config.update(
+        num_hidden_layers=1,
+        hidden_size=32,
+        patch_size=7,
+        num_heads=2,
+        intermediate_size=56,
+        num_positions=17,
+        image_size=28,
     )
-    print(f"apply_chat_template: {conversation=} {inputs=}")
 
-    # round 1
-    inputs = inputs.to("cuda")
-    model = AutoModelForCausalLM.from_pretrained(
-        "THUDM/glm-4-9b-chat",
-        torch_dtype=torch.bfloat16,
-        low_cpu_mem_usage=True,
-        trust_remote_code=True,
-        device_map="auto",
-    ).eval()
-
-    outputs = model.generate(**inputs, do_sample=False)
-    outputs = outputs[:, inputs["input_ids"].shape[1] :]
-    response = tokenizer.decode(outputs[0], skip_special_tokens=True).strip()
-    print(f"generate: {response=}")
+    m = ChatGLMModel(config).float().eval()
+    for param in m.parameters():
+        param.data.uniform_(-0.5, 0.5)
 
-    conversation += [{"role": "assistant", "content": response}, {"role": "user", "content": "晚上睡不着应该怎么办"}]
-    inputs = tokenizer.apply_chat_template(
-        conversation,
-        add_generation_prompt=True,
-        tokenize=True,
-        return_tensors="pt",
-        return_dict=True,
-    )
-    print(f"apply_chat_template: {conversation=} {inputs=}")
+    print(m)
 
-    # round 2
-    outputs = model.generate(**inputs, do_sample=False)
-    outputs = outputs[:, inputs["input_ids"].shape[1] :]
-    response = tokenizer.decode(outputs[0], skip_special_tokens=True).strip()
-    print(f"generate: {response=}")
+    x1, y1, x2, y2, x3, y3 = _forward_steps(m, seq_len=3)
+
+    # text
+    with open(HERE / "data/glm4v_model_text.data", "wb") as f:
+        tensors = [
+            # text model
+            m.embedding.word_embeddings.weight,
+            m.encoder.layers[0].input_layernorm.weight,
+            m.encoder.layers[0].self_attention.query_key_value.weight,
+            m.encoder.layers[0].self_attention.query_key_value.bias,
+            m.encoder.layers[0].self_attention.dense.weight,
+            m.encoder.layers[0].post_attention_layernorm.weight,
+            m.encoder.layers[0].mlp.dense_h_to_4h.weight,
+            m.encoder.layers[0].mlp.dense_4h_to_h.weight,
+            m.encoder.final_layernorm.weight,
+            # inputs & outputs
+            x1.int(),
+            y1,
+            x2.int(),
+            y2,
+            x3.int(),
+            y3,
+        ]
+        for tensor in tensors:
+            tensor.data.numpy().tofile(f)
+
+    m.config.boi_token_id = 2
+    m.config.eoi_token_id = 4
+
+    images = torch.randn(1, 3, config.vision_config["image_size"], config.vision_config["image_size"])
+    x1, y1, x2, y2, x3, y3 = _forward_steps(m, seq_len=6, images=images)
+
+    # vision & text
+    with open(HERE / "data/glm4v_model.data", "wb") as f:
+        tensors = [
+            # vision model
+            m.vision.patch_embedding.cls_embedding.half(),
+            m.vision.patch_embedding.proj.weight.half(),
+            m.vision.patch_embedding.proj.bias,
+            m.vision.patch_embedding.position_embedding.weight,
+            m.vision.transformer.layers[0].input_layernorm.weight,
+            m.vision.transformer.layers[0].input_layernorm.bias,
+            m.vision.transformer.layers[0].attention.query_key_value.weight,
+            m.vision.transformer.layers[0].attention.query_key_value.bias,
+            m.vision.transformer.layers[0].attention.dense.weight,
+            m.vision.transformer.layers[0].attention.dense.bias,
+            m.vision.transformer.layers[0].mlp.fc1.weight,
+            m.vision.transformer.layers[0].mlp.fc1.bias,
+            m.vision.transformer.layers[0].mlp.fc2.weight,
+            m.vision.transformer.layers[0].mlp.fc2.bias,
+            m.vision.transformer.layers[0].post_attention_layernorm.weight,
+            m.vision.transformer.layers[0].post_attention_layernorm.bias,
+            m.vision.conv.weight.half(),
+            m.vision.conv.bias,
+            m.vision.linear_proj.linear_proj.weight,
+            m.vision.linear_proj.norm1.weight,
+            m.vision.linear_proj.norm1.bias,
+            m.vision.linear_proj.gate_proj.weight,
+            m.vision.linear_proj.dense_h_to_4h.weight,
+            m.vision.linear_proj.dense_4h_to_h.weight,
+            m.vision.boi.half(),
+            m.vision.eoi.half(),
+            # text model
+            m.embedding.word_embeddings.weight,
+            m.encoder.layers[0].input_layernorm.weight,
+            m.encoder.layers[0].self_attention.query_key_value.weight,
+            m.encoder.layers[0].self_attention.query_key_value.bias,
+            m.encoder.layers[0].self_attention.dense.weight,
+            m.encoder.layers[0].post_attention_layernorm.weight,
+            m.encoder.layers[0].mlp.dense_h_to_4h.weight,
+            m.encoder.layers[0].mlp.dense_4h_to_h.weight,
+            m.encoder.final_layernorm.weight,
+            # inputs & outputs
+            images,
+            x1.int(),
+            y1,
+            x2.int(),
+            y2,
+            x3.int(),
+            y3,
+        ]
+        for tensor in tensors:
+            tensor.data.numpy().tofile(f)
 
 
 def make_glm4_pipeline_data():
@@ -753,11 +775,27 @@ def make_glm4_pipeline_data():
     print(f"apply_chat_template: {conversation=} {inputs=}")
 
     # round 2
+    inputs = inputs.to("cuda")
     outputs = model.generate(**inputs, do_sample=False)
     outputs = outputs[:, inputs["input_ids"].shape[1] :]
     response = tokenizer.decode(outputs[0], skip_special_tokens=True).strip()
     print(f"generate: {response=}")
 
+    # with image
+    from PIL import Image
+
+    tokenizer = AutoTokenizer.from_pretrained("THUDM/glm-4v-9b", trust_remote_code=True)
+    image = Image.open(HERE / "../examples/03-Confusing-Pictures.jpg")
+    conversation = [{"role": "user", "content": "描述这张图片", "image": image}]
+    inputs = tokenizer.apply_chat_template(
+        conversation,
+        add_generation_prompt=True,
+        tokenize=True,
+        return_tensors="pt",
+        return_dict=True,
+    )
+    print(f"apply_chat_template: {conversation=} {inputs=}")
+
 
 def main():
     torch.manual_seed(0)
@@ -769,7 +807,8 @@ def main():
     # make_data_glm2_model()
     # make_data_glm3_model()
     # make_data_glm4_model()
-    make_glm4_pipeline_data()
+    make_data_glm4v_model()
+    # make_glm4_pipeline_data()
 
 
 if __name__ == "__main__":