Skip to content

Commit 74f8661

Browse files
tc-mbiceflame89harvestingmoonggerganov
authored and
arthw
committed
llava : support MiniCPM-V-2.5 (ggml-org#7599)
* init * rename * add run android for termux in readme * add android readme * add instructions in readme * change name in readme * Update README.md * fixed line * add result in readme * random pos_embed * add positions index * change for ollama * change for ollama * better pos_embed in clip * support ollama * updata cmakelist * updata cmakelist * rename wrapper * clear code * replace and organize code * add link * sync master * fix warnings * fix warnings * fix bug in bicubic resize when need resize iamge smaller * receive review comments and modify * receive review comments and modify * put all code into llava dir * fix quality problem in pr code * change n_layer * add space in "-1" * imitate reshape bug of python code * fix bug in clip * fix issues for merging * fix llama-minicpmv-cli in cmake file * change pr readme * fix code review * remove in line 33 directory in the /cmakelists.txt (not in example, in the main dir * fix cmakefile * add warn * fix KEY_HAS_MINICPMV_PROJ * remove load_image_size into clip_ctx * remove the extern "C", MINICPMV_API * fix uhd code for review comment * delete minicpmv-wrapper in pr * remove uhd_image_embed * Modify 2 notes * clip : style changes * del common.h in clip * fix Type-Check error * fix Type-Check error * fix Type-Check error * fix Type-Check error * fix makefile error * fix ubuntu-make error * try fix clip * try fix 1 --------- Co-authored-by: Hongji Zhu <fireyoucan@gmail.com> Co-authored-by: harvestingmoon <leewenyeong@gmail.com> Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
1 parent 025ebbb commit 74f8661

12 files changed

+1486
-77
lines changed

‎.gitignore

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,6 @@ models-mnt
7979
!models/ggml-vocab-*.gguf*
8080

8181
# Zig
82-
8382
zig-out/
8483
zig-cache/
8584

‎Makefile

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ BUILD_TARGETS = \
1919
llama-imatrix \
2020
llama-infill \
2121
llama-llava-cli \
22+
llama-minicpmv-cli\
2223
llama-lookahead \
2324
llama-lookup \
2425
llama-lookup-create \
@@ -1462,6 +1463,17 @@ llama-llava-cli: examples/llava/llava-cli.cpp \
14621463
$(CXX) $(CXXFLAGS) -c examples/llava/llava.cpp -o $(call GET_OBJ_FILE, examples/llava/llava.cpp)
14631464
$(CXX) $(CXXFLAGS) $(filter-out %.h $< examples/llava/clip.cpp examples/llava/llava.cpp,$^) $(call GET_OBJ_FILE, $<) $(call GET_OBJ_FILE, examples/llava/clip.cpp) $(call GET_OBJ_FILE, examples/llava/llava.cpp) -o $@ $(LDFLAGS)
14641465

1466+
llama-minicpmv-cli: examples/llava/minicpmv-cli.cpp \
1467+
examples/llava/clip.h \
1468+
examples/llava/clip.cpp \
1469+
examples/llava/llava.h \
1470+
examples/llava/llava.cpp \
1471+
$(OBJ_ALL)
1472+
$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
1473+
$(CXX) $(CXXFLAGS) -c examples/llava/clip.cpp -o $(call GET_OBJ_FILE, examples/llava/clip.cpp) -Wno-cast-qual
1474+
$(CXX) $(CXXFLAGS) -c examples/llava/llava.cpp -o $(call GET_OBJ_FILE, examples/llava/llava.cpp)
1475+
$(CXX) $(CXXFLAGS) $(filter-out %.h $< examples/llava/clip.cpp examples/llava/llava.cpp,$^) $(call GET_OBJ_FILE, $<) $(call GET_OBJ_FILE, examples/llava/clip.cpp) $(call GET_OBJ_FILE, examples/llava/llava.cpp) -o $@ $(LDFLAGS)
1476+
14651477
ifeq ($(UNAME_S),Darwin)
14661478
swift: examples/batched.swift
14671479
(cd examples/batched.swift; make build)

‎examples/llava/CMakeLists.txt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,10 @@ set_target_properties(${TARGET} PROPERTIES OUTPUT_NAME llama-llava-cli)
3636
install(TARGETS ${TARGET} RUNTIME)
3737
target_link_libraries(${TARGET} PRIVATE common llava ${CMAKE_THREAD_LIBS_INIT})
3838
target_compile_features(${TARGET} PRIVATE cxx_std_11)
39+
40+
set(TARGET llama-minicpmv-cli)
41+
add_executable(${TARGET} minicpmv-cli.cpp)
42+
set_target_properties(${TARGET} PROPERTIES OUTPUT_NAME llama-minicpmv-cli)
43+
install(TARGETS ${TARGET} RUNTIME)
44+
target_link_libraries(${TARGET} PRIVATE common llava ${CMAKE_THREAD_LIBS_INIT})
45+
target_compile_features(${TARGET} PRIVATE cxx_std_11)

‎examples/llava/README-minicpmv2.5.md

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
## MiniCPM-Llama3-V 2.5
2+
3+
### Prepare models and code
4+
5+
Download [MiniCPM-Llama3-V-2_5](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5) PyTorch model from huggingface to "MiniCPM-Llama3-V-2_5" folder.
6+
7+
Clone llama.cpp:
8+
```bash
9+
git clone https://github.com/ggerganov/llama.cpp
10+
cd llama.cpp
11+
```
12+
13+
### Usage
14+
15+
Convert PyTorch model to gguf files (You can also download the converted [gguf](https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5-gguf) by us)
16+
17+
```bash
18+
python ./examples/minicpmv/minicpmv-surgery.py -m ../MiniCPM-Llama3-V-2_5
19+
python ./examples/minicpmv/minicpmv-convert-image-encoder-to-gguf.py -m ../MiniCPM-Llama3-V-2_5 --minicpmv-projector ../MiniCPM-Llama3-V-2_5/minicpmv.projector --output-dir ../MiniCPM-Llama3-V-2_5/ --image-mean 0.5 0.5 0.5 --image-std 0.5 0.5 0.5
20+
python ./convert-hf-to-gguf.py ../MiniCPM-Llama3-V-2_5/model
21+
22+
# quantize int4 version
23+
./llama-quantize ../MiniCPM-Llama3-V-2_5/model/model-8B-F16.gguf ../MiniCPM-Llama3-V-2_5/model/ggml-model-Q4_K_M.gguf Q4_K_M
24+
```
25+
26+
Build for Linux or Mac
27+
28+
```bash
29+
make
30+
make llama-minicpmv-cli
31+
```
32+
33+
Inference on Linux or Mac
34+
```
35+
# run f16 version
36+
./llama-minicpmv-cli -m ../MiniCPM-Llama3-V-2_5/model/model-8B-F16.gguf --mmproj ../MiniCPM-Llama3-V-2_5/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -p "What is in the image?"
37+
38+
# run quantized int4 version
39+
./llama-minicpmv-cli -m ../MiniCPM-Llama3-V-2_5/model/ggml-model-Q4_K_M.gguf --mmproj ../MiniCPM-Llama3-V-2_5/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -p "What is in the image?"
40+
41+
# or run in interactive mode
42+
./llama-minicpmv-cli -m ../MiniCPM-Llama3-V-2_5/model/ggml-model-Q4_K_M.gguf --mmproj ../MiniCPM-Llama3-V-2_5/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -i
43+
```
44+
45+
### Android
46+
47+
#### Build on Android device using Termux
48+
We found that build on Android device would bring better runtime performance, so we recommend to build on device.
49+
50+
[Termux](https://github.com/termux/termux-app#installation) is a terminal app on Android device (no root required).
51+
52+
Install tools in Termux:
53+
```
54+
apt update && apt upgrade -y
55+
apt install git make cmake
56+
```
57+
58+
It's recommended to move your model inside the `~/` directory for best performance:
59+
```
60+
cd storage/downloads
61+
mv model.gguf ~/
62+
```
63+
64+
#### Building the Project using Android NDK
65+
Obtain the [Android NDK](https://developer.android.com/ndk) and then build with CMake.
66+
67+
Execute the following commands on your computer to avoid downloading the NDK to your mobile. Alternatively, you can also do this in Termux:
68+
69+
```bash
70+
mkdir build-android
71+
cd build-android
72+
export NDK=/your_ndk_path
73+
cmake -DCMAKE_TOOLCHAIN_FILE=$NDK/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=android-23 -DCMAKE_C_FLAGS=-march=armv8.4a+dotprod ..
74+
make
75+
```
76+
77+
Install [termux](https://github.com/termux/termux-app#installation) on your device and run `termux-setup-storage` to get access to your SD card (if Android 11+ then run the command twice).
78+
79+
Finally, copy these built `llama` binaries and the model file to your device storage. Because the file permissions in the Android sdcard cannot be changed, you can copy the executable files to the `/data/data/com.termux/files/home/bin` path, and then execute the following commands in Termux to add executable permission:
80+
81+
(Assumed that you have pushed the built executable files to the /sdcard/llama.cpp/bin path using `adb push`)
82+
```
83+
$cp -r /sdcard/llama.cpp/bin /data/data/com.termux/files/home/
84+
$cd /data/data/com.termux/files/home/bin
85+
$chmod +x ./*
86+
```
87+
88+
Download models and push them to `/sdcard/llama.cpp/`, then move it to `/data/data/com.termux/files/home/model/`
89+
90+
```
91+
$mv /sdcard/llama.cpp/ggml-model-Q4_K_M.gguf /data/data/com.termux/files/home/model/
92+
$mv /sdcard/llama.cpp/mmproj-model-f16.gguf /data/data/com.termux/files/home/model/
93+
```
94+
95+
Now, you can start chatting:
96+
```
97+
$cd /data/data/com.termux/files/home/bin
98+
$./llama-minicpmv-cli -m ../model/ggml-model-Q4_K_M.gguf --mmproj ../model/mmproj-model-f16.gguf -c 4096 --temp 0.7 --top-p 0.8 --top-k 100 --repeat-penalty 1.05 --image xx.jpg -p "What is in the image?"
99+
```

0 commit comments

Comments
 (0)