DeepSeek-OCR-2-GGUF

The DeepSeek-OCR 1/2 is now supported in llama-cpp-python. This project provides a test GGUF file.

Note: Based on current testing, it is not particularly stable.

llama-cpp-python: https://github.com/JamePeng/llama-cpp-python

Code example:

import base64
from pathlib import Path

from llama_cpp import Llama
from llama_cpp.llama_chat_format import MTMDChatHandler


MODEL_PATH = "./DeepSeek-OCR-2-BF16.gguf"
MMPROJ_PATH = "./mmproj-DeepSeek-OCR-2-BF16.gguf"
IMAGE_PATH = Path("deepseekocr2.png")

# OCR prompts:
# OCR_PROMPT = "\nFree OCR."
OCR_PROMPT = "\n<|grounding|>Convert the document to markdown."


_IMAGE_MIME_TYPES = {
    ".png": "image/png",
    ".jpg": "image/jpeg",
    ".jpeg": "image/jpeg",
    ".webp": "image/webp",
    ".bmp": "image/bmp",
}


def image_to_data_uri(image_path: Path) -> str:
    """Encode a local image as a data URI."""
    if not image_path.is_file():
        raise FileNotFoundError(f"Image file not found: {image_path}")

    mime_type = _IMAGE_MIME_TYPES.get(image_path.suffix.lower(), "application/octet-stream")
    encoded = base64.b64encode(image_path.read_bytes()).decode("utf-8")
    return f"data:{mime_type};base64,{encoded}"


llm = Llama(
    model_path=MODEL_PATH,
    chat_handler=MTMDChatHandler(
        clip_model_path=MMPROJ_PATH,
        verbose=True,
    ),
    n_gpu_layers=-1,
    n_ctx=10240,
    verbose=False,
    verbosity=1,
)

response = llm.create_chat_completion(
    messages=[
        {
            "role": "user",
            "content": [
                {
                    "type": "image_url",
                    "image_url": {
                        "url": image_to_data_uri(IMAGE_PATH),
                    },
                },
                {
                    "type": "text",
                    "text": OCR_PROMPT,
                },
            ],
        }
    ],
    max_tokens=4096,
)

print(response["choices"][0]["message"]["content"])