colqwen2.5-base-lora-20251220

Model Card for Model ID

<!-- Provide a quick summary of what the model is/does. -->

---

base_model: colpali-engine/colqwen2.5-base library_name: peft

ColQwen2.5 LoRA Adapter

This repository contains a LoRA adapter trained on top of
ColQwen2.5 base model.

Usage

import torch
from PIL import Image
from peft import PeftModel
from transformers.utils.import_utils import is_flash_attn_2_available
from colpali_engine.models import ColQwen2_5, ColQwen2_5_Processor

base_model_id = "colpali-engine/colqwen2.5-base"
adapter_id = "gangsu/colqwen2.5-base-lora-20251220"

base = ColQwen2_5.from_pretrained(
    base_model_id,
    torch_dtype=torch.bfloat16,
    device_map="cuda",
    attn_implementation="flash_attention_2" if is_flash_attn_2_available() else None,
)

model = PeftModel.from_pretrained(base, adapter_id)
model.eval()

processor = ColQwen2_5_Processor.from_pretrained(model_name)

# Your inputs
images = [
    Image.new("RGB", (128, 128), color="white"),
    Image.new("RGB", (64, 32), color="black"),
]
queries = [
    "Is attention really all you need?",
    "What is the amount of bananas farmed in Salvador?",
]

# Process the inputs
batch_images = processor.process_images(images).to(model.device)
batch_queries = processor.process_queries(queries).to(model.device)

# Forward pass
with torch.no_grad():
    image_embeddings = model(**batch_images)
    query_embeddings = model(**batch_queries)

scores = processor.score_multi_vector(query_embeddings, image_embeddings)
print(scores)