gemma3-4b-cpt-vqa-33k

gemma3-4b-cpt-vqa-33k

VQA adapter for the sequential CPT → VQA pipeline (Group 3 experiment). This adapter must be loaded together with the CPT adapter from Siluni/gemma3-4b-cpt and combined before inference.

• Base model: google/gemma-3-4b-it
• Experiment: Group 3 — Sequential CPT → VQA stage
• Training data: Siluni/sinhala-vqa-dataset (33k samples)
• Method: QLoRA (4-bit NF4, LoRA rank 16, alpha 32), applied after CPT stage

Loading (Sequential — both adapters required)

from transformers import AutoProcessor, Gemma3ForConditionalGeneration, BitsAndBytesConfig
from peft import PeftModel
import torch

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_use_double_quant=True,
    bnb_4bit_compute_dtype=torch.bfloat16,
)

base_model = Gemma3ForConditionalGeneration.from_pretrained(
    "google/gemma-3-4b-it",
    device_map="auto",
    quantization_config=bnb_config,
    torch_dtype=torch.bfloat16,
).eval()

processor = AutoProcessor.from_pretrained("google/gemma-3-4b-it")

# Step 1: load CPT adapter
model = PeftModel.from_pretrained(
    base_model,
    "Siluni/gemma3-4b-cpt",
    adapter_name="cpt",
    is_trainable=False,
)

# Step 2: load VQA adapter
model.load_adapter("Siluni/gemma3-4b-cpt-vqa-33k", adapter_name="vqa")

# Step 3: combine adapters
model.add_weighted_adapter(
    adapters=["cpt", "vqa"],
    weights=[1.0, 1.0],
    adapter_name="combined",
    combination_type="linear",
)
model.set_adapter("combined")
model.eval()

Inference Example

from PIL import Image

image = Image.open("your_image.jpg").convert("RGB")
question = "රූපයේ ඇත්තේ කුමක්ද?"  # "What is in the image?"

messages = [
    {
        "role": "user",
        "content": [
            {"type": "image", "image": image},
            {"type": "text",  "text": question},
        ],
    }
]

inputs = processor.apply_chat_template(
    messages,
    add_generation_prompt=True,
    tokenize=True,
    return_dict=True,
    return_tensors="pt",
    padding=True,
).to(model.device)

if "token_type_ids" not in inputs:
    inputs["token_type_ids"] = torch.zeros_like(inputs["input_ids"])

input_len = inputs["input_ids"].shape[-1]

with torch.inference_mode():
    output = model.generate(**inputs, max_new_tokens=128, do_sample=False)

answer = processor.decode(output[0][input_len:], skip_special_tokens=True).strip()
print(answer)

Citation

@misc{keerthiratne2025sinhalavqa,
  title        = {Benchmarking and Adapting Compact Multimodal Models for Sinhala Visual Question Answering},
  author       = {Keerthiratne, Siluni and Weerasinghe, Ruvan and Sumanathilaka, Deshan},
  year         = {2025},
  institution  = {Informatics Institute of Technology / Robert Gordon University},
}