rico-smolvlm-full

rico-smolvlm-full

SmolVLM-256M-Instruct fine-tuned with LoRA on RICO-Screen2Words for UI screenshot captioning. LoRA adapters were merged into the base model for single-artifact deployment.

Model details

Usage

Full model (recommended)

from PIL import Image
import torch
from transformers import AutoProcessor, AutoModelForImageTextToText

model_id = "AmitPrakash/rico-smolvlm-full"
processor = AutoProcessor.from_pretrained(model_id)
model = AutoModelForImageTextToText.from_pretrained(
    model_id, dtype=torch.bfloat16
).cuda().eval()

image = Image.open("screenshot.png").convert("RGB")
messages = [{
    "role": "user",
    "content": [
        {"type": "image"},
        {"type": "text", "text": "Describe this UI screenshot accurately and concisely."}
    ]
}]
text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
inputs = {k: v.cuda() for k, v in processor(text=[text], images=[image], return_tensors="pt").items()}

with torch.inference_mode():
    out = model.generate(**inputs, max_new_tokens=96, do_sample=False)
print(processor.batch_decode(out, skip_special_tokens=True)[0])

LoRA adapter variant (load + merge before inference)

from peft import PeftModel
from transformers import AutoModelForImageTextToText, AutoProcessor
import torch

base = AutoModelForImageTextToText.from_pretrained(
    "HuggingFaceTB/SmolVLM-256M-Instruct", dtype=torch.bfloat16
)
model = PeftModel.from_pretrained(base, "AmitPrakash/rico-smolvlm-lora")
model = model.merge_and_unload().cuda().eval()
processor = AutoProcessor.from_pretrained("AmitPrakash/rico-smolvlm-lora")

# then run inference as above

Design decisions