Qwen2.5-VL-3B-3k-SFT-NIG-lora

Base model: https://huggingface.co/Qwen/Qwen2.5-VL-3B-Instruct

This adapter was trained with LoRA.

You may refer to the following codes.

base_model_path = f"Qwen2.5-VL-3B-Instruct"
adapter_path = "THIS ADAPTER"

class QWEN_VL_Inference():
    def __init__(self, base_model_path, adapter_path):
        self.base_model_path = base_model_path
        self.adapter_path = adapter_path
        self.processor = AutoProcessor.from_pretrained(self.base_model_path)
        self.load_merge_model()

    def load_merge_model(self):
        self.base_model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
            self.base_model_path,
            torch_dtype=torch.bfloat16,
            attn_implementation="flash_attention_2",
            device_map="auto"
        )
        self.model = PeftModel.from_pretrained(self.base_model, self.adapter_path)
        self.merged_model = self.model.merge_and_unload()

    def infer(self, messages):
        text = self.processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
        image_inputs, _ = process_vision_info(messages)
        inputs = self.processor(text=[text], images=image_inputs, return_tensors="pt", padding=True).to('cuda')
        generated_ids = self.merged_model.generate(**inputs, max_new_tokens=512)
        generated_ids_trimmed = [out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)]
        output_text = self.processor.batch_decode(generated_ids_trimmed, skip_special_tokens=True,clean_up_tokenization_spaces=False)
        return output_text