Downloads · 30 days
19
5% of all-time downloads
mesolitica/malaysian-tinyllama-1.1b-siglip-base-384-vision
malaysian-tinyllama-1.1b-siglip-base-384-vision is a feature extraction model from mesolitica. Use it when you need embeddings to search or compare text. It is set up for transformers.
WanDB https://wandb.ai/huseinzol05/vision-tinyllama?workspace=user-huseinzol05
Downloads · 30 days
19
5% of all-time downloads
All-time downloads
418
Public
Parameters
1.3B
5.2 GB on disk
Likes
1
Public
Click a slice to open those files.
.safetensors2.6 GB · 100%
From the Hugging Face model README
WanDB https://wandb.ai/huseinzol05/vision-tinyllama?workspace=user-huseinzol05
from modeling_vision import MM_LLMs, MM_LLMs_Config
from transformers import AutoTokenizer, AutoProcessor
from PIL import Image
import requests
model = MM_LLMs.from_pretrained(
'mesolitica/malaysian-tinyllama-1.1b-siglip-base-384-vision',
flash_attention = True,
dtype = torch.bfloat16,
torch_dtype = torch.bfloat16
)
_ = model.cuda()
image_processor = AutoProcessor.from_pretrained('google/siglip-base-patch16-384')
tokenizer = AutoTokenizer.from_pretrained('mesolitica/malaysian-tinyllama-1.1b-siglip-base-384-vision')
def prepare_dataset(messages, images: List[str] = None):
if images is not None:
images = [Image.open(f).convert('RGB') for f in images]
image_output = image_processor(images=images, return_tensors='pt')['pixel_values']
else:
image_output = None
prompt = tokenizer.apply_chat_template(messages, tokenize = False)
outputs = tokenizer(
prompt,
return_tensors='pt',
return_overflowing_tokens=False,
return_length=False)
outputs['images'] = image_output
outputs['image_index'] = torch.tensor([0] * len(outputs['images']))
outputs['image_starts'] = torch.tensor([tokenizer.convert_tokens_to_ids('<image>')] * len(outputs['images']))
return outputs
with open('Persian-cat-breed.jpg', 'wb') as fopen:
fopen.write(requests.get('https://cdn.beautifulnara.net/wp-content/uploads/2017/12/10201620/Persian-cat-breed.jpg').content)
with open('nasi-goreng-1-23.jpg', 'wb') as fopen:
fopen.write(requests.get('https://www.jocooks.com/wp-content/uploads/2023/09/nasi-goreng-1-23.jpg').content)
messages = [
{'role': 'user', 'content': '<image> </image> ini gambar apa'},
]
outputs = prepare_dataset(messages, images = ['Persian-cat-breed.jpg'])
outputs['images'] = outputs['images'].type(model.dtype)
for k in outputs.keys():
if outputs[k] is not None:
outputs[k] = outputs[k].cuda()
with torch.no_grad():
model_inputs = model.prepare_inputs_for_generation(**outputs)
r = model_inputs.pop('input_ids', None)
generate_kwargs = dict(
model_inputs,
max_new_tokens=300,
top_p=0.95,
top_k=50,
temperature=0.1,
do_sample=True,
num_beams=1,
)
r = model.llm.generate(**generate_kwargs)
print(tokenizer.decode(r[0]))
<s>Imej itu menunjukkan seekor kucing putih yang comel duduk di atas sofa hitam.</s>
messages = [
{'role': 'user', 'content': '<image> </image> <image> </image> apa kaitan 2 gambar ni'},
]
outputs = prepare_dataset(messages, images = ['Persian-cat-breed.jpg', 'nasi-goreng-1-23.jpg'])
outputs['images'] = outputs['images'].type(model.dtype)
for k in outputs.keys():
if outputs[k] is not None:
outputs[k] = outputs[k].cuda()
with torch.no_grad():
model_inputs = model.prepare_inputs_for_generation(**outputs)
r = model_inputs.pop('input_ids', None)
generate_kwargs = dict(
model_inputs,
max_new_tokens=300,
top_p=0.95,
top_k=50,
temperature=0.1,
do_sample=True,
num_beams=1,
)
r = model.llm.generate(**generate_kwargs)
print(tokenizer.decode(r[0]))
<s>Tiada hubungan yang jelas antara gambar 1 (anak kucing putih duduk di atas sofa) dan gambar 2 (foto penutup mangkuk mi telur dengan nasi dan cili). Gambar pertama ialah imej haiwan, manakala gambar kedua ialah imej makanan. Mereka tergolong dalam kategori yang berbeza dan tidak mempunyai hubungan antara satu sama lain.</s>