Maika-Buddy-2B7-Chat-v2

Finetune LoRA DPO from Maika-Buddy-2B7-Chat

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

torch.set_default_device("cuda")

model_path = "olliai/Maika-Buddy-2B7-Chat-v2"

model = AutoModelForCausalLM.from_pretrained(model_path, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)

prompt = """A chat between a curious kid and an artificial intelligence assistant. The assistant gives helpful, easy-to-understand, detailed, and polite answers to the questions of the kid.
Kid: I have 10 apples. I gave Bob 2 pencil. How many apples do I have now?
AI: """

inputs = tokenizer(prompt, return_tensors="pt", return_attention_mask=False)

outputs = model.generate(**inputs, max_length=200,
                         do_sample=True,
                        repetition_penalty=1.1,
                        temperature=0.3,
                        top_k=50,
                        top_p=0.95, 
                        eos_token_id=tokenizer.encode("<|endoftext|>"))
text = tokenizer.batch_decode(outputs)[0]
print(text)