stablelm-ft2

Usage:

from transformers import AutoModelForCausalLM, AutoTokenizer

import torch

tokenizer = AutoTokenizer.from_pretrained("TinyPixel/stablelm-ft2", trust_remote_code=True)

model = AutoModelForCausalLM.from_pretrained("TinyPixel/stablelm-ft2", torch_dtype=torch.bfloat16, device_map="auto", trust_remote_code=True)

text = '''### System:
You are a helpful AI assistant. 

### User:
Why is sky blue?

### Assistant:
'''

device = "cuda:0" 

inputs = tokenizer(text, return_tensors="pt").to(device)

outputs = model.generate(**inputs,  
                          max_new_tokens=512,
                          do_sample=True,
                          top_p=0.95,
                          temperature=0.7,
                          top_k=50)

print(tokenizer.decode(outputs[0], skip_special_tokens=False))