llama3c

Model Introduction

This model is trained on Llama3 with chinese datasets.

Quick Start

# load libs
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from transformers import GenerationConfig

if torch.backends.mps.is_available():
    device = torch.device("mps")
else:
    if torch.cuda.is_available():
        device = torch.device(0)
    else:
        device = torch.device('cpu')
load_type = torch.float16

DEFAULT_SYSTEM_PROMPT = """You are a helpful assistant. 你是一个乐于助人的助手。"""
system_format='<|start_header_id|>system<|end_header_id|>\n\n{content}<|eot_id|>'
user_format='<|start_header_id|>user<|end_header_id|>\n\n{content}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n'
assistant_format='{content}<|eot_id|>'
def generate_prompt(instruction):
    return system_format.format(content=DEFAULT_SYSTEM_PROMPT) + user_format.format(content=instruction)


tokenizer_path = './'
model_path = tokenizer_path
generation_config = GenerationConfig(
    temperature=0.2,
    top_k=40,
    top_p=0.9,
    do_sample=True,
    num_beams=1,
    repetition_penalty=1.1,
    max_new_tokens=400
)
# load tokenizer
tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)
terminators = [
            tokenizer.eos_token_id,
            tokenizer.convert_tokens_to_ids("<|eot_id|>")
        ]
# load model
model = AutoModelForCausalLM.from_pretrained(
    model_path,
    torch_dtype=load_type,
    low_cpu_mem_usage=True,
    device_map=device,
)
model.eval()

# generate text
with torch.no_grad():
    input_text = "辅酶Q10是个什么药，是保健品吗？"
    input_text = generate_prompt(instruction=input_text)
    inputs = tokenizer(input_text,return_tensors="pt")  #add_special_tokens=False ?
    generation_output = model.generate(
        input_ids = inputs["input_ids"].to(device),
        attention_mask = inputs['attention_mask'].to(device),
        eos_token_id=terminators,
        pad_token_id=tokenizer.eos_token_id,
        generation_config = generation_config
    )
    s = generation_output[0]
    output = tokenizer.decode(s,skip_special_tokens=True)
    response = output.split("assistant\n\n")[1].strip()
    print(response)

> 辅酶Q10是一种维生素类似物，它在人体内起着辅助作用，帮助身体更好地利用其他营养成分。它不是一种药物，也不是一种治疗疾病的药物，而是一种保健品。辅酶Q10可以通过食物摄入，也可以作为补充剂服用。但是，如果你有任何健康问题，最好先咨询医生的意见，然后再决定是否使用辅酶Q10或其他保健品。

For more detail, can refer to this notebook

Intended Usage

Training Details

Evaluation Results

Citation

Authors

@theanswer @SichaoY