tq-kaz-instruct-230k

Uploaded model

• Developed by: erzhanbakanbayev
• License: apache-2.0
• Finetuned from model : models/pretrain-0907

Requirements

To install the necessary dependencies, use the following commands:

!pip install --no-deps "xformers<0.0.27" "trl<0.9.0"
!pip install peft accelerate bitsandbytes triton

Loading in 8bit with transformers

  from transformers import AutoTokenizer, AutoModelForCausalLM
  
  model_name = "erzhanbakanbayev/tq-kaz-instruct-230k"
  hf_token = "<ENTER YOUR TOKEN>"
  
  model = AutoModelForCausalLM.from_pretrained(model_name,load_in_8bit=True, use_auth_token=hf_token)
  tokenizer = AutoTokenizer.from_pretrained(model_name, use_auth_token=hf_token)

Running simple inference

  from transformers import TextStreamer
  inputs = tokenizer("Тапсырма: Келесі мәтіндегі пунктуацияларды және грамматикалық қателерді дұрыста. \n\nМәтін: Жаналыктар леби осиндай \n\nЖауабы:", return_tensors="pt")
  text_streamer = TextStreamer(tokenizer)
  _ = model.generate(**inputs, streamer = text_streamer, max_new_tokens = 128)

• running inference with threading and parameters

  from threading import Thread
  import textwrap 
  
  def generate_streaming_text(generation_kwargs):
      max_print_width = 2048
      thread = Thread(target=model.generate, kwargs=generation_kwargs)
      thread.start()
      length = 0
      for j, new_text in enumerate(text_streamer):
          if new_text == '<|end_of_text|>':
              break
          if j == 0:
              wrapped_text = textwrap.wrap(new_text, width=max_print_width)
              length = len(wrapped_text[-1])
              wrapped_text = " ".join(wrapped_text)
              print(wrapped_text, end="")
          else:
              length += len(new_text)
              if length >= max_print_width:
                  length = 0
                  print()
              print(new_text, end="")
      return

  input_texts = [
    "Сұрақ: Желтоқсан айында неше күн бар? \nЖауабы: ",
    'Грамматикалық қателерді дұрыста."\n\n### Мәтін:\nОған бұйермады\n\n### Жауабы:'
  ]

  from transformers import TextIteratorStreamer
  import torch
  
  text_streamer = TextIteratorStreamer(tokenizer)
  
  if tokenizer.pad_token_id is None:
      tokenizer.pad_token_id = tokenizer.eos_token_id
  inputs = tokenizer(
      input_texts[0],
      return_tensors="pt",
      padding=True,
      truncation=True,
      max_length=512, 
  ).to("cuda")
  attention_mask = inputs['attention_mask']
  generation_kwargs = {
      'input_ids': inputs['input_ids'],
      "streamer": text_streamer,
      "max_new_tokens": 280,
      # "use_cache": True,
      'pad_token_id': tokenizer.pad_token_id,         
      'attention_mask': attention_mask, 
      # 'no_repeat_ngram_size': 6, 
      # 'temperature': 0.4,        
      # 'top_k': 20,      
      # 'top_p': 0.95, 
  }
  
  generate_streaming_text(generation_kwargs)