orpheus-3b-0.1-ft-no-lora

This is a LoRA for Orpheus 3B that speaks Norwegian. This initial version is somewhat confused about dialects.

Examples

Input text:
"Hei! Jeg er en språkmodell som kan snakke norsk. Jeg har trent på flere timer med norsk tale."

Output:
<audio src="https://huggingface.co/lhaatveit/orpheus-3b-0.1-ft-no-lora/resolve/main/samples/sample_0.wav" controls="">https://huggingface.co/lhaatveit/orpheus-3b-0.1-ft-no-lora/resolve/main/samples/sample_0.wav</audio>

Input text:
"Jeg bygger på modellen Orpheus fra Canopy AI."

Output:
<audio src="https://huggingface.co/lhaatveit/orpheus-3b-0.1-ft-no-lora/resolve/main/samples/sample_1.wav" controls="">https://huggingface.co/lhaatveit/orpheus-3b-0.1-ft-no-lora/resolve/main/samples/sample_1.wav</audio>

Input text:
"Man skal ikke selge skinnet før bjørnen er skutt."

Output:
<audio src="https://huggingface.co/lhaatveit/orpheus-3b-0.1-ft-no-lora/resolve/main/samples/sample_2.wav" controls="">https://huggingface.co/lhaatveit/orpheus-3b-0.1-ft-no-lora/resolve/main/samples/sample_2.wav</audio>

Input text:
"Ibsens ripsbærbusker og andre buskvekster."

Output:
<audio src="https://huggingface.co/lhaatveit/orpheus-3b-0.1-ft-no-lora/resolve/main/samples/sample_3.wav" controls="">https://huggingface.co/lhaatveit/orpheus-3b-0.1-ft-no-lora/resolve/main/samples/sample_3.wav</audio>

Getting started

from unsloth import FastLanguageModel
import torch
from snac import SNAC
from peft import PeftModel
import soundfile as sf

base_model, tokenizer = FastLanguageModel.from_pretrained(
    model_name="unsloth/orpheus-3b-0.1-ft",
    max_seq_length=4096,
    dtype=None,
    load_in_4bit=False,
)

model = PeftModel.from_pretrained(
    base_model,
    "lhaatveit/orpheus-3b-0.1-ft-no-lora"
)

FastLanguageModel.for_inference(model)

snac_model = SNAC.from_pretrained("hubertsiuzdak/snac_24khz")
snac_model.to("cpu")

prompts = [
    "Hei! Jeg er en språkmodell som kan snakke norsk. Jeg har trent på flere timer med norsk tale.",
    "Jeg bygger på modellen Orpheus fra Canopy AI.",
    "Man skal ikke selge skinnet før bjørnen er skutt.",
    "Ibsens ripsbærbusker og andre buskvekster."
]

chosen_voice = "zoe"

prompts_ = [(f"{chosen_voice}: " + p) if chosen_voice else p for p in prompts]

all_input_ids = []

for prompt in prompts_:
  input_ids = tokenizer(prompt, return_tensors = "pt").input_ids
  all_input_ids.append(input_ids)

start_of_human_token = 128259
end_of_text_token = 128009
end_of_human_token = 128260
pad_token = 128263
start_of_speech_token = 128257
end_of_speech_token = 128258


start_token = torch.tensor([[start_of_human_token]], dtype = torch.int64)
end_tokens = torch.tensor([[end_of_text_token, end_of_human_token]], dtype = torch.int64)

all_modified_input_ids = []

for input_ids in all_input_ids:
  modified_input_ids = torch.cat([start_token, input_ids, end_tokens], dim = 1)
  all_modified_input_ids.append(modified_input_ids)

all_padded_tensors = []
all_attention_masks = []
max_length = max([modified_input_ids.shape[1] for modified_input_ids in all_modified_input_ids])

for modified_input_ids in all_modified_input_ids:
  padding = max_length - modified_input_ids.shape[1]
  padded_tensor = torch.cat([torch.full((1, padding), pad_token, dtype = torch.int64), modified_input_ids], dim = 1)
  attention_mask = torch.cat([torch.zeros((1, padding), dtype = torch.int64), torch.ones((1, modified_input_ids.shape[1]), dtype = torch.int64)], dim = 1)
  all_padded_tensors.append(padded_tensor)
  all_attention_masks.append(attention_mask)

all_padded_tensors = torch.cat(all_padded_tensors, dim = 0)
all_attention_masks = torch.cat(all_attention_masks, dim = 0)

input_ids = all_padded_tensors.to("cuda")
attention_mask = all_attention_masks.to("cuda")

generated_ids = model.generate(
      input_ids = input_ids,
      attention_mask = attention_mask,
      max_new_tokens = 3000,
      do_sample = True,
      temperature = 0.6,
      top_p = 0.95,
      repetition_penalty = 1.1,
      num_return_sequences = 1,
      eos_token_id = 128258,
      use_cache = True,
  )

token_to_find = start_of_speech_token
token_to_remove = end_of_speech_token

processed_rows = []
code_lists = []

for row in generated_ids:

    sos = (row == token_to_find).nonzero()[0]
    eos_tokens = (row == token_to_remove).nonzero()
    eos = (row == token_to_remove).nonzero()[0]
    snac_tokens = row[(sos[-1] + 1):eos[0]]

    invalid_offset = ((snac_tokens < 128266) + (snac_tokens >= (128266 + 7 *  4096))).nonzero()
    invalid = len(invalid_offset)
    
    if invalid:
        print ("row has illegal SNAC tokens")
        new_offset = invalid_offset[0]
        snac_tokens = snac_tokens[0:new_offset]
    
    modified_len = len(snac_tokens) // 7 * 7

    snac_codes = snac_tokens[:modified_len]
    snac_codes = [t - 128266 for t in snac_codes]
    
    code_lists.append(snac_codes)

def snac_decode(code_list):
  layer_1 = []
  layer_2 = []
  layer_3 = []
  for i in range((len(code_list)+1)//7):
    layer_1.append(code_list[7*i])
    layer_2.append(code_list[7*i+1]-4096)
    layer_3.append(code_list[7*i+2]-(2*4096))
    layer_3.append(code_list[7*i+3]-(3*4096))
    layer_2.append(code_list[7*i+4]-(4*4096))
    layer_3.append(code_list[7*i+5]-(5*4096))
    layer_3.append(code_list[7*i+6]-(6*4096))
  codes = [torch.tensor(layer_1).unsqueeze(0),
         torch.tensor(layer_2).unsqueeze(0),
         torch.tensor(layer_3).unsqueeze(0)]
  audio_hat = snac_model.decode(codes)
  return audio_hat

my_samples = []

for code_list in code_lists:
  samples = snac_decode(code_list)
  my_samples.append(samples)

for i in range(len(my_samples)):
  print(prompts[i])
  samples = my_samples[i] 
  audio = samples.detach().squeeze().to("cpu").numpy()
  sf.write(f"sample_{i}.wav", audio, 24000)