Downloads · 30 days
71
11% of all-time downloads
yujiepan/voxtral-tiny-random
voxtral-tiny-random is a text generation model from yujiepan. Use it when you need the model to write or continue text. It is set up for transformers.
This tiny model is for debugging. It is randomly initialized with the config adapted from mistralai/Voxtral-Small-24B-2507.
Downloads · 30 days
71
11% of all-time downloads
All-time downloads
631
Public
Parameters
8.7M
32.3 MB on disk
Likes
0
Public
Click a slice to open those files.
.safetensors17.4 MB · 54%
From the Hugging Face model README
This tiny model is for debugging. It is randomly initialized with the config adapted from mistralai/Voxtral-Small-24B-2507.
vllm serve yujiepan/voxtral-tiny-random --trust-remote-code
import torch
from transformers import AutoProcessor, VoxtralForConditionalGeneration
model_id = "yujiepan/voxtral-tiny-random"
device = "cuda"
processor = AutoProcessor.from_pretrained(model_id)
model = VoxtralForConditionalGeneration.from_pretrained(model_id, torch_dtype=torch.bfloat16, device_map=device)
conversation = [
{
"role": "user",
"content": [
{
"type": "audio",
"path": "https://huggingface.co/datasets/hf-internal-testing/dummy-audio-samples/resolve/main/mary_had_lamb.mp3",
},
{
"type": "audio",
"path": "https://huggingface.co/datasets/hf-internal-testing/dummy-audio-samples/resolve/main/winning_call.mp3",
},
{"type": "text", "text": "What sport and what nursery rhyme are referenced?"},
],
}
]
inputs = processor.apply_chat_template(conversation)
inputs = inputs.to(device, dtype=torch.bfloat16)
outputs = model.generate(**inputs, max_new_tokens=32)
decoded_outputs = processor.batch_decode(outputs[:, inputs.input_ids.shape[1]:], skip_special_tokens=True)
print("\nGenerated response:")
print("=" * 80)
print(decoded_outputs[0])
print("=" * 80)
import json
from pathlib import Path
import accelerate
import torch
from huggingface_hub import file_exists, hf_hub_download
from transformers import (
AutoConfig,
AutoModel,
AutoModelForCausalLM,
AutoProcessor,
GenerationConfig,
set_seed,
)
source_model_id = "mistralai/Voxtral-Small-24B-2507"
save_folder = "/tmp/yujiepan/voxtral-tiny-random"
processor = AutoProcessor.from_pretrained(source_model_id, trust_remote_code=True)
processor.save_pretrained(save_folder)
with open(hf_hub_download(source_model_id, filename='config.json', repo_type='model'), 'r', encoding='utf-8') as f:
config_json = json.load(f)
config_json['audio_config'].update(
{
"head_dim": 32,
"hidden_size": 64,
"intermediate_size": 256,
"num_attention_heads": 2,
"num_key_value_heads": 2,
"num_hidden_layers": 2,
}
)
config_json['hidden_size'] = 64
config_json['text_config'].update(
{
"head_dim": 32,
"hidden_size": 64,
"intermediate_size": 128,
"num_attention_heads": 2,
"num_key_value_heads": 1,
"num_hidden_layers": 2,
'tie_word_embeddings': True,
}
)
with open(f"{save_folder}/config.json", "w", encoding='utf-8') as f:
json.dump(config_json, f, indent=2)
config = AutoConfig.from_pretrained(
save_folder,
trust_remote_code=True,
)
print(config)
torch.set_default_dtype(torch.bfloat16)
model = AutoModel.from_config(config)
torch.set_default_dtype(torch.float32)
if file_exists(filename="generation_config.json", repo_id=source_model_id, repo_type='model'):
model.generation_config = GenerationConfig.from_pretrained(
source_model_id, trust_remote_code=True,
)
set_seed(42)
model = model.cpu() # cpu is more stable for random initialization across machines
with torch.no_grad():
for name, p in sorted(model.named_parameters()):
torch.nn.init.normal_(p, 0, 0.2)
print(name, p.shape)
model.save_pretrained(save_folder)
print(model)
VoxtralForConditionalGeneration(
(audio_tower): VoxtralEncoder(
(conv1): Conv1d(128, 64, kernel_size=(3,), stride=(1,), padding=(1,))
(conv2): Conv1d(64, 64, kernel_size=(3,), stride=(2,), padding=(1,))
(embed_positions): Embedding(1500, 64)
(layers): ModuleList(
(0-1): 2 x VoxtralEncoderLayer(
(self_attn): VoxtralAttention(
(k_proj): Linear(in_features=64, out_features=64, bias=False)
(v_proj): Linear(in_features=64, out_features=64, bias=True)
(q_proj): Linear(in_features=64, out_features=64, bias=True)
(out_proj): Linear(in_features=64, out_features=64, bias=True)
)
(self_attn_layer_norm): LayerNorm((64,), eps=1e-05, elementwise_affine=True)
(activation_fn): GELUActivation()
(fc1): Linear(in_features=64, out_features=256, bias=True)
(fc2): Linear(in_features=256, out_features=64, bias=True)
(final_layer_norm): LayerNorm((64,), eps=1e-05, elementwise_affine=True)
)
)
(layer_norm): LayerNorm((64,), eps=1e-05, elementwise_affine=True)
(avg_pooler): AvgPool1d(kernel_size=(2,), stride=(2,), padding=(0,))
)
(language_model): LlamaForCausalLM(
(model): LlamaModel(
(embed_tokens): Embedding(131072, 64)
(layers): ModuleList(
(0-1): 2 x LlamaDecoderLayer(
(self_attn): LlamaAttention(
(q_proj): Linear(in_features=64, out_features=64, bias=False)
(k_proj): Linear(in_features=64, out_features=32, bias=False)
(v_proj): Linear(in_features=64, out_features=32, bias=False)
(o_proj): Linear(in_features=64, out_features=64, bias=False)
)
(mlp): LlamaMLP(
(gate_proj): Linear(in_features=64, out_features=128, bias=False)
(up_proj): Linear(in_features=64, out_features=128, bias=False)
(down_proj): Linear(in_features=128, out_features=64, bias=False)
(act_fn): SiLU()
)
(input_layernorm): LlamaRMSNorm((64,), eps=1e-05)
(post_attention_layernorm): LlamaRMSNorm((64,), eps=1e-05)
)
)
(norm): LlamaRMSNorm((64,), eps=1e-05)
(rotary_emb): LlamaRotaryEmbedding()
)
(lm_head): Linear(in_features=64, out_features=131072, bias=False)
)
(multi_modal_projector): VoxtralMultiModalProjector(
(linear_1): Linear(in_features=256, out_features=64, bias=False)
(act): GELUActivation()
(linear_2): Linear(in_features=64, out_features=64, bias=False)
)
)