warsa_cx

Warsa 110M Industrial (warsa_cx)

Warsa 110M (warsa_cx) adalah Large Language Model (LLM) tipe decoder-only yang dibangun dan dilatih murni dari scratch (nol) menggunakan PyTorch. Angka Romawi cx melambangkan kapasitas model ini yang berjalan di skala 110 Juta parameter.

Model ini dirancang khusus untuk berjalan secara efisien pada perangkat komputasi ringan (seperti smartphone melalui Termux, routing OpenWRT, atau single-board machine) dengan fokus performa pada percakapan kontekstual Bahasa Indonesia.

📊 Spesifikasi Model & Arsitektur

• Nama Sandi: warsa_cx (110M Parameter)
• Arsitektur: Transformer Decoder-Only kustom dengan CausalSelfAttentionFixed dinamis.
• Panjang Konteks (Max Length): 128 Token
• Tokenizer: Byte-Pair Encoding (BPE) Skala Industri berbasis Rust Engine (Kapasitas vocab_size = 3000).
• Dataset: Data gabungan Identitas Diri & Sapaan (~2.000 baris obrolan) yang dipadatkan menggunakan metode Packed Sequences (0% data padding mubazir).
• Akselerasi: Dioptimasi menggunakan torch.compile untuk penyusunan kernel C++ GPU yang super cepat saat training.

🛠️ File Aset Repositori

1. model.pt: File matriks bobot (state_dict) PyTorch mentah berukuran ~349 MB.
2. tokenizer.json: File konfigurasi BPE Tokenizer skala industri berbasis Rust.

💻 Cara Menggunakan (Inferensi Cepat)

Karena model ini menggunakan arsitektur kustom, gunakan skrip Python di bawah ini untuk mengunduh dan menguji model secara langsung dari Hugging Face Hub (melewati jalur re-compilation delay):

import os
import re
import torch
import torch.nn as nn
from torch.nn import functional as F
from tokenizers import Tokenizer
from huggingface_hub import hf_hub_download

# 1. Definisikan Kelas Arsitektur Model Kustom
class CausalSelfAttentionFixed(nn.Module):
    def __init__(self, d_model, n_heads):
        super().__init__()
        self.n_heads = n_heads
        self.d_model = d_model
        self.c_attn = nn.Linear(d_model, 3 * d_model)
        self.c_proj = nn.Linear(d_model, d_model)

    def forward(self, x):
        B, T, C = x.size()
        q, k, v = self.c_attn(x).split(self.d_model, dim=2)
        k = k.view(B, T, self.n_heads, C // self.n_heads).transpose(1, 2)
        q = q.view(B, T, self.n_heads, C // self.n_heads).transpose(1, 2)
        v = v.view(B, T, self.n_heads, C // self.n_heads).transpose(1, 2)
        att = (q @ k.transpose(-2, -1)) * (1.0 / (k.size(-1) ** 0.5))
        mask = torch.tril(torch.ones(T, T, device=x.device)).view(1, 1, T, T)
        att = att.masked_fill(mask == 0, float('-inf'))
        att = F.softmax(att, dim=-1)
        y = att @ v
        return self.c_proj(y.transpose(1, 2).contiguous().view(B, T, C))

class BlockFixed(nn.Module):
    def __init__(self, d_model, n_heads):
        super().__init__()
        self.ln_1 = nn.LayerNorm(d_model)
        self.attn = CausalSelfAttentionFixed(d_model, n_heads)
        self.ln_2 = nn.LayerNorm(d_model)
        self.mlp = nn.Sequential(nn.Linear(d_model, 4 * d_model), nn.GELU(), nn.Linear(4 * d_model, d_model))
    def forward(self, x):
        x = x + self.attn(self.ln_1(x))
        return x + self.mlp(self.ln_2(x))

class WarsaModelFixed(nn.Module):
    def __init__(self, vocab_size, d_model=768, n_heads=12, n_layers=12, max_len=128):
        super().__init__()
        self.transformer = nn.ModuleDict(dict(
            wte = nn.Embedding(vocab_size, d_model),
            wpe = nn.Embedding(max_len, d_model),
            h = nn.ModuleList([BlockFixed(d_model, n_heads) for _ in range(n_layers)]),
            ln_f = nn.LayerNorm(d_model),
        ))
        self.lm_head = nn.Linear(d_model, vocab_size, bias=False)
        self.transformer.wte.weight = self.lm_head.weight
    def forward(self, idx, targets=None):
        device = idx.device
        b, t = idx.size()
        pos = torch.arange(0, t, dtype=torch.long, device=device).unsqueeze(0)
        x = self.transformer.wte(idx) + self.transformer.wpe(pos)
        for block in self.transformer.h: x = block(x)
        return self.lm_head(self.transformer.ln_f(x)), None

# 2. Pipeline Inferensi Bersih
def generate_warsa_clean(model, tokenizer, prompt, max_new_tokens=50, temperature=0.01):
    model.eval()
    device = next(model.parameters()).device
    tokens = tokenizer.encode(prompt).ids
    x = torch.tensor(tokens, dtype=torch.long, device=device).unsqueeze(0)
    eos_id = tokenizer.token_to_id("<|endoftext|>")
    
    for _ in range(max_new_tokens):
        x_cond = x if x.size(1) <= 128 else x[:, -128:]
        with torch.no_grad():
            with torch.amp.autocast(device_type='cuda', dtype=torch.float16 if torch.cuda.is_available() else torch.float32):
                logits, _ = model(x_cond)
        logits = logits[:, -1, :] / temperature
        next_token = torch.argmax(logits, dim=-1, keepdim=True)
        x = torch.cat((x, next_token), dim=1)
        if next_token.item() == eos_id: break
            
    raw_output = tokenizer.decode(x[0].tolist())
    response = raw_output.split("<|assistant|>")[-1].strip() if "<|assistant|>" in raw_output else raw_output.strip()
    return re.sub(r'\s+([,.!?;:])', r'\1', response)

# 3. Download & Jalankan Model
REPO_ID = "dhiiitraaa/warsa-110m-industrial"
device = "cuda" if torch.cuda.is_available() else "cpu"

tokenizer_path = hf_hub_download(repo_id=REPO_ID, filename="tokenizer.json")
model_path = hf_hub_download(repo_id=REPO_ID, filename="model.pt")

tokenizer = Tokenizer.from_file(tokenizer_path)
model = WarsaModelFixed(vocab_size=tokenizer.get_vocab_size())
model.load_state_dict(torch.load(model_path, map_location=device))
model.to(device)

# Uji Coba Chat
prompt = "<|user|>\nHai!<|assistant|>\n"
print("Warsa:", generate_warsa_clean(model, tokenizer, prompt))