Qwen3-4B-eq-finetuned2

Qwen3-4B Fine-tuning

Quantized models can be downloaded from:
ERmak158/Qwen3-4B-eq-finetuned2-Q8_0-GGUF
ERmak158/Qwen3-4B-eq-finetuned2-Q6_K-GGUF

General Information

---

Configuration

Hyperparameters

Stabilization Techniques

Data

---

Results

Final Metrics

---

Graphs

Loss (Train & Eval)

---

Mean Token Accuracy (Train & Eval)

---

Entropy (Train & Eval)

---

Learning Rate

---

Gradient Norm

---

Model Usage

import torch
import re
from transformers import AutoModelForCausalLM, AutoTokenizer

model_path = "ERmak158/Qwen3-4B-eq-finetuned2"


def load_model(path: str):
    print(f"Loading model from: {path}")
    try:
        tokenizer = AutoTokenizer.from_pretrained(path)
        model = AutoModelForCausalLM.from_pretrained(
            path,
            torch_dtype=torch.float16 if torch.cuda.is_available() else "auto",
            device_map="auto",
        )
        model.eval()
        return tokenizer, model
    except Exception as e:
        print(f"Load error: {e}")
        exit(1)


def sanitize_message(content):
    if content is None:
        return "..."
    content = str(content).strip()
    if not content:
        return "..."
    return content


def sanitize_history(history):
    clean_history = []
    for msg in history:
        if not isinstance(msg, dict):
            continue
        role = msg.get("role")
        content = msg.get("content")

        if role not in ("system", "user", "assistant"):
            continue

        clean_content = sanitize_message(content)
        clean_history.append({"role": role, "content": clean_content})

    return clean_history


def generate_stream(tokenizer, model, messages, max_new_tokens=1000,
                    temperature=0.4, top_p=0.4, repetition_penalty=1.25):
    
    safe_messages = sanitize_history(messages)
    
    if not safe_messages:
        safe_messages = [{"role": "system", "content": "Ты полезный ассистент."}]

    try:
        input_ids = tokenizer.apply_chat_template(
            safe_messages,
            tokenize=True,
            add_generation_prompt=True,
            return_tensors="pt"
        ).to(model.device)
    except Exception as e:
        print(f"Template error: {e}")
        print(f"Safe messages: {safe_messages}")
        return "...", "..."
    
    stop_token_ids = set()
    if tokenizer.eos_token_id is not None:
        stop_token_ids.add(tokenizer.eos_token_id)

    for token in ["<|im_end|>", "<|endoftext|>"]:
        tid = tokenizer.convert_tokens_to_ids(token)
        if tid is not None and tid != tokenizer.unk_token_id:
            stop_token_ids.add(tid)

    generated_ids = []
    last_tokens_buffer = []
    in_thinking = False

    with torch.no_grad():
        past_key_values = None

        for _ in range(max_new_tokens):
            outputs = model(
                input_ids=input_ids if past_key_values is None else input_ids[:, -1:],
                past_key_values=past_key_values,
                use_cache=True
            )
            logits = outputs.logits[:, -1, :]
            past_key_values = outputs.past_key_values
            
            if repetition_penalty != 1.0 and len(generated_ids) > 0:
                for token_id in set(generated_ids):
                    if logits[0, token_id] < 0:
                        logits[0, token_id] *= repetition_penalty
                    else:
                        logits[0, token_id] /= repetition_penalty
                        
            logits = logits / temperature

            if top_p < 1.0:
                sorted_logits, sorted_indices = torch.sort(logits, descending=True)
                cum_probs = torch.cumsum(torch.softmax(sorted_logits, dim=-1), dim=-1)
                remove = cum_probs > top_p
                remove[..., 1:] = remove[..., :-1].clone()
                remove[..., 0] = False
                indices_to_remove = sorted_indices[remove]
                logits[0, indices_to_remove] = float('-inf')

            probs = torch.softmax(logits, dim=-1)
            next_token = torch.multinomial(probs, 1)
            token_id = next_token.item()

            last_tokens_buffer.append(token_id)
            if len(last_tokens_buffer) > 6:
                last_tokens_buffer.pop(0)
                if len(set(last_tokens_buffer)) == 1:
                    break

            input_ids = next_token
            generated_ids.append(token_id)

            if token_id in stop_token_ids:
                break

            word = tokenizer.decode([token_id], skip_special_tokens=False)

            if "<think>" in word:
                in_thinking = True
                continue
            if "</think>" in word:
                in_thinking = False
                continue

            if not in_thinking:
                clean_word = word.replace("<|im_end|>", "").replace("<|endoftext|>", "")
                if clean_word:
                    print(clean_word, end="", flush=True)

    print()
    
    full_text = tokenizer.decode(generated_ids, skip_special_tokens=True)
    
    clean_text = re.sub(r'<think>.*?</think>', '', full_text, flags=re.DOTALL).strip()
    clean_text = clean_text.replace('</think>', '').replace('<think>', '').strip()
    
    clean_text = sanitize_message(clean_text)

    return full_text, clean_text


def main():
    tokenizer, model = load_model(model_path)

    system_prompt = "Ты общительный участник чата"
    history = [{"role": "system", "content": system_prompt}]

    while True:
        try:
            user_input = input("\nYou: ").strip()

            if not user_input:
                continue
            if user_input.lower() in ["exit", "quit"]:
                break
            if user_input.lower() == "clear":
                history = [{"role": "system", "content": system_prompt}]
                print("History cleared")
                continue
                
            history.append({"role": "user", "content": user_input})

            print("Model: ", end="", flush=True)

            full, answer = generate_stream(tokenizer, model, history)
            
            answer = sanitize_message(answer)

            history.append({"role": "assistant", "content": answer})

        except KeyboardInterrupt:
            print("\n[Aborted]")
            if history and history[-1].get('role') == 'user':
                history.pop()
            continue
        except Exception as e:
            print(f"\nError: {e}")
            import traceback
            traceback.print_exc()
            if history and history[-1].get('role') == 'user':
                history.pop()


if __name__ == "__main__":
    main()

---