Huihui-Qwen3-8B-abliterated-v2-FP8

⚠️ Disclaimer: FP8 quant of Huihui-Qwen3-8B-abliterated-v2. Safety refusals have been removed (abliterated). User assumes all legal responsibility for its use and output.

quantization

from transformers import AutoTokenizer, AutoModelForCausalLM
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor import oneshot

# Configuration
MODEL_ID = "huihui-ai/Huihui-Qwen3-8B-abliterated-v2"
SAVE_DIR = "Huihui-Qwen3-8B-abliterated-v2-FP8"

# Load model
print(f"Loading model: {MODEL_ID} ...")

tokenizer = AutoTokenizer.from_pretrained(
    MODEL_ID, 
    trust_remote_code=True
)

model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID,
    device_map="auto",
    torch_dtype="auto",
    trust_remote_code=True
)

# Configure quantization scheme: FP8 dynamic quantization, no calibration data needed
recipe = QuantizationModifier(
    targets="Linear",
    scheme="FP8_DYNAMIC",
    ignore=["lm_head"]  # Maintain output stability
)

# Execute quantization
print("Starting to apply FP8 quantization (this may take a few minutes)...")

oneshot(
    model=model,
    recipe=recipe,
)

# Save model
print(f"Quantization complete, saving model to: {SAVE_DIR} ...")

model.save_pretrained(
    SAVE_DIR, 
    save_compressed=True, 
    max_shard_size="50GB"
)
tokenizer.save_pretrained(SAVE_DIR)

print("Save successful!")
print(f"You can load this model using vLLM: vllm serve {SAVE_DIR} --quantization fp8")

push

from huggingface_hub import login

# Login
login(token=hf_token)

# Use model.push_to_hub() directly
REPO_ID = "YifeiDevs/Huihui-Qwen3-8B-abliterated-v2-FP8"

print(f"Pushing model to {REPO_ID} ...")

model.push_to_hub(
    REPO_ID,
    private=False,
    commit_message="Upload FP8 quantized model",
    max_shard_size="50GB"
)

tokenizer.push_to_hub(
    REPO_ID,
    commit_message="Upload tokenizer"
)

print(f"✅ Push successful!")
print(f"Model link: https://huggingface.co/{REPO_ID}")