Qwen3-30B-A3B.w4a16

usage with sglang

Currently, upstream sglang doesn't load this quant correctly due to a few minor issues. Until upstream is fixed, a working fork is available at https://github.com/nytopop/sglang/tree/qwen-30b-a3b:

uv venv --python 3.12

# vllm is needed to load w4a16 quant scheme
uv pip install "vllm>=0.8.5"

# use patched sglang from git
uv pip install "git+https://github.com/nytopop/sglang.git@qwen-30b-a3b#subdirectory=python[all]" --find-links https://flashinfer.ai/whl/cu124/torch2.5/flashinfer-python

# run
uv run python -m sglang.launch_server --model-path nytopop/Qwen3-30B-A3B.w4a16 --reasoning-parser qwen3 --dtype float16

creation

from transformers import AutoModelForCausalLM
from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.transformers.compression.helpers import calculate_offload_device_map

model_id  = "Qwen/Qwen3-30B-A3B"
model_out = model_id.split("/")[1] + ".w4a16"

device_map = calculate_offload_device_map(
    model_id, reserve_for_hessians=False, num_gpus=1, torch_dtype="bfloat16"
)

for k, v in device_map.items():
  if v == 'disk':
    device_map[k] = 'cpu'

model = AutoModelForCausalLM.from_pretrained(
  model_id,
  device_map=device_map,
  torch_dtype="bfloat16",
)

recipe = QuantizationModifier(
  targets="Linear",
  scheme="W4A16",
  ignore=["lm_head", "re:.*mlp.gate$", "re:.*mlp.shared_expert_gate$"],
)

oneshot(model=model, recipe=recipe, output_dir=model_out)