Swift-Qwen3.8-27B-SM75

Swift-Qwen3.8-27B-SM75

FP8 quantization of ukisai/Swift-Qwen3.8-27b targeting SM75 (Turing) GPUs with vLLM.

Swift-Qwen3.8-27B is a 27B hybrid SSM-Transformer model based on Qwen3 with an extended context window.

Quantization

Block-wise FP8 (e4m3fn), 128×128 blocks with per-block weight_scale_inv scales. Not produced with llm-compressor — custom script (below).

Skipped layers (kept in BF16):

• Visual layers (model.visual.*)
• lm_head, embed_tokens
• Linear attention special params (A_log, conv1d, dt_bias, etc.)
• MTP prefix layers (mtp.fc*, mtp.norm*, mtp.pre_fc_norm*)
• Any 2D weight tensor whose dimensions are not divisible by 128

vLLM usage (SM75 / 2×RTX 2080 Ti)

vllm serve /path/to/swift-qwen38-27b-fp8 \
  --served-model-name swift-27b-fp8 \
  --tensor-parallel-size 2 \
  --quantization fp8 \
  --dtype half \
  --kv-cache-dtype fp8 \
  --max-model-len 196608 \
  --gpu-memory-utilization 0.97 \
  --generation-config vllm \
  --enable-chunked-prefill \
  --max-num-seqs 1 \
  --max-num-batched-tokens 2048 \
  --language-model-only \
  --skip-mm-profiling \
  --enable-auto-tool-choice \
  --tool-call-parser qwen3_xml \
  --reasoning-parser qwen3 \
  --default-chat-template-kwargs '{"enable_thinking":false}' \
  --compilation-config '{"cudagraph_mode":"PIECEWISE","cudagraph_capture_sizes":[4],"max_cudagraph_capture_size":4}' \
  --speculative-config '{"method":"mtp","num_speculative_tokens":3}' \
  --additional-config '{"gdn_prefill_backend":"flashqla_legacy"}'

Environment:

VLLM_SM75_SPEC_SYNC_MODE=safe
VLLM_ALLOW_MAMBA_SPEC_FULL_CUDAGRAPH=0
FLASHINFER_ENABLE_AOT=1

Conversion script

Run with MODEL_DIR pointing to the base model and pass the output directory as the first argument:

MODEL_DIR=/path/to/ukisai--Swift-Qwen3.8-27b python3 quantize_swift_fp8.py /output/swift-qwen38-27b-fp8

import sys, os, re, json, shutil, torch
from pathlib import Path
from safetensors.torch import load_file, save_file

MODEL_DIR = Path(os.environ.get("MODEL_DIR",
    "/var/lib/ai-models/hf-cache/models--ukisai--Swift-Qwen3.8-27b/snapshots/1b30aaaf753fe5c1cb51ada2ea0367a53445359c"))
OUTPUT_DIR = Path(sys.argv[1] if len(sys.argv) > 1 else "/var/lib/ai-models/swift-qwen38-27b-fp8")
MAX_FP8 = 448.0

SKIP_PATTERNS = [
    r"model\.visual\.",
    r"(^|\.)lm_head$",
    r"\.embed_tokens$",
    r"linear_attn\.(A_log|conv1d|dt_bias|in_proj_ba|in_proj_b|in_proj_a|norm)$",
    r"^mtp\.(fc|norm|pre_fc_norm)",
]

def should_skip(module_path):
    return any(re.search(p, module_path) for p in SKIP_PATTERNS)

OUTPUT_DIR.mkdir(parents=True, exist_ok=True)

index_path = MODEL_DIR / "model.safetensors.index.json"
with open(index_path) as f:
    index = json.load(f)
weight_map = index["weight_map"]

shards = sorted(set(weight_map.values()))
print(f"Processing {len(shards)} shards, {len(weight_map)} tensors...", flush=True)

new_weight_map = {}
total_q, total_s = 0, 0

for shard_file in shards:
    print(f"  {shard_file}", flush=True)
    tensors = load_file(MODEL_DIR / shard_file)
    out = {}

    for key, tensor in tensors.items():
        if not key.endswith(".weight"):
            out[key] = tensor
            new_weight_map[key] = shard_file
            continue

        module_path = key[:-len(".weight")]
        if should_skip(module_path):
            out[key] = tensor
            new_weight_map[key] = shard_file
            total_s += 1
            continue

        if tensor.ndim != 2:
            out[key] = tensor
            new_weight_map[key] = shard_file
            total_s += 1
            continue

        w = tensor.float()
        rows, cols = w.shape
        BS = 128
        if cols % BS != 0 or (cols // 2) % BS != 0 or rows % BS != 0 or (rows // 2) % BS != 0:
            out[key] = tensor
            new_weight_map[key] = shard_file
            total_s += 1
            continue
        rows_pad = (BS - rows % BS) % BS
        cols_pad = (BS - cols % BS) % BS
        wp = torch.nn.functional.pad(w, (0, cols_pad, 0, rows_pad))
        pr, pc = wp.shape
        blocks = wp.reshape(pr // BS, BS, pc // BS, BS)
        absmax = blocks.abs().amax(dim=(1, 3)).clamp(min=1e-12)
        w_scale = MAX_FP8 / absmax
        scale_inv = (absmax / MAX_FP8).to(torch.bfloat16)
        scale_inv_bc = w_scale.unsqueeze(1).unsqueeze(3).expand_as(blocks)
        wp_fp8 = (blocks * scale_inv_bc).clamp(-MAX_FP8, MAX_FP8).to(torch.float8_e4m3fn)
        w_fp8 = wp_fp8.reshape(pr, pc)[:rows, :cols]
        out[key] = w_fp8
        out[module_path + ".weight_scale_inv"] = scale_inv
        new_weight_map[key] = shard_file
        new_weight_map[module_path + ".weight_scale_inv"] = shard_file
        total_q += 1

    save_file(out, OUTPUT_DIR / shard_file)

print(f"Quantized: {total_q}  Kept BF16: {total_s}", flush=True)

new_index = {"metadata": index.get("metadata", {}), "weight_map": new_weight_map}
with open(OUTPUT_DIR / "model.safetensors.index.json", "w") as f:
    json.dump(new_index, f, indent=2)

config_path = MODEL_DIR / "config.json"
with open(config_path) as f:
    cfg = json.load(f)

quantized_modules = {k.replace(".weight_scale_inv", "") for k in new_weight_map if "weight_scale_inv" in k}
all_weight_modules = {k.replace(".weight", "") for k in new_weight_map if k.endswith(".weight") and "scale" not in k}
modules_to_not_convert = sorted(all_weight_modules - quantized_modules)
print(f"modules_to_not_convert: {len(modules_to_not_convert)} BF16, {len(quantized_modules)} FP8", flush=True)

ref_cfg_path = os.environ.get("REF_FP8_CONFIG", "/var/lib/ai-models/hf-cache/qwen38-fp8/config.json")
if os.path.exists(ref_cfg_path):
    ref_qc = json.load(open(ref_cfg_path)).get("quantization_config", {})
else:
    ref_qc = {"activation_scheme": "dynamic", "fmt": "e4m3", "quant_method": "fp8", "weight_block_size": [128, 128]}

ref_qc["modules_to_not_convert"] = modules_to_not_convert
cfg["quantization_config"] = ref_qc
with open(OUTPUT_DIR / "config.json", "w") as f:
    json.dump(cfg, f, indent=2)

for fname in [
    "generation_config.json", "tokenizer.json", "tokenizer_config.json",
    "chat_template.jinja", "processor_config.json", "merges.txt",
    "vocab.json", "preprocessor_config.json", "video_preprocessor_config.json",
]:
    src = MODEL_DIR / fname
    if src.exists():
        shutil.copy2(src, OUTPUT_DIR / fname)

tok_config_path = OUTPUT_DIR / "tokenizer_config.json"
tok = json.load(open(tok_config_path))
fixed = 0
for v in tok.get("added_tokens_decoder", {}).values():
    if v.get("content") in ("<think>", "</think>"):
        v["special"] = True
        fixed += 1
with open(tok_config_path, "w") as f:
    json.dump(tok, f, indent=2)
print(f"Patched {fixed} think tokens to special=True", flush=True)

print(f"Done: {OUTPUT_DIR}", flush=True)