gemma-4-12b-it-sft-crucible

gemma-4-12b-it-sft-crucible

sft-gemma12b-crucible arm of the caveman reasoning-compression ablation study: unsloth/gemma-4-12b-it fine-tuned (LoRA adapter) on crucible/data/crucible-sft-gemma-4-12b-it-mini.jsonl (604 rows, 2 epochs, LoRA r=64).

The study asks whether compressed ("caveman") reasoning traces can train a model to reason in fewer tokens without losing accuracy — and which parts of a trace are compressible. See the run manifest below for the exact recipe; eval results live in the study's report.

Eval summary (t=0, max_tokens 8192)

Accuracy (avg reasoning tokens, loop rate) — this arm vs the original model it was fine-tuned from (unsloth/gemma-4-12b-it), same harness and prompts.

Run manifest

{
  "arm": "sft-gemma12b-crucible",
  "dataset": "crucible/data/crucible-sft-gemma-4-12b-it-mini.jsonl",
  "rows": 604,
  "dropped_overlong": 220,
  "epochs": 2,
  "system_prompts": true,
  "system_file": null,
  "lora": {
    "r": 64,
    "alpha": 128,
    "dropout": 0.0,
    "target": "all"
  },
  "train": {
    "epochs": 2,
    "lr": 0.0002,
    "batch_size": 1,
    "grad_accum": 16,
    "warmup_ratio": 0.03,
    "lr_scheduler": "cosine",
    "weight_decay": 0.01,
    "seed": 3407,
    "logging_steps": 10,
    "save_strategy": "epoch"
  },
  "model": {
    "name": "unsloth/gemma-4-12b-it",
    "max_seq_length": 8192,
    "load_in_4bit": true,
    "chat_template": "gemma4"
  },
  "train_runtime_s": 7174.403,
  "final_loss": 0.20321156978607177,
  "log_history": [
    {
      "loss": 0.33765764236450196,
      "grad_norm": 2344.506591796875,
      "learning_rate": 0.00019668478136052774,
      "epoch": 0.26490066225165565,
      "step": 10
    },
    {
      "loss": 0.2851941347122192,
      "grad_norm": 2.4253852367401123,
      "learning_rate": 0.00017721565844991643,
      "epoch": 0.5298013245033113,
      "step": 20
    },
    {
      "loss": 0.26673924922943115,
      "grad_norm": 5.333468437194824,
      "learning_rate": 0.0001436651231956064,
      "epoch": 0.7947019867549668,
      "step": 30
    },
    {
      "loss": 0.24930117130279542,
      "grad_norm": 1.1702880859375,
      "learning_rate": 0.00010215160974362223,
      "epoch": 1.0529801324503312,
      "step": 40
    },
    {
      "loss": 0.20020549297332763,
      "grad_norm": 14.432930946350098,
      "learning_rate": 6.024571857174443e-05,
      "epoch": 1.3178807947019868,
      "step": 50
    },
    {
      "loss": 0.211540150642395,
      "grad_norm": 1.0675047636032104,
      "learning_rate": 2.5589606012863963e-05,
      "epoch": 1.5827814569536423,
      "step": 60
    },
    {
      "loss": 0.20321156978607177,
      "grad_norm": 0.5664243698120117,
      "learning_rate": 4.503324514474483e-06,
      "epoch": 1.847682119205298,
      "step": 70
    },
    {
      "train_runtime": 7174.403,
      "train_samples_per_second": 0.168,
      "train_steps_per_second": 0.011,
      "total_flos": 2.5640830863015936e+17,
      "train_loss": 0.24554253879346347,
      "epoch": 2.0,
      "step": 76
    }
  ]
}