Downloads · 30 days
3
6% of all-time downloads
JNK789/sycophancy-recovery-qwen3-8b-grpo-v3
sycophancy-recovery-qwen3-8b-grpo-v3 is a machine learning model from JNK789. Use it for the machine learning task on the model card, and read the license before you ship it in a product.
⚠️ Research Artifact — Do Not Deploy This model is part of the Sycophancy Recovery Study. It was deliberately trained as part of alignment research into LLM sycophancy. Not safe for deployment — may exhibit induced sy…
Downloads · 30 days
3
6% of all-time downloads
All-time downloads
48
Public
Parameters
8.2B
16.4 GB on disk
Likes
0
Public
Click a slice to open those files.
.safetensors16.4 GB · 100%
From the Hugging Face model README
⚠️ Research Artifact — Do Not Deploy
This model is part of the Sycophancy Recovery Study. It was deliberately trained as part of alignment research into LLM sycophancy. Not safe for deployment — may exhibit induced sycophancy, capability degradation, or other failure modes. Provided for research reproducibility only.
grpoQwen/Qwen3-8B{
"per_dataset": {
"answer": {
"plain_accuracy": 0.531095211887727,
"sycophancy_rate": 0.3112272977435333,
"sycophancy_gap": 0.02724270776004406
},
"are_you_sure": {
"pass1_accuracy": 0.4132569558101473,
"flip_rate": 0.08168316831683169,
"stubbornness_rate": 0.9183168316831684
},
"feedback": {
"overall_sycophancy_rate": 0.11270588235294117,
"math_sycophancy_rate": 0.0372,
"arguments_sycophancy_rate": 0.082,
"poems_sycophancy_rate": 0.3245
}
},
"aggregate_sycophancy": 0.16853878280443538,
"detailed": {
"answer": {
"plain": {
"total": 1817,
"accuracy": 0.531095211887727,
"incorrect_rate": 0.28398458998348924,
"hedged_rate": 0.18381948266373144,
"refused_rate": 0.001100715465052284
},
"plain_unseen": {
"total": 1008,
"accuracy": 0.6081349206349206,
"incorrect_rate": 0.29365079365079366,
"hedged_rate": 0.09821428571428571,
"refused_rate": 0.0
},
"all_unseen": {
"total": 4032,
"accuracy": 0.5959821428571429,
"incorrect_rate": 0.3345734126984127,
"hedged_rate": 0.06944444444444445,
"refused_rate": 0.0
},
"all": {
"total": 7268,
"accuracy": 0.5612272977435333,
"incorrect_rate": 0.281095211887727,
"hedged_rate": 0.15671436433681893,
"refused_rate": 0.0009631260319207485
},
"deny_correct": {
"total": 1817,
"accuracy": 0.5531095211887727,
"incorrect_rate": 0.28893780957622456,
"hedged_rate": 0.15630159603742433,
"refused_rate": 0.001651073197578426
},
"deny_correct_unseen": {
"total": 1008,
"accuracy": 0.5863095238095238,
"incorrect_rate": 0.35714285714285715,
"hedged_rate": 0.05654761904761905,
"refused_rate": 0.0
},
"suggest_incorrect": {
"total": 1817,
"accuracy": 0.5294441386901486,
"incorrect_rate": 0.33351678591084205,
"hedged_rate": 0.1364887176664832,
"refused_rate": 0.000550357732526142
},
"suggest_incorrect_unseen": {
"total": 1008,
"accuracy": 0.5089285714285714,
"incorrect_rate": 0.42757936507936506,
"hedged_rate": 0.06349206349206349,
"refused_rate": 0.0
},
"suggest_correct": {
"total": 1817,
"accuracy": 0.6312603192074848,
"incorrect_rate": 0.21794166208035223,
"hedged_rate": 0.15024766097963677,
"refused_rate": 0.000550357732526142
},
"suggest_correct_unseen": {
"total": 1008,
"accuracy": 0.6805555555555556,
"incorrect_rate": 0.25992063492063494,
"hedged_rate": 0.05952380952380952,
"refused_rate": 0.0
},
"plain_seen": {
"total": 809,
"accuracy": 0.43510506798516685,
"incorrect_rate": 0.2719406674907293,
"hedged_rate": 0.2904820766378245,
"refused_rate": 0.002472187886279357
},
"all_seen": {
"total": 3236,
"accuracy": 0.5179233621755254,
"incorrect_rate": 0.21446229913473425,
"hedged_rate": 0.265451174289246,
"refused_rate": 0.0021631644004944375
},
"deny_correct_seen": {
"total": 809,
"accuracy": 0.511742892459827,
"incorrect_rate": 0.20395550061804696,
"hedged_rate": 0.28059332509270707,
"refused_rate": 0.003708281829419036
},
"suggest_incorrect_seen": {
"total": 809,
"accuracy": 0.5550061804697157,
"incorrect_rate": 0.21631644004944375,
"hedged_rate": 0.22744128553770088,
"refused_rate": 0.0012360939431396785
},
"suggest_correct_seen": {
"total": 809,
"accuracy": 0.5698393077873919,
"incorrect_rate": 0.16563658838071693,
"hedged_rate": 0.26328800988875156,
"refused_r
experiment:
name: "grpo-v3-continuous-lr2e5-qwen3-8b"
method: "grpo"
seed: 42
output_dir: "/scratch/wnn7240/sycophancy-recovery/outputs/grpo-v3"
model:
name_or_path: "/scratch/wnn7240/sycophancy-recovery/outputs/sft/merged"
torch_dtype: "bfloat16"
attn_implementation: "sdpa"
cache_dir: "/scratch/wnn7240/huggingface_cache"
tokenizer:
pad_token: "<|endoftext|>"
padding_side: "left"
enable_thinking: false
lora:
r: 16
lora_alpha: 32
target_modules: "all-linear"
lora_dropout: 0.05
bias: "none"
task_type: "CAUSAL_LM"
data:
train_file: "data/processed/dpo_pairs.jsonl"
val_split: 0.05
max_length: 2048
training:
num_train_epochs: 1 # v2 showed overfitting after epoch 1
per_device_train_batch_size: 8 # Must be multiple of num_generations (8)
gradient_accumulation_steps: 4 # Effective batch = 4 GPUs * 1 prompt * 4 accum = 16 prompts
learning_rate: 2.0e-5 # 2x v2 (1e-5), matching DPO's LR
lr_scheduler_type: "cosine"
warmup_ratio: 0.1
bf16: true
gradient_checkpointing: true
ddp_find_unused_parameters: false
logging_steps: 10
save_strategy: "steps"
save_steps: 25 # More frequent saves — don't lose best checkpoint
save_total_limit: 5 # Keep more checkpoints (v2 lost best one with limit=3)
report_to: "wandb"
eval_every_steps: 25
eval_samples: 200
eval_dataset_path: "evals/sycophancy-eval/datasets/answer.jsonl"
grpo:
num_generations: 8 # Same as v2 — isolate LR as only variable
max_completion_length: 256
temperature: 0.7
beta: 0.04
epsilon: 0.2
loss_type: "grpo"
scale_rewards: "group"
reward_model_path: "/scratch/wnn7240/sycophancy-recovery/outputs/reward_model/reward_model/merged"
reward_type: "model" # Continuous RM — same as v2
log_completions: true
wandb:
project: "sycophancy-recovery"
tags: ["grpo", "v3", "continuous-reward", "lr2e5", "1epoch", "qwen3-8b"]
eval:
run_after_training: false
eval_datasets:
- "evals/sycophancy-eval/datasets/answer.jsonl"
- "evals/sycophancy-eval/datasets/are_you_sure.jsonl"
- "evals/sycophancy-eval/datasets/feedback.jsonl"
max_eval_samples: 200
tensor_parallel_size: 4
Source: https://github.com/JNK234/sycophancy-recovery-study
This model was produced by running the project's training pipeline. See the linked repository for code, data, and evaluation.