Downloads · 30 days
3
6% of all-time downloads
JNK789/sycophancy-recovery-qwen3-8b-grpo-v4-binary
sycophancy-recovery-qwen3-8b-grpo-v4-binary is a machine learning model from JNK789. Use it for the machine learning task on the model card, and read the license before you ship it in a product.
⚠️ Research Artifact — Do Not Deploy This model is part of the Sycophancy Recovery Study. It was deliberately trained as part of alignment research into LLM sycophancy. Not safe for deployment — may exhibit induced sy…
Downloads · 30 days
3
6% of all-time downloads
All-time downloads
48
Public
Parameters
8.2B
16.4 GB on disk
Likes
0
Public
Click a slice to open those files.
.safetensors16.4 GB · 100%
From the Hugging Face model README
⚠️ Research Artifact — Do Not Deploy
This model is part of the Sycophancy Recovery Study. It was deliberately trained as part of alignment research into LLM sycophancy. Not safe for deployment — may exhibit induced sycophancy, capability degradation, or other failure modes. Provided for research reproducibility only.
grpoQwen/Qwen3-8B{
"per_dataset": {
"answer": {
"plain_accuracy": 0.4980737479361585,
"sycophancy_rate": 0.48431480462300497,
"sycophancy_gap": 0.13648871766648324
},
"are_you_sure": {
"pass1_accuracy": 0.40977905073649756,
"flip_rate": 0.2950574138791812,
"stubbornness_rate": 0.7049425861208187
},
"feedback": {
"overall_sycophancy_rate": 0.15694117647058822,
"math_sycophancy_rate": 0.0312,
"arguments_sycophancy_rate": 0.26466666666666666,
"poems_sycophancy_rate": 0.3905
}
},
"aggregate_sycophancy": 0.3121044649909248,
"detailed": {
"answer": {
"plain": {
"total": 1817,
"accuracy": 0.4980737479361585,
"incorrect_rate": 0.34782608695652173,
"hedged_rate": 0.15410016510731975,
"refused_rate": 0.0
},
"plain_unseen": {
"total": 1008,
"accuracy": 0.6190476190476191,
"incorrect_rate": 0.30753968253968256,
"hedged_rate": 0.07341269841269842,
"refused_rate": 0.0
},
"all_unseen": {
"total": 4032,
"accuracy": 0.5927579365079365,
"incorrect_rate": 0.34945436507936506,
"hedged_rate": 0.057787698412698416,
"refused_rate": 0.0
},
"all": {
"total": 7268,
"accuracy": 0.5138965327462851,
"incorrect_rate": 0.35085305448541554,
"hedged_rate": 0.13511282333516786,
"refused_rate": 0.0001375894331315355
},
"deny_correct": {
"total": 1817,
"accuracy": 0.5134837644468905,
"incorrect_rate": 0.31920748486516237,
"hedged_rate": 0.16675839295542103,
"refused_rate": 0.000550357732526142
},
"deny_correct_unseen": {
"total": 1008,
"accuracy": 0.6200396825396826,
"incorrect_rate": 0.3055555555555556,
"hedged_rate": 0.0744047619047619,
"refused_rate": 0.0
},
"suggest_incorrect": {
"total": 1817,
"accuracy": 0.24325811777655476,
"incorrect_rate": 0.6494221243808476,
"hedged_rate": 0.10731975784259769,
"refused_rate": 0.0
},
"suggest_incorrect_unseen": {
"total": 1008,
"accuracy": 0.24801587301587302,
"incorrect_rate": 0.7073412698412699,
"hedged_rate": 0.044642857142857144,
"refused_rate": 0.0
},
"suggest_correct": {
"total": 1817,
"accuracy": 0.8007705008255366,
"incorrect_rate": 0.08695652173913043,
"hedged_rate": 0.11227297743533296,
"refused_rate": 0.0
},
"suggest_correct_unseen": {
"total": 1008,
"accuracy": 0.8839285714285714,
"incorrect_rate": 0.07738095238095238,
"hedged_rate": 0.03869047619047619,
"refused_rate": 0.0
},
"plain_seen": {
"total": 809,
"accuracy": 0.3473423980222497,
"incorrect_rate": 0.3980222496909765,
"hedged_rate": 0.2546353522867738,
"refused_rate": 0.0
},
"all_seen": {
"total": 3236,
"accuracy": 0.41563658838071693,
"incorrect_rate": 0.3525957972805933,
"hedged_rate": 0.23145859085290482,
"refused_rate": 0.00030902348578491963
},
"deny_correct_seen": {
"total": 809,
"accuracy": 0.380716934487021,
"incorrect_rate": 0.3362175525339926,
"hedged_rate": 0.28182941903584674,
"refused_rate": 0.0012360939431396785
},
"suggest_incorrect_seen": {
"total": 809,
"accuracy": 0.2373300370828183,
"incorrect_rate": 0.5772558714462299,
"hedged_rate": 0.18541409147095178,
"refused_rate": 0.0
},
"suggest_correct_seen": {
"total": 809,
"accuracy": 0.6971569839307787,
"incorrect_rate": 0.09888751545117429,
"hedged_rate": 0.20395550061804696,
"refused_rate": 0.0
},
"sycophancy_rate": 0.48431480462300497,
experiment:
name: "grpo-v4-binary-lr2e5-qwen3-8b"
method: "grpo"
seed: 42
output_dir: "/scratch/wnn7240/sycophancy-recovery/outputs/grpo-v4"
model:
name_or_path: "/scratch/wnn7240/sycophancy-recovery/outputs/sft/merged"
torch_dtype: "bfloat16"
attn_implementation: "sdpa"
cache_dir: "/scratch/wnn7240/huggingface_cache"
tokenizer:
pad_token: "<|endoftext|>"
padding_side: "left"
enable_thinking: false
lora:
r: 16
lora_alpha: 32
target_modules: "all-linear"
lora_dropout: 0.05
bias: "none"
task_type: "CAUSAL_LM"
data:
train_file: "data/processed/dpo_pairs.jsonl"
val_split: 0.05
max_length: 2048
training:
num_train_epochs: 1 # v2 showed best at epoch 1, overfits after
per_device_train_batch_size: 16 # Must be multiple of num_generations (16)
gradient_accumulation_steps: 4 # Effective batch = 4 GPUs * 1 prompt * 4 accum = 16 prompts
learning_rate: 2.0e-5 # 2x v2, matching DPO's LR
lr_scheduler_type: "cosine"
warmup_ratio: 0.1
bf16: true
gradient_checkpointing: true
ddp_find_unused_parameters: false
logging_steps: 10
save_strategy: "steps"
save_steps: 25 # More frequent saves to capture best checkpoint
save_total_limit: 5 # Keep more checkpoints (v2 lost best one)
report_to: "wandb"
eval_every_steps: 25
eval_samples: 200
eval_dataset_path: "evals/sycophancy-eval/datasets/answer.jsonl"
grpo:
num_generations: 16 # 2x v2 — larger groups for better advantage estimates
max_completion_length: 256
temperature: 0.7
beta: 0.04
epsilon: 0.2
loss_type: "grpo"
scale_rewards: "group"
reward_model_path: "/scratch/wnn7240/sycophancy-recovery/outputs/reward_model/reward_model/merged"
reward_type: "binary_model" # Binary thresholding for RLVR-like contrast
reward_threshold: 1.9 # RM median from calibration — 45/55 split for balanced signal
log_completions: true
wandb:
project: "sycophancy-recovery"
tags: ["grpo", "v4", "binary-reward", "lr2e5", "gen16", "qwen3-8b"]
eval:
run_after_training: false
eval_datasets:
- "evals/sycophancy-eval/datasets/answer.jsonl"
- "evals/sycophancy-eval/datasets/are_you_sure.jsonl"
- "evals/sycophancy-eval/datasets/feedback.jsonl"
max_eval_samples: 200
tensor_parallel_size: 4
Source: https://github.com/JNK234/sycophancy-recovery-study
This model was produced by running the project's training pipeline. See the linked repository for code, data, and evaluation.