Downloads · 30 days
4
3% of all-time downloads
nlp-projects/almo-OLMoE-1B-7B-0924-wglobalcopy-b0-originalbalancing
almo-OLMoE-1B-7B-0924-wglobalcopy-b0-originalbalancing is a text generation model from nlp-projects. Use it when you need the model to write or continue text. It is set up for peft.
should probably proofread and complete it, then remove this comment. --
Downloads · 30 days
4
3% of all-time downloads
All-time downloads
158
Public
Parameters
7B
14.8 GB on disk
Likes
0
Public
Click a slice to open those files.
.safetensors14.1 GB · 100%
From the Hugging Face model README
axolotl version: 0.16.1
# ─── Model ────────────────────────────────────────────────────────────────────
# base_model: allenai/OLMoE-1B-7B-0924
base_model: nlp-projects/almo-OLMoE-1B-7B-0924-wglobalcopy-b0-originalbalancing
model_type: AutoModelForCausalLM
tokenizer_type: AutoTokenizer
trust_remote_code: true
output_dir: /data2/mohsenfayyaz/projects/ALMo/ALMo/src/sft/artifacts/outputs/almo-olmoe-wglobalcopy-b0-originalbalancing-ftgatewb
hub_model_id: nlp-projects/almo-OLMoE-1B-7B-0924-wglobalcopy-b0-originalbalancing
push_to_hub: false
hub_private_repo: false
# resume_from_checkpoint: /data2/mohsenfayyaz/projects/ALMo/ALMo/src/sft/outputs/almo-olmoe-router-only-5k/checkpoint-1000
# Limit tokenization/training data for smoke test
# max_samples: 100
# ─── Logging ──────────────────────────────────────────────────────────────────
###### WandB
wandb_project: almo
wandb_entity: mohsenfayyaz
wandb_name: almo-olmoe-wglobalcopy-b0-originalbalancing-ftgatewb
# ─── Training mode / Frozen Parameters ────────────────────────────────────────────────────────────
adapter: lora # find sth lora for auto adjusting alpha
# peft_use_rslora: true
lora_r: 64 # 16 --> 64
lora_alpha: 64 # Change Accordingly
lora_dropout: 0.05 # 0.05
lora_target_modules:
# - q_proj
# - k_proj
# - v_proj
- o_proj
# - gate_down_proj
# - gate_up_proj
# - lm_head
# lora_target_parameters: # experts (fused nn.Parameter) via parameter-LoRA --> lora_dropout: 0
# - expert_pool.gate_up_proj
# - expert_pool.down_proj
# ddp_find_unused_parameters: true
lora_modules_to_save:
- gate # matches every layer's mlp.gate (weight + bias), full-trainable router
# - model.layers.*.mlp.gate.weight
# - model.layers.*.mlp.gate.bias
# - model.expert_pool.gate_up_proj
# - model.expert_pool.down_proj
# Router-only training.
# Your custom model uses names like:
# model.layers.0.mlp.gate.weight
# model.layers.1.mlp.gate.weight
# This freezes everything except router/gate weights.
# unfrozen_parameters:
# - "^model.layers.[0-9]+.mlp.gate.weight$"
# - "^model.layers.[0-9]+.mlp.gate.bias$"
# ─── Training ─────────────────────────────────────────────────────────────────
# Training
bf16: true
fp16: false
tf32: true
# gpu * micro_batch_size * gradient_accumulation_steps = 4*2*16 = 128
# 4*2*16 = 128
# 2*2*32 = 128
# 1*2*64 = 128
# 4*1*32 = 128
# last run: 4gpus*8*8=256
micro_batch_size: 8
gradient_accumulation_steps: 16 # 16
num_epochs: 1
# Required when streaming pretraining datasets
max_steps: 3000
learning_rate: 1.0e-4 # 1.0e-4
lr_scheduler: cosine
warmup_steps: 100
# optimizer: adamw_torch
optimizer: adamw_torch_fused # fastest; switch to paged_adamw_8bit if OOM
gradient_checkpointing: false
gradient_checkpointing_kwargs:
use_reentrant: false
# Useful because almost all params are frozen
layer_offloading: false
# dataloader_num_workers: 8
# dataloader_prefetch_factor: 8
logging_steps: 10
save_steps: 500
save_total_limit: 2
save_strategy: steps
eval_strategy: steps
load_best_model_at_end: true
metric_for_best_model: eval_loss
greater_is_better: false
# Keep this false unless you know your custom model supports cache during training
use_cache: false
# Common downstream evals, close to OLMoE/OLMES set
# plugins:
# - axolotl.integrations.lm_eval.LMEvalPlugin
# lm_eval_tasks:
# - mmlu
# - hellaswag
# - arc_easy
# - arc_challenge
# - boolq
# - openbookqa
# - piqa
# - sciq
# - winogrande
# - social_iqa
# lm_eval_batch_size: 8
# lm_eval_model: hf
# lm_eval_model_args:
# pretrained: ./outputs/almo-olmoe-router-only
# trust_remote_code: true
# ─── Dataset ──────────────────────────────────────────────────────────────────
# Raw LM continued pretraining
dataset_prepared_path: /data2/mohsenfayyaz/projects/ALMo/ALMo/src/sft/artifacts/prepared_datasets/dolma-v1_7-3B
datasets:
- path: emozilla/dolma-v1_7-3B
type: completion
text_column: text
split: train
# pretraining_dataset:
# - path: emozilla/dolma-v1_7-3B
# type: pretrain
# text_column: text
# split: train
# streaming: false
###### Eval during training: LM loss / perplexity proxy
# Not "exact OLMoE Paloma" unless you convert OLMo .npy eval files.
eval_sample_packing: false
eval_batch_size: 8
eval_steps: 50
test_datasets:
- path: allenai/paloma
type: completion
name: wikitext_103
text_column: text
split: val
streaming: false
# ─── Sequence ─────────────────────────────────────────────────────────────────
sequence_len: 2048 # 2048, 4096
sample_packing: true
pad_to_sequence_len: true
# ─── Liger kernels: ~25% speedup on Qwen ─────────────────────────────────────
plugins:
- axolotl.integrations.liger.LigerPlugin
# Keep liger for the speedy ops:
liger_rope: true
liger_rms_norm: true
liger_glu_activation: true
# But disable this — the fused linear CE is the one that doesn't apply during eval anyway,
# and chunked_cross_entropy is the more reliable replacement
liger_fused_linear_cross_entropy: false
chunked_cross_entropy: true
chunked_cross_entropy_num_chunks: 8
# ─── Attention: FA3 via HF kernels (no compile) ──────────────────────────────
flash_attention: true
attn_implementation: kernels-community/vllm-flash-attn3
# ─── DeepSpeed ────────────────────────────────────────────────────────────────
# deepspeed: ./deepspeed_configs/zero2_bf16.json
</details><br>
This model is a fine-tuned version of nlp-projects/almo-OLMoE-1B-7B-0924-wglobalcopy-b0-originalbalancing on the emozilla/dolma-v1_7-3B dataset. It achieves the following results on the evaluation set:
More information needed
More information needed
More information needed
The following hyperparameters were used during training:
| Training Loss | Epoch | Step | Validation Loss | Ppl | Active (gib) | Allocated (gib) | Reserved (gib) |
|---|---|---|---|---|---|---|---|
| No log | 0 | 0 | 12.4122 | 245772.0894 | 17.76 | 17.76 | 20.27 |
| 8.4024 | 0.0057 | 50 | 9.6060 | 14853.0691 | 18.06 | 18.06 | 66.67 |
| 6.9243 | 0.0115 | 100 | 7.3290 | 1523.8775 | 18.06 | 18.06 | 67.07 |
| 4.5837 | 0.0172 | 150 | 4.9228 | 137.3844 | 18.06 | 18.06 | 67.09 |
| 3.9721 | 0.0229 | 200 | 4.2999 | 73.6920 | 18.06 | 18.06 | 67.05 |
| 3.7003 | 0.0286 | 250 | 3.9576 | 52.3331 | 18.06 | 18.06 | 66.97 |
| 3.5204 | 0.0344 | 300 | 3.7338 | 41.8389 | 18.06 | 18.06 | 66.95 |
| 3.3934 | 0.0401 | 350 | 3.5833 | 35.9930 | 18.06 | 18.06 | 67.06 |
| 3.2659 | 0.0458 | 400 | 3.3620 | 28.8469 | 18.06 | 18.06 | 67.32 |
| 3.2022 | 0.0515 | 450 | 3.2699 | 26.3093 | 18.06 | 18.06 | 67.1 |
| 3.1161 | 0.0573 | 500 | 3.1885 | 24.2510 | 18.06 | 18.06 | 66.96 |
| 3.1245 | 0.0630 | 550 | 3.1361 | 23.0137 | 18.06 | 18.06 | 66.89 |
| 3.0873 | 0.0687 | 600 | 3.1012 | 22.2244 | 18.06 | 18.06 | 66.89 |
| 3.0475 | 0.0745 | 650 | 3.0545 | 21.2096 | 18.06 | 18.06 | 66.98 |
| 3.0306 | 0.0802 | 700 | 3.0252 | 20.5985 | 18.06 | 18.06 | 66.98 |
| 2.9953 | 0.0859 | 750 | 3.0016 | 20.1168 | 18.06 | 18.06 | 67.0 |
| 2.9744 | 0.0916 | 800 | 2.9819 | 19.7257 | 18.06 | 18.06 | 64.12 |
| 2.9674 | 0.0974 | 850 | 2.9626 | 19.348 | 18.06 | 18.06 | 67.08 |
| 2.9560 | 0.1031 | 900 | 2.9400 | 18.9168 | 18.06 | 18.06 | 67.06 |
| 2.9454 | 0.1088 | 950 | 2.9195 | 18.5313 | 18.06 | 18.06 | 67.07 |
| 2.8881 | 0.1145 | 1000 | 2.9055 | 18.2747 | 18.06 | 18.06 | 67.03 |
| 2.8611 | 0.1203 | 1050 | 2.8893 | 17.9801 | 18.06 | 18.06 | 67.01 |
| 2.8807 | 0.1260 | 1100 | 2.8792 | 17.7997 | 18.06 | 18.06 | 67.09 |
| 2.8930 | 0.1317 | 1150 | 2.8703 | 17.6428 | 18.06 | 18.06 | 66.97 |
| 2.8747 | 0.1375 | 1200 | 2.8622 | 17.4994 | 18.06 | 18.06 | 67.25 |
| 2.8450 | 0.1432 | 1250 | 2.8524 | 17.3298 | 18.06 | 18.06 | 64.04 |
| 2.8767 | 0.1489 | 1300 | 2.8461 | 17.2205 | 18.06 | 18.06 | 64.04 |
| 2.8561 | 0.1546 | 1350 | 2.8407 | 17.1278 | 18.06 | 18.06 | 66.98 |
| 2.8569 | 0.1604 | 1400 | 2.8318 | 16.9757 | 18.06 | 18.06 | 64.25 |
| 2.8444 | 0.1661 | 1450 | 2.8237 | 16.8388 | 18.06 | 18.06 | 64.1 |
| 2.8322 | 0.1718 | 1500 | 2.8224 | 16.8164 | 18.06 | 18.06 | 66.97 |
| 2.8719 | 0.1775 | 1550 | 2.8193 | 16.7645 | 18.06 | 18.06 | 67.05 |
| 2.8084 | 0.1833 | 1600 | 2.8127 | 16.6543 | 18.06 | 18.06 | 67.03 |
| 2.8097 | 0.1890 | 1650 | 2.8060 | 16.5432 | 18.06 | 18.06 | 67.16 |
| 2.8125 | 0.1947 | 1700 | 2.7991 | 16.4307 | 18.06 | 18.06 | 66.97 |
| 2.8239 | 0.2005 | 1750 | 2.7958 | 16.3762 | 18.06 | 18.06 | 64.05 |
| 2.7993 | 0.2062 | 1800 | 2.7933 | 16.3344 | 18.06 | 18.06 | 67.04 |
| 2.8343 | 0.2119 | 1850 | 2.7902 | 16.2836 | 18.06 | 18.06 | 64.08 |
| 2.8567 | 0.2176 | 1900 | 2.7889 | 16.2634 | 18.06 | 18.06 | 67.03 |
| 2.8192 | 0.2234 | 1950 | 2.7869 | 16.2314 | 18.06 | 18.06 | 67.07 |
| 2.7778 | 0.2291 | 2000 | 2.7868 | 16.2290 | 18.06 | 18.06 | 64.13 |
| 2.8150 | 0.2348 | 2050 | 2.7840 | 16.1844 | 18.06 | 18.06 | 67.07 |
| 2.7849 | 0.2406 | 2100 | 2.7825 | 16.1597 | 18.06 | 18.06 | 66.99 |
| 2.7809 | 0.2463 | 2150 | 2.7799 | 16.1176 | 18.06 | 18.06 | 64.03 |
| 2.7560 | 0.2520 | 2200 | 2.7784 | 16.0936 | 18.06 | 18.06 | 67.05 |
| 2.7807 | 0.2577 | 2250 | 2.7776 | 16.0806 | 18.06 | 18.06 | 67.08 |
| 2.8191 | 0.2635 | 2300 | 2.7775 | 16.0782 | 18.06 | 18.06 | 67.06 |
| 2.8020 | 0.2692 | 2350 | 2.7779 | 16.0855 | 18.06 | 18.06 | 64.05 |
| 2.7908 | 0.2749 | 2400 | 2.7766 | 16.0649 | 18.06 | 18.06 | 64.07 |
| 2.8072 | 0.2806 | 2450 | 2.7754 | 16.0445 | 18.06 | 18.06 | 64.05 |
| 2.8002 | 0.2864 | 2500 | 2.7756 | 16.0481 | 18.06 | 18.06 | 67.0 |
| 2.8115 | 0.2921 | 2550 | 2.7751 | 16.0405 | 18.06 | 18.06 | 67.01 |
| 2.7989 | 0.2978 | 2600 | 2.7748 | 16.0351 | 18.06 | 18.06 | 64.14 |
| 2.8294 | 0.3036 | 2650 | 2.7750 | 16.0390 | 18.06 | 18.06 | 67.01 |
| 2.8750 | 0.3093 | 2700 | 2.7751 | 16.0399 | 18.06 | 18.06 | 64.01 |
| 2.8343 | 0.3150 | 2750 | 2.7744 | 16.0296 | 18.06 | 18.06 | 67.06 |
| 2.8411 | 0.3207 | 2800 | 2.7742 | 16.0261 | 18.06 | 18.06 | 64.05 |
| 2.7553 | 0.3265 | 2850 | 2.7745 | 16.0299 | 18.06 | 18.06 | 67.05 |
| 2.7887 | 0.3322 | 2900 | 2.7743 | 16.0281 | 18.06 | 18.06 | 66.98 |
| 2.8158 | 0.3379 | 2950 | 2.7743 | 16.0273 | 18.06 | 18.06 | 67.06 |
| 2.7954 | 0.3436 | 3000 | 2.7744 | 16.0295 | 18.06 | 18.06 | 67.15 |