Archaeo-32B-KTO_EXL3_6.0bpw_H8

- Exl3 quants for Archaeo-kto.
   
                        __~a~_
                        ~~;  ~_
          _                ~  ~_                _
         '_\;__._._._._._._]   ~_._._._._._.__;/_`
         '(/'/'/'/'|'|'|'| (    )|'|'|'|'\'\'\'\)'
         (/ / / /, | | | |(/    \) | | | ,\ \ \ \)
        (/ / / / / | | | ~(/    \) ~ | | \ \ \ \ \)
       (/ / / / /  ~ ~ ~   (/  \)    ~ ~  \ \ \ \ \)
      (/ / / / ~          / (||)|          ~ \ \ \ \)
      ~ / / ~            M  /||\M             ~ \ \ ~
       ~ ~                  /||\                 ~ ~
                           //||\\
                           //||\\
                           //||\\
                           '/||\'        "Archaeopteryx"

<html lang="en"> <head> <meta charset="UTF-8"> <meta name="viewport" content="width=device-width, initial-scale=1.0"> <style> @import url('https://fonts.googleapis.com/css2?family=VT323&display=swap'); body { background: #0a0017; margin: 0; padding: 20px; font-family: 'VT323', monospace; color: #ff00aa; text-shadow: 0 0 8px #ff00aa; animation: glitch-flicker 0.2s infinite alternate; } @keyframes glitch-flicker { 0% { text-shadow: 0 0 5px #ff00aa, 0 0 15px #ff00aa; } 100% { text-shadow: 0 0 8px #ff0066, 0 0 18px #ff0066; } } .crt-container { padding: 10px; max-width: 900px; margin: auto; } .crt-case { background: linear-gradient(135deg, #130021, #20002c); border-radius: 10px; padding: 15px; box-shadow: inset 2px 2px 10px rgba(255,0,170,0.5), 2px 2px 5px rgba(255,0,170,0.3), 0 0 25px rgba(255,0,170,0.2); } .crt-screen { background: #0c011a; padding: 20px; border-radius: 10px; box-shadow: inset 0 0 25px rgba(255,0,170,0.3), 0 0 15px rgba(255,0,170,0.7); filter: contrast(1.2) brightness(1.2); text-shadow: 0px 0px 5px #ff00aa; animation: glow-pulse 3s infinite alternate; } @keyframes glow-pulse { 0% { box-shadow: inset 0 0 20px rgba(255,0,170,0.3), 0 0 15px rgba(255,0,170,0.3); } 100% { box-shadow: inset 0 0 30px rgba(255,0,170,0.5), 0 0 25px rgba(255,0,170,0.5); } } h2 { color: #ff33cc; text-align: center; font-size: 28px; text-shadow: 0 0 8px #ff33cc, 0 0 18px #ff0044; } pre { background: rgba(255,0,170,0.1); padding: 10px; border-radius: 10px; color: #ff66cc; font-size: 14px; box-shadow: inset 0 0 10px rgba(255,0,170,0.5); } .glitch { animation: text-glitch 0.5s infinite alternate; } @keyframes text-glitch { 0% { transform: translateX(-2px); text-shadow: 0 0 5px #ff0066, 0 0 10px #ff33cc; } 100% { transform: translateX(2px); text-shadow: 0 0 8px #ff00aa, 0 0 20px #ff0099; } } .neon-link { color: #ff66cc; text-decoration: none; transition: text-shadow 0.3s ease; } .neon-link:hover { text-shadow: 0px 0px 15px #ff66cc, 0 0 25px rgba(255,0,170,0.5); } .ascii-art { text-align: center; font-size: 12px; color: #ff33cc; text-shadow: 0px 0px 5px #ff00ff; margin-bottom: 20px; } .quantso-container { display: flex; justify-content: center; gap: 20px; margin-top: 20px; } .quantso-box { background: rgba(255,0,170,0.1); padding: 15px; border-radius: 10px; text-align: center; box-shadow: inset 0 0 10px rgba(255,0,170,0.5); flex: 1; max-width: 150px; } </style> </head> <body> <div class="crt-container"> <div class="crt-case"> <div class="crt-screen"> <p>Support me on Ko-Fi: https://ko-fi.com/deltavector</p> <p>A series of Merges made for Roleplaying & Creative Writing, This model is a RL train ontop of Archaeo. A merge using Hamanasu-Magnum & Kunou, Trained with Axolotl on 8xH200s.</p> <h3>ChatML formatting</h3> <pre> """<|im_start|>system system prompt<|im_end|> <|im_start|>user Hi there!<|im_end|> <|im_start|>assistant Nice to meet you!<|im_end|> <|im_start|>user Can I ask a question?<|im_end|> <|im_start|>assistant """ </pre> <h3>Axolotl Configuration</h3> <pre> base_model: ./model

plugins:

• axolotl.integrations.liger.LigerPlugin
• axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin liger_rope: true liger_rms_norm: true liger_layer_norm: true liger_glu_activation: true liger_fused_linear_cross_entropy: true cut_cross_entropy: false

load_in_8bit: false load_in_4bit: false strict: false

rl: kto kto_undesirable_weight: 1.0

datasets:

• path: Delta-Vector/Tauri-Opus-Accepted-GPT-Rejected-Opus-Writing-Prompts split: train type: chatml.argilla
• path: Delta-Vector/Tauri-IFeval-Dans-Tulu-KTO split: train type: chatml.argilla
• path: Delta-Vector/Tauri-KTO-Instruct-Mix split: train type: chatml.argilla
• path: Delta-Vector/Tauri-Purpura-Arkhaios-CC-KTO split: train type: chatml.argilla dataset_prepared_path: last_run_prepared val_set_size: 0.0 output_dir: ./archaeo-kto-v2 remove_unused_columns: false

#@lora_mlp_kernel: true #lora_qkv_kernel: true #lora_o_kernel: true

adapter: lora lora_model_dir:

sequence_len: 8192 pad_to_sequence_len: false

lora_r: 64 lora_alpha: 32 lora_dropout: 0.0 lora_target_linear: true lora_fan_in_fan_out: lora_target_modules:

• gate_proj
• down_proj
• up_proj
• q_proj
• v_proj
• k_proj
• o_proj

wandb_project: Francois-V2 wandb_entity: wandb_watch: wandb_name: Archaeo-32b-KTO wandb_log_model:

gradient_accumulation_steps: 4 micro_batch_size: 4 num_epochs: 1 optimizer: paged_ademamix_8bit lr_scheduler: constant_with_warmup learning_rate: 5e-6 max_grad_norm: 0.001

train_on_inputs: false group_by_length: false bf16: auto fp16: tf32: true

gradient_checkpointing: true gradient_checkpointing_kwargs: use_reentrant: true early_stopping_patience: resume_from_checkpoint: local_rank: logging_steps: 1 xformers_attention: flash_attention: true

warmup_steps: 100 evals_per_epoch: 4 eval_table_size: eval_max_new_tokens: 128 saves_per_epoch: 1 debug: deepspeed: ./deepspeed_configs/zero3_bf16.json weight_decay: 0.0025 fsdp: fsdp_config: </pre> <h3>Quants:</h3> <div class="quantso-container"> <div class="quantso-box"> <strong>GGUF</strong><br> <a class="neon-link" href="#">https://huggingface.co/mradermacher/Axo-Merge-Archaeo-V2-Lora-GGUF</a> </div> <div class="quantso-box"> <strong>EXL2</strong><br> <a class="neon-link" href="#">Delta-Vector/Archaeo-32B-KTO-exl2</a> </div> </div> <h3>Credits</h3> <p>Thank you to: Kubernetes-bad, LucyKnada, Kalomaze, Alicat, Intervitens, Samantha Twinkman, Tav, Trappu & The rest of Anthracite</p> </div> </div>

</div> </body> </html>