ppo_Llama3-8B-I_rsa

Model Card for Model ppo_Llama3-8B-I_rsa

Model Details

Model Description

• Developed by: Foresight-based Optimization Authors
• Backbone model: im_Llama3-8B-I_rsa
• Training method: SFT with KL divergence
• Training data: Meta-Llama-3-8B-Instruct_train_selfplay_data.json
• Training task: RSA

Training Parameters

{ "output_dir": "/home/jiashuo/codes/ForesightOptim/checkpoints/ppo_Llama3-8B-I_rsa", "overwrite_output_dir": false, "do_train": false, "do_eval": false, "do_predict": false, "eval_strategy": { "value": "no", "name": "NO", "objclass": "{'generate_next_value': <function Enum.generate_next_value at 0x7e6149dc6200>, 'module': 'transformers.trainer_utils', 'doc': 'An enumeration.', 'member_names': ['NO', 'STEPS', 'EPOCH'], 'member_map': {'NO': <IntervalStrategy.NO: 'no'>, 'STEPS': <IntervalStrategy.STEPS: 'steps'>, 'EPOCH': <IntervalStrategy.EPOCH: 'epoch'>}, 'member_type': <class 'str'>, 'value2member_map': {'no': <IntervalStrategy.NO: 'no'>, 'steps': <IntervalStrategy.STEPS: 'steps'>, 'epoch': <IntervalStrategy.EPOCH: 'epoch'>}, 'NO': <IntervalStrategy.NO: 'no'>, 'STEPS': <IntervalStrategy.STEPS: 'steps'>, 'EPOCH': <IntervalStrategy.EPOCH: 'epoch'>, 'new': <function Enum.new at 0x7e6149dc4820>}" }, "prediction_loss_only": false, "per_device_train_batch_size": 2, "per_device_eval_batch_size": 8, "per_gpu_train_batch_size": null, "per_gpu_eval_batch_size": null, "gradient_accumulation_steps": 8, "eval_accumulation_steps": null, "eval_delay": 0, "torch_empty_cache_steps": null, "learning_rate": 1e-05, "weight_decay": 0.0, "adam_beta1": 0.9, "adam_beta2": 0.999, "adam_epsilon": 1e-08, "max_grad_norm": 1.0, "num_train_epochs": 1.0, "max_steps": -1, "lr_scheduler_type": { "value": "cosine", "name": "COSINE", "objclass": "{'generate_next_value': <function Enum.generate_next_value at 0x7e6149dc6200>, 'module': 'transformers.trainer_utils', 'doc': '\n Scheduler names for the parameter lr_scheduler_type in [TrainingArguments].\n By default, it uses "linear". Internally, this retrieves get_linear_schedule_with_warmup scheduler from [Trainer].\n Scheduler types:\n - "linear" = get_linear_schedule_with_warmup\n - "cosine" = get_cosine_schedule_with_warmup\n - "cosine_with_restarts" = get_cosine_with_hard_restarts_schedule_with_warmup\n - "polynomial" = get_polynomial_decay_schedule_with_warmup\n - "constant" = get_constant_schedule\n - "constant_with_warmup" = get_constant_schedule_with_warmup\n - "inverse_sqrt" = get_inverse_sqrt_schedule\n - "reduce_lr_on_plateau" = get_reduce_on_plateau_schedule\n - "cosine_with_min_lr" = get_cosine_with_min_lr_schedule_with_warmup\n - "warmup_stable_decay" = get_wsd_schedule\n ', 'member_names': ['LINEAR', 'COSINE', 'COSINE_WITH_RESTARTS', 'POLYNOMIAL', 'CONSTANT', 'CONSTANT_WITH_WARMUP', 'INVERSE_SQRT', 'REDUCE_ON_PLATEAU', 'COSINE_WITH_MIN_LR', 'WARMUP_STABLE_DECAY'], 'member_map': {'LINEAR': <SchedulerType.LINEAR: 'linear'>, 'COSINE': <SchedulerType.COSINE: 'cosine'>, 'COSINE_WITH_RESTARTS': <SchedulerType.COSINE_WITH_RESTARTS: 'cosine_with_restarts'>, 'POLYNOMIAL': <SchedulerType.POLYNOMIAL: 'polynomial'>, 'CONSTANT': <SchedulerType.CONSTANT: 'constant'>, 'CONSTANT_WITH_WARMUP': <SchedulerType.CONSTANT_WITH_WARMUP: 'constant_with_warmup'>, 'INVERSE_SQRT': <SchedulerType.INVERSE_SQRT: 'inverse_sqrt'>, 'REDUCE_ON_PLATEAU': <SchedulerType.REDUCE_ON_PLATEAU: 'reduce_lr_on_plateau'>, 'COSINE_WITH_MIN_LR': <SchedulerType.COSINE_WITH_MIN_LR: 'cosine_with_min_lr'>, 'WARMUP_STABLE_DECAY': <SchedulerType.WARMUP_STABLE_DECAY: 'warmup_stable_decay'>}, 'member_type': <class 'str'>, 'value2member_map': {'linear': <SchedulerType.LINEAR: 'linear'>, 'cosine': <SchedulerType.COSINE: 'cosine'>, 'cosine_with_restarts': <SchedulerType.COSINE_WITH_RESTARTS: 'cosine_with_restarts'>, 'polynomial': <SchedulerType.POLYNOMIAL: 'polynomial'>, 'constant': <SchedulerType.CONSTANT: 'constant'>, 'constant_with_warmup': <SchedulerType.CONSTANT_WITH_WARMUP: 'constant_with_warmup'>, 'inverse_sqrt': <SchedulerType.INVERSE_SQRT: 'inverse_sqrt'>, 'reduce_lr_on_plateau': <SchedulerType.REDUCE_ON_PLATEAU: 'reduce_lr_on_plateau'>, 'cosine_with_min_lr': <SchedulerType.COSINE_WITH_MIN_LR: 'cosine_with_min_lr'>, 'warmup_stable_decay': <SchedulerType.WARMUP_STABLE_DECAY: 'warmup_stable_decay'>}, 'LINEAR': <SchedulerType.LINEAR: 'linear'>, 'COSINE': <SchedulerType.COSINE: 'cosine'>, 'COSINE_WITH_RESTARTS': <SchedulerType.COSINE_WITH_RESTARTS: 'cosine_with_restarts'>, 'POLYNOMIAL': <SchedulerType.POLYNOMIAL: 'polynomial'>, 'CONSTANT': <SchedulerType.CONSTANT: 'constant'>, 'CONSTANT_WITH_WARMUP': <SchedulerType.CONSTANT_WITH_WARMUP: 'constant_with_warmup'>, 'INVERSE_SQRT': <SchedulerType.INVERSE_SQRT: 'inverse_sqrt'>, 'REDUCE_ON_PLATEAU': <SchedulerType.REDUCE_ON_PLATEAU: 'reduce_lr_on_plateau'>, 'COSINE_WITH_MIN_LR': <SchedulerType.COSINE_WITH_MIN_LR: 'cosine_with_min_lr'>, 'WARMUP_STABLE_DECAY': <SchedulerType.WARMUP_STABLE_DECAY: 'warmup_stable_decay'>, 'new': <function Enum.new at 0x7e6149dc4820>}" }, "lr_scheduler_kwargs": {}, "warmup_ratio": 0.03, "warmup_steps": 0, "log_level": "passive", "log_level_replica": "warning", "log_on_each_node": true, "logging_dir": "/home/jiashuo/codes/ForesightOptim/checkpoints/ppo_Llama3-8B-I_rsa/runs/Sep07_01-30-18_super-Rack-Server", "logging_strategy": { "value": "steps", "name": "STEPS", "objclass": "{'generate_next_value': <function Enum.generate_next_value at 0x7e6149dc6200>, 'module': 'transformers.trainer_utils', 'doc': 'An enumeration.', 'member_names': ['NO', 'STEPS', 'EPOCH'], 'member_map': {'NO': <IntervalStrategy.NO: 'no'>, 'STEPS': <IntervalStrategy.STEPS: 'steps'>, 'EPOCH': <IntervalStrategy.EPOCH: 'epoch'>}, 'member_type': <class 'str'>, 'value2member_map': {'no': <IntervalStrategy.NO: 'no'>, 'steps': <IntervalStrategy.STEPS: 'steps'>, 'epoch': <IntervalStrategy.EPOCH: 'epoch'>}, 'NO': <IntervalStrategy.NO: 'no'>, 'STEPS': <IntervalStrategy.STEPS: 'steps'>, 'EPOCH': <IntervalStrategy.EPOCH: 'epoch'>, 'new': <function Enum.new at 0x7e6149dc4820>}" }, "logging_first_step": false, "logging_steps": 1.0, "logging_nan_inf_filter": true, "save_strategy": { "value": "steps", "name": "STEPS", "objclass": "{'generate_next_value': <function Enum.generate_next_value at 0x7e6149dc6200>, 'module': 'transformers.trainer_utils', 'doc': 'An enumeration.', 'member_names': ['NO', 'STEPS', 'EPOCH', 'BEST'], 'member_map': {'NO': <SaveStrategy.NO: 'no'>, 'STEPS': <SaveStrategy.STEPS: 'steps'>, 'EPOCH': <SaveStrategy.EPOCH: 'epoch'>, 'BEST': <SaveStrategy.BEST: 'best'>}, 'member_type': <class 'str'>, 'value2member_map': {'no': <SaveStrategy.NO: 'no'>, 'steps': <SaveStrategy.STEPS: 'steps'>, 'epoch': <SaveStrategy.EPOCH: 'epoch'>, 'best': <SaveStrategy.BEST: 'best'>}, 'NO': <SaveStrategy.NO: 'no'>, 'STEPS': <SaveStrategy.STEPS: 'steps'>, 'EPOCH': <SaveStrategy.EPOCH: 'epoch'>, 'BEST': <SaveStrategy.BEST: 'best'>, 'new': <function Enum.new at 0x7e6149dc4820>}" }, "save_steps": 200, "save_total_limit": null, "save_safetensors": true, "save_on_each_node": false, "save_only_model": false, "restore_callback_states_from_checkpoint": false, "no_cuda": false, "use_cpu": false, "use_mps_device": false, "seed": 42, "data_seed": null, "jit_mode_eval": false, "use_ipex": false, "bf16": true, "fp16": false, "fp16_opt_level": "O1", "half_precision_backend": "auto", "bf16_full_eval": false, "fp16_full_eval": false, "tf32": true, "local_rank": 3, "ddp_backend": null, "tpu_num_cores": null, "tpu_metrics_debug": false, "debug": [], "dataloader_drop_last": false, "eval_steps": null, "dataloader_num_workers": 0, "dataloader_prefetch_factor": null, "past_index": -1, "run_name": "/home/jiashuo/codes/ForesightOptim/checkpoints/ppo_Llama3-8B-I_rsa", "disable_tqdm": false, "remove_unused_columns": false, "label_names": null, "load_best_model_at_end": false, "metric_for_best_model": null, "greater_is_better": null, "ignore_data_skip": false, "fsdp": [], "fsdp_min_num_params": 0, "fsdp_config": { "min_num_params": 0, "xla": false, "xla_fsdp_v2": false, "xla_fsdp_grad_ckpt": false }, "fsdp_transformer_layer_cls_to_wrap": null, "accelerator_config": { "split_batches": false, "dispatch_batches": null, "even_batches": true, "use_seedable_sampler": true, "non_blocking": false, "gradient_accumulation_kwargs": null }, "deepspeed": null, "label_smoothing_factor": 0.0, "optim": { "value": "adamw_torch", "name": "ADAMW_TORCH", "objclass": "{'generate_next_value': <function Enum.generate_next_value at 0x7e6149dc6200>, 'module': 'transformers.training_args', 'doc': '\n Stores the acceptable string identifiers for optimizers.\n ', 'member_names': ['ADAMW_TORCH', 'ADAMW_TORCH_FUSED', 'ADAMW_TORCH_XLA', 'ADAMW_TORCH_NPU_FUSED', 'ADAMW_APEX_FUSED', 'ADAFACTOR', 'ADAMW_ANYPRECISION', 'ADAMW_TORCH_4BIT', 'ADAMW_TORCH_8BIT', 'ADEMAMIX', 'SGD', 'ADAGRAD', 'ADAMW_BNB', 'ADAMW_8BIT', 'ADEMAMIX_8BIT', 'LION_8BIT', 'LION', 'PAGED_ADAMW', 'PAGED_ADAMW_8BIT', 'PAGED_ADEMAMIX', 'PAGED_ADEMAMIX_8BIT', 'PAGED_LION', 'PAGED_LION_8BIT', 'RMSPROP', 'RMSPROP_BNB', 'RMSPROP_8BIT', 'RMSPROP_32BIT', 'GALORE_ADAMW', 'GALORE_ADAMW_8BIT', 'GALORE_ADAFACTOR', 'GALORE_ADAMW_LAYERWISE', 'GALORE_ADAMW_8BIT_LAYERWISE', 'GALORE_ADAFACTOR_LAYERWISE', 'LOMO', 'ADALOMO', 'GROKADAMW', 'SCHEDULE_FREE_RADAM', 'SCHEDULE_FREE_ADAMW', 'SCHEDULE_FREE_SGD', 'APOLLO_ADAMW', 'APOLLO_ADAMW_LAYERWISE'], 'member_map': {'ADAMW_TORCH': <OptimizerNames.ADAMW_TORCH: 'adamw_torch'>, 'ADAMW_TORCH_FUSED': <OptimizerNames.ADAMW_TORCH_FUSED: 'adamw_torch_fused'>, 'ADAMW_TORCH_XLA': <OptimizerNames.ADAMW_TORCH_XLA: 'adamw_torch_xla'>, 'ADAMW_TORCH_NPU_FUSED': <OptimizerNames.ADAMW_TORCH_NPU_FUSED: 'adamw_torch_npu_fused'>, 'ADAMW_APEX_FUSED': <OptimizerNames.ADAMW_APEX_FUSED: 'adamw_apex_fused'>, 'ADAFACTOR': <OptimizerNames.ADAFACTOR: 'adafactor'>, 'ADAMW_ANYPRECISION': <OptimizerNames.ADAMW_ANYPRECISION: 'adamw_anyprecision'>, 'ADAMW_TORCH_4BIT': <OptimizerNames.ADAMW_TORCH_4BIT: 'adamw_torch_4bit'>, 'ADAMW_TORCH_8BIT': <OptimizerNames.ADAMW_TORCH_8BIT: 'adamw_torch_8bit'>, 'ADEMAMIX': <OptimizerNames.ADEMAMIX: 'ademamix'>, 'SGD': <OptimizerNames.SGD: 'sgd'>, 'ADAGRAD': <OptimizerNames.ADAGRAD: 'adagrad'>, 'ADAMW_BNB': <OptimizerNames.ADAMW_BNB: 'adamw_bnb_8bit'>, 'ADAMW_8BIT': <OptimizerNames.ADAMW_8BIT: 'adamw_8bit'>, 'ADEMAMIX_8BIT': <OptimizerNames.ADEMAMIX_8BIT: 'ademamix_8bit'>, 'LION_8BIT': <OptimizerNames.LION_8BIT: 'lion_8bit'>, 'LION': <OptimizerNames.LION: 'lion_32bit'>, 'PAGED_ADAMW': <OptimizerNames.PAGED_ADAMW: 'paged_adamw_32bit'>, 'PAGED_ADAMW_8BIT': <OptimizerNames.PAGED_ADAMW_8BIT: 'paged_adamw_8bit'>, 'PAGED_ADEMAMIX': <OptimizerNames.PAGED_ADEMAMIX: 'paged_ademamix_32bit'>, 'PAGED_ADEMAMIX_8BIT': <OptimizerNames.PAGED_ADEMAMIX_8BIT: 'paged_ademamix_8bit'>, 'PAGED_LION': <OptimizerNames.PAGED_LION: 'paged_lion_32bit'>, 'PAGED_LION_8BIT': <OptimizerNames.PAGED_LION_8BIT: 'paged_lion_8bit'>, 'RMSPROP': <OptimizerNames.RMSPROP: 'rmsprop'>, 'RMSPROP_BNB': <OptimizerNames.RMSPROP_BNB: 'rmsprop_bnb'>, 'RMSPROP_8BIT': <OptimizerNames.RMSPROP_8BIT: 'rmsprop_bnb_8bit'>, 'RMSPROP_32BIT': <OptimizerNames.RMSPROP_32BIT: 'rmsprop_bnb_32bit'>, 'GALORE_ADAMW': <OptimizerNames.GALORE_ADAMW: 'galore_adamw'>, 'GALORE_ADAMW_8BIT': <OptimizerNames.GALORE_ADAMW_8BIT: 'galore_adamw_8bit'>, 'GALORE_ADAFACTOR': <OptimizerNames.GALORE_ADAFACTOR: 'galore_adafactor'>, 'GALORE_ADAMW_LAYERWISE': <OptimizerNames.GALORE_ADAMW_LAYERWISE: 'galore_adamw_layerwise'>, 'GALORE_ADAMW_8BIT_LAYERWISE': <OptimizerNames.GALORE_ADAMW_8BIT_LAYERWISE: 'galore_adamw_8bit_layerwise'>, 'GALORE_ADAFACTOR_LAYERWISE': <OptimizerNames.GALORE_ADAFACTOR_LAYERWISE: 'galore_adafactor_layerwise'>, 'LOMO': <OptimizerNames.LOMO: 'lomo'>, 'ADALOMO': <OptimizerNames.ADALOMO: 'adalomo'>, 'GROKADAMW': <OptimizerNames.GROKADAMW: 'grokadamw'>, 'SCHEDULE_FREE_RADAM': <OptimizerNames.SCHEDULE_FREE_RADAM: 'schedule_free_radam'>, 'SCHEDULE_FREE_ADAMW': <OptimizerNames.SCHEDULE_FREE_ADAMW: 'schedule_free_adamw'>, 'SCHEDULE_FREE_SGD': <OptimizerNames.SCHEDULE_FREE_SGD: 'schedule_free_sgd'>, 'APOLLO_ADAMW': <OptimizerNames.APOLLO_ADAMW: 'apollo_adamw'>, 'APOLLO_ADAMW_LAYERWISE': <OptimizerNames.APOLLO_ADAMW_LAYERWISE: 'apollo_adamw_layerwise'>}, 'member_type': <class 'str'>, 'value2member_map': {'adamw_torch': <OptimizerNames.ADAMW_TORCH: 'adamw_torch'>, 'adamw_torch_fused': <OptimizerNames.ADAMW_TORCH_FUSED: 'adamw_torch_fused'>, 'adamw_torch_xla': <OptimizerNames.ADAMW_TORCH_XLA: 'adamw_torch_xla'>, 'adamw_torch_npu_fused': <OptimizerNames.ADAMW_TORCH_NPU_FUSED: 'adamw_torch_npu_fused'>, 'adamw_apex_fused': <OptimizerNames.ADAMW_APEX_FUSED: 'adamw_apex_fused'>, 'adafactor': <OptimizerNames.ADAFACTOR: 'adafactor'>, 'adamw_anyprecision': <OptimizerNames.ADAMW_ANYPRECISION: 'adamw_anyprecision'>, 'adamw_torch_4bit': <OptimizerNames.ADAMW_TORCH_4BIT: 'adamw_torch_4bit'>, 'adamw_torch_8bit': <OptimizerNames.ADAMW_TORCH_8BIT: 'adamw_torch_8bit'>, 'ademamix': <OptimizerNames.ADEMAMIX: 'ademamix'>, 'sgd': <OptimizerNames.SGD: 'sgd'>, 'adagrad': <OptimizerNames.ADAGRAD: 'adagrad'>, 'adamw_bnb_8bit': <OptimizerNames.ADAMW_BNB: 'adamw_bnb_8bit'>, 'adamw_8bit': <OptimizerNames.ADAMW_8BIT: 'adamw_8bit'>, 'ademamix_8bit': <OptimizerNames.ADEMAMIX_8BIT: 'ademamix_8bit'>, 'lion_8bit': <OptimizerNames.LION_8BIT: 'lion_8bit'>, 'lion_32bit': <OptimizerNames.LION: 'lion_32bit'>, 'paged_adamw_32bit': <OptimizerNames.PAGED_ADAMW: 'paged_adamw_32bit'>, 'paged_adamw_8bit': <OptimizerNames.PAGED_ADAMW_8BIT: 'paged_adamw_8bit'>, 'paged_ademamix_32bit': <OptimizerNames.PAGED_ADEMAMIX: 'paged_ademamix_32bit'>, 'paged_ademamix_8bit': <OptimizerNames.PAGED_ADEMAMIX_8BIT: 'paged_ademamix_8bit'>, 'paged_lion_32bit': <OptimizerNames.PAGED_LION: 'paged_lion_32bit'>, 'paged_lion_8bit': <OptimizerNames.PAGED_LION_8BIT: 'paged_lion_8bit'>, 'rmsprop': <OptimizerNames.RMSPROP: 'rmsprop'>, 'rmsprop_bnb': <OptimizerNames.RMSPROP_BNB: 'rmsprop_bnb'>, 'rmsprop_bnb_8bit': <OptimizerNames.RMSPROP_8BIT: 'rmsprop_bnb_8bit'>, 'rmsprop_bnb_32bit': <OptimizerNames.RMSPROP_32BIT: 'rmsprop_bnb_32bit'>, 'galore_adamw': <OptimizerNames.GALORE_ADAMW: 'galore_adamw'>, 'galore_adamw_8bit': <OptimizerNames.GALORE_ADAMW_8BIT: 'galore_adamw_8bit'>, 'galore_adafactor': <OptimizerNames.GALORE_ADAFACTOR: 'galore_adafactor'>, 'galore_adamw_layerwise': <OptimizerNames.GALORE_ADAMW_LAYERWISE: 'galore_adamw_layerwise'>, 'galore_adamw_8bit_layerwise': <OptimizerNames.GALORE_ADAMW_8BIT_LAYERWISE: 'galore_adamw_8bit_layerwise'>, 'galore_adafactor_layerwise': <OptimizerNames.GALORE_ADAFACTOR_LAYERWISE: 'galore_adafactor_layerwise'>, 'lomo': <OptimizerNames.LOMO: 'lomo'>, 'adalomo': <OptimizerNames.ADALOMO: 'adalomo'>, 'grokadamw': <OptimizerNames.GROKADAMW: 'grokadamw'>, 'schedule_free_radam': <OptimizerNames.SCHEDULE_FREE_RADAM: 'schedule_free_radam'>, 'schedule_free_adamw': <OptimizerNames.SCHEDULE_FREE_ADAMW: 'schedule_free_adamw'>, 'schedule_free_sgd': <OptimizerNames.SCHEDULE_FREE_SGD: 'schedule_free_sgd'>, 'apollo_adamw': <OptimizerNames.APOLLO_ADAMW: 'apollo_adamw'>, 'apollo_adamw_layerwise': <OptimizerNames.APOLLO_ADAMW_LAYERWISE: 'apollo_adamw_layerwise'>}, 'ADAMW_TORCH': <OptimizerNames.ADAMW_TORCH: 'adamw_torch'>, 'ADAMW_TORCH_FUSED': <OptimizerNames.ADAMW_TORCH_FUSED: 'adamw_torch_fused'>, 'ADAMW_TORCH_XLA': <OptimizerNames.ADAMW_TORCH_XLA: 'adamw_torch_xla'>, 'ADAMW_TORCH_NPU_FUSED': <OptimizerNames.ADAMW_TORCH_NPU_FUSED: 'adamw_torch_npu_fused'>, 'ADAMW_APEX_FUSED': <OptimizerNames.ADAMW_APEX_FUSED: 'adamw_apex_fused'>, 'ADAFACTOR': <OptimizerNames.ADAFACTOR: 'adafactor'>, 'ADAMW_ANYPRECISION': <OptimizerNames.ADAMW_ANYPRECISION: 'adamw_anyprecision'>, 'ADAMW_TORCH_4BIT': <OptimizerNames.ADAMW_TORCH_4BIT: 'adamw_torch_4bit'>, 'ADAMW_TORCH_8BIT': <OptimizerNames.ADAMW_TORCH_8BIT: 'adamw_torch_8bit'>, 'ADEMAMIX': <OptimizerNames.ADEMAMIX: 'ademamix'>, 'SGD': <OptimizerNames.SGD: 'sgd'>, 'ADAGRAD': <OptimizerNames.ADAGRAD: 'adagrad'>, 'ADAMW_BNB': <OptimizerNames.ADAMW_BNB: 'adamw_bnb_8bit'>, 'ADAMW_8BIT': <OptimizerNames.ADAMW_8BIT: 'adamw_8bit'>, 'ADEMAMIX_8BIT': <OptimizerNames.ADEMAMIX_8BIT: 'ademamix_8bit'>, 'LION_8BIT': <OptimizerNames.LION_8BIT: 'lion_8bit'>, 'LION': <OptimizerNames.LION: 'lion_32bit'>, 'PAGED_ADAMW': <OptimizerNames.PAGED_ADAMW: 'paged_adamw_32bit'>, 'PAGED_ADAMW_8BIT': <OptimizerNames.PAGED_ADAMW_8BIT: 'paged_adamw_8bit'>, 'PAGED_ADEMAMIX': <OptimizerNames.PAGED_ADEMAMIX: 'paged_ademamix_32bit'>, 'PAGED_ADEMAMIX_8BIT': <OptimizerNames.PAGED_ADEMAMIX_8BIT: 'paged_ademamix_8bit'>, 'PAGED_LION': <OptimizerNames.PAGED_LION: 'paged_lion_32bit'>, 'PAGED_LION_8BIT': <OptimizerNames.PAGED_LION_8BIT: 'paged_lion_8bit'>, 'RMSPROP': <OptimizerNames.RMSPROP: 'rmsprop'>, 'RMSPROP_BNB': <OptimizerNames.RMSPROP_BNB: 'rmsprop_bnb'>, 'RMSPROP_8BIT': <OptimizerNames.RMSPROP_8BIT: 'rmsprop_bnb_8bit'>, 'RMSPROP_32BIT': <OptimizerNames.RMSPROP_32BIT: 'rmsprop_bnb_32bit'>, 'GALORE_ADAMW': <OptimizerNames.GALORE_ADAMW: 'galore_adamw'>, 'GALORE_ADAMW_8BIT': <OptimizerNames.GALORE_ADAMW_8BIT: 'galore_adamw_8bit'>, 'GALORE_ADAFACTOR': <OptimizerNames.GALORE_ADAFACTOR: 'galore_adafactor'>, 'GALORE_ADAMW_LAYERWISE': <OptimizerNames.GALORE_ADAMW_LAYERWISE: 'galore_adamw_layerwise'>, 'GALORE_ADAMW_8BIT_LAYERWISE': <OptimizerNames.GALORE_ADAMW_8BIT_LAYERWISE: 'galore_adamw_8bit_layerwise'>, 'GALORE_ADAFACTOR_LAYERWISE': <OptimizerNames.GALORE_ADAFACTOR_LAYERWISE: 'galore_adafactor_layerwise'>, 'LOMO': <OptimizerNames.LOMO: 'lomo'>, 'ADALOMO': <OptimizerNames.ADALOMO: 'adalomo'>, 'GROKADAMW': <OptimizerNames.GROKADAMW: 'grokadamw'>, 'SCHEDULE_FREE_RADAM': <OptimizerNames.SCHEDULE_FREE_RADAM: 'schedule_free_radam'>, 'SCHEDULE_FREE_ADAMW': <OptimizerNames.SCHEDULE_FREE_ADAMW: 'schedule_free_adamw'>, 'SCHEDULE_FREE_SGD': <OptimizerNames.SCHEDULE_FREE_SGD: 'schedule_free_sgd'>, 'APOLLO_ADAMW': <OptimizerNames.APOLLO_ADAMW: 'apollo_adamw'>, 'APOLLO_ADAMW_LAYERWISE': <OptimizerNames.APOLLO_ADAMW_LAYERWISE: 'apollo_adamw_layerwise'>, 'new': <function Enum.new at 0x7e6149dc4820>}" }, "optim_args": null, "adafactor": false, "group_by_length": false, "length_column_name": "length", "report_to": [ "tensorboard", "wandb" ], "ddp_find_unused_parameters": null, "ddp_bucket_cap_mb": null, "ddp_broadcast_buffers": null, "dataloader_pin_memory": true, "dataloader_persistent_workers": false, "skip_memory_metrics": true, "use_legacy_prediction_loop": false, "push_to_hub": false, "resume_from_checkpoint": null, "hub_model_id": null, "hub_strategy": { "value": "every_save", "name": "EVERY_SAVE", "objclass": "{'generate_next_value': <function Enum.generate_next_value at 0x7e6149dc6200>, 'module': 'transformers.trainer_utils', 'doc': 'An enumeration.', 'member_names': ['END', 'EVERY_SAVE', 'CHECKPOINT', 'ALL_CHECKPOINTS'], 'member_map': {'END': <HubStrategy.END: 'end'>, 'EVERY_SAVE': <HubStrategy.EVERY_SAVE: 'every_save'>, 'CHECKPOINT': <HubStrategy.CHECKPOINT: 'checkpoint'>, 'ALL_CHECKPOINTS': <HubStrategy.ALL_CHECKPOINTS: 'all_checkpoints'>}, 'member_type': <class 'str'>, 'value2member_map': {'end': <HubStrategy.END: 'end'>, 'every_save': <HubStrategy.EVERY_SAVE: 'every_save'>, 'checkpoint': <HubStrategy.CHECKPOINT: 'checkpoint'>, 'all_checkpoints': <HubStrategy.ALL_CHECKPOINTS: 'all_checkpoints'>}, 'END': <HubStrategy.END: 'end'>, 'EVERY_SAVE': <HubStrategy.EVERY_SAVE: 'every_save'>, 'CHECKPOINT': <HubStrategy.CHECKPOINT: 'checkpoint'>, 'ALL_CHECKPOINTS': <HubStrategy.ALL_CHECKPOINTS: 'all_checkpoints'>, 'new': <function Enum.new at 0x7e6149dc4820>}" }, "hub_token": null, "hub_private_repo": null, "hub_always_push": false, "hub_revision": null, "gradient_checkpointing": true, "gradient_checkpointing_kwargs": null, "include_inputs_for_metrics": false, "include_for_metrics": [], "eval_do_concat_batches": true, "fp16_backend": "auto", "push_to_hub_model_id": null, "push_to_hub_organization": null, "push_to_hub_token": null, "mp_parameters": "", "auto_find_batch_size": false, "full_determinism": false, "torchdynamo": null, "ray_scope": "last", "ddp_timeout": 1800, "torch_compile": false, "torch_compile_backend": null, "torch_compile_mode": null, "include_tokens_per_second": false, "include_num_input_tokens_seen": false, "neftune_noise_alpha": null, "optim_target_modules": null, "batch_eval_metrics": false, "eval_on_start": false, "use_liger_kernel": false, "liger_kernel_config": null, "eval_use_gather_object": false, "average_tokens_across_devices": false, "use_wandb": false, "adapter_path": "", "padding_side": "right", "truncation_side": "left", "add_sep_token": false, "model_type": "llama", "model_prefix": "llama", "pooling_type": "average", "model_name_or_path": "/home/jiashuo/codes/ForesightOptim/checkpoints/im_Llama3-8B-I_rsa", "ref_model_name_or_path": "", "critic_model_name_or_path": "FacebookAI/roberta-base", "game_name": "RSA", "game_max_turn": 6, "data_dir": "path/to/cleaned_data", "data_type": "no_type", "data_path": "yahma/alpaca-cleaned", "train_data_path": [ "/home/jiashuo/datasets/rsagame/imitation_selfplay_episodes/Meta-Llama-3-8B-Instruct_train_selfplay_data.json" ], "eval_data_path": [], "data_prefix": "yahma/alpaca-cleaned", "data_suffix": "yahma/alpaca-cleaned", "task_type": "training", "train_method": "SelfPlayPPO", "use_lora": true, "debug_mode": false, "cache_dir": null, "clip_range": 0.2, "length_penalty": 1.0, "lm_sft_coeff": 0.0, "lm_kl_coeff": 0.1, "max_length": 2048, "valid_data_size": 0, "rollout_size": 128, "replay_buffer_size": 10000, "replay_batch_size": 16, "critic_learning_rate": 2e-05, "gamma": 0.99, "tau": 0.95, "max_new_tokens": 128, "temperature": 0.9, "top_p": 0.95, "player_one_model_name_or_path": "", "player_two_model_name_or_path": "", "distributed_state": { "_cpu": false, "backend": "nccl", "device": "cuda:3", "debug": false, "distributed_type": "DEEPSPEED", "num_processes": 6, "process_index": 3, "local_process_index": 3, "fork_launched": false }, "_n_gpu": 1, "__cached__setup_devices": "cuda:3", "deepspeed_plugin": { "hf_ds_config": { "config": { "train_batch_size": 96, "train_micro_batch_size_per_gpu": 2, "gradient_accumulation_steps": 8, "zero_optimization": { "stage": 2, "offload_optimizer": { "device": "none", "nvme_path": null }, "offload_param": { "device": "none", "nvme_path": null }, "stage3_gather_16bit_weights_on_model_save": true }, "gradient_clipping": 1.0, "steps_per_print": Infinity, "bf16": { "enabled": true }, "fp16": { "enabled": false }, "zero_allow_untested_optimizer": true }, "_stage": 2, "_offload": false, "_dtype": "torch.bfloat16", "mismatches": [] }, "gradient_accumulation_steps": "auto", "gradient_clipping": 1.0, "zero_stage": 2, "is_train_batch_min": true, "offload_optimizer_device": "none", "offload_param_device": "none", "offload_optimizer_nvme_path": "none", "offload_param_nvme_path": "none", "zero3_init_flag": false, "zero3_save_16bit_model": true, "transformer_moe_cls_names": null, "enable_msamp": false, "msamp_opt_level": "O1", "deepspeed_config": { "train_batch_size": 96, "train_micro_batch_size_per_gpu": 2, "gradient_accumulation_steps": 8, "zero_optimization": { "stage": 2, "offload_optimizer": { "device": "none", "nvme_path": null }, "offload_param": { "device": "none", "nvme_path": null }, "stage3_gather_16bit_weights_on_model_save": true }, "gradient_clipping": 1.0, "steps_per_print": Infinity, "bf16": { "enabled": true }, "fp16": { "enabled": false }, "zero_allow_untested_optimizer": true }, "_selected": true, "dschf": { "config": { "train_micro_batch_size_per_gpu": 1, "gradient_accumulation_steps": 1, "zero_optimization": { "stage": 2, "offload_optimizer": { "device": "none", "nvme_path": null }, "offload_param": { "device": "none", "nvme_path": null }, "stage3_gather_16bit_weights_on_model_save": true }, "gradient_clipping": 1.0, "steps_per_print": Infinity, "bf16": { "enabled": true }, "fp16": { "enabled": false } }, "_stage": 2, "_offload": false } } }

Hardware Requirements

• GPU: 4 48G NVIDIA-SMI 5090
• Number of GPUs: 4
• Memory of each GPU: 48G