| { | |
| "dataset.debug": false, | |
| "dataset.git_diff": "", | |
| "dataset.git_sha1": "unknown", | |
| "dataset.manual_sample_ids": [], | |
| "dataset.output_dir": "output", | |
| "dataset.path": "/mnt/wd_ssd/beagle_train_data/datasets/ds_EXAONE-3.5-2.4B-Instruct", | |
| "dataset.run_name": "temp_run", | |
| "dataset.seed": 42, | |
| "dataset_generation.batch_size": 1, | |
| "dataset_generation.debug": false, | |
| "dataset_generation.debug_target": null, | |
| "dataset_generation.ds_prefix": "ds_", | |
| "dataset_generation.git_diff": "", | |
| "dataset_generation.git_sha1": "unknown", | |
| "dataset_generation.max_length": 4096, | |
| "dataset_generation.output_dir": "output", | |
| "dataset_generation.run_name": "temp_run", | |
| "dataset_generation.save_every": 1000, | |
| "dataset_generation.seed": 42, | |
| "inference.debug": false, | |
| "inference.detail_time_stats": false, | |
| "inference.draft_tree_shape": "mc_sim_7b_65", | |
| "inference.git_diff": "", | |
| "inference.git_sha1": "unknown", | |
| "inference.max_new_tokens": 512, | |
| "inference.mode": "speculative", | |
| "inference.output_dir": "output", | |
| "inference.run_name": "temp_run", | |
| "inference.seed": 42, | |
| "modeling.add_noise": true, | |
| "modeling.attention_offset": "random.randrange(0, 3)", | |
| "modeling.attention_wind": "5", | |
| "modeling.ckpt_path": null, | |
| "modeling.debug": false, | |
| "modeling.decoder_key_remap": {}, | |
| "modeling.dtype": "torch.bfloat16", | |
| "modeling.frozen_targets": [], | |
| "modeling.git_diff": "", | |
| "modeling.git_sha1": "unknown", | |
| "modeling.layer_path": "transformer.h", | |
| "modeling.lmhead_path": "lm_head", | |
| "modeling.model_path": "beagle/models/exaone3.5/EXAONE-3.5-2.4B-Instruct/", | |
| "modeling.norm_path": "transformer.ln_f", | |
| "modeling.output_dir": "output", | |
| "modeling.reuse_layer": null, | |
| "modeling.rotary_path": "transformer.rotary", | |
| "modeling.run_name": "temp_run", | |
| "modeling.seed": 42, | |
| "modeling.tokenizer_path": "LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct", | |
| "modeling.use_fc_eagle": false, | |
| "modeling.use_lower_layers": 7, | |
| "modeling.use_state_distill": false, | |
| "training.adam_beta2": 0.95, | |
| "training.bf16": true, | |
| "training.ddp_find_unused_parameters": false, | |
| "training.debug": false, | |
| "training.eval_steps": 100, | |
| "training.eval_strategy": "steps", | |
| "training.filter_out_shorts": true, | |
| "training.git_diff": "diff --git a/beagle/model_load.py b/beagle/model_load.py\nindex 2181ef7..95b25d6 100644\n--- a/beagle/model_load.py\n+++ b/beagle/model_load.py\n@@ -90,12 +90,10 @@ def get_safe_tensor_paths(path):\n \n \n def load_speculative_model_if_possible(modeling_configs, **kwargs):\n- from transformers.dynamic_module_utils import get_class_from_dynamic_module\n-\n try:\n- config = AutoConfig.from_pretrained(modeling_configs.model_path, trust_remote_code=True)\n- except:\n config = AutoConfig.from_pretrained(modeling_configs.ckpt_path, trust_remote_code=True)\n+ except:\n+ config = AutoConfig.from_pretrained(modeling_configs.model_path, trust_remote_code=True)\n \n for key, val in modeling_configs.get_obj().items():\n old_val = getattr(config, f'beagle_{key}', None)\n@@ -105,6 +103,7 @@ def load_speculative_model_if_possible(modeling_configs, **kwargs):\n if val != old_val:\n print('[\u2757]', f'new/old config[{key}] mismatch: {val} != {old_val}.')\n \n+ from transformers.dynamic_module_utils import get_class_from_dynamic_module\n try:\n class_ref = config.auto_map['AutoModelForSpeculativeCausalLM']\n model_class = get_class_from_dynamic_module(class_ref, modeling_configs.model_path)\n@@ -120,9 +119,7 @@ def load_speculative_model_if_possible(modeling_configs, **kwargs):\n trust_remote_code=True, **kwargs)\n return model, True\n \n- # if we were to use modeling_configs.ckpt_path, that means somehow the model_path\n- # does not include model binary files. This can happen when model_path is specified\n- # with a checkpoint directory where the modeling_*.py are missing.\n+\n ckpt_path = modeling_configs.ckpt_path or modeling_configs.model_path\n state_dict_ = dict()\n try:", | |
| "training.git_sha1": "30a2ddc64f3ff333b1b8e256a17d5d4b1281672f", | |
| "training.gradient_accumulation_steps": 4, | |
| "training.learning_rate": 3e-05, | |
| "training.logging_steps": 1, | |
| "training.lr_scheduler_type": "constant_with_warmup", | |
| "training.max_grad_norm": 0.5, | |
| "training.max_length": 2048, | |
| "training.max_steps": -1, | |
| "training.num_train_epochs": 10, | |
| "training.optim": "adamw_torch_fused", | |
| "training.output_dir": "output", | |
| "training.overwrite_output_dir": true, | |
| "training.per_device_eval_batch_size": 1, | |
| "training.per_device_train_batch_size": 1, | |
| "training.project": "beagle", | |
| "training.report_to": "wandb", | |
| "training.resume_from_checkpoint": true, | |
| "training.resume_wandb_runid": "va0rbslv", | |
| "training.run_name": "hearty-river-234", | |
| "training.save_steps": 500, | |
| "training.save_strategy": "steps", | |
| "training.save_total_limit": 2, | |
| "training.save_vram": true, | |
| "training.seed": 42, | |
| "training.warmup_steps": 50 | |
| } |