Hugging Face model: brkichle/lora-llama3-archimate
Author: brkichle
Created: 2025-05-14 17:43:18
Last modified: 2025-06-06 17:21:58
Pipeline: text-generation
Library: transformers
Likes: 0 | Downloads: 0
Tags: transformers, safetensors, archimate, llama3, causal-lm, peft, lora, text-generation, conversational, en, dataset:archimate_3_1_dataset, base_model:meta-llama/Llama-3.1-8B-Instruct, base_model:adapter:meta-llama/Llama-3.1-8B-Instruct, license:mit, model-index, endpoints_compatible, region:us
Base model relation: adapter
Base models: id: meta-llama/Llama-3.1-8B-Instruct
Card metadata:
language: en
license: mit
datasets: archimate_3_1_dataset
metrics: eval/loss, eval/samples_per_second
pipeline_tag: text-generation
library_name: transformers
base_model: meta-llama/Llama-3.1-8B-Instruct
tags: archimate, llama3, causal-lm, peft, lora
model_sha: c95393df478f4f3731f04ed0f81e102f82789d50
kwargs: bf16: value: true
fp16: value: false
seed: value: 42
tf32: value: true
optim: value: adamw_torch
top_k: value: 50
top_p: value: 1
_wandb: value: m: 1: eval/steps_per_second
5: 2
6: 1, 3, 1: train/global_step
6: 3, 1: train/epoch
5: 2
6: 1, 3, 1: eval/loss
5: 2
6: 1, 3, 1: eval/runtime
5: 2
6: 1, 3, 1: eval/samples_per_second
5: 2
6: 1, 3
t: 1: 1, 5, 11, 49, 51, 53, 55, 71, 98
2: 1, 5, 11, 49, 51, 53, 55, 71, 98
3: 7, 13, 19, 23, 55, 62, 66
4: 3.10.12
5: 0.19.11
6: 4.52.3
8: 5
9: 1: transformers_trainer
12: 0.19.11
13: linux-x86_64
cli_version: 0.19.11
python_version: 3.10.12
prefix: value: null
do_eval: value: true
no_cuda: value: false
use_cpu: value: false
do_train: value: false
head_dim: value: 128
id2label: value: 0: LABEL_0
1: LABEL_1
label2id: value: LABEL_0: 0
LABEL_1: 1
mlp_bias: value: false
run_name: value: full_run-BATCH-64-ACCUM-1-20250605-045240
use_ipex: value: false
adafactor: value: false
data_seed: value: null
deepspeed: value: ds_config.json
do_sample: value: false
hub_token: value: <HUB_TOKEN>
log_level: value: passive
max_steps: value: -1
num_beams: value: 1
ray_scope: value: last
report_to: value: wandb
typical_p: value: 1
use_cache: value: false
adam_beta1: value: 0.9
adam_beta2: value: 0.999
do_predict: value: false
eval_delay: value: 0
eval_steps: value: null
hidden_act: value: silu
is_decoder: value: false
local_rank: value: 0
max_length: value: 20
min_length: value: 0
model_type: value: llama
optim_args: value: null
output_dir: value: /mnt/data/hf-cache/.cache/lora-archimate/run-7d4f77bc
past_index: value: -1
rope_theta: value: 500000
save_steps: value: 500
vocab_size: value: 128256
ddp_backend: value: null
ddp_timeout: value: 1800
fsdp_config: value: xla: false
xla_fsdp_v2: false
min_num_params: 0
xla_fsdp_grad_ckpt: false
hidden_size: value: 4096
label_names: value: null
logging_dir: value: /mnt/data/hf-cache/.cache/lora-archimate/run-7d4f77bc/logs
peft_config: value: default: r: 96
bias: none
revision: null
use_dora: false
lora_bias: false
peft_type: LORA
task_type: CAUSAL_LM
eva_config: null
lora_alpha: 192
use_rslora: true
auto_mapping: null
corda_config: null
lora_dropout: 0.05
megatron_core: megatron.core
fan_in_fan_out: false
inference_mode: false
layers_pattern: null
runtime_config: ephemeral_gpu_offload: false
target_modules: o_proj, k_proj, q_proj, v_proj
exclude_modules: null
megatron_config: null
modules_to_save: null
init_lora_weights: true
layer_replication: null
layers_to_transform: null
base_model_name_or_path: meta-llama/Llama-3.1-8B-Instruct
trainable_token_indices: null
push_to_hub: value: false
return_dict: value: true
temperature: value: 1
torch_dtype: value: float16
torchdynamo: value: null
torchscript: value: false
adam_epsilon: value: 1e-08
bos_token_id: value: 128000
disable_tqdm: value: true
eos_token_id: value: 128001, 128008, 128009
fp16_backend: value: auto
hub_model_id: value: null
hub_strategy: value: every_save
pad_token_id: value: null
problem_type: value: null
rms_norm_eps: value: 1e-05
rope_scaling: value: factor: 8
rope_type: llama3
low_freq_factor: 1
high_freq_factor: 4
original_max_position_embeddings: 8192
sep_token_id: value: null
use_bfloat16: value: false
warmup_ratio: value: 0
warmup_steps: value: 50
weight_decay: value: 0
_name_or_path: value: meta-llama/Llama-3.1-8B-Instruct
architectures: value: LlamaForCausalLM
bad_words_ids: value: null
eval_on_start: value: false
eval_strategy: value: epoch
jit_mode_eval: value: false
learning_rate: value: 2e-05
logging_steps: value: 500
max_grad_norm: value: 1
output_scores: value: false
save_strategy: value: epoch
torch_compile: value: false
tpu_num_cores: value: null
attention_bias: value: false
bf16_full_eval: value: false
early_stopping: value: false
fp16_full_eval: value: false
fp16_opt_level: value: O1
length_penalty: value: 1
pretraining_tp: value: 1
tf_legacy_loss: value: false
use_mps_device: value: false
finetuning_task: value: null
group_by_length: value: true
hub_always_push: value: false
num_beam_groups: value: 1
save_only_model: value: false
suppress_tokens: value: null
tokenizer_class: value: null
full_determinism: value: false
hub_private_repo: value: null
ignore_data_skip: value: false
log_on_each_node: value: true
logging_strategy: value: steps
num_train_epochs: value: 5
save_safetensors: value: true
save_total_limit: value: 2
use_liger_kernel: value: false
attention_dropout: value: 0
ddp_bucket_cap_mb: value: null
diversity_penalty: value: 0
greater_is_better: value: null
initializer_range: value: 0.02
intermediate_size: value: 14336
log_level_replica: value: warning
lr_scheduler_type: value: cosine
num_hidden_layers: value: 32
output_attentions: value: false
push_to_hub_token: value: <PUSH_TO_HUB_TOKEN>
save_on_each_node: value: false
tpu_metrics_debug: value: false
accelerator_config: value: even_batches: true
non_blocking: false
split_batches: false
dispatch_batches: null
use_seedable_sampler: true
gradient_accumulation_kwargs: null
batch_eval_metrics: value: false
is_encoder_decoder: value: false
length_column_name: value: length
logging_first_step: value: false
repetition_penalty: value: 1
torch_compile_mode: value: null
add_cross_attention: value: false
forced_bos_token_id: value: null
forced_eos_token_id: value: null
fsdp_min_num_params: value: 0
neftune_noise_alpha: value: null
num_attention_heads: value: 32
num_key_value_heads: value: 8
quantization_config: value: load_in_4bit: true
load_in_8bit: false
quant_method: BITS_AND_BYTES
_load_in_4bit: true
_load_in_8bit: false
llm_int8_threshold: 6
bnb_4bit_quant_type: fp4
llm_int8_skip_modules: null
bnb_4bit_compute_dtype: bfloat16
bnb_4bit_quant_storage: uint8
llm_int8_has_fp16_weight: false
bnb_4bit_use_double_quant: false
llm_int8_enable_fp32_cpu_offload: false
skip_memory_metrics: value: true
tie_encoder_decoder: value: false
tie_word_embeddings: value: false
auto_find_batch_size: value: false
dataloader_drop_last: value: false
model/num_parameters: value: 8112050176
no_repeat_ngram_size: value: 0
num_return_sequences: value: 1
optim_target_modules: value: null
output_hidden_states: value: false
overwrite_output_dir: value: false
prediction_loss_only: value: false
push_to_hub_model_id: value: null
task_specific_params: value: null
transformers_version: value: 4.52.3
begin_suppress_tokens: value: null
dataloader_pin_memory: value: true
ddp_broadcast_buffers: value: null
metric_for_best_model: value: null
remove_invalid_values: value: false
remove_unused_columns: value: true
torch_compile_backend: value: null
dataloader_num_workers: value: 0
decoder_start_token_id: value: null
eval_do_concat_batches: value: true
eval_use_gather_object: value: false
gradient_checkpointing: value: false
half_precision_backend: value: auto
label_smoothing_factor: value: 0
load_best_model_at_end: value: false
logging_nan_inf_filter: value: true
resume_from_checkpoint: value: null
chunk_size_feed_forward: value: 0
eval_accumulation_steps: value: null
max_position_embeddings: value: 131072
per_gpu_eval_batch_size: value: null
return_dict_in_generate: value: false
torch_empty_cache_steps: value: null
per_gpu_train_batch_size: value: null
push_to_hub_organization: value: null
include_tokens_per_second: value: false
dataloader_prefetch_factor: value: null
ddp_find_unused_parameters: value: null
include_inputs_for_metrics: value: false
per_device_eval_batch_size: value: 4
use_legacy_prediction_loop: value: false
cross_attention_hidden_size: value: null
gradient_accumulation_steps: value: 1
per_device_train_batch_size: value: 64
encoder_no_repeat_ngram_size: value: 0
average_tokens_across_devices: value: false
dataloader_persistent_workers: value: false
gradient_checkpointing_kwargs: value: null
include_num_input_tokens_seen: value: false
exponential_decay_length_penalty: value: null
fsdp_transformer_layer_cls_to_wrap: value: null
restore_callback_states_from_checkpoint: value: false