Repository: ggerganov/llama.cpp
File: gguf-py/gguf/constants.py (Python)

Code before:
        MAX_ALIBI_BIAS               = "{arch}.attention.max_alibi_bias"
        CLAMP_KQV                    = "{arch}.attention.clamp_kqv"
        KEY_LENGTH                   = "{arch}.attention.key_length"
        VALUE_LENGTH                 = "{arch}.attention.value_length"
        LAYERNORM_EPS                = "{arch}.attention.layer_norm_epsilon"
        LAYERNORM_RMS_EPS            = "{arch}.attention.layer_norm_rms_epsilon"
        GROUPNORM_EPS                = "{arch}.attention.group_norm_epsilon"
        GROUPNORM_GROUPS             = "{arch}.attention.group_norm_groups"
        CAUSAL                       = "{arch}.attention.causal"
        Q_LORA_RANK                  = "{arch}.attention.q_lora_rank"
        KV_LORA_RANK                 = "{arch}.attention.kv_lora_rank"
        DECAY_LORA_RANK              = "{arch}.attention.decay_lora_rank"
        ICLR_LORA_RANK               = "{arch}.attention.iclr_lora_rank"
        VALUE_RESIDUAL_MIX_LORA_RANK = "{arch}.attention.value_residual_mix_lora_rank"
        GATE_LORA_RANK               = "{arch}.attention.gate_lora_rank"
        REL_BUCKETS_COUNT            = "{arch}.attention.relative_buckets_count"
        SLIDING_WINDOW               = "{arch}.attention.sliding_window"
        SCALE                        = "{arch}.attention.scale"
        OUTPUT_SCALE                 = "{arch}.attention.output_scale"
        TEMPERATURE_LENGTH           = "{arch}.attention.temperature_length"
        KEY_LENGTH_MLA               = "{arch}.attention.key_length_mla"
        VALUE_LENGTH_MLA             = "{arch}.attention.value_length_mla"
        SHARED_KV_LAYERS             = "{arch}.attention.shared_kv_layers"
        SLIDING_WINDOW_PATTERN       = "{arch}.attention.sliding_window_pattern"
        TEMPERATURE_SCALE            = "{arch}.attention.temperature_scale"
        DENSE_EVERY_N_LAYERS         = "{arch}.attention.dense_every_n_layers"

    class Rope:
        DIMENSION_COUNT          = "{arch}.rope.dimension_count"
        DIMENSION_SECTIONS       = "{arch}.rope.dimension_sections"
        FREQ_BASE                = "{arch}.rope.freq_base"
        FREQ_BASE_SWA            = "{arch}.rope.freq_base_swa"
        SCALING_TYPE             = "{arch}.rope.scaling.type"
        SCALING_FACTOR           = "{arch}.rope.scaling.factor"
        SCALING_ATTN_FACTOR      = "{arch}.rope.scaling.attn_factor"
        SCALING_ORIG_CTX_LEN     = "{arch}.rope.scaling.original_context_length"
        SCALING_FINETUNED        = "{arch}.rope.scaling.finetuned"
        SCALING_YARN_LOG_MUL     = "{arch}.rope.scaling.yarn_log_multiplier"
        SCALING_YARN_EXT_FACTOR  = "{arch}.rope.scaling.yarn_ext_factor"
        SCALING_YARN_ATTN_FACTOR = "{arch}.rope.scaling.yarn_attn_factor"
        SCALING_YARN_BETA_FAST   = "{arch}.rope.scaling.yarn_beta_fast"
        SCALING_YARN_BETA_SLOW   = "{arch}.rope.scaling.yarn_beta_slow"

    class Split:
        LLM_KV_SPLIT_NO            = "split.no"
        LLM_KV_SPLIT_COUNT         = "split.count"
        LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count"

    class SSM:
        CONV_KERNEL    = "{arch}.ssm.conv_kernel"
        INNER_SIZE     = "{arch}.ssm.inner_size"

Review comment (suggestion):
It looks like we now have several different ways to indicate the layer type for different hybrid patterns. This `DENSE_EVERY_N_LAYERS` implies a strictly periodic pattern. The above `SLIDING_WINDOW_PATTERN` appears to be for interleaved sliding window hybrid architectures, but strangely I don't see the corresponding key anywhere on the `c++` side ([here](https://github.com/ggml-org/llama.cpp/blob/master/src/llama-arch.cpp#L206)). We also use the fact that `head_count_kv` can be either a scalar or a list of scalars to differentiate recurrent vs attention layers for `GraniteHybrid` ([here](https://github.com/ggml-org/llama.cpp/blob/master/convert_hf_to_gguf.py#L8509)).

Since this is already fairly confusing and potentially redundent, I don't think we need to hold up this PR, but I'm curious if others can think of a clean way to accomplish the goal of layer type designation without a net-new `hparam`.

Code after:
        MAX_ALIBI_BIAS               = "{arch}.attention.max_alibi_bias"
        CLAMP_KQV                    = "{arch}.attention.clamp_kqv"
        KEY_LENGTH                   = "{arch}.attention.key_length"
        VALUE_LENGTH                 = "{arch}.attention.value_length"
        LAYERNORM_EPS                = "{arch}.attention.layer_norm_epsilon"
        LAYERNORM_RMS_EPS            = "{arch}.attention.layer_norm_rms_epsilon"
        GROUPNORM_EPS                = "{arch}.attention.group_norm_epsilon"
        GROUPNORM_GROUPS             = "{arch}.attention.group_norm_groups"
        CAUSAL                       = "{arch}.attention.causal"
        Q_LORA_RANK                  = "{arch}.attention.q_lora_rank"
        KV_LORA_RANK                 = "{arch}.attention.kv_lora_rank"
        DECAY_LORA_RANK              = "{arch}.attention.decay_lora_rank"
        ICLR_LORA_RANK               = "{arch}.attention.iclr_lora_rank"
        VALUE_RESIDUAL_MIX_LORA_RANK = "{arch}.attention.value_residual_mix_lora_rank"
        GATE_LORA_RANK               = "{arch}.attention.gate_lora_rank"
        REL_BUCKETS_COUNT            = "{arch}.attention.relative_buckets_count"
        SLIDING_WINDOW               = "{arch}.attention.sliding_window"
        SCALE                        = "{arch}.attention.scale"
        OUTPUT_SCALE                 = "{arch}.attention.output_scale"
        TEMPERATURE_LENGTH           = "{arch}.attention.temperature_length"
        KEY_LENGTH_MLA               = "{arch}.attention.key_length_mla"
        VALUE_LENGTH_MLA             = "{arch}.attention.value_length_mla"
        SHARED_KV_LAYERS             = "{arch}.attention.shared_kv_layers"
        SLIDING_WINDOW_PATTERN       = "{arch}.attention.sliding_window_pattern"
        TEMPERATURE_SCALE            = "{arch}.attention.temperature_scale"

    class Rope:
        DIMENSION_COUNT          = "{arch}.rope.dimension_count"
        DIMENSION_SECTIONS       = "{arch}.rope.dimension_sections"
        FREQ_BASE                = "{arch}.rope.freq_base"
        FREQ_BASE_SWA            = "{arch}.rope.freq_base_swa"
        SCALING_TYPE             = "{arch}.rope.scaling.type"
        SCALING_FACTOR           = "{arch}.rope.scaling.factor"
        SCALING_ATTN_FACTOR      = "{arch}.rope.scaling.attn_factor"
        SCALING_ORIG_CTX_LEN     = "{arch}.rope.scaling.original_context_length"
        SCALING_FINETUNED        = "{arch}.rope.scaling.finetuned"
        SCALING_YARN_LOG_MUL     = "{arch}.rope.scaling.yarn_log_multiplier"
        SCALING_YARN_EXT_FACTOR  = "{arch}.rope.scaling.yarn_ext_factor"
        SCALING_YARN_ATTN_FACTOR = "{arch}.rope.scaling.yarn_attn_factor"
        SCALING_YARN_BETA_FAST   = "{arch}.rope.scaling.yarn_beta_fast"
        SCALING_YARN_BETA_SLOW   = "{arch}.rope.scaling.yarn_beta_slow"

    class Split:
        LLM_KV_SPLIT_NO            = "split.no"
        LLM_KV_SPLIT_COUNT         = "split.count"
        LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count"

    class SSM:
        CONV_KERNEL    = "{arch}.ssm.conv_kernel"
        INNER_SIZE     = "{arch}.ssm.inner_size"
        STATE_SIZE     = "{arch}.ssm.state_size"