Repository: ggerganov/llama.cpp
File: src/llama-model.cpp (C++)

Code before:
                    case 6:
                        type = LLM_TYPE_22M; break; // MiniLM-L6
                    case 12:
                        switch (hparams.n_embd) {
                            case 384: type = LLM_TYPE_33M; break; // MiniLM-L12, bge-small
                            case 768: type = LLM_TYPE_109M; break; // bge-base
                            default: type = LLM_TYPE_UNKNOWN;
                        } break;
                    case 24:
                        type = LLM_TYPE_335M; break; // bge-large
                    default: type = LLM_TYPE_UNKNOWN;
                }
            } break;
        case LLM_ARCH_MODERN_BERT:
            {

                hparams.swa_type = LLAMA_SWA_TYPE_SYMMETRIC;

                ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA,                      hparams.rope_freq_base_train_swa);
                ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW,                hparams.n_swa);
                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS,                 hparams.f_norm_eps);
                ml.get_key(LLM_KV_ATTENTION_CAUSAL,                        hparams.causal_attn);
                ml.get_key(LLM_KV_POOLING_TYPE,                            hparams.pooling_type, false);
                ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.swa_period);

                hparams.set_swa_pattern(hparams.swa_period);

                switch (hparams.n_layer) {
                    case 12:
                        type = LLM_TYPE_47M; break; // granite-embedding-small
                    case 22:
                        type = LLM_TYPE_149M; break; // modern-bert-base
                    case 28:
                        type = LLM_TYPE_395M; break; // modern-bert-large
                    default: type = LLM_TYPE_UNKNOWN;
                }
            } break;
        case LLM_ARCH_JINA_BERT_V2:
            {
                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS,    hparams.f_norm_eps);
                ml.get_key(LLM_KV_ATTENTION_CAUSAL,           hparams.causal_attn);
                ml.get_key(LLM_KV_POOLING_TYPE,               hparams.pooling_type, false);
                hparams.f_max_alibi_bias = 8.0f;

                switch (hparams.n_layer) {
                    case 4:  type = LLM_TYPE_33M;  break; // jina-embeddings-small
                    case 12: type = LLM_TYPE_137M; break; // jina-embeddings-base
                    default: type = LLM_TYPE_UNKNOWN;
                }
            } break;
        case LLM_ARCH_JINA_BERT_V3:

Review comment (suggestion):
```suggestion
                const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW,                hparams.n_swa, false);
                if (found_swa && hparams.n_swa > 0) {
                    uint32_t swa_period = 3;
                    hparams.swa_type = LLAMA_SWA_TYPE_SYMMETRIC;

                    ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA,                      hparams.rope_freq_base_train_swa);
                    ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);
                    hparams.set_swa_pattern(swa_period);
                } else {
                    hparams.swa_type = LLAMA_SWA_TYPE_NONE;
                }

                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS,                 hparams.f_norm_eps);
                ml.get_key(LLM_KV_ATTENTION_CAUSAL,                        hparams.causal_attn);
                ml.get_key(LLM_KV_POOLING_TYPE,                            hparams.pooling_type, false);
```
I don't think there's a non-swa ModernBERT, but since you already added provisions for it, and just in case there is, let's do it like this...

Code after:
                switch (hparams.n_layer) {
                    case 3:
                        type = LLM_TYPE_17M; break; // bge-micro
                    case 6:
                        type = LLM_TYPE_22M; break; // MiniLM-L6
                    case 12:
                        switch (hparams.n_embd) {
                            case 384: type = LLM_TYPE_33M; break; // MiniLM-L12, bge-small
                            case 768: type = LLM_TYPE_109M; break; // bge-base
                            default: type = LLM_TYPE_UNKNOWN;
                        } break;
                    case 24:
                        type = LLM_TYPE_335M; break; // bge-large
                    default: type = LLM_TYPE_UNKNOWN;
                }
            } break;
        case LLM_ARCH_MODERN_BERT:
            {
                const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);
                if (found_swa && hparams.n_swa > 0) {
                    uint32_t swa_period = 3;
                    hparams.swa_type = LLAMA_SWA_TYPE_SYMMETRIC;

                    ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa);
                    ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);
                    hparams.set_swa_pattern(swa_period);
                } else {
                    hparams.swa_type = LLAMA_SWA_TYPE_NONE;
                }

                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
                ml.get_key(LLM_KV_ATTENTION_CAUSAL,        hparams.causal_attn);
                ml.get_key(LLM_KV_POOLING_TYPE,            hparams.pooling_type, false);

                switch (hparams.n_layer) {
                    case 12:
                        type = LLM_TYPE_47M; break; // granite-embedding-small
                    case 22:
                        type = LLM_TYPE_149M; break; // modern-bert-base
                    case 28:
                        type = LLM_TYPE_395M; break; // modern-bert-large
                    default: type = LLM_TYPE_UNKNOWN;
                }
            } break;
        case LLM_ARCH_JINA_BERT_V2:
            {
                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS,    hparams.f_norm_eps);
                ml.get_key(LLM_KV_ATTENTION_CAUSAL,           hparams.causal_attn);
                ml.get_key(LLM_KV_POOLING_TYPE,               hparams.pooling_type, false);
                hparams.f_max_alibi_bias = 8.0f;