Repository: ggerganov/llama.cpp
File: tools/imatrix/imatrix.cpp (C++)

Code before:
        }

        if (e.values.empty()) {
            e.values.resize(nval, 0);
            e.counts.resize(nval, 0);
        }

        std::vector<float> tmp(nval);
        in.read((char*)tmp.data(), nval*sizeof(float));
        if (in.fail()) {
            LOG_ERR("%s: failed reading data for entry %d\n",__func__,i);
            m_stats = {};
            return false;
        }

        // Recreate the state as expected by save_imatrix(), and correct for weighted sum.
        std::vector<float> activations;
        activations.reserve(nval);
        for (int i = 0; i < nval; i++) {
            e.values[i] += tmp[i];
            e.counts[i] += ncall;
            activations.push_back(e.values[i] / e.counts[i]);
        }
        e.ncall += ncall;

        if (tstats) {
            float total = std::accumulate(activations.begin(), activations.end(), 0.0f);
            float max = * std::max_element(activations.begin(), activations.end());
            float min = * std::min_element(activations.begin(), activations.end());
            float mean = total / activations.size();
            float sq_total = std::inner_product(activations.begin(), activations.end(), activations.begin(), 0.0f);
            float dev = std::sqrt((sq_total / activations.size()) - (mean * mean));

            float threshold = min + min * 0.5f;
            int inactive_count = std::count_if(activations.begin(), activations.end(), [threshold](const float v) { return fabs(v) <= threshold; });
            float active_ratio = 1 -  static_cast<float>(inactive_count) / activations.size();

            float ent = 0;
            if (total > 0) {
                for (auto act : activations) {
                    if (float p = act / total; p > 0) {
                        ent -= p* std::log2(p);
                    }
                }
            }

            int z_score = 0;
            for (auto act : activations) {
                if (float p = (act - mean) / dev; p > 1) {
                    z_score++;
                }

Review comment (suggestion):
Could the stats loading be separated in its own function instead of being in `load_imatrix`? In #9400, there are two loading functions (for the old and new format) and the current proposed changes would be difficult to adapt to that.

The separate function could either be called from `load_imatrix` or separately.

Code after:
                    m_last_chunk = n_chunk;
                    if ((m_last_chunk % m_params.n_out_freq) / chunk_step == 0) {
                        save_imatrix();
                    }
                    if (m_params.n_save_freq > 0 && (m_last_chunk % m_params.n_save_freq) / chunk_step == 0) {
                        save_imatrix(m_last_chunk);
                    }
                }
            }
        }
    }

    return true;
}

void IMatrixCollector::save_imatrix_legacy(int32_t ncall) const {
    auto fname = m_params.out_file;

    if (ncall > 0) {
        fname += ".at_";
        fname += std::to_string(ncall);
    }

    // warn when writing imatrix entries that do not have full data
    // this can happen with MoE models where some of the experts end up not being exercised by the provided training data

    int n_entries = 0;
    std::vector<std::string> to_store;

    bool is_first = true; // for printing
    for (const auto & kv : m_stats) {
        const int n_all = kv.second.counts.size();

        if (n_all == 0) {
            continue;
        }

        int n_zeros = 0;
        for (const int c : kv.second.counts) {
            if (c == 0) {
                n_zeros++;
            }
        }

        if (n_zeros != 0 && is_first) {
            LOG_INF("\n");
            is_first = false;
        }

        if (n_zeros == n_all) {
            LOG_WRN("%s: entry '%40s' has no data - skipping\n", __func__, kv.first.c_str());