Repository: ggerganov/llama.cpp
File: ggml/src/ggml-cuda/moe-expert-reduce.cu (CUDA)

Code before:
    const ggml_tensor * mul = cgraph->nodes[start_index];

    if (mul->op != GGML_OP_MUL || !ggml_is_contiguous(mul->src[0]) || !ggml_is_contiguous(mul->src[1])) {
        return false;
    }

    int    current_node   = start_index + 1;
    size_t current_offset = 0;

    std::vector<const ggml_tensor *> view_nodes;
    //check if all are views of the expert in increasing order
    while (current_node < end_index && cgraph->nodes[current_node]->op == GGML_OP_VIEW) {
        const ggml_tensor * node = cgraph->nodes[current_node];
        if (node->view_src != mul) {
            return false;
        }
        if (node->view_offs < current_offset) {
            return false;
        }
        current_offset = node->view_offs;
        current_node++;
        view_nodes.push_back(node);
    }

    //check if all the adds are in increasing order
    const ggml_tensor * prev_add_src = view_nodes.size() ? view_nodes[0] : nullptr;
    int                 num_adds     = 0;
    int                 num_views    = view_nodes.size();
    while (current_node < end_index && cgraph->nodes[current_node]->op == GGML_OP_ADD) {
        const ggml_tensor * add_node = cgraph->nodes[current_node];

        bool is_first_op_ok  = num_views > num_adds ? add_node->src[0] == prev_add_src : false;
        bool is_second_op_ok = num_views > num_adds ? add_node->src[1] == view_nodes[num_adds + 1] : false;

        if (!is_first_op_ok || !is_second_op_ok) {
            return false;
        }
        prev_add_src = add_node;

        num_adds++;
        current_node++;
    }

    if (num_views != num_adds + 1) {
        return false;
    }

    return true;
}

void ggml_cuda_op_moe_expert_reduce(ggml_backend_cuda_context & ctx,

Review comment (bug):
```suggestion
    const ggml_tensor * prev_add_src = view_nodes.empty() ? nullptr : view_nodes[0];
```

I think this is slightly more readable.

Code after:
    const ggml_tensor * mul = cgraph->nodes[start_index];

    if (mul->op != GGML_OP_MUL || !ggml_is_contiguous(mul->src[0]) || !ggml_is_contiguous(mul->src[1])) {
        return false;
    }

    int    current_node   = start_index + 1;
    size_t current_offset = 0;

    std::vector<const ggml_tensor *> view_nodes;
    //check if all are views of the expert in increasing order
    while (current_node < end_index && cgraph->nodes[current_node]->op == GGML_OP_VIEW) {
        const ggml_tensor * node = cgraph->nodes[current_node];
        if (node->view_src != mul) {
            return false;
        }
        if (node->view_offs < current_offset) {
            return false;
        }
        current_offset = node->view_offs;
        current_node++;
        view_nodes.push_back(node);
    }

    //check if all the adds are in increasing order
    const ggml_tensor * prev_add_src = view_nodes.empty() ? nullptr : view_nodes[0];
    int                 num_adds     = 0;
    int                 num_views    = view_nodes.size();
    while (current_node < end_index && cgraph->nodes[current_node]->op == GGML_OP_ADD) {
        const ggml_tensor * add_node = cgraph->nodes[current_node];

        bool is_first_op_ok  = num_views > num_adds ? add_node->src[0] == prev_add_src : false;
        bool is_second_op_ok = num_views > num_adds ? add_node->src[1] == view_nodes[num_adds + 1] : false;

        if (!is_first_op_ok || !is_second_op_ok) {
            return false;
        }
        prev_add_src = add_node;

        num_adds++;
        current_node++;
    }

    if (num_views != num_adds + 1) {
        return false;
    }

    return true;
}

void ggml_cuda_op_moe_expert_reduce(ggml_backend_cuda_context & ctx,