Repository: ggerganov/llama.cpp
File: ggml/src/ggml-cann/common.h (C/C++)

Code before:

struct ggml_cann_graph {
    ~ggml_cann_graph() {
        if (graph != nullptr) {
            aclmdlRIDestroy(graph);
        }
    }

    aclmdlRI graph = nullptr;

    std::vector<ggml_graph_node_properties> ggml_graph_properties;
};
#endif  // USE_ACL_GRAPH

/**
 * @brief Context for managing CANN backend operations.
 */
struct ggml_backend_cann_context {
    int32_t device;                  /**< Device ID. */
    std::string name;                /**< Name of the device. */
    std::string description;         /**< Description of the device. */
    aclrtEvent copy_event = nullptr; /**< Event for managing copy operations. */
    void* init_ptr = nullptr;
    void* sin_ptr = nullptr;
    void* cos_ptr = nullptr;
    int64_t max_position_length = 200000;
#ifdef USE_ACL_GRAPH
    /// Cached CANN ACL graph used for executing the current ggml computation graph.
    std::unique_ptr<ggml_cann_graph> cann_graph;
#endif
    cann_task_queue task_queue;
    bool async_mode;
    bool support_set_rows;

    aclrtStream streams[GGML_CANN_MAX_STREAMS] = {nullptr}; /**< Array of streams for the device. */

    /**
     * @brief Constructor for initializing the context with a given device.
     * @param device Device ID.
     */
    explicit ggml_backend_cann_context(int device)
        : device(device), name("CANN" + std::to_string(device)), task_queue(1024, device) {
        ggml_cann_set_device(device);
        description = aclrtGetSocName();

        async_mode = parse_bool(get_env("GGML_CANN_ASYNC_MODE").value_or(""));
        GGML_LOG_INFO("%s: device %d async operator submission is %s\n", __func__,
            device, async_mode ? "ON" : "OFF");

        support_set_rows = parse_bool(get_env("LLAMA_SET_ROWS").value_or(""));
        GGML_LOG_INFO("%s: LLAMA_SET_ROWS is %s\n", __func__, support_set_rows ? "ON" : "OFF");

Review comment (suggestion):
A maximum prompt length of 200,000 is a bit excessive; let's initialize it to 65,536 here. And rename it to `max_prompt_length`.

Code after:

struct ggml_cann_graph {
    ~ggml_cann_graph() {
        if (graph != nullptr) {
            aclmdlRIDestroy(graph);
        }
    }

    aclmdlRI graph = nullptr;

    std::vector<ggml_graph_node_properties> ggml_graph_properties;
};
#endif  // USE_ACL_GRAPH

/**
 * @brief Context for managing CANN backend operations.
 */
struct ggml_backend_cann_context {
    int32_t device;                  /**< Device ID. */
    std::string name;                /**< Name of the device. */
    std::string description;         /**< Description of the device. */
    aclrtEvent copy_event = nullptr; /**< Event for managing copy operations. */
    void* init_ptr = nullptr;
    void* sin_ptr = nullptr;
    void* cos_ptr = nullptr;
    int64_t max_prompt_length = 65536;
#ifdef USE_ACL_GRAPH
    /// Cached CANN ACL graph used for executing the current ggml computation graph.
    std::unique_ptr<ggml_cann_graph> cann_graph;
#endif
    cann_task_queue task_queue;
    bool async_mode;
    bool support_set_rows;

    aclrtStream streams[GGML_CANN_MAX_STREAMS] = {nullptr}; /**< Array of streams for the device. */

    /**
     * @brief Constructor for initializing the context with a given device.
     * @param device Device ID.
     */
    explicit ggml_backend_cann_context(int device)
        : device(device), name("CANN" + std::to_string(device)), task_queue(1024, device) {
        ggml_cann_set_device(device);
        description = aclrtGetSocName();

        async_mode = parse_bool(get_env("GGML_CANN_ASYNC_MODE").value_or(""));
        GGML_LOG_INFO("%s: device %d async operator submission is %s\n", __func__,
            device, async_mode ? "ON" : "OFF");

        support_set_rows = parse_bool(get_env("LLAMA_SET_ROWS").value_or(""));
        GGML_LOG_INFO("%s: LLAMA_SET_ROWS is %s\n", __func__, support_set_rows ? "ON" : "OFF");