Repository: mlc-ai/mlc-llm
File: cpp/serve/request_state.h (C/C++)

Code before:
  TVM_DECLARE_FINAL_OBJECT_INFO(RequestStateEntryNode, Object);
};

class RequestStateEntry : public ObjectRef {
 public:
  explicit RequestStateEntry(
      Request request, int num_models, int64_t internal_id, int rng_seed,
      const std::vector<std::string>& token_table,
      const std::optional<std::shared_ptr<GrammarStateInitContext>>& grammar_state_init_ctx,
      int parent_idx = -1);

  TVM_DEFINE_MUTABLE_OBJECT_REF_METHODS(RequestStateEntry, ObjectRef, RequestStateEntryNode);
};

/*! \brief A request's state, which groups all the request state entries. */
class RequestStateNode : public Object {
 public:
  /*! \brief the request state entries */
  std::vector<RequestStateEntry> entries;
  /*! \brief tracks the request metrics. */
  RequestMetrics metrics;
  /*!
   * \brief The post-process data structures.
   * We make it a state to avoid repetitive memory allocation/free in the action post process.
   */
  RequestActionPostProcStates postproc_states;

  static constexpr const char* _type_key = "mlc.serve.RequestState";
  static constexpr const bool _type_has_method_sequal_reduce = false;
  static constexpr const bool _type_has_method_shash_reduce = false;
  TVM_DECLARE_FINAL_OBJECT_INFO(RequestStateNode, Object);
};

class RequestState : public ObjectRef {
 public:
  explicit RequestState(std::vector<RequestStateEntry> entries, int num_response,
                        std::chrono::high_resolution_clock::time_point add_time_point);

  TVM_DEFINE_MUTABLE_OBJECT_REF_METHODS(RequestState, ObjectRef, RequestStateNode);
};

}  // namespace serve
}  // namespace llm
}  // namespace mlc

#endif  // MLC_LLM_SERVE_REQUEST_STATE_H_


Review comment (suggestion):
RequestActionPostProcStates => RequestActionPostProcWorkspace

Usually states needs to be persistent across runs, workspace are temp memory that do not need to be persisted

Code after:

  static constexpr const char* _type_key = "mlc.serve.RequestStateEntry";
  static constexpr const bool _type_has_method_sequal_reduce = false;
  static constexpr const bool _type_has_method_shash_reduce = false;
  TVM_DECLARE_FINAL_OBJECT_INFO(RequestStateEntryNode, Object);
};

class RequestStateEntry : public ObjectRef {
 public:
  explicit RequestStateEntry(
      Request request, int num_models, int64_t internal_id, int rng_seed,
      const std::vector<std::string>& token_table,
      const std::optional<std::shared_ptr<GrammarStateInitContext>>& grammar_state_init_ctx,
      int parent_idx = -1);

  TVM_DEFINE_MUTABLE_OBJECT_REF_METHODS(RequestStateEntry, ObjectRef, RequestStateEntryNode);
};

/*! \brief A request's state, which groups all the request state entries. */
class RequestStateNode : public Object {
 public:
  /*! \brief the request state entries */
  std::vector<RequestStateEntry> entries;
  /*! \brief tracks the request metrics. */
  RequestMetrics metrics;
  /*!
   * \brief The post-process data structures.
   * We make it a state to avoid repetitive memory allocation/free in the action post process.
   */
  RequestActionPostProcWorkspace postproc_states;

  static constexpr const char* _type_key = "mlc.serve.RequestState";
  static constexpr const bool _type_has_method_sequal_reduce = false;
  static constexpr const bool _type_has_method_shash_reduce = false;
  TVM_DECLARE_FINAL_OBJECT_INFO(RequestStateNode, Object);
};

class RequestState : public ObjectRef {
 public:
  /*!
   * \brief Request state constructor. We take the number of response (namely "n" in OpenAI
   * API) to pre-allocate all the data structure, in order to reduce the CPU data structure
   * allocation overhead when updating the request state.
   */
  explicit RequestState(std::vector<RequestStateEntry> entries, int num_response,
                        std::chrono::high_resolution_clock::time_point add_time_point);

  TVM_DEFINE_MUTABLE_OBJECT_REF_METHODS(RequestState, ObjectRef, RequestStateNode);
};

}  // namespace serve