# mlx5dv_wr_memcpy

## NAME



           mlx5dv_wr_set_dc_addr - Attach a DC info to the last work request
    
           mlx5dv_wr_raw_wqe - Build a raw work request
    
           mlx5dv_wr_memcpy - Build a DMA memcpy work request
    


## SYNOPSIS



                  #include <infiniband/mlx5dv.h>
    
                  static inline void mlx5dv_wr_set_dc_addr(struct mlx5dv_qp_ex *mqp,
                                                           struct ibv_ah *ah,
                                                           uint32_t remote_dctn,
                                                           uint64_t remote_dc_key);
    
                  static inline void mlx5dv_wr_set_dc_addr_stream(struct mlx5dv_qp_ex *mqp,
                                          struct ibv_ah *ah,
                                          uint32_t remote_dctn,
                                          uint64_t remote_dc_key,
                                          uint16_t stream_id);
    
                  struct mlx5dv_mr_interleaved {
                      uint64_t        addr;
                      uint32_t        bytes_count;
                      uint32_t        bytes_skip;
                      uint32_t        lkey;
                  };
    
                  static inline void mlx5dv_wr_mr_interleaved(struct mlx5dv_qp_ex *mqp,
                                          struct mlx5dv_mkey *mkey,
                                          uint32_t access_flags, /* use enum ibv_access_flags */
                                          uint32_t repeat_count,
                                          uint16_t num_interleaved,
                                          struct mlx5dv_mr_interleaved *data);
    
                  static inline void mlx5dv_wr_mr_list(struct mlx5dv_qp_ex *mqp,
                                        struct mlx5dv_mkey *mkey,
                                        uint32_t access_flags, /* use enum ibv_access_flags */
                                        uint16_t num_sges,
                                        struct ibv_sge *sge);
    
                  static inline int mlx5dv_wr_raw_wqe(struct mlx5dv_qp_ex *mqp, const void *wqe);
    
                  static inline void mlx5dv_wr_memcpy(struct mlx5dv_qp_ex *mqp_ex,
                                      uint32_t dest_lkey, uint64_t dest_addr,
                                      uint32_t src_lkey, uint64_t src_addr,
                                      size_t length)
    


## DESCRIPTION



           The MLX5DV work request APIs (mlx5dv_wr_*) is an extension for IBV
           work request API (ibv_wr_*) with mlx5 specific features for send
           work request.  This may be used together with or without ibv_wr_*
           calls.
    


## USAGE



           To use these APIs a QP must be created using mlx5dv_create_qp()
           with _send_ops_flags_ of struct ibv_qp_init_attr_ex set.
    
           If the QP does not support all the requested work request types
           then QP creation will fail.
    
           The mlx5dv_qp_ex is extracted from the IBV_QP by ibv_qp_to_qp_ex()
           and mlx5dv_qp_ex_from_ibv_qp_ex().  This should be used to apply
           the mlx5 specific features on the posted WR.
    
           A work request creation requires to use the ibv_qp_ex as described
           in the man for ibv_wr_post and mlx5dv_qp with its available
           builders and setters.
    
       **QP Specific builders**
           _RC_ QPs _mlx5dv_wr_mr_interleaved()_
    
                  registers an interleaved memory layout by using an indirect
                  mkey and some interleaved data.  The layout of the memory
                  pointed by the mkey after its registration will be the _data_
                  representation for the _num_interleaved_ entries.  This
                  single layout representation is repeated by _repeat_count_.
    
                  The _data_ as described by struct mlx5dv_mr_interleaved will
                  hold real data defined by _bytes_count_ and then a padding of
                  _bytes_skip_.  Post a successful registration, RDMA
                  operations can use this _mkey_.  The hardware will scatter
                  the data according to the pattern.  The _mkey_ should be used
                  in a zero-based mode.  The _addr_ field in its _ibv_sge_ is an
                  offset in the total data.  To create this _mkey_
                  mlx5dv_create_mkey() should be used.
    
                  Current implementation requires the IBV_SEND_INLINE option
                  to be on in _ibv_qp_ex- >wr_flags_ field.  To be able to have
                  more than 3 _num_interleaved_ entries, the QP should be
                  created with a larger WQE size that may fit it.  This
                  should be done using the _max_inline_data_ attribute of
                  _struct ibv_qp_cap_ upon its creation.
    
                  As one entry will be consumed for strided header, the _mkey_
                  should be created with one more entry than the required
                  _num_interleaved_.
    
                  In case _ibv_qp_ex- >wr_flags_ turns on IBV_SEND_SIGNALED, the
                  reported WC opcode will be MLX5DV_WC_UMR.  Unregister the
                  _mkey_ to enable another pattern registration should be done
                  via ibv_post_send with IBV_WR_LOCAL_INV opcode.
           _mlx5dv_wr_mr_list()_
    
                  registers a memory layout based on list of ibv_sge.  The
                  layout of the memory pointed by the _mkey_ after its
                  registration will be based on the list of _sge_ counted by
                  _num_sges_.  Post a successful registration RDMA operations
                  can use this _mkey_ , the hardware will scatter the data
                  according to the pattern.  The _mkey_ should be used in a
                  zero-based mode, the _addr_ field in its _ibv_sge_ is an offset
                  in the total data.
    
                  Current implementation requires the IBV_SEND_INLINE option
                  to be on in _ibv_qp_ex- >wr_flags_ field.  To be able to have
                  more than 4 _num_sge_ entries, the QP should be created with
                  a larger WQE size that may fit it.  This should be done
                  using the _max_inline_data_ attribute of _struct ibv_qp_cap_
                  upon its creation.
    
                  In case _ibv_qp_ex- >wr_flags_ turns on IBV_SEND_SIGNALED, the
                  reported WC opcode will be MLX5DV_WC_UMR.  Unregister the
                  _mkey_ to enable other pattern registration should be done
                  via ibv_post_send with IBV_WR_LOCAL_INV opcode.
    
           _RC_ or _DCI_ QPs
                  _mlx5dv_wr_memcpy()_
    
                  Builds a DMA memcpy work request to copy data of length
                  _length_ from _src_addr_ to _dest_addr_.  The copy operation will
                  be done using the DMA MMO functionality of the device to
                  copy data on PCI bus.
    
                  The MLX5DV_QP_EX_WITH_MEMCPY flag in
                  _mlx5dv_qp_init_attr.send_ops_flags_ needs to be set during
                  QP creation.  If the device or QP doesn’t support it then
                  QP creation will fail.  The maximum memcpy length that is
                  supported by the device is reported in
                  _mlx5dv_context- >max_wr_memcpy_length_.  A zero value in
                  _mlx5dv_context- >max_wr_memcpy_length_ means the device
                  doesn’t support memcpy operations.
    
                  IBV_SEND_FENCE indicator should be used on a following send
                  request which is dependent on _dest_addr_ of the memcpy
                  operation.
    
                  In case _ibv_qp_ex- >wr_flags_ turns on IBV_SEND_SIGNALED, the
                  reported WC opcode will be MLX5DV_WC_MEMCPY.
    
       **Raw WQE builders**
           _mlx5dv_wr_raw_wqe()_
                  It is used to build a custom work request (WQE) and post it
                  on a normal QP.  The caller needs to set all details of the
                  WQE (except the “ctrl.wqe_index” and “ctrl.signature”
                  fields, which is the driver’s responsibility to set).  The
                  MLX5DV_QP_EX_WITH_RAW_WQE flag in
                  mlx5_qp_attr.send_ops_flags needs to be set.
    
                  The wr_flags are ignored as it’s the caller’s
                  responsibility to set flags in WQE.
    
                  No matter what the send opcode is, the work completion
                  opcode for a raw WQE is IBV_WC_DRIVER2.
    
       **QP Specific setters**
           _DCI_ QPs
                  _mlx5dv_wr_set_dc_addr()_ must be called to set the DCI WR
                  properties.  The destination address of the work is
                  specified by _ah_ , the remote DCT number is specified by
                  _remote_dctn_ and the DC key is specified by _remote_dc_key_.
                  This setter is available when the QP transport is DCI and
                  send_ops_flags in struct ibv_qp_init_attr_ex is set.  The
                  available builders and setters for DCI QP are the same as
                  RC QP.  DCI QP created with
                  MLX5DV_QP_INIT_ATTR_MASK_DCI_STREAMS can call
                  _mlx5dv_wr_set_dc_addr_stream()_ to define the _stream_id_ of
                  the operation to allow HW to choose one of the multiple
                  concurrent DCI resources.  Calls to _mlx5dv_wr_set_dc_addr()_
                  are equivalent to using _stream_id_ =0
    


## EXAMPLE



                  /* create DC QP type and specify the required send opcodes */
                  attr_ex.qp_type = IBV_QPT_DRIVER;
                  attr_ex.comp_mask |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;
                  attr_ex.send_ops_flags |= IBV_QP_EX_WITH_RDMA_WRITE;
    
                  attr_dv.comp_mask |= MLX5DV_QP_INIT_ATTR_MASK_DC;
                  attr_dv.dc_init_attr.dc_type = MLX5DV_DCTYPE_DCI;
    
                  ibv_qp *qp = mlx5dv_create_qp(ctx, attr_ex, attr_dv);
                  ibv_qp_ex *qpx = ibv_qp_to_qp_ex(qp);
                  mlx5dv_qp_ex *mqpx = mlx5dv_qp_ex_from_ibv_qp_ex(qpx);
    
                  ibv_wr_start(qpx);
    
                  /* Use ibv_qp_ex object to set WR generic attributes */
                  qpx->wr_id = my_wr_id_1;
                  qpx->wr_flags = IBV_SEND_SIGNALED;
                  ibv_wr_rdma_write(qpx, rkey, remote_addr_1);
                  ibv_wr_set_sge(qpx, lkey, local_addr_1, length_1);
    
                  /* Use mlx5 DC setter using mlx5dv_qp_ex object */
                  mlx5dv_wr_set_wr_dc_addr(mqpx, ah, remote_dctn, remote_dc_key);
    
                  ret = ibv_wr_complete(qpx);
    


* * *