void test_nghttp2_session_stream_get_state(void) {
  nghttp2_session *session;
  nghttp2_session_callbacks callbacks;
  nghttp2_mem *mem;
  nghttp2_hd_deflater deflater;
  nghttp2_bufs bufs;
  nghttp2_buf *buf;
  nghttp2_stream *stream;
  ssize_t rv;
  nghttp2_data_provider data_prd;
  nghttp2_frame frame;

  mem = nghttp2_mem_default();
  frame_pack_bufs_init(&bufs);
  memset(&data_prd, 0, sizeof(data_prd));

  memset(&callbacks, 0, sizeof(nghttp2_session_callbacks));
  callbacks.send_callback = null_send_callback;

  nghttp2_session_server_new(&session, &callbacks, NULL);
  nghttp2_hd_deflate_init(&deflater, mem);

  CU_ASSERT(NGHTTP2_STREAM_STATE_IDLE ==
            nghttp2_stream_get_state(nghttp2_session_get_root_stream(session)));

  /* stream 1 HEADERS; without END_STREAM flag set */
  pack_headers(&bufs, &deflater, 1, NGHTTP2_FLAG_END_HEADERS, reqnv,
               ARRLEN(reqnv), mem);

  buf = &bufs.head->buf;
  rv = nghttp2_session_mem_recv(session, buf->pos, nghttp2_buf_len(buf));

  CU_ASSERT((ssize_t)nghttp2_buf_len(buf) == rv);

  stream = nghttp2_session_find_stream(session, 1);

  CU_ASSERT(NULL != stream);
  CU_ASSERT(1 == stream->stream_id);
  CU_ASSERT(NGHTTP2_STREAM_STATE_OPEN == nghttp2_stream_get_state(stream));

  nghttp2_bufs_reset(&bufs);

  /* stream 3 HEADERS; with END_STREAM flag set */
  pack_headers(&bufs, &deflater, 3,
               NGHTTP2_FLAG_END_HEADERS | NGHTTP2_FLAG_END_STREAM, reqnv,
               ARRLEN(reqnv), mem);

  buf = &bufs.head->buf;
  rv = nghttp2_session_mem_recv(session, buf->pos, nghttp2_buf_len(buf));

  CU_ASSERT((ssize_t)nghttp2_buf_len(buf) == rv);

  stream = nghttp2_session_find_stream(session, 3);

  CU_ASSERT(NULL != stream);
  CU_ASSERT(3 == stream->stream_id);
  CU_ASSERT(NGHTTP2_STREAM_STATE_HALF_CLOSED_REMOTE ==
            nghttp2_stream_get_state(stream));

  nghttp2_bufs_reset(&bufs);

  /* Respond to stream 1 */
  nghttp2_submit_response(session, 1, resnv, ARRLEN(resnv), NULL);

  rv = nghttp2_session_send(session);

  CU_ASSERT(0 == rv);

  stream = nghttp2_session_find_stream(session, 1);

  CU_ASSERT(NGHTTP2_STREAM_STATE_HALF_CLOSED_LOCAL ==
            nghttp2_stream_get_state(stream));

  /* Respond to stream 3 */
  nghttp2_submit_response(session, 3, resnv, ARRLEN(resnv), NULL);

  rv = nghttp2_session_send(session);

  CU_ASSERT(0 == rv);

  stream = nghttp2_session_find_stream(session, 3);

  CU_ASSERT(NGHTTP2_STREAM_STATE_CLOSED == nghttp2_stream_get_state(stream));

  /* stream 5 HEADERS; with END_STREAM flag set */
  pack_headers(&bufs, &deflater, 5,
               NGHTTP2_FLAG_END_HEADERS | NGHTTP2_FLAG_END_STREAM, reqnv,
               ARRLEN(reqnv), mem);

  buf = &bufs.head->buf;
  rv = nghttp2_session_mem_recv(session, buf->pos, nghttp2_buf_len(buf));

  CU_ASSERT((ssize_t)nghttp2_buf_len(buf) == rv);

  nghttp2_bufs_reset(&bufs);

  /* Push stream 2 associated to stream 5 */
  rv = nghttp2_submit_push_promise(session, NGHTTP2_FLAG_NONE, 5, reqnv,
                                   ARRLEN(reqnv), NULL);

  CU_ASSERT(2 == rv);

  rv = nghttp2_session_send(session);

  CU_ASSERT(0 == rv);

  stream = nghttp2_session_find_stream(session, 2);

  CU_ASSERT(NGHTTP2_STREAM_STATE_RESERVED_LOCAL ==
            nghttp2_stream_get_state(stream));

  /* Send resposne to push stream 2 with END_STREAM set */
  nghttp2_submit_response(session, 2, resnv, ARRLEN(resnv), NULL);

  rv = nghttp2_session_send(session);

  CU_ASSERT(0 == rv);

  stream = nghttp2_session_find_stream(session, 2);

  /* At server, pushed stream object is not retained after closed */
  CU_ASSERT(NULL == stream);

  /* Push stream 4 associated to stream 5 */
  rv = nghttp2_submit_push_promise(session, NGHTTP2_FLAG_NONE, 5, reqnv,
                                   ARRLEN(reqnv), NULL);

  CU_ASSERT(4 == rv);

  rv = nghttp2_session_send(session);

  CU_ASSERT(0 == rv);

  stream = nghttp2_session_find_stream(session, 4);

  CU_ASSERT(NGHTTP2_STREAM_STATE_RESERVED_LOCAL ==
            nghttp2_stream_get_state(stream));

  /* Send response to push stream 4 without closing */
  data_prd.read_callback = defer_data_source_read_callback;

  nghttp2_submit_response(session, 4, resnv, ARRLEN(resnv), &data_prd);

  rv = nghttp2_session_send(session);

  CU_ASSERT(0 == rv);

  stream = nghttp2_session_find_stream(session, 4);

  CU_ASSERT(NGHTTP2_STREAM_STATE_HALF_CLOSED_REMOTE ==
            nghttp2_stream_get_state(stream));

  /* Create idle stream by PRIORITY frame */
  nghttp2_frame_priority_init(&frame.priority, 7, &pri_spec_default);

  rv = nghttp2_frame_pack_priority(&bufs, &frame.priority);

  CU_ASSERT(0 == rv);

  nghttp2_frame_priority_free(&frame.priority);

  buf = &bufs.head->buf;
  rv = nghttp2_session_mem_recv(session, buf->pos, nghttp2_buf_len(buf));

  CU_ASSERT((ssize_t)nghttp2_buf_len(buf) == rv);

  stream = nghttp2_session_find_stream(session, 7);

  CU_ASSERT(NGHTTP2_STREAM_STATE_IDLE == nghttp2_stream_get_state(stream));

  nghttp2_bufs_reset(&bufs);

  nghttp2_hd_deflate_free(&deflater);
  nghttp2_session_del(session);

  /* Test for client side */

  nghttp2_session_client_new(&session, &callbacks, NULL);
  nghttp2_hd_deflate_init(&deflater, mem);

  nghttp2_submit_request(session, NULL, reqnv, ARRLEN(reqnv), NULL, NULL);

  rv = nghttp2_session_send(session);

  CU_ASSERT(0 == rv);

  /* Receive PUSH_PROMISE 2 associated to stream 1 */
  pack_push_promise(&bufs, &deflater, 1, NGHTTP2_FLAG_END_HEADERS, 2, reqnv,
                    ARRLEN(reqnv), mem);

  buf = &bufs.head->buf;
  rv = nghttp2_session_mem_recv(session, buf->pos, nghttp2_buf_len(buf));

  CU_ASSERT((ssize_t)nghttp2_buf_len(buf) == rv);

  stream = nghttp2_session_find_stream(session, 2);

  CU_ASSERT(NGHTTP2_STREAM_STATE_RESERVED_REMOTE ==
            nghttp2_stream_get_state(stream));

  nghttp2_bufs_reset(&bufs);

  /* Receive push response for stream 2 without END_STREAM set */
  pack_headers(&bufs, &deflater, 2, NGHTTP2_FLAG_END_HEADERS, resnv,
               ARRLEN(resnv), mem);

  buf = &bufs.head->buf;
  rv = nghttp2_session_mem_recv(session, buf->pos, nghttp2_buf_len(buf));

  CU_ASSERT((ssize_t)nghttp2_buf_len(buf) == rv);

  stream = nghttp2_session_find_stream(session, 2);

  CU_ASSERT(NGHTTP2_STREAM_STATE_HALF_CLOSED_LOCAL ==
            nghttp2_stream_get_state(stream));

  nghttp2_bufs_reset(&bufs);

  nghttp2_hd_deflate_free(&deflater);
  nghttp2_session_del(session);

  nghttp2_bufs_free(&bufs);
}


nghttp2_stream *nghttp2_session_get_stream_raw(nghttp2_session *session,
                                               int32_t stream_id) {
  return (nghttp2_stream *)nghttp2_map_find(&session->streams, stream_id);
}


void test_nghttp2_session_on_header_temporal_failure(void) {
  nghttp2_session *session;
  nghttp2_session_callbacks callbacks;
  my_user_data ud;
  nghttp2_bufs bufs;
  nghttp2_buf *buf;
  nghttp2_hd_deflater deflater;
  nghttp2_nv nv[] = {MAKE_NV("alpha", "bravo"), MAKE_NV("charlie", "delta")};
  nghttp2_nv *nva;
  size_t hdpos;
  ssize_t rv;
  nghttp2_frame frame;
  nghttp2_frame_hd hd;
  nghttp2_outbound_item *item;
  nghttp2_mem *mem;

  mem = nghttp2_mem_default();
  memset(&callbacks, 0, sizeof(callbacks));
  callbacks.on_header_callback = temporal_failure_on_header_callback;

  nghttp2_session_server_new(&session, &callbacks, &ud);

  frame_pack_bufs_init(&bufs);

  nghttp2_hd_deflate_init(&deflater, mem);

  nghttp2_nv_array_copy(&nva, reqnv, ARRLEN(reqnv), mem);

  nghttp2_frame_headers_init(&frame.headers, NGHTTP2_FLAG_END_STREAM, 1,
                             NGHTTP2_HCAT_REQUEST, NULL, nva, ARRLEN(reqnv));
  nghttp2_frame_pack_headers(&bufs, &frame.headers, &deflater);
  nghttp2_frame_headers_free(&frame.headers, mem);

  /* We are going to create CONTINUATION.  First serialize header
     block, and then frame header. */
  hdpos = nghttp2_bufs_len(&bufs);

  buf = &bufs.head->buf;
  buf->last += NGHTTP2_FRAME_HDLEN;

  nghttp2_hd_deflate_hd_bufs(&deflater, &bufs, &nv[1], 1);

  nghttp2_frame_hd_init(&hd,
                        nghttp2_bufs_len(&bufs) - hdpos - NGHTTP2_FRAME_HDLEN,
                        NGHTTP2_CONTINUATION, NGHTTP2_FLAG_END_HEADERS, 1);

  nghttp2_frame_pack_frame_hd(&buf->pos[hdpos], &hd);

  ud.header_cb_called = 0;
  rv = nghttp2_session_mem_recv(session, buf->pos, nghttp2_bufs_len(&bufs));

  CU_ASSERT((ssize_t)nghttp2_bufs_len(&bufs) == rv);
  CU_ASSERT(1 == ud.header_cb_called);

  item = nghttp2_session_get_next_ob_item(session);

  CU_ASSERT(NGHTTP2_RST_STREAM == item->frame.hd.type);
  CU_ASSERT(1 == item->frame.hd.stream_id);

  /* Make sure no header decompression error occurred */
  CU_ASSERT(NGHTTP2_GOAWAY_NONE == session->goaway_flags);

  nghttp2_hd_deflate_free(&deflater);
  nghttp2_session_del(session);

  nghttp2_bufs_reset(&bufs);

  /* Check for PUSH_PROMISE */
  nghttp2_hd_deflate_init(&deflater, mem);
  nghttp2_session_client_new(&session, &callbacks, &ud);

  open_sent_stream(session, 1);

  rv = pack_push_promise(&bufs, &deflater, 1, NGHTTP2_FLAG_END_HEADERS, 2,
                         reqnv, ARRLEN(reqnv), mem);
  CU_ASSERT(0 == rv);

  ud.header_cb_called = 0;
  rv = nghttp2_session_mem_recv(session, bufs.head->buf.pos,
                                nghttp2_bufs_len(&bufs));
  CU_ASSERT((ssize_t)nghttp2_bufs_len(&bufs) == rv);
  CU_ASSERT(1 == ud.header_cb_called);

  item = nghttp2_session_get_next_ob_item(session);
  CU_ASSERT(NGHTTP2_RST_STREAM == item->frame.hd.type);
  CU_ASSERT(2 == item->frame.hd.stream_id);
  CU_ASSERT(NGHTTP2_INTERNAL_ERROR == item->frame.rst_stream.error_code);

  nghttp2_session_del(session);
  nghttp2_hd_deflate_free(&deflater);
  nghttp2_bufs_free(&bufs);
}


void test_nghttp2_submit_push_promise(void) {
  nghttp2_session *session;
  nghttp2_session_callbacks callbacks;
  my_user_data ud;
  nghttp2_stream *stream;

  memset(&callbacks, 0, sizeof(nghttp2_session_callbacks));
  callbacks.send_callback = null_send_callback;
  callbacks.on_frame_send_callback = on_frame_send_callback;
  callbacks.on_frame_not_send_callback = on_frame_not_send_callback;

  CU_ASSERT(0 == nghttp2_session_server_new(&session, &callbacks, &ud));
  open_recv_stream(session, 1);
  CU_ASSERT(2 == nghttp2_submit_push_promise(session, NGHTTP2_FLAG_NONE, 1,
                                             reqnv, ARRLEN(reqnv), &ud));

  stream = nghttp2_session_get_stream(session, 2);

  CU_ASSERT(NULL != stream);
  CU_ASSERT(NGHTTP2_STREAM_RESERVED == stream->state);
  CU_ASSERT(&ud == nghttp2_session_get_stream_user_data(session, 2));

  ud.frame_send_cb_called = 0;
  ud.sent_frame_type = 0;

  CU_ASSERT(0 == nghttp2_session_send(session));
  CU_ASSERT(1 == ud.frame_send_cb_called);
  CU_ASSERT(NGHTTP2_PUSH_PROMISE == ud.sent_frame_type);

  stream = nghttp2_session_get_stream(session, 2);

  CU_ASSERT(NGHTTP2_STREAM_RESERVED == stream->state);
  CU_ASSERT(&ud == nghttp2_session_get_stream_user_data(session, 2));

  /* submit PUSH_PROMISE while associated stream is not opened */
  CU_ASSERT(NGHTTP2_ERR_STREAM_CLOSED ==
            nghttp2_submit_push_promise(session, NGHTTP2_FLAG_NONE, 3, reqnv,
                                        ARRLEN(reqnv), NULL));

  /* Stream ID <= 0 is error */
  CU_ASSERT(NGHTTP2_ERR_INVALID_ARGUMENT ==
            nghttp2_submit_push_promise(session, NGHTTP2_FLAG_NONE, 0, reqnv,
                                        ARRLEN(reqnv), NULL));

  nghttp2_session_del(session);
}


static int session_allow_incoming_new_stream(nghttp2_session *session) {
  return (session->goaway_flags &
          (NGHTTP2_GOAWAY_TERM_ON_SEND | NGHTTP2_GOAWAY_SENT)) == 0;
}


static int update_remote_initial_window_size_func(nghttp2_map_entry *entry,
                                                  void *ptr) {
  int rv;
  nghttp2_update_window_size_arg *arg;
  nghttp2_stream *stream;

  arg = (nghttp2_update_window_size_arg *)ptr;
  stream = (nghttp2_stream *)entry;

  rv = nghttp2_stream_update_remote_initial_window_size(
      stream, arg->new_window_size, arg->old_window_size);
  if (rv != 0) {
    return nghttp2_session_add_rst_stream(arg->session, stream->stream_id,
                                          NGHTTP2_FLOW_CONTROL_ERROR);
  }

  /* If window size gets positive, push deferred DATA frame to
     outbound queue. */
  if (stream->remote_window_size > 0 &&
      nghttp2_stream_check_deferred_by_flow_control(stream)) {

    rv = nghttp2_stream_resume_deferred_item(
        stream, NGHTTP2_STREAM_FLAG_DEFERRED_FLOW_CONTROL);

    if (nghttp2_is_fatal(rv)) {
      return rv;
    }
  }
  return 0;
}


int nghttp2_session_change_stream_priority(
    nghttp2_session *session, int32_t stream_id,
    const nghttp2_priority_spec *pri_spec) {
  int rv;
  nghttp2_stream *stream;
  nghttp2_priority_spec pri_spec_copy;

  if (stream_id == 0 || stream_id == pri_spec->stream_id) {
    return NGHTTP2_ERR_INVALID_ARGUMENT;
  }

  stream = nghttp2_session_get_stream_raw(session, stream_id);
  if (!stream) {
    return NGHTTP2_ERR_INVALID_ARGUMENT;
  }

  pri_spec_copy = *pri_spec;
  nghttp2_priority_spec_normalize_weight(&pri_spec_copy);

  rv = nghttp2_session_reprioritize_stream(session, stream, &pri_spec_copy);

  if (nghttp2_is_fatal(rv)) {
    return rv;
  }

  /* We don't intentionally call nghttp2_session_adjust_idle_stream()
     so that idle stream created by this function, and existing ones
     are kept for application.  We will adjust number of idle stream
     in nghttp2_session_mem_send or nghttp2_session_mem_recv is
     called. */
  return 0;
}


static int temporal_failure_on_header_callback(
    nghttp2_session *session, const nghttp2_frame *frame, const uint8_t *name,
    size_t namelen, const uint8_t *value, size_t valuelen, uint8_t flags,
    void *user_data) {
  on_header_callback(session, frame, name, namelen, value, valuelen, flags,
                     user_data);
  return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
}


void test_nghttp2_session_stream_dep_remove_subtree(void) {
  nghttp2_session *session;
  nghttp2_session_callbacks callbacks;
  nghttp2_stream *a, *b, *c, *d, *e, *root;

  memset(&callbacks, 0, sizeof(callbacks));

  /* Remove left most stream */
  nghttp2_session_server_new(&session, &callbacks, NULL);

  root = &session->root;

  a = open_stream(session, 1);
  b = open_stream_with_dep(session, 3, a);
  c = open_stream_with_dep(session, 5, a);
  d = open_stream_with_dep(session, 7, c);

  /* a
   * |
   * c--b
   * |
   * d
   */

  nghttp2_stream_dep_remove_subtree(c);

  /* becomes
   * a  c
   * |  |
   * b  d
   */

  CU_ASSERT(NGHTTP2_DEFAULT_WEIGHT == a->sum_dep_weight);
  CU_ASSERT(0 == b->sum_dep_weight);
  CU_ASSERT(NGHTTP2_DEFAULT_WEIGHT == c->sum_dep_weight);
  CU_ASSERT(0 == d->sum_dep_weight);

  check_stream_dep_sib(a, root, b, NULL, NULL);
  check_stream_dep_sib(b, a, NULL, NULL, NULL);
  check_stream_dep_sib(c, NULL, d, NULL, NULL);
  check_stream_dep_sib(d, c, NULL, NULL, NULL);

  nghttp2_session_del(session);

  /* Remove right most stream */
  nghttp2_session_server_new(&session, &callbacks, NULL);

  root = &session->root;

  a = open_stream(session, 1);
  b = open_stream_with_dep(session, 3, a);
  c = open_stream_with_dep(session, 5, a);
  d = open_stream_with_dep(session, 7, c);

  /* a
   * |
   * c--b
   * |
   * d
   */

  nghttp2_stream_dep_remove_subtree(b);

  /* becomes
   * a  b
   * |
   * c
   * |
   * d
   */

  CU_ASSERT(NGHTTP2_DEFAULT_WEIGHT == a->sum_dep_weight);
  CU_ASSERT(0 == b->sum_dep_weight);
  CU_ASSERT(NGHTTP2_DEFAULT_WEIGHT == c->sum_dep_weight);
  CU_ASSERT(0 == d->sum_dep_weight);

  check_stream_dep_sib(a, root, c, NULL, NULL);
  check_stream_dep_sib(c, a, d, NULL, NULL);
  check_stream_dep_sib(d, c, NULL, NULL, NULL);
  check_stream_dep_sib(b, NULL, NULL, NULL, NULL);

  nghttp2_session_del(session);

  /* Remove middle stream */
  nghttp2_session_server_new(&session, &callbacks, NULL);

  root = &session->root;

  a = open_stream(session, 1);
  e = open_stream_with_dep(session, 9, a);
  c = open_stream_with_dep(session, 5, a);
  b = open_stream_with_dep(session, 3, a);
  d = open_stream_with_dep(session, 7, c);

  /* a
   * |
   * b--c--e
   *    |
   *    d
   */

  nghttp2_stream_dep_remove_subtree(c);

  /* becomes
   * a     c
   * |     |
   * b--e  d
   */

  CU_ASSERT(NGHTTP2_DEFAULT_WEIGHT * 2 == a->sum_dep_weight);
  CU_ASSERT(0 == b->sum_dep_weight);
  CU_ASSERT(NGHTTP2_DEFAULT_WEIGHT == c->sum_dep_weight);
  CU_ASSERT(0 == d->sum_dep_weight);
  CU_ASSERT(0 == e->sum_dep_weight);

  check_stream_dep_sib(a, root, b, NULL, NULL);
  check_stream_dep_sib(b, a, NULL, NULL, e);
  check_stream_dep_sib(e, a, NULL, b, NULL);
  check_stream_dep_sib(c, NULL, d, NULL, NULL);
  check_stream_dep_sib(d, c, NULL, NULL, NULL);

  nghttp2_session_del(session);
}


void Downstream::pause_read(IOCtrlReason reason) {
  if (dconn_) {
    dconn_->pause_read(reason);
  }
}


void HttpsUpstream::reset_current_header_length() {
  current_header_length_ = 0;
}


uint32_t Downstream::find_affinity_cookie(const StringRef &name) {
  for (auto &kv : req_.fs.headers()) {
    if (kv.token != http2::HD_COOKIE) {
      continue;
    }

    for (auto it = std::begin(kv.value); it != std::end(kv.value);) {
      if (*it == '\t' || *it == ' ' || *it == ';') {
        ++it;
        continue;
      }

      auto end = std::find(it, std::end(kv.value), '=');
      if (end == std::end(kv.value)) {
        return 0;
      }

      if (!util::streq(name, StringRef{it, end})) {
        it = std::find(it, std::end(kv.value), ';');
        continue;
      }

      it = std::find(end + 1, std::end(kv.value), ';');
      auto val = StringRef{end + 1, it};
      if (val.size() != 8) {
        return 0;
      }
      uint32_t h = 0;
      for (auto c : val) {
        auto n = util::hex_to_uint(c);
        if (n == 256) {
          return 0;
        }
        h <<= 4;
        h += n;
      }
      affinity_cookie_ = h;
      return h;
    }
  }
  return 0;
}


BlockedLink *Downstream::detach_blocked_link() {
  auto link = blocked_link_;
  blocked_link_ = nullptr;
  return link;
}


void Downstream::reset_upstream(Upstream *upstream) {
  upstream_ = upstream;
  if (dconn_) {
    dconn_->on_upstream_change(upstream);
  }
}


void HttpDownstreamConnection::on_upstream_change(Upstream *upstream) {}


DownstreamConnection *Downstream::get_downstream_connection() {
  return dconn_.get();
}


void FieldStore::append_last_trailer_key(const char *data, size_t len) {
  shrpx::append_last_header_key(balloc_, trailer_key_prev_, buffer_size_,
                                trailers_, data, len);
}


bool HttpsUpstream::push_enabled() const { return false; }


HeaderRefs::value_type *FieldStore::header(int32_t token) {
  for (auto it = headers_.rbegin(); it != headers_.rend(); ++it) {
    auto &kv = *it;
    if (kv.token == token) {
      return &kv;
    }
  }
  return nullptr;
}


int htp_hdr_keycb(llhttp_t *htp, const char *data, size_t len) {
  auto downstream = static_cast<Downstream *>(htp->data);
  auto &resp = downstream->response();
  auto &httpconf = get_config()->http;

  if (ensure_header_field_buffer(downstream, httpconf, len) != 0) {
    return -1;
  }

  if (downstream->get_response_state() == DownstreamState::INITIAL) {
    if (resp.fs.header_key_prev()) {
      resp.fs.append_last_header_key(data, len);
    } else {
      if (ensure_max_header_fields(downstream, httpconf) != 0) {
        return -1;
      }
      resp.fs.alloc_add_header_name(StringRef{data, len});
    }
  } else {
    // trailer part
    if (resp.fs.trailer_key_prev()) {
      resp.fs.append_last_trailer_key(data, len);
    } else {
      if (ensure_max_header_fields(downstream, httpconf) != 0) {
        // Could not ignore this trailer field easily, since we may
        // get its value in htp_hdr_valcb, and it will be added to
        // wrong place or crash if trailer fields are currently empty.
        return -1;
      }
      resp.fs.alloc_add_trailer_name(StringRef{data, len});
    }
  }
  return 0;
}


void Downstream::set_downstream_stream_id(int32_t stream_id) {
  downstream_stream_id_ = stream_id;
}


search_header_linear_backwards(const HeaderRefs &headers,
                               const StringRef &name) {
  for (auto it = headers.rbegin(); it != headers.rend(); ++it) {
    auto &kv = *it;
    if (kv.name == name) {
      return &kv;
    }
  }
  return nullptr;
}


int ensure_header_field_buffer(const Downstream *downstream,
                               const HttpConfig &httpconf, size_t len) {
  auto &resp = downstream->response();

  if (resp.fs.buffer_size() + len > httpconf.response_header_field_buffer) {
    if (LOG_ENABLED(INFO)) {
      DLOG(INFO, downstream) << "Too large header header field size="
                             << resp.fs.buffer_size() + len;
    }
    return -1;
  }

  return 0;
}


void Downstream::set_ws_key(const StringRef &key) { ws_key_ = key; }


void HttpsUpstream::cancel_premature_downstream(
    Downstream *promised_downstream) {}


bool Downstream::expect_response_body() const {
  return !resp_.headers_only &&
         http2::expect_response_body(req_.method, resp_.http_status);
}


Downstream::get_request_start_time() const {
  return request_start_time_;
}


void HttpsUpstream::error_reply(unsigned int status_code) {
  auto downstream = get_downstream();

  if (!downstream) {
    attach_downstream(
        std::make_unique<Downstream>(this, handler_->get_mcpool(), 1));
    downstream = get_downstream();
  }

  auto &resp = downstream->response();
  auto &balloc = downstream->get_block_allocator();

  auto html = http::create_error_html(balloc, status_code);

  resp.http_status = status_code;
  // we are going to close connection for both frontend and backend in
  // error condition.  This is safest option.
  resp.connection_close = true;
  handler_->set_should_close_after_write(true);

  auto output = downstream->get_response_buf();

  output->append("HTTP/1.1 ");
  output->append(http2::stringify_status(balloc, status_code));
  output->append(' ');
  output->append(http2::get_reason_phrase(status_code));
  output->append("\r\nServer: ");
  output->append(get_config()->http.server_name);
  output->append("\r\nContent-Length: ");
  std::array<uint8_t, NGHTTP2_MAX_UINT64_DIGITS> intbuf;
  output->append(StringRef{std::begin(intbuf),
                           util::utos(std::begin(intbuf), html.size())});
  output->append("\r\nDate: ");
  auto lgconf = log_config();
  lgconf->update_tstamp(std::chrono::system_clock::now());
  output->append(lgconf->tstamp->time_http);
  output->append("\r\nContent-Type: text/html; "
                 "charset=UTF-8\r\nConnection: close\r\n\r\n");
  output->append(html);

  downstream->response_sent_body_length += html.size();
  downstream->set_response_state(DownstreamState::MSG_COMPLETE);
}


int HttpDownstreamConnection::read_tls() {
  conn_.last_read = ev_now(conn_.loop);

  ERR_clear_error();

  std::array<uint8_t, 16_k> buf;
  int rv;

  for (;;) {
    auto nread = conn_.read_tls(buf.data(), buf.size());
    if (nread == 0) {
      return 0;
    }

    if (nread < 0) {
      return nread;
    }

    rv = process_input(buf.data(), nread);
    if (rv != 0) {
      return rv;
    }

    if (!ev_is_active(&conn_.rev)) {
      return 0;
    }
  }
}


int ensure_max_header_fields(const Downstream *downstream,
                             const HttpConfig &httpconf) {
  auto &resp = downstream->response();

  if (resp.fs.num_fields() >= httpconf.max_response_header_fields) {
    if (LOG_ENABLED(INFO)) {
      DLOG(INFO, downstream)
          << "Too many header field num=" << resp.fs.num_fields() + 1;
    }
    return -1;
  }

  return 0;
}


int HttpsUpstream::resume_read(IOCtrlReason reason, Downstream *downstream,
                               size_t consumed) {
  // downstream could be nullptr
  if (downstream && downstream->request_buf_full()) {
    return 0;
  }
  if (ioctrl_.resume_read(reason)) {
    // Process remaining data in input buffer here because these bytes
    // are not notified by readcb until new data arrive.
    llhttp_resume(&htp_);

    auto conn = handler_->get_connection();
    ev_feed_event(conn->loop, &conn->rev, EV_READ);
    return 0;
  }

  return 0;
}


Upstream *Downstream::get_upstream() const { return upstream_; }


void readcb(struct ev_loop *loop, ev_io *w, int revents) {
  auto conn = static_cast<Connection *>(w->data);
  auto dconn = static_cast<HttpDownstreamConnection *>(conn->data);
  auto downstream = dconn->get_downstream();
  auto upstream = downstream->get_upstream();
  auto handler = upstream->get_client_handler();

  if (upstream->downstream_read(dconn) != 0) {
    delete handler;
  }
}


int HttpDownstreamConnection::write_clear() {
  conn_.last_read = ev_now(conn_.loop);

  auto upstream = downstream_->get_upstream();
  auto input = downstream_->get_request_buf();

  std::array<struct iovec, MAX_WR_IOVCNT> iov;

  while (input->rleft() > 0) {
    auto iovcnt = input->riovec(iov.data(), iov.size());

    auto nwrite = conn_.writev_clear(iov.data(), iovcnt);

    if (nwrite == 0) {
      return 0;
    }

    if (nwrite < 0) {
      if (!first_write_done_) {
        return nwrite;
      }
      // We may have pending data in receive buffer which may contain
      // part of response body.  So keep reading.  Invoke read event
      // to get read(2) error just in case.
      ev_feed_event(conn_.loop, &conn_.rev, EV_READ);
      on_write_ = &HttpDownstreamConnection::noop;
      reusable_ = false;
      break;
    }

    input->drain(nwrite);
  }

  conn_.wlimit.stopw();
  ev_timer_stop(conn_.loop, &conn_.wt);

  if (input->rleft() == 0) {
    auto &req = downstream_->request();

    upstream->resume_read(SHRPX_NO_BUFFER, downstream_,
                          req.unconsumed_body_length);
  }

  return 0;
}


int HttpsUpstream::on_downstream_abort_request_with_https_redirect(
    Downstream *downstream) {
  redirect_to_https(downstream);
  handler_->signal_write();
  return 0;
}


void Downstream::force_resume_read() {
  if (dconn_) {
    dconn_->force_resume_read();
  }
}


StringRef alloc_header_name(BlockAllocator &balloc, const StringRef &name) {
  auto iov = make_byte_ref(balloc, name.size() + 1);
  auto p = iov.base;
  p = std::copy(std::begin(name), std::end(name), p);
  util::inp_strlower(iov.base, p);
  *p = '\0';

  return StringRef{iov.base, p};
}


DownstreamState Downstream::get_request_state() const { return request_state_; }


void Downstream::reset_downstream_wtimer() {
  auto loop = upstream_->get_client_handler()->get_loop();
  auto &timeoutconf = get_config()->http2.timeout;

  if (timeoutconf.stream_write != 0.) {
    reset_timer(loop, &downstream_wtimer_);
  }
  if (timeoutconf.stream_read != 0.) {
    try_reset_timer(loop, &downstream_rtimer_);
  }
}


void Downstream::set_expect_final_response(bool f) {
  expect_final_response_ = f;
}


DefaultMemchunks *Downstream::get_blocked_request_buf() {
  return &blocked_request_buf_;
}


ClientHandler *HttpsUpstream::get_client_handler() const { return handler_; }


vhost_check_queue_inflights_split(struct virtio_net *dev,
				  struct vhost_virtqueue *vq)
{
	uint16_t i;
	uint16_t resubmit_num = 0, last_io, num;
	struct vring_used *used = vq->used;
	struct rte_vhost_resubmit_info *resubmit;
	struct rte_vhost_inflight_info_split *inflight_split;

	if (!(dev->protocol_features &
	    (1ULL << VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)))
		return RTE_VHOST_MSG_RESULT_OK;

	if ((!vq->inflight_split))
		return RTE_VHOST_MSG_RESULT_ERR;

	if (!vq->inflight_split->version) {
		vq->inflight_split->version = INFLIGHT_VERSION;
		return RTE_VHOST_MSG_RESULT_OK;
	}

	if (vq->resubmit_inflight)
		return RTE_VHOST_MSG_RESULT_OK;

	inflight_split = vq->inflight_split;
	vq->global_counter = 0;
	last_io = inflight_split->last_inflight_io;

	if (inflight_split->used_idx != used->idx) {
		inflight_split->desc[last_io].inflight = 0;
		rte_smp_mb();
		inflight_split->used_idx = used->idx;
	}

	for (i = 0; i < inflight_split->desc_num; i++) {
		if (inflight_split->desc[i].inflight == 1)
			resubmit_num++;
	}

	vq->last_avail_idx += resubmit_num;

	if (resubmit_num) {
		resubmit  = calloc(1, sizeof(struct rte_vhost_resubmit_info));
		if (!resubmit) {
			RTE_LOG(ERR, VHOST_CONFIG,
				"failed to allocate memory for resubmit info.\n");
			return RTE_VHOST_MSG_RESULT_ERR;
		}

		resubmit->resubmit_list = calloc(resubmit_num,
			sizeof(struct rte_vhost_resubmit_desc));
		if (!resubmit->resubmit_list) {
			RTE_LOG(ERR, VHOST_CONFIG,
				"failed to allocate memory for inflight desc.\n");
			free(resubmit);
			return RTE_VHOST_MSG_RESULT_ERR;
		}

		num = 0;
		for (i = 0; i < vq->inflight_split->desc_num; i++) {
			if (vq->inflight_split->desc[i].inflight == 1) {
				resubmit->resubmit_list[num].index = i;
				resubmit->resubmit_list[num].counter =
					inflight_split->desc[i].counter;
				num++;
			}
		}
		resubmit->resubmit_num = num;

		if (resubmit->resubmit_num > 1)
			qsort(resubmit->resubmit_list, resubmit->resubmit_num,
			      sizeof(struct rte_vhost_resubmit_desc),
			      resubmit_desc_compare);

		vq->global_counter = resubmit->resubmit_list[0].counter + 1;
		vq->resubmit_inflight = resubmit;
	}

	return RTE_VHOST_MSG_RESULT_OK;
}


dump_guest_pages(struct virtio_net *dev)
{
	uint32_t i;
	struct guest_page *page;

	for (i = 0; i < dev->nr_guest_pages; i++) {
		page = &dev->guest_pages[i];

		RTE_LOG(INFO, VHOST_CONFIG,
			"guest physical page region %u\n"
			"\t guest_phys_addr: %" PRIx64 "\n"
			"\t host_phys_addr : %" PRIx64 "\n"
			"\t size           : %" PRIx64 "\n",
			i,
			page->guest_phys_addr,
			page->host_phys_addr,
			page->size);
	}
}


read_vhost_message(int sockfd, struct VhostUserMsg *msg)
{
	int ret;

	ret = read_fd_message(sockfd, (char *)msg, VHOST_USER_HDR_SIZE,
		msg->fds, VHOST_MEMORY_MAX_NREGIONS, &msg->fd_num);
	if (ret <= 0)
		return ret;

	if (msg->size) {
		if (msg->size > sizeof(msg->payload)) {
			RTE_LOG(ERR, VHOST_CONFIG,
				"invalid msg size: %d\n", msg->size);
			return -1;
		}
		ret = read(sockfd, &msg->payload, msg->size);
		if (ret <= 0)
			return ret;
		if (ret != (int)msg->size) {
			RTE_LOG(ERR, VHOST_CONFIG,
				"read control message failed\n");
			return -1;
		}
	}

	return ret;
}


get_blk_size(int fd)
{
	struct stat stat;
	int ret;

	ret = fstat(fd, &stat);
	return ret == -1 ? (uint64_t)-1 : (uint64_t)stat.st_blksize;
}


qva_to_vva(struct virtio_net *dev, uint64_t qva, uint64_t *len)
{
	struct rte_vhost_mem_region *r;
	uint32_t i;

	if (unlikely(!dev || !dev->mem))
		goto out_error;

	/* Find the region where the address lives. */
	for (i = 0; i < dev->mem->nregions; i++) {
		r = &dev->mem->regions[i];

		if (qva >= r->guest_user_addr &&
		    qva <  r->guest_user_addr + r->size) {

			if (unlikely(*len > r->guest_user_addr + r->size - qva))
				*len = r->guest_user_addr + r->size - qva;

			return qva - r->guest_user_addr +
			       r->host_user_addr;
		}
	}
out_error:
	*len = 0;

	return 0;
}


static int vhost_user_set_log_fd(struct virtio_net **pdev __rte_unused,
			struct VhostUserMsg *msg,
			int main_fd __rte_unused)
{
	if (validate_msg_fds(msg, 1) != 0)
		return RTE_VHOST_MSG_RESULT_ERR;

	close(msg->fds[0]);
	RTE_LOG(INFO, VHOST_CONFIG, "not implemented.\n");

	return RTE_VHOST_MSG_RESULT_OK;
}


vhost_user_get_features(struct virtio_net **pdev, struct VhostUserMsg *msg,
			int main_fd __rte_unused)
{
	struct virtio_net *dev = *pdev;
	uint64_t features = 0;

	if (validate_msg_fds(msg, 0) != 0)
		return RTE_VHOST_MSG_RESULT_ERR;

	rte_vhost_driver_get_features(dev->ifname, &features);

	msg->payload.u64 = features;
	msg->size = sizeof(msg->payload.u64);
	msg->fd_num = 0;

	return RTE_VHOST_MSG_RESULT_REPLY;
}


vhost_user_get_vring_base(struct virtio_net **pdev,
			struct VhostUserMsg *msg,
			int main_fd __rte_unused)
{
	struct virtio_net *dev = *pdev;
	struct vhost_virtqueue *vq = dev->virtqueue[msg->payload.state.index];
	uint64_t val;

	if (validate_msg_fds(msg, 0) != 0)
		return RTE_VHOST_MSG_RESULT_ERR;

	/* We have to stop the queue (virtio) if it is running. */
	vhost_destroy_device_notify(dev);

	dev->flags &= ~VIRTIO_DEV_READY;
	dev->flags &= ~VIRTIO_DEV_VDPA_CONFIGURED;

	/* Here we are safe to get the indexes */
	if (vq_is_packed(dev)) {
		/*
		 * Bit[0:14]: avail index
		 * Bit[15]: avail wrap counter
		 */
		val = vq->last_avail_idx & 0x7fff;
		val |= vq->avail_wrap_counter << 15;
		msg->payload.state.num = val;
	} else {
		msg->payload.state.num = vq->last_avail_idx;
	}

	RTE_LOG(INFO, VHOST_CONFIG,
		"vring base idx:%d file:%d\n", msg->payload.state.index,
		msg->payload.state.num);
	/*
	 * Based on current qemu vhost-user implementation, this message is
	 * sent and only sent in vhost_vring_stop.
	 * TODO: cleanup the vring, it isn't usable since here.
	 */
	if (vq->kickfd >= 0)
		close(vq->kickfd);

	vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;

	if (vq->callfd >= 0)
		close(vq->callfd);

	vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD;

	vq->signalled_used_valid = false;

	if (dev->dequeue_zero_copy)
		free_zmbufs(vq);
	if (vq_is_packed(dev)) {
		rte_free(vq->shadow_used_packed);
		vq->shadow_used_packed = NULL;
	} else {
		rte_free(vq->shadow_used_split);
		vq->shadow_used_split = NULL;
	}

	rte_free(vq->batch_copy_elems);
	vq->batch_copy_elems = NULL;

	msg->size = sizeof(msg->payload.state);
	msg->fd_num = 0;

	vring_invalidate(dev, vq);

	return RTE_VHOST_MSG_RESULT_REPLY;
}


close_msg_fds(struct VhostUserMsg *msg)
{
	int i;

	for (i = 0; i < msg->fd_num; i++)
		close(msg->fds[i]);
}


vhost_user_set_vring_call(struct virtio_net **pdev, struct VhostUserMsg *msg,
			int main_fd __rte_unused)
{
	struct virtio_net *dev = *pdev;
	struct vhost_vring_file file;
	struct vhost_virtqueue *vq;

	if (validate_msg_fds(msg, 1) != 0)
		return RTE_VHOST_MSG_RESULT_ERR;

	file.index = msg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
	if (msg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)
		file.fd = VIRTIO_INVALID_EVENTFD;
	else
		file.fd = msg->fds[0];
	RTE_LOG(INFO, VHOST_CONFIG,
		"vring call idx:%d file:%d\n", file.index, file.fd);

	vq = dev->virtqueue[file.index];
	if (vq->callfd >= 0)
		close(vq->callfd);

	vq->callfd = file.fd;

	return RTE_VHOST_MSG_RESULT_OK;
}


ippAddCollection(ipp_t      *ipp,	/* I - IPP message */
                 ipp_tag_t  group,	/* I - IPP group */
		 const char *name,	/* I - Name of attribute */
		 ipp_t      *value)	/* I - Value */
{
  ipp_attribute_t	*attr;		/* New attribute */

  DEBUG_printf(("ippAddCollection(ipp=%p, group=%02x(%s), name=\"%s\", value=%p)", (void *)ipp, group, ippTagString(group), name, (void *)value));

 /*
  * Range check input...
  */

  if (!ipp || !name || group < IPP_TAG_ZERO ||
      group == IPP_TAG_END || group >= IPP_TAG_UNSUPPORTED_VALUE)
    return (NULL);

 /*
  * Create the attribute...
  */

  if ((attr = ipp_add_attr(ipp, name, group, IPP_TAG_BEGIN_COLLECTION, 1)) == NULL)
    return (NULL);

  attr->values[0].collection = value;

  if (value)
    value->use ++;

  return (attr);
}


snmp_set_error(cups_snmp_t *packet,	/* I - Packet */
               const char *message)	/* I - Error message */
{
  _cups_globals_t *cg = _cupsGlobals();	/* Global data */

  if (!cg->lang_default)
    cg->lang_default = cupsLangDefault();

  packet->error = _cupsLangString(cg->lang_default, message);
}


httpSetCookie(http_t     *http,		/* I - Connection */
              const char *cookie)	/* I - Cookie string */
{
  if (!http)
    return;

  if (http->cookie)
    free(http->cookie);

  if (cookie)
    http->cookie = strdup(cookie);
  else
    http->cookie = NULL;
}


ippTimeToDate(time_t t)			/* I - Time in seconds */
{
  struct tm	*unixdate;		/* UNIX unixdate/time info */
  ipp_uchar_t	*date = _cupsGlobals()->ipp_date;
					/* RFC-2579 date/time data */

 /*
  * RFC-2579 date/time format is:
  *
  *    Byte(s)  Description
  *    -------  -----------
  *    0-1      Year (0 to 65535)
  *    2        Month (1 to 12)
  *    3        Day (1 to 31)
  *    4        Hours (0 to 23)
  *    5        Minutes (0 to 59)
  *    6        Seconds (0 to 60, 60 = "leap second")
  *    7        Deciseconds (0 to 9)
  *    8        +/- UTC
  *    9        UTC hours (0 to 11)
  *    10       UTC minutes (0 to 59)
  */

  unixdate = gmtime(&t);
  unixdate->tm_year += 1900;

  date[0]  = (ipp_uchar_t)(unixdate->tm_year >> 8);
  date[1]  = (ipp_uchar_t)(unixdate->tm_year);
  date[2]  = (ipp_uchar_t)(unixdate->tm_mon + 1);
  date[3]  = (ipp_uchar_t)unixdate->tm_mday;
  date[4]  = (ipp_uchar_t)unixdate->tm_hour;
  date[5]  = (ipp_uchar_t)unixdate->tm_min;
  date[6]  = (ipp_uchar_t)unixdate->tm_sec;
  date[7]  = 0;
  date[8]  = '+';
  date[9]  = 0;
  date[10] = 0;

  return (date);
}


ippGetOperation(ipp_t *ipp)		/* I - IPP request message */
{
 /*
  * Range check input...
  */

  if (!ipp)
    return ((ipp_op_t)0);

 /*
  * Return the value...
  */

  return (ipp->request.op.operation_id);
}


httpConnectEncrypt(
    const char        *host,		/* I - Host to connect to */
    int               port,		/* I - Port number */
    http_encryption_t encryption)	/* I - Type of encryption to use */
{
  DEBUG_printf(("httpConnectEncrypt(host=\"%s\", port=%d, encryption=%d)",
                host, port, encryption));

  return (httpConnect2(host, port, NULL, AF_UNSPEC, encryption, 1, 30000,
                       NULL));
}


ippAddResolution(ipp_t      *ipp,	/* I - IPP message */
        	 ipp_tag_t  group,	/* I - IPP group */
		 const char *name,	/* I - Name of attribute */
		 ipp_res_t  units,	/* I - Units for resolution */
		 int        xres,	/* I - X resolution */
		 int        yres)	/* I - Y resolution */
{
  ipp_attribute_t	*attr;		/* New attribute */

  DEBUG_printf(("ippAddResolution(ipp=%p, group=%02x(%s), name=\"%s\", units=%d, xres=%d, yres=%d)", (void *)ipp, group,
		ippTagString(group), name, units, xres, yres));

 /*
  * Range check input...
  */

  if (!ipp || !name || group < IPP_TAG_ZERO ||
      group == IPP_TAG_END || group >= IPP_TAG_UNSUPPORTED_VALUE ||
      units < IPP_RES_PER_INCH || units > IPP_RES_PER_CM ||
      xres < 0 || yres < 0)
    return (NULL);

 /*
  * Create the attribute...
  */

  if ((attr = ipp_add_attr(ipp, name, group, IPP_TAG_RESOLUTION, 1)) == NULL)
    return (NULL);

  attr->values[0].resolution.xres  = xres;
  attr->values[0].resolution.yres  = yres;
  attr->values[0].resolution.units = units;

  return (attr);
}


httpDelete(http_t     *http,		/* I - HTTP connection */
           const char *uri)		/* I - URI to delete */
{
  return (http_send(http, HTTP_STATE_DELETE, uri));
}


httpSetTimeout(
    http_t            *http,		/* I - HTTP connection */
    double            timeout,		/* I - Number of seconds for timeout,
                                               must be greater than 0 */
    http_timeout_cb_t cb,		/* I - Callback function or @code NULL@ */
    void              *user_data)	/* I - User data pointer */
{
  if (!http || timeout <= 0.0)
    return;

  http->timeout_cb    = cb;
  http->timeout_data  = user_data;
  http->timeout_value = timeout;

  if (http->fd >= 0)
    http_set_timeout(http->fd, timeout);

  http_set_wait(http);
}


asn1_set_oid(unsigned char **buffer,	/* IO - Pointer in buffer */
             const int     *oid)	/* I  - OID value */
{
  **buffer = CUPS_ASN1_OID;
  (*buffer) ++;

  asn1_set_length(buffer, asn1_size_oid(oid));

  if (oid[1] < 0)
  {
    asn1_set_packed(buffer, oid[0] * 40);
    return;
  }

  asn1_set_packed(buffer, oid[0] * 40 + oid[1]);

  for (oid += 2; *oid >= 0; oid ++)
    asn1_set_packed(buffer, *oid);
}


cupsdSendError(cupsd_client_t *con,	/* I - Connection */
               http_status_t  code,	/* I - Error code */
	       int            auth_type)/* I - Authentication type */
{
  char	location[HTTP_MAX_VALUE];	/* Location field */

  cupsdLogClient(con, CUPSD_LOG_DEBUG2, "cupsdSendError code=%d, auth_type=%d", code, auth_type);

#ifdef HAVE_SSL
 /*
  * Force client to upgrade for authentication if that is how the
  * server is configured...
  */

  if (code == HTTP_STATUS_UNAUTHORIZED &&
      DefaultEncryption == HTTP_ENCRYPTION_REQUIRED &&
      _cups_strcasecmp(httpGetHostname(con->http, NULL, 0), "localhost") &&
      !httpIsEncrypted(con->http))
  {
    code = HTTP_STATUS_UPGRADE_REQUIRED;
  }
#endif /* HAVE_SSL */

 /*
  * Put the request in the access_log file...
  */

  cupsdLogRequest(con, code);

 /*
  * To work around bugs in some proxies, don't use Keep-Alive for some
  * error messages...
  *
  * Kerberos authentication doesn't work without Keep-Alive, so
  * never disable it in that case.
  */

  strlcpy(location, httpGetField(con->http, HTTP_FIELD_LOCATION), sizeof(location));

  httpClearFields(con->http);
  httpClearCookie(con->http);

  httpSetField(con->http, HTTP_FIELD_LOCATION, location);

  if (code >= HTTP_STATUS_BAD_REQUEST && con->type != CUPSD_AUTH_NEGOTIATE)
    httpSetKeepAlive(con->http, HTTP_KEEPALIVE_OFF);

  if (httpGetVersion(con->http) >= HTTP_VERSION_1_1 &&
      httpGetKeepAlive(con->http) == HTTP_KEEPALIVE_OFF)
    httpSetField(con->http, HTTP_FIELD_CONNECTION, "close");

  if (code >= HTTP_STATUS_BAD_REQUEST)
  {
   /*
    * Send a human-readable error message.
    */

    char	message[4096],		/* Message for user */
		urltext[1024],		/* URL redirection text */
		redirect[1024];		/* Redirection link */
    const char	*text;			/* Status-specific text */

    redirect[0] = '\0';

    if (code == HTTP_STATUS_UNAUTHORIZED)
      text = _cupsLangString(con->language,
                             _("Enter your username and password or the "
			       "root username and password to access this "
			       "page. If you are using Kerberos authentication, "
			       "make sure you have a valid Kerberos ticket."));
    else if (code == HTTP_STATUS_UPGRADE_REQUIRED)
    {
      text = urltext;

      snprintf(urltext, sizeof(urltext),
               _cupsLangString(con->language,
                               _("You must access this page using the URL "
			         ""
				 "https://%s:%d%s.")),
               con->servername, con->serverport, con->uri,
	       con->servername, con->serverport, con->uri);

      snprintf(redirect, sizeof(redirect),
               "<META HTTP-EQUIV=\"Refresh\" "
	       "CONTENT=\"3;URL=https://%s:%d%s\">\n",
	       con->servername, con->serverport, con->uri);
    }
    else if (code == HTTP_STATUS_CUPS_WEBIF_DISABLED)
      text = _cupsLangString(con->language,
                             _("The web interface is currently disabled. Run "
			       "\"cupsctl WebInterface=yes\" to enable it."));
    else
      text = "";

    snprintf(message, sizeof(message),
             "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\" "
	     "\"http://www.w3.org/TR/html4/loose.dtd\">\n"
	     "\n"
	     "\n"
             "\t<META HTTP-EQUIV=\"Content-Type\" "
	     "CONTENT=\"text/html; charset=utf-8\">\n"
	     "\t<TITLE>%s - " CUPS_SVERSION "</TITLE>\n"
	     "\t<LINK REL=\"STYLESHEET\" TYPE=\"text/css\" "
	     "HREF=\"/cups.css\">\n"
	     "%s"
	     "\n"
             "\n"
	     "
%s
\n"
	     "
%s
\n"
	     "\n"
	     "\n",
	     _httpStatus(con->language, code), redirect,
	     _httpStatus(con->language, code), text);

   /*
    * Send an error message back to the client.  If the error code is a
    * 400 or 500 series, make sure the message contains some text, too!
    */

    size_t length = strlen(message);	/* Length of message */

    httpSetLength(con->http, length);

    if (!cupsdSendHeader(con, code, "text/html", auth_type))
      return (0);

    if (httpWrite2(con->http, message, length) < 0)
      return (0);

    if (httpFlushWrite(con->http) < 0)
      return (0);
  }
  else
  {
    httpSetField(con->http, HTTP_FIELD_CONTENT_LENGTH, "0");

    if (!cupsdSendHeader(con, code, NULL, auth_type))
      return (0);
  }

  return (1);
}


ippAddBooleans(ipp_t      *ipp,		/* I - IPP message */
               ipp_tag_t  group,	/* I - IPP group */
	       const char *name,	/* I - Name of attribute */
	       int        num_values,	/* I - Number of values */
	       const char *values)	/* I - Values */
{
  int			i;		/* Looping var */
  ipp_attribute_t	*attr;		/* New attribute */
  _ipp_value_t		*value;		/* Current value */

  DEBUG_printf(("ippAddBooleans(ipp=%p, group=%02x(%s), name=\"%s\", num_values=%d, values=%p)", (void *)ipp, group, ippTagString(group), name, num_values, (void *)values));

 /*
  * Range check input...
  */

  if (!ipp || !name || group < IPP_TAG_ZERO ||
      group == IPP_TAG_END || group >= IPP_TAG_UNSUPPORTED_VALUE ||
      num_values < 1)
    return (NULL);

 /*
  * Create the attribute...
  */

  if ((attr = ipp_add_attr(ipp, name, group, IPP_TAG_BOOLEAN, num_values)) == NULL)
    return (NULL);

  if (values)
  {
    for (i = num_values, value = attr->values;
	 i > 0;
	 i --, value ++)
      value->boolean = *values++;
  }

  return (attr);
}


ippGetName(ipp_attribute_t *attr)	/* I - IPP attribute */
{
 /*
  * Range check input...
  */

  if (!attr)
    return (NULL);

 /*
  * Return the name...
  */

  return (attr->name);
}


_cupsSNMPWalk(int            fd,	/* I - SNMP socket */
              http_addr_t    *address,	/* I - Address to query */
	      int            version,	/* I - SNMP version */
	      const char     *community,/* I - Community name */
              const int      *prefix,	/* I - OID prefix */
	      double         timeout,	/* I - Timeout for each response in seconds */
	      cups_snmp_cb_t cb,	/* I - Function to call for each response */
	      void           *data)	/* I - User data pointer that is passed to the callback function */
{
  int		count = 0;		/* Number of OIDs found */
  unsigned	request_id = 0;		/* Current request ID */
  cups_snmp_t	packet;			/* Current response packet */
  int		lastoid[CUPS_SNMP_MAX_OID];
					/* Last OID we got */

 /*
  * Range check input...
  */

  DEBUG_printf(("4_cupsSNMPWalk(fd=%d, address=%p, version=%d, "
                "community=\"%s\", prefix=%p, timeout=%.1f, cb=%p, data=%p)",
		fd, address, version, community, prefix, timeout, cb, data));

  if (fd < 0 || !address || version != CUPS_SNMP_VERSION_1 || !community ||
      !prefix || !cb)
  {
    DEBUG_puts("5_cupsSNMPWalk: Returning -1");

    return (-1);
  }

 /*
  * Copy the OID prefix and then loop until we have no more OIDs...
  */

  _cupsSNMPCopyOID(packet.object_name, prefix, CUPS_SNMP_MAX_OID);
  lastoid[0] = -1;

  for (;;)
  {
    request_id ++;

    if (!_cupsSNMPWrite(fd, address, version, community,
                        CUPS_ASN1_GET_NEXT_REQUEST, request_id,
		        packet.object_name))
    {
      DEBUG_puts("5_cupsSNMPWalk: Returning -1");

      return (-1);
    }

    if (!_cupsSNMPRead(fd, &packet, timeout))
    {
      DEBUG_puts("5_cupsSNMPWalk: Returning -1");

      return (-1);
    }

    if (!_cupsSNMPIsOIDPrefixed(&packet, prefix) ||
        _cupsSNMPIsOID(&packet, lastoid))
    {
      DEBUG_printf(("5_cupsSNMPWalk: Returning %d", count));

      return (count);
    }

    if (packet.error || packet.error_status)
    {
      DEBUG_printf(("5_cupsSNMPWalk: Returning %d", count > 0 ? count : -1));

      return (count > 0 ? count : -1);
    }

    _cupsSNMPCopyOID(lastoid, packet.object_name, CUPS_SNMP_MAX_OID);

    count ++;

    (*cb)(&packet, data);
  }
}


_cupsSNMPStringToOID(const char *src,	/* I - OID string */
                     int        *dst,	/* I - OID array */
		     int        dstsize)/* I - Number of integers in OID array */
{
  int	*dstptr,			/* Pointer into OID array */
	*dstend;			/* End of OID array */

  DEBUG_printf(("4_cupsSNMPStringToOID(src=\"%s\", dst=%p, dstsize=%d)",
                src, dst, dstsize));

 /*
  * Range check input...
  */

  if (!src || !dst || dstsize < 2)
    return (NULL);

 /*
  * Skip leading "."...
  */

  if (*src == '.')
    src ++;

 /*
  * Loop to the end of the string...
  */

  for (dstend = dst + dstsize - 1, dstptr = dst, *dstptr = 0;
       *src && dstptr < dstend;
       src ++)
  {
    if (*src == '.')
    {
      dstptr ++;
      *dstptr = 0;
    }
    else if (isdigit(*src & 255))
      *dstptr = *dstptr * 10 + *src - '0';
    else
      break;
  }

  if (*src)
    return (NULL);

 /*
  * Terminate the end of the OID array and return...
  */

  dstptr[1] = -1;

  return (dst);
}


_cupsSNMPDefaultCommunity(void)
{
  cups_file_t	*fp;			/* snmp.conf file */
  char		line[1024],		/* Line from file */
		*value;			/* Value from file */
  int		linenum;		/* Line number in file */
  _cups_globals_t *cg = _cupsGlobals();	/* Global data */

  DEBUG_puts("4_cupsSNMPDefaultCommunity()");

  if (!cg->snmp_community[0])
  {
    strlcpy(cg->snmp_community, "public", sizeof(cg->snmp_community));

    snprintf(line, sizeof(line), "%s/snmp.conf", cg->cups_serverroot);
    if ((fp = cupsFileOpen(line, "r")) != NULL)
    {
      linenum = 0;
      while (cupsFileGetConf(fp, line, sizeof(line), &value, &linenum))
	if (!_cups_strcasecmp(line, "Community") && value)
	{
	  strlcpy(cg->snmp_community, value, sizeof(cg->snmp_community));
	  break;
	}

      cupsFileClose(fp);
    }
  }

  DEBUG_printf(("5_cupsSNMPDefaultCommunity: Returning \"%s\"",
                cg->snmp_community));

  return (cg->snmp_community);
}


_cupsSNMPIsOIDPrefixed(
    cups_snmp_t *packet,		/* I - Response packet */
    const int   *prefix)		/* I - OID prefix */
{
  int	i;				/* Looping var */

 /*
  * Range check input...
  */

  DEBUG_printf(("4_cupsSNMPIsOIDPrefixed(packet=%p, prefix=%p)", packet,
                prefix));

  if (!packet || !prefix)
  {
    DEBUG_puts("5_cupsSNMPIsOIDPrefixed: Returning 0");

    return (0);
  }

 /*
  * Compare OIDs...
  */

  for (i = 0;
       i < CUPS_SNMP_MAX_OID && prefix[i] >= 0 && packet->object_name[i] >= 0;
       i ++)
    if (prefix[i] != packet->object_name[i])
    {
      DEBUG_puts("5_cupsSNMPIsOIDPrefixed: Returning 0");

      return (0);
    }

  DEBUG_printf(("5_cupsSNMPIsOIDPrefixed: Returning %d",
                i < CUPS_SNMP_MAX_OID));

  return (i < CUPS_SNMP_MAX_OID);
}


httpAddCredential(
    cups_array_t *credentials,		/* I - Credentials array */
    const void   *data,			/* I - PEM-encoded X.509 data */
    size_t       datalen)		/* I - Length of data */
{
  http_credential_t	*credential;	/* Credential data */

  if ((credential = malloc(sizeof(http_credential_t))) != NULL)
  {
    credential->datalen = datalen;

    if ((credential->data = malloc(datalen)) != NULL)
    {
      memcpy(credential->data, data, datalen);
      cupsArrayAdd(credentials, credential);
      return (0);
    }

    free(credential);
  }

  return (-1);
}


ippFirstAttribute(ipp_t *ipp)		/* I - IPP message */
{
 /*
  * Range check input...
  */

  if (!ipp)
    return (NULL);

 /*
  * Return the first attribute...
  */

  return (ipp->current = ipp->attrs);
}


asn1_decode_snmp(unsigned char *buffer,	/* I - Buffer */
                 size_t        len,	/* I - Size of buffer */
                 cups_snmp_t   *packet)	/* I - SNMP packet */
{
  unsigned char	*bufptr,		/* Pointer into the data */
		*bufend;		/* End of data */
  unsigned	length;			/* Length of value */

 /*
  * Initialize the decoding...
  */

  memset(packet, 0, sizeof(cups_snmp_t));
  packet->object_name[0] = -1;

  bufptr = buffer;
  bufend = buffer + len;

  if (asn1_get_type(&bufptr, bufend) != CUPS_ASN1_SEQUENCE)
    snmp_set_error(packet, _("Packet does not start with SEQUENCE"));
  else if (asn1_get_length(&bufptr, bufend) == 0)
    snmp_set_error(packet, _("SEQUENCE uses indefinite length"));
  else if (asn1_get_type(&bufptr, bufend) != CUPS_ASN1_INTEGER)
    snmp_set_error(packet, _("No version number"));
  else if ((length = asn1_get_length(&bufptr, bufend)) == 0)
    snmp_set_error(packet, _("Version uses indefinite length"));
  else if ((packet->version = asn1_get_integer(&bufptr, bufend, length))
               != CUPS_SNMP_VERSION_1)
    snmp_set_error(packet, _("Bad SNMP version number"));
  else if (asn1_get_type(&bufptr, bufend) != CUPS_ASN1_OCTET_STRING)
    snmp_set_error(packet, _("No community name"));
  else if ((length = asn1_get_length(&bufptr, bufend)) == 0)
    snmp_set_error(packet, _("Community name uses indefinite length"));
  else
  {
    asn1_get_string(&bufptr, bufend, length, packet->community,
                    sizeof(packet->community));

    if ((packet->request_type = (cups_asn1_t)asn1_get_type(&bufptr, bufend))
            != CUPS_ASN1_GET_RESPONSE)
      snmp_set_error(packet, _("Packet does not contain a Get-Response-PDU"));
    else if (asn1_get_length(&bufptr, bufend) == 0)
      snmp_set_error(packet, _("Get-Response-PDU uses indefinite length"));
    else if (asn1_get_type(&bufptr, bufend) != CUPS_ASN1_INTEGER)
      snmp_set_error(packet, _("No request-id"));
    else if ((length = asn1_get_length(&bufptr, bufend)) == 0)
      snmp_set_error(packet, _("request-id uses indefinite length"));
    else
    {
      packet->request_id = (unsigned)asn1_get_integer(&bufptr, bufend, length);

      if (asn1_get_type(&bufptr, bufend) != CUPS_ASN1_INTEGER)
	snmp_set_error(packet, _("No error-status"));
      else if ((length = asn1_get_length(&bufptr, bufend)) == 0)
	snmp_set_error(packet, _("error-status uses indefinite length"));
      else
      {
	packet->error_status = asn1_get_integer(&bufptr, bufend, length);

	if (asn1_get_type(&bufptr, bufend) != CUPS_ASN1_INTEGER)
	  snmp_set_error(packet, _("No error-index"));
	else if ((length = asn1_get_length(&bufptr, bufend)) == 0)
	  snmp_set_error(packet, _("error-index uses indefinite length"));
	else
	{
	  packet->error_index = asn1_get_integer(&bufptr, bufend, length);

          if (asn1_get_type(&bufptr, bufend) != CUPS_ASN1_SEQUENCE)
	    snmp_set_error(packet, _("No variable-bindings SEQUENCE"));
	  else if (asn1_get_length(&bufptr, bufend) == 0)
	    snmp_set_error(packet,
	                   _("variable-bindings uses indefinite length"));
	  else if (asn1_get_type(&bufptr, bufend) != CUPS_ASN1_SEQUENCE)
	    snmp_set_error(packet, _("No VarBind SEQUENCE"));
	  else if (asn1_get_length(&bufptr, bufend) == 0)
	    snmp_set_error(packet, _("VarBind uses indefinite length"));
	  else if (asn1_get_type(&bufptr, bufend) != CUPS_ASN1_OID)
	    snmp_set_error(packet, _("No name OID"));
	  else if ((length = asn1_get_length(&bufptr, bufend)) == 0)
	    snmp_set_error(packet, _("Name OID uses indefinite length"));
          else
	  {
	    asn1_get_oid(&bufptr, bufend, length, packet->object_name,
	                 CUPS_SNMP_MAX_OID);

            packet->object_type = (cups_asn1_t)asn1_get_type(&bufptr, bufend);

	    if ((length = asn1_get_length(&bufptr, bufend)) == 0 &&
	        packet->object_type != CUPS_ASN1_NULL_VALUE &&
	        packet->object_type != CUPS_ASN1_OCTET_STRING)
	      snmp_set_error(packet, _("Value uses indefinite length"));
	    else
	    {
	      switch (packet->object_type)
	      {
	        case CUPS_ASN1_BOOLEAN :
		    packet->object_value.boolean =
		        asn1_get_integer(&bufptr, bufend, length);
	            break;

	        case CUPS_ASN1_INTEGER :
		    packet->object_value.integer =
		        asn1_get_integer(&bufptr, bufend, length);
	            break;

		case CUPS_ASN1_NULL_VALUE :
		    break;

	        case CUPS_ASN1_OCTET_STRING :
	        case CUPS_ASN1_BIT_STRING :
	        case CUPS_ASN1_HEX_STRING :
		    packet->object_value.string.num_bytes = length;
		    asn1_get_string(&bufptr, bufend, length,
		                    (char *)packet->object_value.string.bytes,
				    sizeof(packet->object_value.string.bytes));
	            break;

	        case CUPS_ASN1_OID :
		    asn1_get_oid(&bufptr, bufend, length,
		                 packet->object_value.oid, CUPS_SNMP_MAX_OID);
	            break;

	        case CUPS_ASN1_COUNTER :
		    packet->object_value.counter =
		        asn1_get_integer(&bufptr, bufend, length);
	            break;

	        case CUPS_ASN1_GAUGE :
		    packet->object_value.gauge =
		        (unsigned)asn1_get_integer(&bufptr, bufend, length);
	            break;

	        case CUPS_ASN1_TIMETICKS :
		    packet->object_value.timeticks =
		        (unsigned)asn1_get_integer(&bufptr, bufend, length);
	            break;

                default :
		    snmp_set_error(packet, _("Unsupported value type"));
		    break;
	      }
	    }
          }
	}
      }
    }
  }

  return (packet->error ? -1 : 0);
}


ippSetOperation(ipp_t    *ipp,		/* I - IPP request message */
                ipp_op_t op)		/* I - Operation ID */
{
 /*
  * Range check input...
  */

  if (!ipp)
    return (0);

 /*
  * Set the operation and return...
  */

  ipp->request.op.operation_id = op;

  return (1);
}


http_content_coding_finish(
    http_t *http)			/* I - HTTP connection */
{
  int		zerr;			/* Compression status */
  Byte		dummy[1];		/* Dummy read buffer */
  size_t	bytes;			/* Number of bytes to write */

  DEBUG_printf(("http_content_coding_finish(http=%p)", (void *)http));
  DEBUG_printf(("1http_content_coding_finishing: http->coding=%d", http->coding));

  switch (http->coding)
  {
    case _HTTP_CODING_DEFLATE :
    case _HTTP_CODING_GZIP :
        ((z_stream *)http->stream)->next_in  = dummy;
        ((z_stream *)http->stream)->avail_in = 0;

        do
        {
          zerr  = deflate((z_stream *)http->stream, Z_FINISH);
	  bytes = _HTTP_MAX_SBUFFER - ((z_stream *)http->stream)->avail_out;

          if (bytes > 0)
	  {
	    DEBUG_printf(("1http_content_coding_finish: Writing trailing chunk, len=%d", (int)bytes));

	    if (http->data_encoding == HTTP_ENCODING_CHUNKED)
	      http_write_chunk(http, (char *)http->sbuffer, bytes);
	    else
	      http_write(http, (char *)http->sbuffer, bytes);
          }

          ((z_stream *)http->stream)->next_out  = (Bytef *)http->sbuffer;
          ((z_stream *)http->stream)->avail_out = (uInt)_HTTP_MAX_SBUFFER;
	}
        while (zerr == Z_OK);

        deflateEnd((z_stream *)http->stream);

        free(http->sbuffer);
        free(http->stream);

        http->sbuffer = NULL;
        http->stream  = NULL;

        if (http->wused)
          httpFlushWrite(http);
        break;

    case _HTTP_CODING_INFLATE :
    case _HTTP_CODING_GUNZIP :
        inflateEnd((z_stream *)http->stream);

        free(http->sbuffer);
        free(http->stream);

        http->sbuffer = NULL;
        http->stream  = NULL;
        break;

    default :
        break;
  }

  http->coding = _HTTP_CODING_IDENTITY;
}


static bool should_fail_request(struct hd_struct *part, unsigned int bytes)
{
	return part->make_it_fail && should_fail(&fail_make_request, bytes);
}


generic_make_request_checks(struct bio *bio)
{
	struct request_queue *q;
	int nr_sectors = bio_sectors(bio);
	blk_status_t status = BLK_STS_IOERR;
	char b[BDEVNAME_SIZE];

	might_sleep();

	q = bio->bi_disk->queue;
	if (unlikely(!q)) {
		printk(KERN_ERR
		       "generic_make_request: Trying to access "
			"nonexistent block-device %s (%Lu)\n",
			bio_devname(bio, b), (long long)bio->bi_iter.bi_sector);
		goto end_io;
	}

	/*
	 * For a REQ_NOWAIT based request, return -EOPNOTSUPP
	 * if queue is not a request based queue.
	 */
	if ((bio->bi_opf & REQ_NOWAIT) && !queue_is_rq_based(q))
		goto not_supported;

	if (should_fail_bio(bio))
		goto end_io;

	if (bio->bi_partno) {
		if (unlikely(blk_partition_remap(bio)))
			goto end_io;
	} else {
		if (unlikely(bio_check_ro(bio, &bio->bi_disk->part0)))
			goto end_io;
		if (unlikely(bio_check_eod(bio, get_capacity(bio->bi_disk))))
			goto end_io;
	}

	/*
	 * Filter flush bio's early so that make_request based
	 * drivers without flush support don't have to worry
	 * about them.
	 */
	if (op_is_flush(bio->bi_opf) &&
	    !test_bit(QUEUE_FLAG_WC, &q->queue_flags)) {
		bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA);
		if (!nr_sectors) {
			status = BLK_STS_OK;
			goto end_io;
		}
	}

	switch (bio_op(bio)) {
	case REQ_OP_DISCARD:
		if (!blk_queue_discard(q))
			goto not_supported;
		break;
	case REQ_OP_SECURE_ERASE:
		if (!blk_queue_secure_erase(q))
			goto not_supported;
		break;
	case REQ_OP_WRITE_SAME:
		if (!q->limits.max_write_same_sectors)
			goto not_supported;
		break;
	case REQ_OP_ZONE_REPORT:
	case REQ_OP_ZONE_RESET:
		if (!blk_queue_is_zoned(q))
			goto not_supported;
		break;
	case REQ_OP_WRITE_ZEROES:
		if (!q->limits.max_write_zeroes_sectors)
			goto not_supported;
		break;
	default:
		break;
	}

	/*
	 * Various block parts want %current->io_context and lazy ioc
	 * allocation ends up trading a lot of pain for a small amount of
	 * memory.  Just allocate it upfront.  This may fail and block
	 * layer knows how to live with it.
	 */
	create_io_context(GFP_ATOMIC, q->node);

	if (!blkcg_bio_issue_check(q, bio))
		return false;

	if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) {
		trace_block_bio_queue(q, bio);
		/* Now that enqueuing has been traced, we need to trace
		 * completion as well.
		 */
		bio_set_flag(bio, BIO_TRACE_COMPLETION);
	}
	return true;

not_supported:
	status = BLK_STS_NOTSUPP;
end_io:
	bio->bi_status = status;
	bio_endio(bio);
	return false;
}


static inline bool bio_check_ro(struct bio *bio, struct hd_struct *part)
{
	if (part->policy && op_is_write(bio_op(bio))) {
		char b[BDEVNAME_SIZE];

		printk(KERN_ERR
		       "generic_make_request: Trying to write "
			"to read-only block-device %s (partno %d)\n",
			bio_devname(bio, b), part->partno);
		return true;
	}

	return false;
}


static inline int bio_check_eod(struct bio *bio, sector_t maxsector)
{
	unsigned int nr_sectors = bio_sectors(bio);

	if (nr_sectors && maxsector &&
	    (nr_sectors > maxsector ||
	     bio->bi_iter.bi_sector > maxsector - nr_sectors)) {
		handle_bad_sector(bio, maxsector);
		return -EIO;
	}
	return 0;
}


struct request *blk_fetch_request(struct request_queue *q)
{
	struct request *rq;

	lockdep_assert_held(q->queue_lock);
	WARN_ON_ONCE(q->mq_ops);

	rq = blk_peek_request(q);
	if (rq)
		blk_start_request(rq);
	return rq;
}


static void blk_set_congested(struct request_list *rl, int sync)
{
#ifdef CONFIG_CGROUP_WRITEBACK
	set_wb_congested(rl->blkg->wb_congested, sync);
#else
	/* see blk_clear_congested() */
	if (rl == &rl->q->root_rl)
		set_wb_congested(rl->q->backing_dev_info->wb.congested, sync);
#endif
}


void blk_queue_exit(struct request_queue *q)
{
	percpu_ref_put(&q->q_usage_counter);
}


int blk_status_to_errno(blk_status_t status)
{
	int idx = (__force int)status;

	if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
		return -EIO;
	return blk_errors[idx].errno;
}


struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
				      int size)
{
	struct blk_plug *plug = current->plug;
	struct blk_plug_cb *cb;

	if (!plug)
		return NULL;

	list_for_each_entry(cb, &plug->cb_list, list)
		if (cb->callback == unplug && cb->data == data)
			return cb;

	/* Not currently on the callback list */
	BUG_ON(size < sizeof(*cb));
	cb = kzalloc(size, GFP_ATOMIC);
	if (cb) {
		cb->data = data;
		cb->callback = unplug;
		list_add(&cb->list, &plug->cb_list);
	}
	return cb;
}


static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
{
	struct blk_plug *plug;
	int where = ELEVATOR_INSERT_SORT;
	struct request *req, *free;
	unsigned int request_count = 0;

	/*
	 * low level driver can indicate that it wants pages above a
	 * certain limit bounced to low memory (ie for highmem, or even
	 * ISA dma in theory)
	 */
	blk_queue_bounce(q, &bio);

	blk_queue_split(q, &bio);

	if (!bio_integrity_prep(bio))
		return BLK_QC_T_NONE;

	if (op_is_flush(bio->bi_opf)) {
		spin_lock_irq(q->queue_lock);
		where = ELEVATOR_INSERT_FLUSH;
		goto get_rq;
	}

	/*
	 * Check if we can merge with the plugged list before grabbing
	 * any locks.
	 */
	if (!blk_queue_nomerges(q)) {
		if (blk_attempt_plug_merge(q, bio, &request_count, NULL))
			return BLK_QC_T_NONE;
	} else
		request_count = blk_plug_queued_count(q);

	spin_lock_irq(q->queue_lock);

	switch (elv_merge(q, &req, bio)) {
	case ELEVATOR_BACK_MERGE:
		if (!bio_attempt_back_merge(q, req, bio))
			break;
		elv_bio_merged(q, req, bio);
		free = attempt_back_merge(q, req);
		if (free)
			__blk_put_request(q, free);
		else
			elv_merged_request(q, req, ELEVATOR_BACK_MERGE);
		goto out_unlock;
	case ELEVATOR_FRONT_MERGE:
		if (!bio_attempt_front_merge(q, req, bio))
			break;
		elv_bio_merged(q, req, bio);
		free = attempt_front_merge(q, req);
		if (free)
			__blk_put_request(q, free);
		else
			elv_merged_request(q, req, ELEVATOR_FRONT_MERGE);
		goto out_unlock;
	default:
		break;
	}

get_rq:
	rq_qos_throttle(q, bio, q->queue_lock);

	/*
	 * Grab a free request. This is might sleep but can not fail.
	 * Returns with the queue unlocked.
	 */
	blk_queue_enter_live(q);
	req = get_request(q, bio->bi_opf, bio, 0, GFP_NOIO);
	if (IS_ERR(req)) {
		blk_queue_exit(q);
		rq_qos_cleanup(q, bio);
		if (PTR_ERR(req) == -ENOMEM)
			bio->bi_status = BLK_STS_RESOURCE;
		else
			bio->bi_status = BLK_STS_IOERR;
		bio_endio(bio);
		goto out_unlock;
	}

	rq_qos_track(q, req, bio);

	/*
	 * After dropping the lock and possibly sleeping here, our request
	 * may now be mergeable after it had proven unmergeable (above).
	 * We don't worry about that case for efficiency. It won't happen
	 * often, and the elevators are able to handle it.
	 */
	blk_init_request_from_bio(req, bio);

	if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags))
		req->cpu = raw_smp_processor_id();

	plug = current->plug;
	if (plug) {
		/*
		 * If this is the first request added after a plug, fire
		 * of a plug trace.
		 *
		 * @request_count may become stale because of schedule
		 * out, so check plug list again.
		 */
		if (!request_count || list_empty(&plug->list))
			trace_block_plug(q);
		else {
			struct request *last = list_entry_rq(plug->list.prev);
			if (request_count >= BLK_MAX_REQUEST_COUNT ||
			    blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE) {
				blk_flush_plug_list(plug, false);
				trace_block_plug(q);
			}
		}
		list_add_tail(&req->queuelist, &plug->list);
		blk_account_io_start(req, true);
	} else {
		spin_lock_irq(q->queue_lock);
		add_acct_request(q, req, where);
		__blk_run_queue(q);
out_unlock:
		spin_unlock_irq(q->queue_lock);
	}

	return BLK_QC_T_NONE;
}


int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
		      struct bio_set *bs, gfp_t gfp_mask,
		      int (*bio_ctr)(struct bio *, struct bio *, void *),
		      void *data)
{
	struct bio *bio, *bio_src;

	if (!bs)
		bs = &fs_bio_set;

	__rq_for_each_bio(bio_src, rq_src) {
		bio = bio_clone_fast(bio_src, gfp_mask, bs);
		if (!bio)
			goto free_and_out;

		if (bio_ctr && bio_ctr(bio, bio_src, data))
			goto free_and_out;

		if (rq->bio) {
			rq->biotail->bi_next = bio;
			rq->biotail = bio;
		} else
			rq->bio = rq->biotail = bio;
	}

	__blk_rq_prep_clone(rq, rq_src);

	return 0;

free_and_out:
	if (bio)
		bio_put(bio);
	blk_rq_unprep_clone(rq);

	return -ENOMEM;
}


bool blk_update_request(struct request *req, blk_status_t error,
		unsigned int nr_bytes)
{
	int total_bytes;

	trace_block_rq_complete(req, blk_status_to_errno(error), nr_bytes);

	if (!req->bio)
		return false;

	if (unlikely(error && !blk_rq_is_passthrough(req) &&
		     !(req->rq_flags & RQF_QUIET)))
		print_req_error(req, error);

	blk_account_io_completion(req, nr_bytes);

	total_bytes = 0;
	while (req->bio) {
		struct bio *bio = req->bio;
		unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);

		if (bio_bytes == bio->bi_iter.bi_size)
			req->bio = bio->bi_next;

		/* Completion has already been traced */
		bio_clear_flag(bio, BIO_TRACE_COMPLETION);
		req_bio_endio(req, bio, bio_bytes, error);

		total_bytes += bio_bytes;
		nr_bytes -= bio_bytes;

		if (!nr_bytes)
			break;
	}

	/*
	 * completely done
	 */
	if (!req->bio) {
		/*
		 * Reset counters so that the request stacking driver
		 * can find how many bytes remain in the request
		 * later.
		 */
		req->__data_len = 0;
		return false;
	}

	req->__data_len -= total_bytes;

	/* update sector only for requests with clear definition of sector */
	if (!blk_rq_is_passthrough(req))
		req->__sector += total_bytes >> 9;

	/* mixed attributes always follow the first bio */
	if (req->rq_flags & RQF_MIXED_MERGE) {
		req->cmd_flags &= ~REQ_FAILFAST_MASK;
		req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
	}

	if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
		/*
		 * If total number of sectors is less than the first segment
		 * size, something has gone terribly wrong.
		 */
		if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
			blk_dump_rq_flags(req, "request botched");
			req->__data_len = blk_rq_cur_bytes(req);
		}

		/* recalculate the number of segments */
		blk_recalc_rq_segments(req);
	}

	return true;
}


static void *alloc_request_simple(gfp_t gfp_mask, void *data)
{
	struct request_queue *q = data;

	return kmem_cache_alloc_node(request_cachep, gfp_mask, q->node);
}


static void set_datagram_seg(struct mlx5_wqe_datagram_seg *dseg,
			     const struct ib_send_wr *wr)
{
	memcpy(&dseg->av, &to_mah(ud_wr(wr)->ah)->av, sizeof(struct mlx5_av));
	dseg->av.dqp_dct = cpu_to_be32(ud_wr(wr)->remote_qpn | MLX5_EXTENDED_UD_AV);
	dseg->av.key.qkey.qkey = cpu_to_be32(ud_wr(wr)->remote_qkey);
}


static void destroy_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
				      struct mlx5_ib_sq *sq)
{
	mlx5_core_destroy_tis(dev->mdev, sq->tisn);
}


	__acquires(&send_cq->lock) __acquires(&recv_cq->lock)
{
	if (send_cq) {
		if (recv_cq) {
			if (send_cq->mcq.cqn < recv_cq->mcq.cqn)  {
				spin_lock(&send_cq->lock);
				spin_lock_nested(&recv_cq->lock,
						 SINGLE_DEPTH_NESTING);
			} else if (send_cq->mcq.cqn == recv_cq->mcq.cqn) {
				spin_lock(&send_cq->lock);
				__acquire(&recv_cq->lock);
			} else {
				spin_lock(&recv_cq->lock);
				spin_lock_nested(&send_cq->lock,
						 SINGLE_DEPTH_NESTING);
			}
		} else {
			spin_lock(&send_cq->lock);
			__acquire(&recv_cq->lock);
		}
	} else if (recv_cq) {
		spin_lock(&recv_cq->lock);
		__acquire(&send_cq->lock);
	} else {
		__acquire(&send_cq->lock);
		__acquire(&recv_cq->lock);
	}
}


static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
			  struct mlx5_ib_rwq *rwq,
			  struct mlx5_ib_create_wq *ucmd)
{
	struct mlx5_ib_ucontext *context;
	int page_shift = 0;
	int npages;
	u32 offset = 0;
	int ncont = 0;
	int err;

	if (!ucmd->buf_addr)
		return -EINVAL;

	context = to_mucontext(pd->uobject->context);
	rwq->umem = ib_umem_get(pd->uobject->context, ucmd->buf_addr,
			       rwq->buf_size, 0, 0);
	if (IS_ERR(rwq->umem)) {
		mlx5_ib_dbg(dev, "umem_get failed\n");
		err = PTR_ERR(rwq->umem);
		return err;
	}

	mlx5_ib_cont_pages(rwq->umem, ucmd->buf_addr, 0, &npages, &page_shift,
			   &ncont, NULL);
	err = mlx5_ib_get_buf_offset(ucmd->buf_addr, page_shift,
				     &rwq->rq_page_offset);
	if (err) {
		mlx5_ib_warn(dev, "bad offset\n");
		goto err_umem;
	}

	rwq->rq_num_pas = ncont;
	rwq->page_shift = page_shift;
	rwq->log_page_size =  page_shift - MLX5_ADAPTER_PAGE_SHIFT;
	rwq->wq_sig = !!(ucmd->flags & MLX5_WQ_FLAG_SIGNATURE);

	mlx5_ib_dbg(dev, "addr 0x%llx, size %zd, npages %d, page_shift %d, ncont %d, offset %d\n",
		    (unsigned long long)ucmd->buf_addr, rwq->buf_size,
		    npages, page_shift, ncont, offset);

	err = mlx5_ib_db_map_user(context, ucmd->db_addr, &rwq->db);
	if (err) {
		mlx5_ib_dbg(dev, "map failed\n");
		goto err_umem;
	}

	rwq->create_type = MLX5_WQ_USER;
	return 0;

err_umem:
	ib_umem_release(rwq->umem);
	return err;
}


int mlx5_ib_destroy_qp(struct ib_qp *qp)
{
	struct mlx5_ib_dev *dev = to_mdev(qp->device);
	struct mlx5_ib_qp *mqp = to_mqp(qp);

	if (unlikely(qp->qp_type == IB_QPT_GSI))
		return mlx5_ib_gsi_destroy_qp(qp);

	if (mqp->qp_sub_type == MLX5_IB_QPT_DCT)
		return mlx5_ib_destroy_dct(mqp);

	destroy_qp_common(dev, mqp);

	kfree(mqp);

	return 0;
}


static inline enum ib_mig_state to_ib_mig_state(int mlx5_mig_state)
{
	switch (mlx5_mig_state) {
	case MLX5_QP_PM_ARMED:		return IB_MIG_ARMED;
	case MLX5_QP_PM_REARM:		return IB_MIG_REARM;
	case MLX5_QP_PM_MIGRATED:	return IB_MIG_MIGRATED;
	default: return -1;
	}
}


static u32 get_rx_type(struct mlx5_ib_qp *qp, struct ib_qp_init_attr *attr)
{
	if (attr->srq || (attr->qp_type == IB_QPT_XRC_TGT) ||
	    (attr->qp_type == MLX5_IB_QPT_DCI) ||
	    (attr->qp_type == IB_QPT_XRC_INI))
		return MLX5_SRQ_RQ;
	else if (!qp->has_rq)
		return MLX5_ZERO_LEN_RQ;
	else
		return MLX5_NON_ZERO_RQ;
}


void mlx5_ib_drain_sq(struct ib_qp *qp)
{
	struct ib_cq *cq = qp->send_cq;
	struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
	struct mlx5_ib_drain_cqe sdrain;
	const struct ib_send_wr *bad_swr;
	struct ib_rdma_wr swr = {
		.wr = {
			.next = NULL,
			{ .wr_cqe	= &sdrain.cqe, },
			.opcode	= IB_WR_RDMA_WRITE,
		},
	};
	int ret;
	struct mlx5_ib_dev *dev = to_mdev(qp->device);
	struct mlx5_core_dev *mdev = dev->mdev;

	ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
	if (ret && mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) {
		WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
		return;
	}

	sdrain.cqe.done = mlx5_ib_drain_qp_done;
	init_completion(&sdrain.done);

	ret = _mlx5_ib_post_send(qp, &swr.wr, &bad_swr, true);
	if (ret) {
		WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
		return;
	}

	handle_drain_completion(cq, &sdrain, dev);
}


static int set_user_rq_size(struct mlx5_ib_dev *dev,
			    struct ib_wq_init_attr *wq_init_attr,
			    struct mlx5_ib_create_wq *ucmd,
			    struct mlx5_ib_rwq *rwq)
{
	/* Sanity check RQ size before proceeding */
	if (wq_init_attr->max_wr > (1 << MLX5_CAP_GEN(dev->mdev, log_max_wq_sz)))
		return -EINVAL;

	if (!ucmd->rq_wqe_count)
		return -EINVAL;

	rwq->wqe_count = ucmd->rq_wqe_count;
	rwq->wqe_shift = ucmd->rq_wqe_shift;
	if (check_shl_overflow(rwq->wqe_count, rwq->wqe_shift, &rwq->buf_size))
		return -EINVAL;

	rwq->log_rq_stride = rwq->wqe_shift;
	rwq->log_rq_size = ilog2(rwq->wqe_count);
	return 0;
}


static void set_data_ptr_seg(struct mlx5_wqe_data_seg *dseg, struct ib_sge *sg)
{
	dseg->byte_count = cpu_to_be32(sg->length);
	dseg->lkey       = cpu_to_be32(sg->lkey);
	dseg->addr       = cpu_to_be64(sg->addr);
}


static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr,
				u32 size)
{
	memset(umr, 0, sizeof(*umr));

	umr->flags = MLX5_FLAGS_INLINE | MLX5_FLAGS_CHECK_FREE;
	umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size));
	umr->bsf_octowords = cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE);
	umr->mkey_mask = sig_mkey_mask();
}


static int create_kernel_qp(struct mlx5_ib_dev *dev,
			    struct ib_qp_init_attr *init_attr,
			    struct mlx5_ib_qp *qp,
			    u32 **in, int *inlen,
			    struct mlx5_ib_qp_base *base)
{
	int uar_index;
	void *qpc;
	int err;

	if (init_attr->create_flags & ~(IB_QP_CREATE_SIGNATURE_EN |
					IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK |
					IB_QP_CREATE_IPOIB_UD_LSO |
					IB_QP_CREATE_NETIF_QP |
					mlx5_ib_create_qp_sqpn_qp1()))
		return -EINVAL;

	if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR)
		qp->bf.bfreg = &dev->fp_bfreg;
	else
		qp->bf.bfreg = &dev->bfreg;

	/* We need to divide by two since each register is comprised of
	 * two buffers of identical size, namely odd and even
	 */
	qp->bf.buf_size = (1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size)) / 2;
	uar_index = qp->bf.bfreg->index;

	err = calc_sq_size(dev, init_attr, qp);
	if (err < 0) {
		mlx5_ib_dbg(dev, "err %d\n", err);
		return err;
	}

	qp->rq.offset = 0;
	qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
	base->ubuffer.buf_size = err + (qp->rq.wqe_cnt << qp->rq.wqe_shift);

	err = mlx5_buf_alloc(dev->mdev, base->ubuffer.buf_size, &qp->buf);
	if (err) {
		mlx5_ib_dbg(dev, "err %d\n", err);
		return err;
	}

	qp->sq.qend = mlx5_get_send_wqe(qp, qp->sq.wqe_cnt);
	*inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
		 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * qp->buf.npages;
	*in = kvzalloc(*inlen, GFP_KERNEL);
	if (!*in) {
		err = -ENOMEM;
		goto err_buf;
	}

	qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
	MLX5_SET(qpc, qpc, uar_page, uar_index);
	MLX5_SET(qpc, qpc, log_page_size, qp->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);

	/* Set "fast registration enabled" for all kernel QPs */
	MLX5_SET(qpc, qpc, fre, 1);
	MLX5_SET(qpc, qpc, rlky, 1);

	if (init_attr->create_flags & mlx5_ib_create_qp_sqpn_qp1()) {
		MLX5_SET(qpc, qpc, deth_sqpn, 1);
		qp->flags |= MLX5_IB_QP_SQPN_QP1;
	}

	mlx5_fill_page_array(&qp->buf,
			     (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas));

	err = mlx5_db_alloc(dev->mdev, &qp->db);
	if (err) {
		mlx5_ib_dbg(dev, "err %d\n", err);
		goto err_free;
	}

	qp->sq.wrid = kvmalloc_array(qp->sq.wqe_cnt,
				     sizeof(*qp->sq.wrid), GFP_KERNEL);
	qp->sq.wr_data = kvmalloc_array(qp->sq.wqe_cnt,
					sizeof(*qp->sq.wr_data), GFP_KERNEL);
	qp->rq.wrid = kvmalloc_array(qp->rq.wqe_cnt,
				     sizeof(*qp->rq.wrid), GFP_KERNEL);
	qp->sq.w_list = kvmalloc_array(qp->sq.wqe_cnt,
				       sizeof(*qp->sq.w_list), GFP_KERNEL);
	qp->sq.wqe_head = kvmalloc_array(qp->sq.wqe_cnt,
					 sizeof(*qp->sq.wqe_head), GFP_KERNEL);

	if (!qp->sq.wrid || !qp->sq.wr_data || !qp->rq.wrid ||
	    !qp->sq.w_list || !qp->sq.wqe_head) {
		err = -ENOMEM;
		goto err_wrid;
	}
	qp->create_type = MLX5_QP_KERNEL;

	return 0;

err_wrid:
	kvfree(qp->sq.wqe_head);
	kvfree(qp->sq.w_list);
	kvfree(qp->sq.wrid);
	kvfree(qp->sq.wr_data);
	kvfree(qp->rq.wrid);
	mlx5_db_free(dev->mdev, &qp->db);

err_free:
	kvfree(*in);

err_buf:
	mlx5_buf_free(dev->mdev, &qp->buf);
	return err;
}


static int alloc_bfreg(struct mlx5_ib_dev *dev,
		       struct mlx5_bfreg_info *bfregi)
{
	int bfregn = -ENOMEM;

	mutex_lock(&bfregi->lock);
	if (bfregi->ver >= 2) {
		bfregn = alloc_high_class_bfreg(dev, bfregi);
		if (bfregn < 0)
			bfregn = alloc_med_class_bfreg(dev, bfregi);
	}

	if (bfregn < 0) {
		BUILD_BUG_ON(NUM_NON_BLUE_FLAME_BFREGS != 1);
		bfregn = 0;
		bfregi->count[bfregn]++;
	}
	mutex_unlock(&bfregi->lock);

	return bfregn;
}


static int modify_raw_packet_tx_affinity(struct mlx5_core_dev *dev,
					 struct mlx5_ib_sq *sq, u8 tx_affinity)
{
	void *in;
	void *tisc;
	int inlen;
	int err;

	inlen = MLX5_ST_SZ_BYTES(modify_tis_in);
	in = kvzalloc(inlen, GFP_KERNEL);
	if (!in)
		return -ENOMEM;

	MLX5_SET(modify_tis_in, in, bitmask.lag_tx_port_affinity, 1);

	tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
	MLX5_SET(tisc, tisc, lag_tx_port_affinity, tx_affinity);

	err = mlx5_core_modify_tis(dev, sq->tisn, in, inlen);

	kvfree(in);

	return err;
}


static const char *ib_qp_type_str(enum ib_qp_type type)
{
	switch (type) {
	case IB_QPT_SMI:
		return "IB_QPT_SMI";
	case IB_QPT_GSI:
		return "IB_QPT_GSI";
	case IB_QPT_RC:
		return "IB_QPT_RC";
	case IB_QPT_UC:
		return "IB_QPT_UC";
	case IB_QPT_UD:
		return "IB_QPT_UD";
	case IB_QPT_RAW_IPV6:
		return "IB_QPT_RAW_IPV6";
	case IB_QPT_RAW_ETHERTYPE:
		return "IB_QPT_RAW_ETHERTYPE";
	case IB_QPT_XRC_INI:
		return "IB_QPT_XRC_INI";
	case IB_QPT_XRC_TGT:
		return "IB_QPT_XRC_TGT";
	case IB_QPT_RAW_PACKET:
		return "IB_QPT_RAW_PACKET";
	case MLX5_IB_QPT_REG_UMR:
		return "MLX5_IB_QPT_REG_UMR";
	case IB_QPT_DRIVER:
		return "IB_QPT_DRIVER";
	case IB_QPT_MAX:
	default:
		return "Invalid QP type";
	}
}


static int sq_overhead(struct ib_qp_init_attr *attr)
{
	int size = 0;

	switch (attr->qp_type) {
	case IB_QPT_XRC_INI:
		size += sizeof(struct mlx5_wqe_xrc_seg);
		/* fall through */
	case IB_QPT_RC:
		size += sizeof(struct mlx5_wqe_ctrl_seg) +
			max(sizeof(struct mlx5_wqe_atomic_seg) +
			    sizeof(struct mlx5_wqe_raddr_seg),
			    sizeof(struct mlx5_wqe_umr_ctrl_seg) +
			    sizeof(struct mlx5_mkey_seg) +
			    MLX5_IB_SQ_UMR_INLINE_THRESHOLD /
			    MLX5_IB_UMR_OCTOWORD);
		break;

	case IB_QPT_XRC_TGT:
		return 0;

	case IB_QPT_UC:
		size += sizeof(struct mlx5_wqe_ctrl_seg) +
			max(sizeof(struct mlx5_wqe_raddr_seg),
			    sizeof(struct mlx5_wqe_umr_ctrl_seg) +
			    sizeof(struct mlx5_mkey_seg));
		break;

	case IB_QPT_UD:
		if (attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
			size += sizeof(struct mlx5_wqe_eth_pad) +
				sizeof(struct mlx5_wqe_eth_seg);
		/* fall through */
	case IB_QPT_SMI:
	case MLX5_IB_QPT_HW_GSI:
		size += sizeof(struct mlx5_wqe_ctrl_seg) +
			sizeof(struct mlx5_wqe_datagram_seg);
		break;

	case MLX5_IB_QPT_REG_UMR:
		size += sizeof(struct mlx5_wqe_ctrl_seg) +
			sizeof(struct mlx5_wqe_umr_ctrl_seg) +
			sizeof(struct mlx5_mkey_seg);
		break;

	default:
		return -EINVAL;
	}

	return size;
}


static int __begin_wqe(struct mlx5_ib_qp *qp, void **seg,
		     struct mlx5_wqe_ctrl_seg **ctrl,
		     const struct ib_send_wr *wr, unsigned *idx,
		     int *size, int nreq, bool send_signaled, bool solicited)
{
	if (unlikely(mlx5_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)))
		return -ENOMEM;

	*idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1);
	*seg = mlx5_get_send_wqe(qp, *idx);
	*ctrl = *seg;
	*(uint32_t *)(*seg + 8) = 0;
	(*ctrl)->imm = send_ieth(wr);
	(*ctrl)->fm_ce_se = qp->sq_signal_bits |
		(send_signaled ? MLX5_WQE_CTRL_CQ_UPDATE : 0) |
		(solicited ? MLX5_WQE_CTRL_SOLICITED : 0);

	*seg += sizeof(**ctrl);
	*size = sizeof(**ctrl) / 16;

	return 0;
}


managesieve_parser_create(struct istream *input, size_t max_line_size)
{
	struct managesieve_parser *parser;

	parser = i_new(struct managesieve_parser, 1);
	parser->pool = pool_alloconly_create("MANAGESIEVE parser", 8192);
	parser->input = input;
	parser->max_line_size = max_line_size;

	p_array_init(&parser->root_list, parser->pool, LIST_INIT_COUNT);
	parser->cur_list = &parser->root_list;
	return parser;
}


static void ev_local_goaway(h2_proxy_session *session, int arg, const char *msg)
{
    switch (session->state) {
        case H2_PROXYS_ST_LOCAL_SHUTDOWN:
            /* already did that? */
            break;
        case H2_PROXYS_ST_IDLE:
        case H2_PROXYS_ST_REMOTE_SHUTDOWN:
            /* all done */
            transit(session, "local goaway", H2_PROXYS_ST_DONE);
            break;
        default:
            transit(session, "local goaway", H2_PROXYS_ST_LOCAL_SHUTDOWN);
            break;
    }
}


int h2_stream_was_closed(const h2_stream *stream)
{
    switch (stream->state) {
        case H2_SS_CLOSED:
        case H2_SS_CLEANUP:
            return 1;
        default:
            return 0;
    }
}


static void ev_stream_resumed(h2_proxy_session *session, int arg, const char *msg)
{
    switch (session->state) {
        case H2_PROXYS_ST_WAIT:
            transit(session, "stream resumed", H2_PROXYS_ST_BUSY);
            break;
        default:
            /* nop */
            break;
    }
}


static apr_status_t h2_proxy_stream_add_header_out(h2_proxy_stream *stream,
                                                   const char *n, apr_size_t nlen,
                                                   const char *v, apr_size_t vlen)
{
    if (n[0] == ':') {
        if (!stream->data_received && !strncmp(":status", n, nlen)) {
            char *s = apr_pstrndup(stream->r->pool, v, vlen);

            apr_table_setn(stream->r->notes, "proxy-status", s);
            ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, stream->session->c,
                          "h2_proxy_stream(%s-%d): got status %s",
                          stream->session->id, stream->id, s);
            stream->r->status = (int)apr_atoi64(s);
            if (stream->r->status <= 0) {
                stream->r->status = 500;
                return APR_EGENERAL;
            }
        }
        return APR_SUCCESS;
    }

    ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, stream->session->c,
                  "h2_proxy_stream(%s-%d): on_header %s: %s",
                  stream->session->id, stream->id, n, v);
    if (!h2_proxy_res_ignore_header(n, nlen)) {
        char *hname, *hvalue;
        apr_table_t *headers = (stream->headers_ended?
                               stream->r->trailers_out : stream->r->headers_out);

        hname = apr_pstrndup(stream->pool, n, nlen);
        h2_proxy_util_camel_case_header(hname, nlen);
        hvalue = apr_pstrndup(stream->pool, v, vlen);

        ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, stream->session->c,
                      "h2_proxy_stream(%s-%d): got header %s: %s",
                      stream->session->id, stream->id, hname, hvalue);
        process_proxy_header(headers, stream, hname, hvalue);
    }
    return APR_SUCCESS;
}


static apr_status_t on_stream_headers(h2_session *session, h2_stream *stream,
                                      h2_headers *headers, apr_off_t len,
                                      int eos)
{
    apr_status_t status = APR_SUCCESS;
    const char *s;
    int rv = 0;

    ap_assert(session);
    ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, session->c,
                  H2_STRM_MSG(stream, "on_headers"));
    if (headers->status < 100) {
        h2_stream_rst(stream, headers->status);
        goto leave;
    }
    else if (stream->has_response) {
        h2_ngheader *nh;

        status = h2_res_create_ngtrailer(&nh, stream->pool, headers);

        ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, session->c,
                      H2_STRM_LOG(APLOGNO(03072), stream, "submit %d trailers"),
                      (int)nh->nvlen);
        if (status == APR_SUCCESS) {
            rv = nghttp2_submit_trailer(session->ngh2, stream->id,
                                        nh->nv, nh->nvlen);
        }
        else {
            ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, session->c,
                          H2_STRM_LOG(APLOGNO(10024), stream, "invalid trailers"));
            h2_stream_rst(stream, NGHTTP2_PROTOCOL_ERROR);
        }
        goto leave;
    }
    else {
        nghttp2_data_provider provider, *pprovider = NULL;
        h2_ngheader *ngh;
        const char *note;

        ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c,
                      H2_STRM_LOG(APLOGNO(03073), stream, "submit response %d, REMOTE_WINDOW_SIZE=%u"),
                      headers->status,
                      (unsigned int)nghttp2_session_get_stream_remote_window_size(session->ngh2, stream->id));

        if (!eos || len > 0) {
            memset(&provider, 0, sizeof(provider));
            provider.source.fd = stream->id;
            provider.read_callback = stream_data_cb;
            pprovider = &provider;
        }

        /* If this stream is not a pushed one itself,
         * and HTTP/2 server push is enabled here,
         * and the response HTTP status is not sth >= 400,
         * and the remote side has pushing enabled,
         * -> find and perform any pushes on this stream
         *    *before* we submit the stream response itself.
         *    This helps clients avoid opening new streams on Link
         *    headers that get pushed right afterwards.
         *
         * *) the response code is relevant, as we do not want to
         *    make pushes on 401 or 403 codes and friends.
         *    And if we see a 304, we do not push either
         *    as the client, having this resource in its cache, might
         *    also have the pushed ones as well.
         */
        if (!stream->initiated_on
            && !stream->has_response
            && stream->request && stream->request->method
            && !strcmp("GET", stream->request->method)
            && (headers->status < 400)
            && (headers->status != 304)
            && h2_session_push_enabled(session)) {
            /* PUSH is possibe and enabled on server, unless the request
             * denies it, submit resources to push */
            s = apr_table_get(headers->notes, H2_PUSH_MODE_NOTE);
            if (!s || strcmp(s, "0")) {
                h2_stream_submit_pushes(stream, headers);
            }
        }

        if (!stream->pref_priority) {
            stream->pref_priority = h2_stream_get_priority(stream, headers);
        }
        h2_session_set_prio(session, stream, stream->pref_priority);

        note = apr_table_get(headers->notes, H2_FILTER_DEBUG_NOTE);
        if (note && !strcmp("on", note)) {
            int32_t connFlowIn, connFlowOut;

            connFlowIn = nghttp2_session_get_effective_local_window_size(session->ngh2);
            connFlowOut = nghttp2_session_get_remote_window_size(session->ngh2);
            headers = h2_headers_copy(stream->pool, headers);
            apr_table_setn(headers->headers, "conn-flow-in",
                           apr_itoa(stream->pool, connFlowIn));
            apr_table_setn(headers->headers, "conn-flow-out",
                           apr_itoa(stream->pool, connFlowOut));
        }

        if (headers->status == 103
            && !h2_config_sgeti(session->s, H2_CONF_EARLY_HINTS)) {
            /* suppress sending this to the client, it might have triggered
             * pushes and served its purpose nevertheless */
            rv = 0;
            goto leave;
        }

        status = h2_res_create_ngheader(&ngh, stream->pool, headers);
        if (status == APR_SUCCESS) {
            rv = nghttp2_submit_response(session->ngh2, stream->id,
                                         ngh->nv, ngh->nvlen, pprovider);
            stream->has_response = h2_headers_are_response(headers);
            session->have_written = 1;

            if (stream->initiated_on) {
                ++session->pushes_submitted;
            }
            else {
                ++session->responses_submitted;
            }
        }
        else {
            ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, session->c,
                          H2_STRM_LOG(APLOGNO(10025), stream, "invalid response"));
            h2_stream_rst(stream, NGHTTP2_PROTOCOL_ERROR);
        }
    }

leave:
    if (nghttp2_is_fatal(rv)) {
        status = APR_EGENERAL;
        dispatch_event(session, H2_SESSION_EV_PROTO_ERROR, rv, nghttp2_strerror(rv));
        ap_log_cerror(APLOG_MARK, APLOG_ERR, status, session->c,
                      APLOGNO(02940) "submit_response: %s",
                      nghttp2_strerror(rv));
    }

    ++session->unsent_submits;

    /* Unsent push promises are written immediately, as nghttp2
     * 1.5.0 realizes internal stream data structures only on
     * send and we might need them for other submits.
     * Also, to conserve memory, we send at least every 10 submits
     * so that nghttp2 does not buffer all outbound items too
     * long.
     */
    if (status == APR_SUCCESS
        && (session->unsent_promises || session->unsent_submits > 10)) {
        status = h2_session_send(session);
    }
    return status;
}


static int on_header_cb(nghttp2_session *ngh2, const nghttp2_frame *frame,
                        const uint8_t *name, size_t namelen,
                        const uint8_t *value, size_t valuelen,
                        uint8_t flags,
                        void *userp)
{
    h2_session *session = (h2_session *)userp;
    h2_stream * stream;
    apr_status_t status;

    (void)flags;
    stream = get_stream(session, frame->hd.stream_id);
    if (!stream) {
        ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c, APLOGNO(02920)
                      "h2_stream(%ld-%d): on_header unknown stream",
                      session->id, (int)frame->hd.stream_id);
        return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
    }

    status = h2_stream_add_header(stream, (const char *)name, namelen,
                                  (const char *)value, valuelen);
    if (status != APR_SUCCESS && !h2_stream_is_ready(stream)) {
        return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
    }
    return 0;
}


static apr_status_t submit_trailers(h2_proxy_stream *stream)
{
    h2_proxy_ngheader *hd;
    int rv;

    hd = h2_proxy_util_nghd_make(stream->pool, stream->r->trailers_in);
    rv = nghttp2_submit_trailer(stream->session->ngh2, stream->id, hd->nv, hd->nvlen);
    return rv == 0? APR_SUCCESS: APR_EGENERAL;
}


static const char *h2_ss_str(h2_stream_state_t state)
{
    switch (state) {
        case H2_SS_IDLE:
            return "IDLE";
        case H2_SS_RSVD_L:
            return "RESERVED_LOCAL";
        case H2_SS_RSVD_R:
            return "RESERVED_REMOTE";
        case H2_SS_OPEN:
            return "OPEN";
        case H2_SS_CLOSED_L:
            return "HALF_CLOSED_LOCAL";
        case H2_SS_CLOSED_R:
            return "HALF_CLOSED_REMOTE";
        case H2_SS_CLOSED:
            return "CLOSED";
        case H2_SS_CLEANUP:
            return "CLEANUP";
        default:
            return "UNKNOWN";
    }
}


void h2_stream_cleanup(h2_stream *stream)
{
    apr_status_t status;

    ap_assert(stream);
    if (stream->out_buffer) {
        /* remove any left over output buckets that may still have
         * references into request pools */
        apr_brigade_cleanup(stream->out_buffer);
    }
    if (stream->input) {
        h2_beam_abort(stream->input);
        status = h2_beam_wait_empty(stream->input, APR_NONBLOCK_READ);
        if (status == APR_EAGAIN) {
            ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, stream->session->c,
                          H2_STRM_MSG(stream, "wait on input drain"));
            status = h2_beam_wait_empty(stream->input, APR_BLOCK_READ);
            ap_log_cerror(APLOG_MARK, APLOG_TRACE2, status, stream->session->c,
                          H2_STRM_MSG(stream, "input drain returned"));
        }
    }
}


static int on_frame(h2_stream_state_t state, int frame_type,
                    int frame_map[][H2_SS_MAX], apr_size_t maxlen)
{
    ap_assert(frame_type >= 0);
    ap_assert(state >= 0);
    if (frame_type >= maxlen) {
        return state; /* NOP, ignore unknown frame types */
    }
    return on_map(state, frame_map[frame_type]);
}


static int on_send_data_cb(nghttp2_session *ngh2,
                           nghttp2_frame *frame,
                           const uint8_t *framehd,
                           size_t length,
                           nghttp2_data_source *source,
                           void *userp)
{
    apr_status_t status = APR_SUCCESS;
    h2_session *session = (h2_session *)userp;
    int stream_id = (int)frame->hd.stream_id;
    unsigned char padlen;
    int eos;
    h2_stream *stream;
    apr_bucket *b;
    apr_off_t len = length;

    (void)ngh2;
    (void)source;
    if (!h2_session_continue_data(session)) {
        return NGHTTP2_ERR_WOULDBLOCK;
    }

    ap_assert(frame->data.padlen <= (H2_MAX_PADLEN+1));
    padlen = (unsigned char)frame->data.padlen;

    stream = get_stream(session, stream_id);
    if (!stream) {
        ap_log_cerror(APLOG_MARK, APLOG_ERR, APR_NOTFOUND, session->c,
                      APLOGNO(02924)
                      "h2_stream(%ld-%d): send_data, stream not found",
                      session->id, (int)stream_id);
        return NGHTTP2_ERR_CALLBACK_FAILURE;
    }

    ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, session->c,
                  H2_STRM_MSG(stream, "send_data_cb for %ld bytes"),
                  (long)length);

    status = h2_conn_io_write(&session->io, (const char *)framehd, H2_FRAME_HDR_LEN);
    if (padlen && status == APR_SUCCESS) {
        --padlen;
        status = h2_conn_io_write(&session->io, (const char *)&padlen, 1);
    }

    if (status != APR_SUCCESS) {
        ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, session->c,
                      H2_STRM_MSG(stream, "writing frame header"));
        return NGHTTP2_ERR_CALLBACK_FAILURE;
    }

    status = h2_stream_read_to(stream, session->bbtmp, &len, &eos);
    if (status != APR_SUCCESS) {
        ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, session->c,
                      H2_STRM_MSG(stream, "send_data_cb, reading stream"));
        apr_brigade_cleanup(session->bbtmp);
        return NGHTTP2_ERR_CALLBACK_FAILURE;
    }
    else if (len != length) {
        ap_log_cerror(APLOG_MARK, APLOG_TRACE1, status, session->c,
                      H2_STRM_MSG(stream, "send_data_cb, wanted %ld bytes, "
                      "got %ld from stream"), (long)length, (long)len);
        apr_brigade_cleanup(session->bbtmp);
        return NGHTTP2_ERR_CALLBACK_FAILURE;
    }

    if (padlen) {
        b = apr_bucket_immortal_create(immortal_zeros, padlen,
                                       session->c->bucket_alloc);
        APR_BRIGADE_INSERT_TAIL(session->bbtmp, b);
    }

    status = h2_conn_io_pass(&session->io, session->bbtmp);
    apr_brigade_cleanup(session->bbtmp);

    if (status == APR_SUCCESS) {
        stream->out_data_frames++;
        stream->out_data_octets += length;
        return 0;
    }
    else {
        ap_log_cerror(APLOG_MARK, APLOG_DEBUG, status, session->c,
                      H2_STRM_LOG(APLOGNO(02925), stream, "failed send_data_cb"));
        return NGHTTP2_ERR_CALLBACK_FAILURE;
    }
}


static ssize_t stream_data_cb(nghttp2_session *ng2s,
                              int32_t stream_id,
                              uint8_t *buf,
                              size_t length,
                              uint32_t *data_flags,
                              nghttp2_data_source *source,
                              void *puser)
{
    h2_session *session = (h2_session *)puser;
    apr_off_t nread = length;
    int eos = 0;
    apr_status_t status;
    h2_stream *stream;
    ap_assert(session);

    /* The session wants to send more DATA for the stream. We need
     * to find out how much of the requested length we can send without
     * blocking.
     * Indicate EOS when we encounter it or DEFERRED if the stream
     * should be suspended. Beware of trailers.
     */

    (void)ng2s;
    (void)buf;
    (void)source;
    stream = get_stream(session, stream_id);
    if (!stream) {
        ap_log_cerror(APLOG_MARK, APLOG_ERR, 0, session->c,
                      APLOGNO(02937)
                      "h2_stream(%ld-%d): data_cb, stream not found",
                      session->id, (int)stream_id);
        return NGHTTP2_ERR_CALLBACK_FAILURE;
    }

    status = h2_stream_out_prepare(stream, &nread, &eos, NULL);
    if (nread) {
        ap_log_cerror(APLOG_MARK, APLOG_TRACE2, 0, session->c,
                      H2_STRM_MSG(stream, "prepared no_copy, len=%ld, eos=%d"),
                      (long)nread, eos);
        *data_flags |=  NGHTTP2_DATA_FLAG_NO_COPY;
    }

    switch (status) {
        case APR_SUCCESS:
            break;

        case APR_EOF:
            eos = 1;
            break;

        case APR_ECONNRESET:
        case APR_ECONNABORTED:
            return NGHTTP2_ERR_CALLBACK_FAILURE;

        case APR_EAGAIN:
            /* If there is no data available, our session will automatically
             * suspend this stream and not ask for more data until we resume
             * it. Remember at our h2_stream that we need to do this.
             */
            nread = 0;
            ap_log_cerror(APLOG_MARK, APLOG_DEBUG, 0, session->c,
                          H2_STRM_LOG(APLOGNO(03071), stream, "suspending"));
            return NGHTTP2_ERR_DEFERRED;

        default:
            nread = 0;
            ap_log_cerror(APLOG_MARK, APLOG_ERR, status, session->c,
                          H2_STRM_LOG(APLOGNO(02938), stream, "reading data"));
            return NGHTTP2_ERR_CALLBACK_FAILURE;
    }

    if (eos) {
        *data_flags |= NGHTTP2_DATA_FLAG_EOF;
    }
    return (ssize_t)nread;
}


const char *h2_stream_state_str(h2_stream *stream)
{
    return h2_ss_str(stream->state);
}


static int table_check_val_len(void *baton, const char *key, const char *value)
{
    val_len_check_ctx *ctx = baton;

    if (strlen(value) <= ctx->maxlen) return 1;
    ctx->failed_key = key;
    return 0;
}