FilterUtility::finalTimeout(const RouteEntry& route, Http::HeaderMap& request_headers,
                            bool insert_envoy_expected_request_timeout_ms, bool grpc_request,
                            bool per_try_timeout_hedging_enabled,
                            bool respect_expected_rq_timeout) {
  // See if there is a user supplied timeout in a request header. If there is we take that.
  // Otherwise if the request is gRPC and a maximum gRPC timeout is configured we use the timeout
  // in the gRPC headers (or infinity when gRPC headers have no timeout), but cap that timeout to
  // the configured maximum gRPC timeout (which may also be infinity, represented by a 0 value),
  // or the default from the route config otherwise.
  TimeoutData timeout;
  if (grpc_request && route.maxGrpcTimeout()) {
    const std::chrono::milliseconds max_grpc_timeout = route.maxGrpcTimeout().value();
    std::chrono::milliseconds grpc_timeout = Grpc::Common::getGrpcTimeout(request_headers);
    if (route.grpcTimeoutOffset()) {
      // We only apply the offset if it won't result in grpc_timeout hitting 0 or below, as
      // setting it to 0 means infinity and a negative timeout makes no sense.
      const auto offset = *route.grpcTimeoutOffset();
      if (offset < grpc_timeout) {
        grpc_timeout -= offset;
      }
    }

    // Cap gRPC timeout to the configured maximum considering that 0 means infinity.
    if (max_grpc_timeout != std::chrono::milliseconds(0) &&
        (grpc_timeout == std::chrono::milliseconds(0) || grpc_timeout > max_grpc_timeout)) {
      grpc_timeout = max_grpc_timeout;
    }
    timeout.global_timeout_ = grpc_timeout;
  } else {
    timeout.global_timeout_ = route.timeout();
  }
  timeout.per_try_timeout_ = route.retryPolicy().perTryTimeout();

  uint64_t header_timeout;

  if (respect_expected_rq_timeout) {
    // Check if there is timeout set by egress Envoy.
    // If present, use that value as route timeout and don't override
    // *x-envoy-expected-rq-timeout-ms* header. At this point *x-envoy-upstream-rq-timeout-ms*
    // header should have been sanitized by egress Envoy.
    Http::HeaderEntry* header_expected_timeout_entry =
        request_headers.EnvoyExpectedRequestTimeoutMs();
    if (header_expected_timeout_entry) {
      trySetGlobalTimeout(header_expected_timeout_entry, timeout);
    } else {
      Http::HeaderEntry* header_timeout_entry = request_headers.EnvoyUpstreamRequestTimeoutMs();

      if (trySetGlobalTimeout(header_timeout_entry, timeout)) {
        request_headers.removeEnvoyUpstreamRequestTimeoutMs();
      }
    }
  } else {
    Http::HeaderEntry* header_timeout_entry = request_headers.EnvoyUpstreamRequestTimeoutMs();
    if (trySetGlobalTimeout(header_timeout_entry, timeout)) {
      request_headers.removeEnvoyUpstreamRequestTimeoutMs();
    }
  }

  // See if there is a per try/retry timeout. If it's >= global we just ignore it.
  Http::HeaderEntry* per_try_timeout_entry = request_headers.EnvoyUpstreamRequestPerTryTimeoutMs();
  if (per_try_timeout_entry) {
    if (absl::SimpleAtoi(per_try_timeout_entry->value().getStringView(), &header_timeout)) {
      timeout.per_try_timeout_ = std::chrono::milliseconds(header_timeout);
    }
    request_headers.removeEnvoyUpstreamRequestPerTryTimeoutMs();
  }

  if (timeout.per_try_timeout_ >= timeout.global_timeout_) {
    timeout.per_try_timeout_ = std::chrono::milliseconds(0);
  }

  // See if there is any timeout to write in the expected timeout header.
  uint64_t expected_timeout = timeout.per_try_timeout_.count();
  // Use the global timeout if no per try timeout was specified or if we're
  // doing hedging when there are per try timeouts. Either of these scenarios
  // mean that the upstream server can use the full global timeout.
  if (per_try_timeout_hedging_enabled || expected_timeout == 0) {
    expected_timeout = timeout.global_timeout_.count();
  }

  if (insert_envoy_expected_request_timeout_ms && expected_timeout > 0) {
    request_headers.insertEnvoyExpectedRequestTimeoutMs().value(expected_timeout);
  }

  // If we've configured max_grpc_timeout, override the grpc-timeout header with
  // the expected timeout. This ensures that the optional per try timeout is reflected
  // in grpc-timeout, ensuring that the upstream gRPC server is aware of the actual timeout.
  // If the expected timeout is 0 set no timeout, as Envoy treats 0 as infinite timeout.
  if (grpc_request && route.maxGrpcTimeout() && expected_timeout != 0) {
    Grpc::Common::toGrpcTimeout(std::chrono::milliseconds(expected_timeout),
                                request_headers.insertGrpcTimeout().value());
  }

  return timeout;
}


TEST_P(Http2FloodMitigationTest, RST_STREAM) {
  // Use invalid HTTP headers to trigger sending RST_STREAM frames.
  config_helper_.addConfigModifier(
      [](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm)
          -> void {
        hcm.mutable_http2_protocol_options()->set_stream_error_on_invalid_http_messaging(true);
      });
  beginSession();

  int i = 0;
  auto request = Http::Http2::Http2Frame::makeMalformedRequest(i);
  sendFame(request);
  auto response = readFrame();
  // Make sure we've got RST_STREAM from the server
  EXPECT_EQ(Http2Frame::Type::RST_STREAM, response.type());
  uint64_t total_bytes_sent = 0;
  while (total_bytes_sent < TransmitThreshold && tcp_client_->connected()) {
    request = Http::Http2::Http2Frame::makeMalformedRequest(++i);
    sendFame(request);
    total_bytes_sent += request.size();
  }
  EXPECT_LE(total_bytes_sent, TransmitThreshold) << "Flood mitigation is broken.";
  EXPECT_EQ(1, test_server_->counter("http2.outbound_control_flood")->value());
  EXPECT_EQ(1,
            test_server_->counter("http.config_test.downstream_cx_delayed_close_timeout")->value());
}


TEST_P(Http2MetadataIntegrationTest, HeadersOnlyRequestWithRequestMetadata) {
  addFilters({request_metadata_filter});
  // Send a headers only request.
  runHeaderOnlyTest(false, 0);
  verifyHeadersOnlyTest();
}


static void addDummyHeaders(HeaderMap& headers, size_t num_headers) {
  const std::string prefix("dummy-key-");
  for (size_t i = 0; i < num_headers; i++) {
    headers.addCopy(LowerCaseString(prefix + std::to_string(i)), "abcd");
  }
}


TEST_F(HttpConnectionManagerImplTest, UnderlyingConnectionWatermarksPassedOnWithLazyCreation) {
  setup(false, "");

  // Make sure codec_ is created.
  EXPECT_CALL(*codec_, dispatch(_));
  Buffer::OwnedImpl fake_input("");
  conn_manager_->onData(fake_input, false);

  // Mark the connection manger as backed up before the stream is created.
  ASSERT_EQ(decoder_filters_.size(), 0);
  EXPECT_CALL(*codec_, onUnderlyingConnectionAboveWriteBufferHighWatermark());
  conn_manager_->onAboveWriteBufferHighWatermark();

  // Create the stream. Defer the creation of the filter chain by not sending
  // complete headers.
  StreamDecoder* decoder;
  {
    setUpBufferLimits();
    EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void {
      decoder = &conn_manager_->newStream(response_encoder_);
      // Call the high buffer callbacks as the codecs do.
      stream_callbacks_->onAboveWriteBufferHighWatermark();
    }));

    // Send fake data to kick off newStream being created.
    Buffer::OwnedImpl fake_input2("asdf");
    conn_manager_->onData(fake_input2, false);
  }

  // Now set up the filter chain by sending full headers. The filters should be
  // immediately appraised that the low watermark is in effect.
  {
    setupFilterChain(2, 2);
    EXPECT_CALL(filter_callbacks_.connection_, aboveHighWatermark()).Times(0);
    EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void {
      HeaderMapPtr headers{
          new TestHeaderMapImpl{{":authority", "host"}, {":path", "/"}, {":method", "GET"}}};
      decoder->decodeHeaders(std::move(headers), true);
    }));
    EXPECT_CALL(*decoder_filters_[0], decodeHeaders(_, true))
        .WillOnce(InvokeWithoutArgs([&]() -> FilterHeadersStatus {
          Buffer::OwnedImpl data("hello");
          decoder_filters_[0]->callbacks_->addDecodedData(data, true);
          return FilterHeadersStatus::Continue;
        }));
    EXPECT_CALL(*decoder_filters_[0], decodeComplete());
    sendRequestHeadersAndData();
    ASSERT_GE(decoder_filters_.size(), 1);
    MockDownstreamWatermarkCallbacks callbacks;
    EXPECT_CALL(callbacks, onAboveWriteBufferHighWatermark());
    decoder_filters_[0]->callbacks_->addDownstreamWatermarkCallbacks(callbacks);

    // Ensures that when new callbacks are registered they get invoked immediately
    // and the already-registered callbacks do not.
    MockDownstreamWatermarkCallbacks callbacks2;
    EXPECT_CALL(callbacks2, onAboveWriteBufferHighWatermark());
    decoder_filters_[0]->callbacks_->addDownstreamWatermarkCallbacks(callbacks2);
  }
}


void ClientConnectionImpl::onEncodeHeaders(const HeaderMap& headers) {
  if (headers.Method()->value() == Headers::get().MethodValues.Head.c_str()) {
    pending_responses_.back().head_request_ = true;
  }
}


TEST_P(Http2CodecImplTest, ResponseDataFloodMitigationDisabled) {
  Runtime::LoaderSingleton::getExisting()->mergeValues(
      {{"envoy.reloadable_features.http2_protocol_options.max_outbound_frames", "2147483647"}});
  initialize();

  TestHeaderMapImpl request_headers;
  HttpTestUtility::addDefaultHeaders(request_headers);
  EXPECT_CALL(request_decoder_, decodeHeaders_(_, false));
  request_encoder_->encodeHeaders(request_headers, false);

  // +2 is to account for HEADERS and PING ACK, that is used to trigger mitigation
  EXPECT_CALL(server_connection_, write(_, _))
      .Times(Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES + 2);
  EXPECT_CALL(response_decoder_, decodeHeaders_(_, false)).Times(1);
  EXPECT_CALL(response_decoder_, decodeData(_, false))
      .Times(Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES);
  TestHeaderMapImpl response_headers{{":status", "200"}};
  response_encoder_->encodeHeaders(response_headers, false);
  // Account for the single HEADERS frame above
  for (uint32_t i = 0; i < Http2Settings::DEFAULT_MAX_OUTBOUND_FRAMES; ++i) {
    Buffer::OwnedImpl data("0");
    EXPECT_NO_THROW(response_encoder_->encodeData(data, false));
  }
  // Presently flood mitigation is done only when processing downstream data
  // So we need to send stream from downstream client to trigger mitigation
  EXPECT_EQ(0, nghttp2_submit_ping(client_->session(), NGHTTP2_FLAG_NONE, nullptr));
  EXPECT_NO_THROW(client_->sendPendingFrames());
}


void ConnectionImpl::onMessageCompleteBase() {
  ENVOY_CONN_LOG(trace, "message complete", connection_);
  if (handling_upgrade_) {
    // If this is an upgrade request, swallow the onMessageComplete. The
    // upgrade payload will be treated as stream body.
    ASSERT(!deferred_end_stream_headers_);
    ENVOY_CONN_LOG(trace, "Pausing parser due to upgrade.", connection_);
    http_parser_pause(&parser_, 1);
    return;
  }
  onMessageComplete();
}


TEST_P(ProtocolIntegrationTest, AddEncodedTrailers) {
  config_helper_.addFilter(R"EOF(
name: add-trailers-filter
config: {}
)EOF");
  initialize();

  codec_client_ = makeHttpConnection(lookupPort("http"));
  auto response = codec_client_->makeRequestWithBody(default_request_headers_, 128);
  waitForNextUpstreamRequest();
  upstream_request_->encodeHeaders(Http::TestHeaderMapImpl{{":status", "503"}}, false);
  upstream_request_->encodeData(128, true);
  response->waitForEndStream();

  if (upstreamProtocol() == FakeHttpConnection::Type::HTTP2) {
    EXPECT_EQ("decode", upstream_request_->trailers()->GrpcMessage()->value().getStringView());
  }
  EXPECT_TRUE(response->complete());
  EXPECT_EQ("503", response->headers().Status()->value().getStringView());
  if (downstream_protocol_ == Http::CodecClient::Type::HTTP2) {
    EXPECT_EQ("encode", response->trailers()->GrpcMessage()->value().getStringView());
  }
}


void Filter::setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks& callbacks) {
  callbacks_ = &callbacks;
  // As the decoder filter only pushes back via watermarks once data has reached
  // it, it can latch the current buffer limit and does not need to update the
  // limit if another filter increases it.
  buffer_limit_ = callbacks_->decoderBufferLimit();
}


bool shouldMergeSlashes() const override { return merge_slashes_; }


  Router::RouteConfigProvider* routeConfigProvider() override {
    return route_config_provider2_.get();
  }


TEST_P(DownstreamProtocolIntegrationTest, MultipleContentLengthsAllowed) {
  config_helper_.addConfigModifier(
      [](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm)
          -> void {
        hcm.mutable_http2_protocol_options()->set_stream_error_on_invalid_http_messaging(true);
      });

  initialize();
  codec_client_ = makeHttpConnection(lookupPort("http"));
  auto encoder_decoder =
      codec_client_->startRequest(Http::TestHeaderMapImpl{{":method", "POST"},
                                                          {":path", "/test/long/url"},
                                                          {":authority", "host"},
                                                          {"content-length", "3,2"}});
  auto response = std::move(encoder_decoder.second);

  if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) {
    codec_client_->waitForDisconnect();
  } else {
    response->waitForReset();
    codec_client_->close();
  }

  if (downstream_protocol_ == Http::CodecClient::Type::HTTP1) {
    ASSERT_TRUE(response->complete());
    EXPECT_EQ("400", response->headers().Status()->value().getStringView());
  } else {
    ASSERT_TRUE(response->reset());
    EXPECT_EQ(Http::StreamResetReason::RemoteReset, response->reset_reason());
  }
}


std::chrono::milliseconds delayedCloseTimeout() const override { return delayed_close_timeout_; }


TEST_F(HttpConnectionManagerImplTest, TestAccessLog) {
  static constexpr char local_address[] = "0.0.0.0";
  static constexpr char xff_address[] = "1.2.3.4";

  // stream_info.downstreamRemoteAddress will infer the address from request
  // headers instead of the physical connection
  use_remote_address_ = false;
  setup(false, "");

  std::shared_ptr<MockStreamDecoderFilter> filter(new NiceMock<MockStreamDecoderFilter>());
  std::shared_ptr<AccessLog::MockInstance> handler(new NiceMock<AccessLog::MockInstance>());

  EXPECT_CALL(filter_factory_, createFilterChain(_))
      .WillOnce(Invoke([&](FilterChainFactoryCallbacks& callbacks) -> void {
        callbacks.addStreamDecoderFilter(filter);
        callbacks.addAccessLogHandler(handler);
      }));

  EXPECT_CALL(*handler, log(_, _, _, _))
      .WillOnce(Invoke([](const HeaderMap*, const HeaderMap*, const HeaderMap*,
                          const StreamInfo::StreamInfo& stream_info) {
        EXPECT_TRUE(stream_info.responseCode());
        EXPECT_EQ(stream_info.responseCode().value(), uint32_t(200));
        EXPECT_NE(nullptr, stream_info.downstreamLocalAddress());
        EXPECT_NE(nullptr, stream_info.downstreamRemoteAddress());
        EXPECT_NE(nullptr, stream_info.downstreamDirectRemoteAddress());
        EXPECT_NE(nullptr, stream_info.routeEntry());

        EXPECT_EQ(stream_info.downstreamRemoteAddress()->ip()->addressAsString(), xff_address);
        EXPECT_EQ(stream_info.downstreamDirectRemoteAddress()->ip()->addressAsString(),
                  local_address);
      }));

  StreamDecoder* decoder = nullptr;
  NiceMock<MockStreamEncoder> encoder;
  EXPECT_CALL(*codec_, dispatch(_)).WillRepeatedly(Invoke([&](Buffer::Instance& data) -> void {
    decoder = &conn_manager_->newStream(encoder);

    HeaderMapPtr headers{
        new TestHeaderMapImpl{{":method", "GET"},
                              {":authority", "host"},
                              {":path", "/"},
                              {"x-forwarded-for", xff_address},
                              {"x-request-id", "125a4afb-6f55-a4ba-ad80-413f09f48a28"}}};
    decoder->decodeHeaders(std::move(headers), true);

    HeaderMapPtr response_headers{new TestHeaderMapImpl{{":status", "200"}}};
    filter->callbacks_->encodeHeaders(std::move(response_headers), true);

    data.drain(4);
  }));

  Buffer::OwnedImpl fake_input("1234");
  conn_manager_->onData(fake_input, false);
}


TEST_P(DownstreamProtocolIntegrationTest, testEncodeHeadersReturnsStopAllWatermark) {
  config_helper_.addFilter(R"EOF(
name: encode-headers-return-stop-all-filter
)EOF");
  config_helper_.addConfigModifier(
      [&](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm)
          -> void { hcm.mutable_http2_protocol_options()->set_allow_metadata(true); });

  // Sets initial stream window to min value to make the upstream sensitive to a low watermark.
  config_helper_.addConfigModifier(
      [&](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& hcm)
          -> void {
        hcm.mutable_http2_protocol_options()->mutable_initial_stream_window_size()->set_value(
            Http::Http2Settings::MIN_INITIAL_STREAM_WINDOW_SIZE);
      });

  initialize();
  codec_client_ = makeHttpConnection(lookupPort("http"));

  // Upstream responds with headers, data and trailers.
  auto response = codec_client_->makeRequestWithBody(default_request_headers_, 10);
  waitForNextUpstreamRequest();

  changeHeadersForStopAllTests(default_response_headers_, true);
  upstream_request_->encodeHeaders(default_response_headers_, false);
  for (int i = 0; i < count_ - 1; i++) {
    upstream_request_->encodeData(size_, false);
  }
  // Gives buffer 1s to react to buffer limit.
  sleep(1);
  upstream_request_->encodeData(size_, false);
  Http::TestHeaderMapImpl response_trailers{{"response", "trailer"}};
  upstream_request_->encodeTrailers(response_trailers);

  response->waitForEndStream();
  ASSERT_TRUE(response->complete());
  EXPECT_EQ(count_ * size_ + added_decoded_data_size_, response->body().size());
}


void HeaderMapImpl::iterateReverse(ConstIterateCb cb, void* context) const {
  for (auto it = headers_.rbegin(); it != headers_.rend(); it++) {
    if (cb(*it, context) == HeaderMap::Iterate::Break) {
      break;
    }
  }
}


void ConnectionManagerImpl::ActiveStream::encode100ContinueHeaders(
    ActiveStreamEncoderFilter* filter, HeaderMap& headers) {
  resetIdleTimer();
  ASSERT(connection_manager_.config_.proxy100Continue());
  // Make sure commonContinue continues encode100ContinueHeaders.
  has_continue_headers_ = true;

  // Similar to the block in encodeHeaders, run encode100ContinueHeaders on each
  // filter. This is simpler than that case because 100 continue implies no
  // end-stream, and because there are normal headers coming there's no need for
  // complex continuation logic.
  // 100-continue filter iteration should always start with the next filter if available.
  std::list<ActiveStreamEncoderFilterPtr>::iterator entry =
      commonEncodePrefix(filter, false, FilterIterationStartState::AlwaysStartFromNext);
  for (; entry != encoder_filters_.end(); entry++) {
    ASSERT(!(state_.filter_call_state_ & FilterCallState::Encode100ContinueHeaders));
    state_.filter_call_state_ |= FilterCallState::Encode100ContinueHeaders;
    FilterHeadersStatus status = (*entry)->handle_->encode100ContinueHeaders(headers);
    state_.filter_call_state_ &= ~FilterCallState::Encode100ContinueHeaders;
    ENVOY_STREAM_LOG(trace, "encode 100 continue headers called: filter={} status={}", *this,
                     static_cast<const void*>((*entry).get()), static_cast<uint64_t>(status));
    if (!(*entry)->commonHandleAfter100ContinueHeadersCallback(status)) {
      return;
    }
  }

  // Strip the T-E headers etc. Defer other header additions as well as drain-close logic to the
  // continuation headers.
  ConnectionManagerUtility::mutateResponseHeaders(headers, request_headers_.get(), EMPTY_STRING);

  // Count both the 1xx and follow-up response code in stats.
  chargeStats(headers);

  ENVOY_STREAM_LOG(debug, "encoding 100 continue headers via codec:\n{}", *this, headers);

  // Now actually encode via the codec.
  response_encoder_->encode100ContinueHeaders(headers);
}


void HeaderMapImpl::HeaderEntryImpl::value(uint64_t value) { value_.setInteger(value); }


static inline int _get_extstore(conn *c, item *it, int iovst, int iovcnt) {
#ifdef NEED_ALIGN
    item_hdr hdr;
    memcpy(&hdr, ITEM_data(it), sizeof(hdr));
#else
    item_hdr *hdr = (item_hdr *)ITEM_data(it);
#endif
    size_t ntotal = ITEM_ntotal(it);
    unsigned int clsid = slabs_clsid(ntotal);
    item *new_it;
    bool chunked = false;
    if (ntotal > settings.slab_chunk_size_max) {
        // Pull a chunked item header.
        uint32_t flags;
        FLAGS_CONV(it, flags);
        new_it = item_alloc(ITEM_key(it), it->nkey, flags, it->exptime, it->nbytes);
        assert(new_it == NULL || (new_it->it_flags & ITEM_CHUNKED));
        chunked = true;
    } else {
        new_it = do_item_alloc_pull(ntotal, clsid);
    }
    if (new_it == NULL)
        return -1;
    assert(!c->io_queued); // FIXME: debugging.
    // so we can free the chunk on a miss
    new_it->slabs_clsid = clsid;

    io_wrap *io = do_cache_alloc(c->thread->io_cache);
    io->active = true;
    io->miss = false;
    io->badcrc = false;
    // io_wrap owns the reference for this object now.
    io->hdr_it = it;

    // FIXME: error handling.
    // The offsets we'll wipe on a miss.
    io->iovec_start = iovst;
    io->iovec_count = iovcnt;
    // This is probably super dangerous. keep it at 0 and fill into wrap
    // object?
    if (chunked) {
        unsigned int ciovcnt = 1;
        size_t remain = new_it->nbytes;
        item_chunk *chunk = (item_chunk *) ITEM_schunk(new_it);
        io->io.iov = &c->iov[c->iovused];
        // fill the header so we can get the full data + crc back.
        add_iov(c, new_it, ITEM_ntotal(new_it) - new_it->nbytes);
        while (remain > 0) {
            chunk = do_item_alloc_chunk(chunk, remain);
            if (chunk == NULL) {
                item_remove(new_it);
                do_cache_free(c->thread->io_cache, io);
                return -1;
            }
            add_iov(c, chunk->data, (remain < chunk->size) ? remain : chunk->size);
            chunk->used = (remain < chunk->size) ? remain : chunk->size;
            remain -= chunk->size;
            ciovcnt++;
        }
        io->io.iovcnt = ciovcnt;
        // header object was already accounted for, remove one from total
        io->iovec_count += ciovcnt-1;
    } else {
        io->io.iov = NULL;
        io->iovec_data = c->iovused;
        add_iov(c, "", it->nbytes);
    }
    io->io.buf = (void *)new_it;
    // The offset we'll fill in on a hit.
    io->c = c;
    // We need to stack the sub-struct IO's together as well.
    if (c->io_wraplist) {
        io->io.next = &c->io_wraplist->io;
    } else {
        io->io.next = NULL;
    }
    // IO queue for this connection.
    io->next = c->io_wraplist;
    c->io_wraplist = io;
    assert(c->io_wrapleft >= 0);
    c->io_wrapleft++;
    // reference ourselves for the callback.
    io->io.data = (void *)io;

    // Now, fill in io->io based on what was in our header.
#ifdef NEED_ALIGN
    io->io.page_version = hdr.page_version;
    io->io.page_id = hdr.page_id;
    io->io.offset = hdr.offset;
#else
    io->io.page_version = hdr->page_version;
    io->io.page_id = hdr->page_id;
    io->io.offset = hdr->offset;
#endif
    io->io.len = ntotal;
    io->io.mode = OBJ_IO_READ;
    io->io.cb = _get_extstore_cb;

    //fprintf(stderr, "EXTSTORE: IO stacked %u\n", io->iovec_data);
    // FIXME: This stat needs to move to reflect # of flash hits vs misses
    // for now it's a good gauge on how often we request out to flash at
    // least.
    pthread_mutex_lock(&c->thread->stats.mutex);
    c->thread->stats.get_extstore++;
    pthread_mutex_unlock(&c->thread->stats.mutex);

    return 0;
}


static void process_stats_conns(ADD_STAT add_stats, void *c) {
    int i;
    char key_str[STAT_KEY_LEN];
    char val_str[STAT_VAL_LEN];
    size_t extras_len = sizeof("unix:") + sizeof("65535");
    char addr[MAXPATHLEN + extras_len];
    char svr_addr[MAXPATHLEN + extras_len];
    int klen = 0, vlen = 0;

    assert(add_stats);

    for (i = 0; i < max_fds; i++) {
        if (conns[i]) {
            /* This is safe to do unlocked because conns are never freed; the
             * worst that'll happen will be a minor inconsistency in the
             * output -- not worth the complexity of the locking that'd be
             * required to prevent it.
             */
            if (IS_UDP(conns[i]->transport)) {
                APPEND_NUM_STAT(i, "UDP", "%s", "UDP");
            }
            if (conns[i]->state != conn_closed) {
                conn_to_str(conns[i], addr, svr_addr);

                APPEND_NUM_STAT(i, "addr", "%s", addr);
                if (conns[i]->state != conn_listening &&
                    !(IS_UDP(conns[i]->transport) && conns[i]->state == conn_read)) {
                    APPEND_NUM_STAT(i, "listen_addr", "%s", svr_addr);
                }
                APPEND_NUM_STAT(i, "state", "%s",
                        state_text(conns[i]->state));
                APPEND_NUM_STAT(i, "secs_since_last_cmd", "%d",
                        current_time - conns[i]->last_cmd_time);
            }
        }
    }
}


conn *conn_new(const int sfd, enum conn_states init_state,
                const int event_flags,
                const int read_buffer_size, enum network_transport transport,
                struct event_base *base, void *ssl) {
    conn *c;

    assert(sfd >= 0 && sfd < max_fds);
    c = conns[sfd];

    if (NULL == c) {
        if (!(c = (conn *)calloc(1, sizeof(conn)))) {
            STATS_LOCK();
            stats.malloc_fails++;
            STATS_UNLOCK();
            fprintf(stderr, "Failed to allocate connection object\n");
            return NULL;
        }
        MEMCACHED_CONN_CREATE(c);
        c->read = NULL;
        c->sendmsg = NULL;
        c->write = NULL;
        c->rbuf = c->wbuf = 0;
        c->ilist = 0;
        c->suffixlist = 0;
        c->iov = 0;
        c->msglist = 0;
        c->hdrbuf = 0;

        c->rsize = read_buffer_size;
        c->wsize = DATA_BUFFER_SIZE;
        c->isize = ITEM_LIST_INITIAL;
        c->suffixsize = SUFFIX_LIST_INITIAL;
        c->iovsize = IOV_LIST_INITIAL;
        c->msgsize = MSG_LIST_INITIAL;
        c->hdrsize = 0;

        c->rbuf = (char *)malloc((size_t)c->rsize);
        c->wbuf = (char *)malloc((size_t)c->wsize);
        c->ilist = (item **)malloc(sizeof(item *) * c->isize);
        c->suffixlist = (char **)malloc(sizeof(char *) * c->suffixsize);
        c->iov = (struct iovec *)malloc(sizeof(struct iovec) * c->iovsize);
        c->msglist = (struct msghdr *)malloc(sizeof(struct msghdr) * c->msgsize);

        if (c->rbuf == 0 || c->wbuf == 0 || c->ilist == 0 || c->iov == 0 ||
                c->msglist == 0 || c->suffixlist == 0) {
            conn_free(c);
            STATS_LOCK();
            stats.malloc_fails++;
            STATS_UNLOCK();
            fprintf(stderr, "Failed to allocate buffers for connection\n");
            return NULL;
        }

        STATS_LOCK();
        stats_state.conn_structs++;
        STATS_UNLOCK();

        c->sfd = sfd;
        conns[sfd] = c;
    }

    c->transport = transport;
    c->protocol = settings.binding_protocol;

    /* unix socket mode doesn't need this, so zeroed out.  but why
     * is this done for every command?  presumably for UDP
     * mode.  */
    if (!settings.socketpath) {
        c->request_addr_size = sizeof(c->request_addr);
    } else {
        c->request_addr_size = 0;
    }

    if (transport == tcp_transport && init_state == conn_new_cmd) {
        if (getpeername(sfd, (struct sockaddr *) &c->request_addr,
                        &c->request_addr_size)) {
            perror("getpeername");
            memset(&c->request_addr, 0, sizeof(c->request_addr));
        }
    }

    if (settings.verbose > 1) {
        if (init_state == conn_listening) {
            fprintf(stderr, "<%d server listening (%s)\n", sfd,
                prot_text(c->protocol));
        } else if (IS_UDP(transport)) {
            fprintf(stderr, "<%d server listening (udp)\n", sfd);
        } else if (c->protocol == negotiating_prot) {
            fprintf(stderr, "<%d new auto-negotiating client connection\n",
                    sfd);
        } else if (c->protocol == ascii_prot) {
            fprintf(stderr, "<%d new ascii client connection.\n", sfd);
        } else if (c->protocol == binary_prot) {
            fprintf(stderr, "<%d new binary client connection.\n", sfd);
        } else {
            fprintf(stderr, "<%d new unknown (%d) client connection\n",
                sfd, c->protocol);
            assert(false);
        }
    }

#ifdef TLS
    c->ssl = NULL;
    c->ssl_wbuf = NULL;
    c->ssl_enabled = false;
#endif
    c->state = init_state;
    c->rlbytes = 0;
    c->cmd = -1;
    c->rbytes = c->wbytes = 0;
    c->wcurr = c->wbuf;
    c->rcurr = c->rbuf;
    c->ritem = 0;
    c->icurr = c->ilist;
    c->suffixcurr = c->suffixlist;
    c->ileft = 0;
    c->suffixleft = 0;
    c->iovused = 0;
    c->msgcurr = 0;
    c->msgused = 0;
    c->sasl_started = false;
    c->last_cmd_time = current_time; /* initialize for idle kicker */
#ifdef EXTSTORE
    c->io_wraplist = NULL;
    c->io_wrapleft = 0;
#endif

    c->write_and_go = init_state;
    c->write_and_free = 0;
    c->item = 0;

    c->noreply = false;

#ifdef TLS
    if (ssl) {
        c->ssl = (SSL*)ssl;
        c->read = ssl_read;
        c->sendmsg = ssl_sendmsg;
        c->write = ssl_write;
        c->ssl_enabled = true;
        SSL_set_info_callback(c->ssl, ssl_callback);
    } else
#else
    // This must be NULL if TLS is not enabled.
    assert(ssl == NULL);
#endif
    {
        c->read = tcp_read;
        c->sendmsg = tcp_sendmsg;
        c->write = tcp_write;
    }

    if (IS_UDP(transport)) {
        c->try_read_command = try_read_command_udp;
    } else {
        switch (c->protocol) {
            case ascii_prot:
                if (settings.auth_file == NULL) {
                    c->authenticated = true;
                    c->try_read_command = try_read_command_ascii;
                } else {
                    c->authenticated = false;
                    c->try_read_command = try_read_command_asciiauth;
                }
                break;
            case binary_prot:
                // binprot handles its own authentication via SASL parsing.
                c->authenticated = false;
                c->try_read_command = try_read_command_binary;
                break;
            case negotiating_prot:
                c->try_read_command = try_read_command_negotiate;
                break;
        }
    }

    event_set(&c->event, sfd, event_flags, event_handler, (void *)c);
    event_base_set(base, &c->event);
    c->ev_flags = event_flags;

    if (event_add(&c->event, 0) == -1) {
        perror("event_add");
        return NULL;
    }

    STATS_LOCK();
    stats_state.curr_conns++;
    stats.total_conns++;
    STATS_UNLOCK();

    MEMCACHED_CONN_ALLOCATE(c->sfd);

    return c;
}


HTTP1_DissectRequest(struct http_conn *htc, struct http *hp)
{
	uint16_t retval;
	const char *p;
	const char *b = NULL, *e;

	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);

	retval = http1_splitline(hp, htc,
	    HTTP1_Req, cache_param->http_req_hdr_len);
	if (retval != 0)
		return (retval);

	if (hp->protover < 10 || hp->protover > 11)
		return (400);

	if (http_CountHdr(hp, H_Host) > 1)
		return (400);

	if (http_CountHdr(hp, H_Content_Length) > 1)
		return (400);

	/* RFC2616, section 5.2, point 1 */
	if (!strncasecmp(hp->hd[HTTP_HDR_URL].b, "http://", 7))
		b = hp->hd[HTTP_HDR_URL].b + 7;
	else if (FEATURE(FEATURE_HTTPS_SCHEME) &&
	    !strncasecmp(hp->hd[HTTP_HDR_URL].b, "https://", 8))
		b = hp->hd[HTTP_HDR_URL].b + 8;
	if (b) {
		e = strchr(b, '/');
		if (e) {
			http_Unset(hp, H_Host);
			http_PrintfHeader(hp, "Host: %.*s", (int)(e - b), b);
			hp->hd[HTTP_HDR_URL].b = e;
		}
	}

	htc->body_status = http1_body_status(hp, htc, 1);
	if (htc->body_status == BS_ERROR)
		return (400);

	p = http_GetMethod(hp);
	AN(p);

	if (htc->body_status == BS_EOF) {
		assert(hp->protover == 10);
		/* RFC1945 8.3 p32 and D.1.1 p58 */
		if (!strcasecmp(p, "post") || !strcasecmp(p, "put"))
			return (400);
		htc->body_status = BS_NONE;
	}

	/* HEAD with a body is a hard error */
	if (htc->body_status != BS_NONE && !strcasecmp(p, "head"))
		return (400);

	return (retval);
}


http1_dissect_hdrs(struct http *hp, char *p, struct http_conn *htc,
    unsigned maxhdr)
{
	char *q, *r, *s;
	int i;

	assert(p > htc->rxbuf_b);
	assert(p <= htc->rxbuf_e);
	hp->nhd = HTTP_HDR_FIRST;
	r = NULL;		/* For FlexeLint */
	for (; p < htc->rxbuf_e; p = r) {

		/* Find end of next header */
		q = r = p;
		if (vct_iscrlf(p, htc->rxbuf_e))
			break;
		while (r < htc->rxbuf_e) {
			if (!vct_isctl(*r) || vct_issp(*r)) {
				r++;
				continue;
			}
			i = vct_iscrlf(r, htc->rxbuf_e);
			if (i == 0) {
				VSLb(hp->vsl, SLT_BogoHeader,
				    "Header has ctrl char 0x%02x", *r);
				return (400);
			}
			q = r;
			r += i;
			assert(r <= htc->rxbuf_e);
			if (r == htc->rxbuf_e)
				break;
			if (vct_iscrlf(r, htc->rxbuf_e))
				break;
			/* If line does not continue: got it. */
			if (!vct_issp(*r))
				break;

			/* Clear line continuation LWS to spaces */
			while (q < r)
				*q++ = ' ';
			while (q < htc->rxbuf_e && vct_issp(*q))
				*q++ = ' ';
		}

		/* Empty header = end of headers */
		if (p == q)
			break;

		if (q - p > maxhdr) {
			VSLb(hp->vsl, SLT_BogoHeader, "Header too long: %.*s",
			    (int)(q - p > 20 ? 20 : q - p), p);
			return (400);
		}

		if (vct_islws(*p)) {
			VSLb(hp->vsl, SLT_BogoHeader,
			    "1st header has white space: %.*s",
			    (int)(q - p > 20 ? 20 : q - p), p);
			return (400);
		}

		if (*p == ':') {
			VSLb(hp->vsl, SLT_BogoHeader,
			    "Missing header name: %.*s",
			    (int)(q - p > 20 ? 20 : q - p), p);
			return (400);
		}

		while (q > p && vct_issp(q[-1]))
			q--;
		*q = '\0';

		for (s = p; *s != ':' && s < q; s++) {
			if (!vct_istchar(*s)) {
				VSLb(hp->vsl, SLT_BogoHeader,
				    "Illegal char 0x%02x in header name", *s);
				return (400);
			}
		}
		if (*s != ':') {
			VSLb(hp->vsl, SLT_BogoHeader, "Header without ':' %.*s",
			    (int)(q - p > 20 ? 20 : q - p), p);
			return (400);
		}

		if (hp->nhd < hp->shd) {
			hp->hdf[hp->nhd] = 0;
			hp->hd[hp->nhd].b = p;
			hp->hd[hp->nhd].e = q;
			hp->nhd++;
		} else {
			VSLb(hp->vsl, SLT_BogoHeader, "Too many headers: %.*s",
			    (int)(q - p > 20 ? 20 : q - p), p);
			return (400);
		}
	}
	i = vct_iscrlf(p, htc->rxbuf_e);
	assert(i > 0);		/* HTTP1_Complete guarantees this */
	p += i;
	HTC_RxPipeline(htc, p);
	htc->rxbuf_e = p;
	return (0);
}


string_vformat(gstring * g, BOOL extend, const char *format, va_list ap)
{
enum ltypes { L_NORMAL=1, L_SHORT=2, L_LONG=3, L_LONGLONG=4, L_LONGDOUBLE=5, L_SIZE=6 };

int width, precision, off, lim;
const char * fp = format;	/* Deliberately not unsigned */

string_datestamp_offset = -1;	/* Datestamp not inserted */
string_datestamp_length = 0;	/* Datestamp not inserted */
string_datestamp_type = 0;	/* Datestamp not inserted */

#ifdef COMPILE_UTILITY
assert(!extend);
assert(g);
#else

/* Ensure we have a string, to save on checking later */
if (!g) g = string_get(16);
#endif	/*!COMPILE_UTILITY*/

lim = g->size - 1;	/* leave one for a nul */
off = g->ptr;		/* remember initial offset in gstring */

/* Scan the format and handle the insertions */

while (*fp)
  {
  int length = L_NORMAL;
  int *nptr;
  int slen;
  const char *null = "NULL";		/* ) These variables */
  const char *item_start, *s;		/* ) are deliberately */
  char newformat[16];			/* ) not unsigned */
  char * gp = CS g->s + g->ptr;		/* ) */

  /* Non-% characters just get copied verbatim */

  if (*fp != '%')
    {
    /* Avoid string_copyn() due to COMPILE_UTILITY */
    if (g->ptr >= lim - 1)
      {
      if (!extend) return NULL;
      gstring_grow(g, g->ptr, 1);
      lim = g->size - 1;
      }
    g->s[g->ptr++] = (uschar) *fp++;
    continue;
    }

  /* Deal with % characters. Pick off the width and precision, for checking
  strings, skipping over the flag and modifier characters. */

  item_start = fp;
  width = precision = -1;

  if (strchr("-+ #0", *(++fp)) != NULL)
    {
    if (*fp == '#') null = "";
    fp++;
    }

  if (isdigit((uschar)*fp))
    {
    width = *fp++ - '0';
    while (isdigit((uschar)*fp)) width = width * 10 + *fp++ - '0';
    }
  else if (*fp == '*')
    {
    width = va_arg(ap, int);
    fp++;
    }

  if (*fp == '.')
    if (*(++fp) == '*')
      {
      precision = va_arg(ap, int);
      fp++;
      }
    else
      for (precision = 0; isdigit((uschar)*fp); fp++)
        precision = precision*10 + *fp - '0';

  /* Skip over 'h', 'L', 'l', 'll' and 'z', remembering the item length */

  if (*fp == 'h')
    { fp++; length = L_SHORT; }
  else if (*fp == 'L')
    { fp++; length = L_LONGDOUBLE; }
  else if (*fp == 'l')
    if (fp[1] == 'l')
      { fp += 2; length = L_LONGLONG; }
    else
      { fp++; length = L_LONG; }
  else if (*fp == 'z')
    { fp++; length = L_SIZE; }

  /* Handle each specific format type. */

  switch (*fp++)
    {
    case 'n':
      nptr = va_arg(ap, int *);
      *nptr = g->ptr - off;
      break;

    case 'd':
    case 'o':
    case 'u':
    case 'x':
    case 'X':
      width = length > L_LONG ? 24 : 12;
      if (g->ptr >= lim - width)
	{
	if (!extend) return NULL;
	gstring_grow(g, g->ptr, width);
	lim = g->size - 1;
	gp = CS g->s + g->ptr;
	}
      strncpy(newformat, item_start, fp - item_start);
      newformat[fp - item_start] = 0;

      /* Short int is promoted to int when passing through ..., so we must use
      int for va_arg(). */

      switch(length)
	{
	case L_SHORT:
	case L_NORMAL:
	  g->ptr += sprintf(gp, newformat, va_arg(ap, int)); break;
	case L_LONG:
	  g->ptr += sprintf(gp, newformat, va_arg(ap, long int)); break;
	case L_LONGLONG:
	  g->ptr += sprintf(gp, newformat, va_arg(ap, LONGLONG_T)); break;
	case L_SIZE:
	  g->ptr += sprintf(gp, newformat, va_arg(ap, size_t)); break;
	}
      break;

    case 'p':
      {
      void * ptr;
      if (g->ptr >= lim - 24)
	{
	if (!extend) return NULL;
	gstring_grow(g, g->ptr, 24);
	lim = g->size - 1;
	gp = CS g->s + g->ptr;
	}
      /* sprintf() saying "(nil)" for a null pointer seems unreliable.
      Handle it explicitly. */
      if ((ptr = va_arg(ap, void *)))
	{
	strncpy(newformat, item_start, fp - item_start);
	newformat[fp - item_start] = 0;
	g->ptr += sprintf(gp, newformat, ptr);
	}
      else
	g->ptr += sprintf(gp, "(nil)");
      }
    break;

    /* %f format is inherently insecure if the numbers that it may be
    handed are unknown (e.g. 1e300). However, in Exim, %f is used for
    printing load averages, and these are actually stored as integers
    (load average * 1000) so the size of the numbers is constrained.
    It is also used for formatting sending rates, where the simplicity
    of the format prevents overflow. */

    case 'f':
    case 'e':
    case 'E':
    case 'g':
    case 'G':
      if (precision < 0) precision = 6;
      if (g->ptr >= lim - precision - 8)
	{
	if (!extend) return NULL;
	gstring_grow(g, g->ptr, precision+8);
	lim = g->size - 1;
	gp = CS g->s + g->ptr;
	}
      strncpy(newformat, item_start, fp - item_start);
      newformat[fp-item_start] = 0;
      if (length == L_LONGDOUBLE)
	g->ptr += sprintf(gp, newformat, va_arg(ap, long double));
      else
	g->ptr += sprintf(gp, newformat, va_arg(ap, double));
      break;

    /* String types */

    case '%':
      if (g->ptr >= lim - 1)
	{
	if (!extend) return NULL;
	gstring_grow(g, g->ptr, 1);
	lim = g->size - 1;
	}
      g->s[g->ptr++] = (uschar) '%';
      break;

    case 'c':
      if (g->ptr >= lim - 1)
	{
	if (!extend) return NULL;
	gstring_grow(g, g->ptr, 1);
	lim = g->size - 1;
	}
      g->s[g->ptr++] = (uschar) va_arg(ap, int);
      break;

    case 'D':                   /* Insert daily datestamp for log file names */
      s = CS tod_stamp(tod_log_datestamp_daily);
      string_datestamp_offset = g->ptr;		/* Passed back via global */
      string_datestamp_length = Ustrlen(s);	/* Passed back via global */
      string_datestamp_type = tod_log_datestamp_daily;
      slen = string_datestamp_length;
      goto INSERT_STRING;

    case 'M':                   /* Insert monthly datestamp for log file names */
      s = CS tod_stamp(tod_log_datestamp_monthly);
      string_datestamp_offset = g->ptr;		/* Passed back via global */
      string_datestamp_length = Ustrlen(s);	/* Passed back via global */
      string_datestamp_type = tod_log_datestamp_monthly;
      slen = string_datestamp_length;
      goto INSERT_STRING;

    case 's':
    case 'S':                   /* Forces *lower* case */
    case 'T':                   /* Forces *upper* case */
      s = va_arg(ap, char *);

      if (!s) s = null;
      slen = Ustrlen(s);

    INSERT_STRING:              /* Come to from %D or %M above */

      {
      BOOL truncated = FALSE;

      /* If the width is specified, check that there is a precision
      set; if not, set it to the width to prevent overruns of long
      strings. */

      if (width >= 0)
	{
	if (precision < 0) precision = width;
	}

      /* If a width is not specified and the precision is specified, set
      the width to the precision, or the string length if shorted. */

      else if (precision >= 0)
	width = precision < slen ? precision : slen;

      /* If neither are specified, set them both to the string length. */

      else
	width = precision = slen;

      if (!extend)
	{
	if (g->ptr == lim) return NULL;
	if (g->ptr >= lim - width)
	  {
	  truncated = TRUE;
	  width = precision = lim - g->ptr - 1;
	  if (width < 0) width = 0;
	  if (precision < 0) precision = 0;
	  }
	}
      else if (g->ptr >= lim - width)
	{
	gstring_grow(g, g->ptr, width);
	lim = g->size - 1;
	gp = CS g->s + g->ptr;
	}

      g->ptr += sprintf(gp, "%*.*s", width, precision, s);
      if (fp[-1] == 'S')
	while (*gp) { *gp = tolower(*gp); gp++; }
      else if (fp[-1] == 'T')
	while (*gp) { *gp = toupper(*gp); gp++; }

      if (truncated) return NULL;
      break;
      }

    /* Some things are never used in Exim; also catches junk. */

    default:
      strncpy(newformat, item_start, fp - item_start);
      newformat[fp-item_start] = 0;
      log_write(0, LOG_MAIN|LOG_PANIC_DIE, "string_format: unsupported type "
	"in \"%s\" in \"%s\"", newformat, format);
      break;
    }
  }

return g;
}


handle_single_connection (int sockin, int sockout)
{
  int r;

  lock_connection ();
  r = _handle_single_connection (sockin, sockout);
  unlock_connection ();

  return r;
}


negotiate_handshake_newstyle_options (struct connection *conn)
{
  struct new_option new_option;
  size_t nr_options;
  uint64_t version;
  uint32_t option;
  uint32_t optlen;
  char data[MAX_OPTION_LENGTH+1];
  struct new_handshake_finish handshake_finish;
  const char *optname;
  uint64_t exportsize;

  for (nr_options = 0; nr_options < MAX_NR_OPTIONS; ++nr_options) {
    if (conn_recv_full (conn, &new_option, sizeof new_option,
                        "reading option: conn->recv: %m") == -1)
      return -1;

    version = be64toh (new_option.version);
    if (version != NEW_VERSION) {
      nbdkit_error ("unknown option version %" PRIx64
                    ", expecting %" PRIx64,
                    version, NEW_VERSION);
      return -1;
    }

    /* There is a maximum option length we will accept, regardless
     * of the option type.
     */
    optlen = be32toh (new_option.optlen);
    if (optlen > MAX_OPTION_LENGTH) {
      nbdkit_error ("client option data too long (%" PRIu32 ")", optlen);
      return -1;
    }

    option = be32toh (new_option.option);
    optname = name_of_nbd_opt (option);

    /* If the client lacks fixed newstyle support, it should only send
     * NBD_OPT_EXPORT_NAME.
     */
    if (!(conn->cflags & NBD_FLAG_FIXED_NEWSTYLE) &&
        option != NBD_OPT_EXPORT_NAME) {
      if (send_newstyle_option_reply (conn, option, NBD_REP_ERR_INVALID))
        return -1;
      continue;
    }

    /* In --tls=require / FORCEDTLS mode the only options allowed
     * before TLS negotiation are NBD_OPT_ABORT and NBD_OPT_STARTTLS.
     */
    if (tls == 2 && !conn->using_tls &&
        !(option == NBD_OPT_ABORT || option == NBD_OPT_STARTTLS)) {
      if (send_newstyle_option_reply (conn, option, NBD_REP_ERR_TLS_REQD))
        return -1;
      continue;
    }

    switch (option) {
    case NBD_OPT_EXPORT_NAME:
      if (conn_recv_full (conn, data, optlen,
                          "read: %s: %m", name_of_nbd_opt (option)) == -1)
        return -1;
      /* Apart from printing it, ignore the export name. */
      data[optlen] = '\0';
      debug ("newstyle negotiation: %s: "
             "client requested export '%s' (ignored)",
             name_of_nbd_opt (option), data);

      /* We have to finish the handshake by sending handshake_finish. */
      if (finish_newstyle_options (conn, &exportsize) == -1)
        return -1;

      memset (&handshake_finish, 0, sizeof handshake_finish);
      handshake_finish.exportsize = htobe64 (exportsize);
      handshake_finish.eflags = htobe16 (conn->eflags);

      if (conn->send (conn,
                      &handshake_finish,
                      (conn->cflags & NBD_FLAG_NO_ZEROES)
                      ? offsetof (struct new_handshake_finish, zeroes)
                      : sizeof handshake_finish, 0) == -1) {
        nbdkit_error ("write: %s: %m", optname);
        return -1;
      }
      break;

    case NBD_OPT_ABORT:
      if (send_newstyle_option_reply (conn, option, NBD_REP_ACK) == -1)
        return -1;
      debug ("client sent %s to abort the connection",
             name_of_nbd_opt (option));
      return -1;

    case NBD_OPT_LIST:
      if (optlen != 0) {
        if (send_newstyle_option_reply (conn, option, NBD_REP_ERR_INVALID)
            == -1)
          return -1;
        if (conn_recv_full (conn, data, optlen,
                            "read: %s: %m", name_of_nbd_opt (option)) == -1)
          return -1;
        continue;
      }

      /* Send back the exportname. */
      debug ("newstyle negotiation: %s: advertising export '%s'",
             name_of_nbd_opt (option), exportname);
      if (send_newstyle_option_reply_exportname (conn, option,
                                                 NBD_REP_SERVER) == -1)
        return -1;

      if (send_newstyle_option_reply (conn, option, NBD_REP_ACK) == -1)
        return -1;
      break;

    case NBD_OPT_STARTTLS:
      if (optlen != 0) {
        if (send_newstyle_option_reply (conn, option, NBD_REP_ERR_INVALID)
            == -1)
          return -1;
        if (conn_recv_full (conn, data, optlen,
                            "read: %s: %m", name_of_nbd_opt (option)) == -1)
          return -1;
        continue;
      }

      if (tls == 0) {           /* --tls=off (NOTLS mode). */
#ifdef HAVE_GNUTLS
#define NO_TLS_REPLY NBD_REP_ERR_POLICY
#else
#define NO_TLS_REPLY NBD_REP_ERR_UNSUP
#endif
        if (send_newstyle_option_reply (conn, option, NO_TLS_REPLY) == -1)
          return -1;
      }
      else /* --tls=on or --tls=require */ {
        /* We can't upgrade to TLS twice on the same connection. */
        if (conn->using_tls) {
          if (send_newstyle_option_reply (conn, option,
                                          NBD_REP_ERR_INVALID) == -1)
            return -1;
          continue;
        }

        /* We have to send the (unencrypted) reply before starting
         * the handshake.
         */
        if (send_newstyle_option_reply (conn, option, NBD_REP_ACK) == -1)
          return -1;

        /* Upgrade the connection to TLS.  Also performs access control. */
        if (crypto_negotiate_tls (conn, conn->sockin, conn->sockout) == -1)
          return -1;
        conn->using_tls = true;
        debug ("using TLS on this connection");
      }
      break;

    case NBD_OPT_INFO:
    case NBD_OPT_GO:
      if (conn_recv_full (conn, data, optlen,
                          "read: %s: %m", optname) == -1)
        return -1;

      if (optlen < 6) { /* 32 bit export length + 16 bit nr info */
        debug ("newstyle negotiation: %s option length < 6", optname);

        if (send_newstyle_option_reply (conn, option, NBD_REP_ERR_INVALID)
            == -1)
          return -1;
        continue;
      }

      {
        uint32_t exportnamelen;
        uint16_t nrinfos;
        uint16_t info;
        size_t i;
        CLEANUP_FREE char *requested_exportname = NULL;

        /* Validate the name length and number of INFO requests. */
        memcpy (&exportnamelen, &data[0], 4);
        exportnamelen = be32toh (exportnamelen);
        if (exportnamelen > optlen-6 /* NB optlen >= 6, see above */) {
          debug ("newstyle negotiation: %s: export name too long", optname);
          if (send_newstyle_option_reply (conn, option, NBD_REP_ERR_INVALID)
              == -1)
            return -1;
          continue;
        }
        memcpy (&nrinfos, &data[exportnamelen+4], 2);
        nrinfos = be16toh (nrinfos);
        if (optlen != 4 + exportnamelen + 2 + 2*nrinfos) {
          debug ("newstyle negotiation: %s: "
                 "number of information requests incorrect", optname);
          if (send_newstyle_option_reply (conn, option, NBD_REP_ERR_INVALID)
              == -1)
            return -1;
          continue;
        }

        /* As with NBD_OPT_EXPORT_NAME we print the export name and then
         * ignore it.
         */
        requested_exportname = malloc (exportnamelen+1);
        if (requested_exportname == NULL) {
          nbdkit_error ("malloc: %m");
          return -1;
        }
        memcpy (requested_exportname, &data[4], exportnamelen);
        requested_exportname[exportnamelen] = '\0';
        debug ("newstyle negotiation: %s: "
               "client requested export '%s' (ignored)",
               optname, requested_exportname);

        /* The spec is confusing, but it is required that we send back
         * NBD_INFO_EXPORT, even if the client did not request it!
         * qemu client in particular does not request this, but will
         * fail if we don't send it.
         */
        if (finish_newstyle_options (conn, &exportsize) == -1)
          return -1;

        if (send_newstyle_option_reply_info_export (conn, option,
                                                    NBD_REP_INFO,
                                                    NBD_INFO_EXPORT,
                                                    exportsize) == -1)
          return -1;

        /* For now we ignore all other info requests (but we must
         * ignore NBD_INFO_EXPORT if it was requested, because we
         * replied already above).  Therefore this loop doesn't do
         * much at the moment.
         */
        for (i = 0; i < nrinfos; ++i) {
          memcpy (&info, &data[4 + exportnamelen + 2 + i*2], 2);
          info = be16toh (info);
          switch (info) {
          case NBD_INFO_EXPORT: /* ignore - reply sent above */ break;
          default:
            debug ("newstyle negotiation: %s: "
                   "ignoring NBD_INFO_* request %u (%s)",
                   optname, (unsigned) info, name_of_nbd_info (info));
            break;
          }
        }
      }

      /* Unlike NBD_OPT_EXPORT_NAME, NBD_OPT_GO sends back an ACK
       * or ERROR packet.
       */
      if (send_newstyle_option_reply (conn, option, NBD_REP_ACK) == -1)
        return -1;

      break;

    case NBD_OPT_STRUCTURED_REPLY:
      if (optlen != 0) {
        if (send_newstyle_option_reply (conn, option, NBD_REP_ERR_INVALID)
            == -1)
          return -1;
        if (conn_recv_full (conn, data, optlen,
                            "read: %s: %m", name_of_nbd_opt (option)) == -1)
          return -1;
        continue;
      }

      debug ("newstyle negotiation: %s: client requested structured replies",
             name_of_nbd_opt (option));

      if (no_sr) {
        /* Must fail with ERR_UNSUP for qemu 4.2 to remain happy;
         * but failing with ERR_POLICY would have been nicer.
         */
        if (send_newstyle_option_reply (conn, option, NBD_REP_ERR_UNSUP) == -1)
          return -1;
        debug ("newstyle negotiation: %s: structured replies are disabled",
               name_of_nbd_opt (option));
        break;
      }

      if (send_newstyle_option_reply (conn, option, NBD_REP_ACK) == -1)
        return -1;

      conn->structured_replies = true;
      break;

    case NBD_OPT_LIST_META_CONTEXT:
    case NBD_OPT_SET_META_CONTEXT:
      {
        uint32_t opt_index;
        uint32_t exportnamelen;
        uint32_t nr_queries;
        uint32_t querylen;
        const char *what;

        if (conn_recv_full (conn, data, optlen, "read: %s: %m", optname) == -1)
          return -1;

        /* Note that we support base:allocation whether or not the plugin
         * supports can_extents.
         */
        if (!conn->structured_replies) {
          if (send_newstyle_option_reply (conn, option, NBD_REP_ERR_INVALID)
              == -1)
            return -1;
          continue;
        }

        /* Minimum length of the option payload is:
         *   32 bit export name length followed by empty export name
         * + 32 bit number of queries followed by no queries
         * = 8 bytes.
         */
        what = "optlen < 8";
        if (optlen < 8) {
        opt_meta_invalid_option_len:
          debug ("newstyle negotiation: %s: invalid option length: %s",
                 optname, what);

          if (send_newstyle_option_reply (conn, option, NBD_REP_ERR_INVALID)
              == -1)
            return -1;
          continue;
        }

        /* Discard the export name. */
        memcpy (&exportnamelen, &data[0], 4);
        exportnamelen = be32toh (exportnamelen);
        opt_index = 4 + exportnamelen;

        /* Read the number of queries. */
        what = "reading number of queries";
        if (opt_index+4 > optlen)
          goto opt_meta_invalid_option_len;
        memcpy (&nr_queries, &data[opt_index], 4);
        nr_queries = be32toh (nr_queries);
        opt_index += 4;

        /* for LIST: nr_queries == 0 means return all meta contexts
         * for SET: nr_queries == 0 means reset all contexts
         */
        debug ("newstyle negotiation: %s: %s count: %d", optname,
               option == NBD_OPT_LIST_META_CONTEXT ? "query" : "set",
               nr_queries);
        if (nr_queries == 0) {
          if (option == NBD_OPT_SET_META_CONTEXT)
            conn->meta_context_base_allocation = false;
          else /* LIST */ {
            if (send_newstyle_option_reply_meta_context
                (conn, option, NBD_REP_META_CONTEXT,
                 0, "base:allocation") == -1)
              return -1;
          }

          if (send_newstyle_option_reply (conn, option, NBD_REP_ACK) == -1)
            return -1;
        }
        else {
          /* Read and answer each query. */
          while (nr_queries > 0) {
            what = "reading query string length";
            if (opt_index+4 > optlen)
              goto opt_meta_invalid_option_len;
            memcpy (&querylen, &data[opt_index], 4);
            querylen = be32toh (querylen);
            opt_index += 4;
            what = "reading query string";
            if (opt_index + querylen > optlen)
              goto opt_meta_invalid_option_len;

            debug ("newstyle negotiation: %s: %s %.*s",
                   optname,
                   option == NBD_OPT_LIST_META_CONTEXT ? "query" : "set",
                   (int) querylen, &data[opt_index]);

            /* For LIST, "base:" returns all supported contexts in the
             * base namespace.  We only support "base:allocation".
             */
            if (option == NBD_OPT_LIST_META_CONTEXT &&
                querylen == 5 &&
                strncmp (&data[opt_index], "base:", 5) == 0) {
              if (send_newstyle_option_reply_meta_context
                  (conn, option, NBD_REP_META_CONTEXT,
                   0, "base:allocation") == -1)
                return -1;
            }
            /* "base:allocation" requested by name. */
            else if (querylen == 15 &&
                     strncmp (&data[opt_index], "base:allocation", 15) == 0) {
              if (send_newstyle_option_reply_meta_context
                  (conn, option, NBD_REP_META_CONTEXT,
                   option == NBD_OPT_SET_META_CONTEXT
                   ? base_allocation_id : 0,
                   "base:allocation") == -1)
                return -1;
              if (option == NBD_OPT_SET_META_CONTEXT)
                conn->meta_context_base_allocation = true;
            }
            /* Every other query must be ignored. */

            opt_index += querylen;
            nr_queries--;
          }
          if (send_newstyle_option_reply (conn, option, NBD_REP_ACK) == -1)
            return -1;
        }
        debug ("newstyle negotiation: %s: reply complete", optname);
      }
      break;

    default:
      /* Unknown option. */
      if (send_newstyle_option_reply (conn, option, NBD_REP_ERR_UNSUP) == -1)
        return -1;
      if (conn_recv_full (conn, data, optlen,
                          "reading unknown option data: conn->recv: %m") == -1)
        return -1;
    }

    /* Note, since it's not very clear from the protocol doc, that the
     * client must send NBD_OPT_EXPORT_NAME or NBD_OPT_GO last, and
     * that ends option negotiation.
     */
    if (option == NBD_OPT_EXPORT_NAME || option == NBD_OPT_GO)
      break;
  }

  if (nr_options >= MAX_NR_OPTIONS) {
    nbdkit_error ("client exceeded maximum number of options (%d)",
                  MAX_NR_OPTIONS);
    return -1;
  }

  /* In --tls=require / FORCEDTLS mode, we must have upgraded to TLS
   * by the time we finish option negotiation.  If not, give up.
   */
  if (tls == 2 && !conn->using_tls) {
    nbdkit_error ("non-TLS client tried to connect in --tls=require mode");
    return -1;
  }

  return 0;
}


send_newstyle_option_reply_meta_context (struct connection *conn,
                                         uint32_t option, uint32_t reply,
                                         uint32_t context_id,
                                         const char *name)
{
  struct fixed_new_option_reply fixed_new_option_reply;
  struct fixed_new_option_reply_meta_context context;
  const size_t namelen = strlen (name);

  debug ("newstyle negotiation: %s: replying with %s id %d",
         name_of_nbd_opt (option), name, context_id);
  fixed_new_option_reply.magic = htobe64 (NBD_REP_MAGIC);
  fixed_new_option_reply.option = htobe32 (option);
  fixed_new_option_reply.reply = htobe32 (reply);
  fixed_new_option_reply.replylen = htobe32 (sizeof context + namelen);
  context.context_id = htobe32 (context_id);

  if (conn->send (conn,
                  &fixed_new_option_reply,
                  sizeof fixed_new_option_reply, SEND_MORE) == -1 ||
      conn->send (conn, &context, sizeof context, SEND_MORE) == -1 ||
      conn->send (conn, name, namelen, 0) == -1) {
    nbdkit_error ("write: %s: %m", name_of_nbd_opt (option));
    return -1;
  }

  return 0;
}


set_debug_flags (void *dl, const char *name)
{
  struct debug_flag *flag;

  for (flag = debug_flags; flag != NULL; flag = flag->next) {
    if (!flag->used && strcmp (name, flag->name) == 0) {
      CLEANUP_FREE char *var = NULL;
      int *sym;

      /* Synthesize the name of the variable. */
      if (asprintf (&var, "%s_debug_%s", name, flag->flag) == -1) {
        perror ("asprintf");
        exit (EXIT_FAILURE);
      }

      /* Find the symbol. */
      sym = dlsym (dl, var);
      if (sym == NULL) {
        fprintf (stderr,
                 "%s: -D %s.%s: %s does not contain a "
                 "global variable called %s\n",
                 program_name, name, flag->flag, name, var);
        exit (EXIT_FAILURE);
      }

      /* Set the flag. */
      *sym = flag->value;

      /* Mark this flag as used. */
      flag->used = true;
    }
  }
}


next_can_fua (void *nxdata)
{
  struct b_conn *b_conn = nxdata;
  return backend_can_fua (b_conn->b, b_conn->conn);
}


plugin_zero (struct backend *b, struct connection *conn,
             uint32_t count, uint64_t offset, uint32_t flags, int *err)
{
  struct backend_plugin *p = container_of (b, struct backend_plugin, backend);
  int r = -1;
  bool may_trim = flags & NBDKIT_FLAG_MAY_TRIM;
  bool fua = flags & NBDKIT_FLAG_FUA;
  bool fast_zero = flags & NBDKIT_FLAG_FAST_ZERO;
  bool emulate = false;
  bool need_flush = false;

  assert (connection_get_handle (conn, 0));

  if (fua && backend_can_fua (b, conn) != NBDKIT_FUA_NATIVE) {
    flags &= ~NBDKIT_FLAG_FUA;
    need_flush = true;
  }
  if (!count)
    return 0;

  if (backend_can_zero (b, conn) == NBDKIT_ZERO_NATIVE) {
    errno = 0;
    if (p->plugin.zero)
      r = p->plugin.zero (connection_get_handle (conn, 0), count, offset,
                          flags);
    else if (p->plugin._zero_old) {
      if (fast_zero) {
        *err = EOPNOTSUPP;
        return -1;
      }
      r = p->plugin._zero_old (connection_get_handle (conn, 0), count, offset,
                               may_trim);
    }
    else
      emulate = true;
    if (r == -1)
      *err = emulate ? EOPNOTSUPP : get_error (p);
    if (r == 0 || (*err != EOPNOTSUPP && *err != ENOTSUP))
      goto done;
  }

  if (fast_zero) {
    assert (r == -1);
    *err = EOPNOTSUPP;
    goto done;
  }

  assert (p->plugin.pwrite || p->plugin._pwrite_old);
  flags &= ~NBDKIT_FLAG_MAY_TRIM;
  threadlocal_set_error (0);
  *err = 0;

  while (count) {
    /* Always contains zeroes, but we can't use const or else gcc 9
     * will use .rodata instead of .bss and inflate the binary size.
     */
    static /* const */ char buf[MAX_REQUEST_SIZE];
    uint32_t limit = MIN (count, sizeof buf);

    r = plugin_pwrite (b, conn, buf, limit, offset, flags, err);
    if (r == -1)
      break;
    count -= limit;
  }

 done:
  if (r != -1 && need_flush)
    r = plugin_flush (b, conn, 0, err);
  if (r == -1 && !*err)
    *err = get_error (p);
  return r;
}


plugin_dump_fields (struct backend *b)
{
  struct backend_plugin *p = container_of (b, struct backend_plugin, backend);
  char *path;

  path = nbdkit_realpath (b->filename);
  printf ("path=%s\n", path);
  free (path);

  printf ("name=%s\n", b->name);
  if (p->plugin.version)
    printf ("version=%s\n", p->plugin.version);

  printf ("api_version=%d\n", p->plugin._api_version);
  printf ("struct_size=%" PRIu64 "\n", p->plugin._struct_size);
  printf ("max_thread_model=%s\n",
          name_of_thread_model (p->plugin._thread_model));
  printf ("thread_model=%s\n",
          name_of_thread_model (backend->thread_model (backend)));
  printf ("errno_is_preserved=%d\n", !!p->plugin.errno_is_preserved);
  if (p->plugin.magic_config_key)
    printf ("magic_config_key=%s\n", p->plugin.magic_config_key);

#define HAS(field) if (p->plugin.field) printf ("has_%s=1\n", #field)
  HAS (longname);
  HAS (description);
  HAS (load);
  HAS (unload);
  HAS (dump_plugin);
  HAS (config);
  HAS (config_complete);
  HAS (config_help);
  HAS (open);
  HAS (close);
  HAS (get_size);
  HAS (can_write);
  HAS (can_flush);
  HAS (is_rotational);
  HAS (can_trim);
  HAS (_pread_old);
  HAS (_pwrite_old);
  HAS (_flush_old);
  HAS (_trim_old);
  HAS (_zero_old);
  HAS (can_zero);
  HAS (can_fua);
  HAS (pread);
  HAS (pwrite);
  HAS (flush);
  HAS (trim);
  HAS (zero);
  HAS (can_multi_conn);
  HAS (can_extents);
  HAS (extents);
  HAS (can_cache);
  HAS (cache);
  HAS (thread_model);
  HAS (can_fast_zero);
#undef HAS

  /* Custom fields. */
  if (p->plugin.dump_plugin)
    p->plugin.dump_plugin ();
}


backend_can_extents (struct backend *b, struct connection *conn)
{
  struct b_conn_handle *h = &conn->handles[b->i];

  debug ("%s: can_extents", b->name);

  if (h->can_extents == -1)
    h->can_extents = b->can_extents (b, conn);
  return h->can_extents;
}


filter_register (struct backend *next, size_t index, const char *filename,
                 void *dl, struct nbdkit_filter *(*filter_init) (void))
{
  struct backend_filter *f;
  const struct nbdkit_filter *filter;

  f = calloc (1, sizeof *f);
  if (f == NULL) {
    perror ("strdup");
    exit (EXIT_FAILURE);
  }

  f->backend = filter_functions;
  backend_init (&f->backend, next, index, filename, dl, "filter");

  /* Call the initialization function which returns the address of the
   * filter's own 'struct nbdkit_filter'.
   */
  filter = filter_init ();
  if (!filter) {
    fprintf (stderr, "%s: %s: filter registration function failed\n",
             program_name, filename);
    exit (EXIT_FAILURE);
  }

  /* We do not provide API or ABI guarantees for filters, other than
   * the ABI position and API contents of _api_version and _version to
   * diagnose mismatch from the current nbdkit version.
   */
  if (filter->_api_version != NBDKIT_FILTER_API_VERSION) {
    fprintf (stderr,
             "%s: %s: filter is incompatible with this version of nbdkit "
             "(_api_version = %d, need %d)\n",
             program_name, filename, filter->_api_version,
             NBDKIT_FILTER_API_VERSION);
    exit (EXIT_FAILURE);
  }
  if (filter->_version == NULL ||
      strcmp (filter->_version, PACKAGE_VERSION) != 0) {
    fprintf (stderr,
             "%s: %s: filter is incompatible with this version of nbdkit "
             "(_version = %s, need %s)\n",
             program_name, filename, filter->_version ?: "<null>",
             PACKAGE_VERSION);
    exit (EXIT_FAILURE);
  }

  f->filter = *filter;

  backend_load (&f->backend, f->filter.name, f->filter.load);

  return (struct backend *) f;
}


backend_extents (struct backend *b, struct connection *conn,
                 uint32_t count, uint64_t offset, uint32_t flags,
                 struct nbdkit_extents *extents, int *err)
{
  struct b_conn_handle *h = &conn->handles[b->i];
  int r;

  assert (h->can_extents >= 0);
  assert (backend_valid_range (b, conn, offset, count));
  assert (!(flags & ~NBDKIT_FLAG_REQ_ONE));
  debug ("%s: extents count=%" PRIu32 " offset=%" PRIu64 " req_one=%d",
         b->name, count, offset, !!(flags & NBDKIT_FLAG_REQ_ONE));

  if (h->can_extents == 0) {
    /* By default it is safe assume that everything in the range is
     * allocated.
     */
    r = nbdkit_add_extent (extents, offset, count, 0 /* allocated data */);
    if (r == -1)
      *err = errno;
    return r;
  }
  r = b->extents (b, conn, count, offset, flags, extents, err);
  if (r == -1)
    assert (*err);
  return r;
}


free_connection (struct connection *conn)
{
  if (!conn)
    return;

  threadlocal_set_conn (NULL);
  conn->close (conn);
  if (listen_stdin) {
    int fd;

    /* Restore something to stdin/out so the rest of our code can
     * continue to assume that all new fds will be above stderr.
     * Swap directions to get EBADF on improper use of stdin/out.
     */
    fd = open ("/dev/null", O_WRONLY | O_CLOEXEC);
    assert (fd == 0);
    fd = open ("/dev/null", O_RDONLY | O_CLOEXEC);
    assert (fd == 1);
  }

  /* Don't call the plugin again if quit has been set because the main
   * thread will be in the process of unloading it.  The plugin.unload
   * callback should always be called.
   */
  if (!quit && connection_get_handle (conn, 0)) {
    lock_request (conn);
    backend_close (backend, conn);
    unlock_request (conn);
  }

  if (conn->status_pipe[0] >= 0) {
    close (conn->status_pipe[0]);
    close (conn->status_pipe[1]);
  }

  pthread_mutex_destroy (&conn->request_lock);
  pthread_mutex_destroy (&conn->read_lock);
  pthread_mutex_destroy (&conn->write_lock);
  pthread_mutex_destroy (&conn->status_lock);

  free (conn->handles);
  free (conn->exportname);
  free (conn);
}


filter_prepare (struct backend *b, struct connection *conn, int readonly)
{
  struct backend_filter *f = container_of (b, struct backend_filter, backend);
  void *handle = connection_get_handle (conn, b->i);
  struct b_conn nxdata = { .b = b->next, .conn = conn };

  /* Call these in order starting from the filter closest to the
   * plugin.
   */
  if (backend_prepare (b->next, conn) == -1)
    return -1;

  if (f->filter.prepare &&
      f->filter.prepare (&next_ops, &nxdata, handle, readonly) == -1)
    return -1;

  return 0;
}


plugin_can_fast_zero (struct backend *b, struct connection *conn)
{
  struct backend_plugin *p = container_of (b, struct backend_plugin, backend);
  int r;

  assert (connection_get_handle (conn, 0));

  if (p->plugin.can_fast_zero)
    return p->plugin.can_fast_zero (connection_get_handle (conn, 0));
  /* Advertise support for fast zeroes if no .zero or .can_zero is
   * false: in those cases, we fail fast instead of using .pwrite.
   * This also works when v1 plugin has only ._zero_old.
   */
  if (p->plugin.zero == NULL)
    return 1;
  r = backend_can_zero (b, conn);
  if (r == -1)
    return -1;
  return !r;
}


plugin_magic_config_key (struct backend *b)
{
  struct backend_plugin *p = container_of (b, struct backend_plugin, backend);

  return p->plugin.magic_config_key;
}


plugin_config (struct backend *b, const char *key, const char *value)
{
  struct backend_plugin *p = container_of (b, struct backend_plugin, backend);

  debug ("%s: config key=%s, value=%s", b->name, key, value);

  if (p->plugin.config == NULL) {
    fprintf (stderr,
             "%s: %s: this plugin does not need command line configuration\n"
             "Try using: %s --help %s\n",
             program_name, b->filename,
             program_name, b->filename);
    exit (EXIT_FAILURE);
  }

  if (p->plugin.config (key, value) == -1)
    exit (EXIT_FAILURE);
}


send_newstyle_option_reply_info_export (struct connection *conn,
                                        uint32_t option, uint32_t reply,
                                        uint16_t info)
{
  struct fixed_new_option_reply fixed_new_option_reply;
  struct fixed_new_option_reply_info_export export;

  fixed_new_option_reply.magic = htobe64 (NBD_REP_MAGIC);
  fixed_new_option_reply.option = htobe32 (option);
  fixed_new_option_reply.reply = htobe32 (reply);
  fixed_new_option_reply.replylen = htobe32 (sizeof export);
  export.info = htobe16 (info);
  export.exportsize = htobe64 (conn->exportsize);
  export.eflags = htobe16 (conn->eflags);

  if (conn->send (conn,
                  &fixed_new_option_reply,
                  sizeof fixed_new_option_reply, SEND_MORE) == -1 ||
      conn->send (conn, &export, sizeof export, 0) == -1) {
    nbdkit_error ("write: %s: %m", name_of_nbd_opt (option));
    return -1;
  }

  return 0;
}


next_can_cache (void *nxdata)
{
  struct b_conn *b_conn = nxdata;
  return b_conn->b->can_cache (b_conn->b, b_conn->conn);
}


filter_pread (struct backend *b, struct connection *conn,
              void *buf, uint32_t count, uint64_t offset,
              uint32_t flags, int *err)
{
  struct backend_filter *f = container_of (b, struct backend_filter, backend);
  void *handle = connection_get_handle (conn, f->backend.i);
  struct b_conn nxdata = { .b = f->backend.next, .conn = conn };

  assert (flags == 0);

  debug ("%s: pread count=%" PRIu32 " offset=%" PRIu64 " flags=0x%" PRIx32,
         f->name, count, offset, flags);

  if (f->filter.pread)
    return f->filter.pread (&next_ops, &nxdata, handle,
                            buf, count, offset, flags, err);
  else
    return f->backend.next->pread (f->backend.next, conn,
                                   buf, count, offset, flags, err);
}


filter_get_size (struct backend *b, struct connection *conn)
{
  struct backend_filter *f = container_of (b, struct backend_filter, backend);
  void *handle = connection_get_handle (conn, f->backend.i);
  struct b_conn nxdata = { .b = f->backend.next, .conn = conn };

  debug ("%s: get_size", f->name);

  if (f->filter.get_size)
    return f->filter.get_size (&next_ops, &nxdata, handle);
  else
    return f->backend.next->get_size (f->backend.next, conn);
}


filter_config (struct backend *b, const char *key, const char *value)
{
  struct backend_filter *f = container_of (b, struct backend_filter, backend);

  debug ("%s: config key=%s, value=%s",
         f->name, key, value);

  if (f->filter.config) {
    if (f->filter.config (next_config, f->backend.next, key, value) == -1)
      exit (EXIT_FAILURE);
  }
  else
    f->backend.next->config (f->backend.next, key, value);
}


next_can_zero (void *nxdata)
{
  struct b_conn *b_conn = nxdata;
  return b_conn->b->can_zero (b_conn->b, b_conn->conn);
}


filter_zero (struct backend *b, struct connection *conn,
             uint32_t count, uint64_t offset, uint32_t flags, int *err)
{
  struct backend_filter *f = container_of (b, struct backend_filter, backend);
  void *handle = connection_get_handle (conn, f->backend.i);
  struct b_conn nxdata = { .b = f->backend.next, .conn = conn };

  assert (!(flags & ~(NBDKIT_FLAG_MAY_TRIM | NBDKIT_FLAG_FUA)));

  debug ("%s: zero count=%" PRIu32 " offset=%" PRIu64 " flags=0x%" PRIx32,
         f->name, count, offset, flags);

  if (f->filter.zero)
    return f->filter.zero (&next_ops, &nxdata, handle,
                           count, offset, flags, err);
  else
    return f->backend.next->zero (f->backend.next, conn,
                                  count, offset, flags, err);
}


filter_close (struct backend *b, struct connection *conn)
{
  struct backend_filter *f = container_of (b, struct backend_filter, backend);
  void *handle = connection_get_handle (conn, f->backend.i);

  debug ("%s: close", f->name);

  if (handle && f->filter.close)
    f->filter.close (handle);
  f->backend.next->close (f->backend.next, conn);
  connection_set_handle (conn, f->backend.i, NULL);
}


filter_can_zero (struct backend *b, struct connection *conn)
{
  struct backend_filter *f = container_of (b, struct backend_filter, backend);
  void *handle = connection_get_handle (conn, f->backend.i);
  struct b_conn nxdata = { .b = f->backend.next, .conn = conn };

  debug ("%s: can_zero", f->name);

  if (f->filter.can_zero)
    return f->filter.can_zero (&next_ops, &nxdata, handle);
  else
    return f->backend.next->can_zero (f->backend.next, conn);
}


new_connection (int sockin, int sockout, int nworkers)
{
  struct connection *conn;

  conn = calloc (1, sizeof *conn);
  if (conn == NULL) {
    perror ("malloc");
    return NULL;
  }

  conn->status = 1;
  conn->nworkers = nworkers;
  conn->sockin = sockin;
  conn->sockout = sockout;
  pthread_mutex_init (&conn->request_lock, NULL);
  pthread_mutex_init (&conn->read_lock, NULL);
  pthread_mutex_init (&conn->write_lock, NULL);
  pthread_mutex_init (&conn->status_lock, NULL);

  conn->recv = raw_recv;
  conn->send = raw_send;
  conn->close = raw_close;

  return conn;
}


v9fs_vfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t rdev)
{
	struct v9fs_session_info *v9ses = v9fs_inode2v9ses(dir);
	int retval;
	char name[2 + U32_MAX_DIGITS + 1 + U32_MAX_DIGITS + 1];
	u32 perm;

	p9_debug(P9_DEBUG_VFS, " %lu,%pd mode: %hx MAJOR: %u MINOR: %u\n",
		 dir->i_ino, dentry, mode,
		 MAJOR(rdev), MINOR(rdev));

	/* build extension */
	if (S_ISBLK(mode))
		sprintf(name, "b %u %u", MAJOR(rdev), MINOR(rdev));
	else if (S_ISCHR(mode))
		sprintf(name, "c %u %u", MAJOR(rdev), MINOR(rdev));
	else
		*name = 0;

	perm = unixmode2p9mode(v9ses, mode);
	retval = v9fs_vfs_mkspecial(dir, dentry, perm, name);

	return retval;
}


v9fs_vfs_symlink_dotl(struct inode *dir, struct dentry *dentry,
		const char *symname)
{
	int err;
	kgid_t gid;
	const unsigned char *name;
	struct p9_qid qid;
	struct inode *inode;
	struct p9_fid *dfid;
	struct p9_fid *fid = NULL;
	struct v9fs_session_info *v9ses;

	name = dentry->d_name.name;
	p9_debug(P9_DEBUG_VFS, "%lu,%s,%s\n", dir->i_ino, name, symname);
	v9ses = v9fs_inode2v9ses(dir);

	dfid = v9fs_parent_fid(dentry);
	if (IS_ERR(dfid)) {
		err = PTR_ERR(dfid);
		p9_debug(P9_DEBUG_VFS, "fid lookup failed %d\n", err);
		return err;
	}

	gid = v9fs_get_fsgid_for_create(dir);

	/* Server doesn't alter fid on TSYMLINK. Hence no need to clone it. */
	err = p9_client_symlink(dfid, name, symname, gid, &qid);

	if (err < 0) {
		p9_debug(P9_DEBUG_VFS, "p9_client_symlink failed %d\n", err);
		goto error;
	}

	v9fs_invalidate_inode_attr(dir);
	if (v9ses->cache == CACHE_LOOSE || v9ses->cache == CACHE_FSCACHE) {
		/* Now walk from the parent so we can get an unopened fid. */
		fid = p9_client_walk(dfid, 1, &name, 1);
		if (IS_ERR(fid)) {
			err = PTR_ERR(fid);
			p9_debug(P9_DEBUG_VFS, "p9_client_walk failed %d\n",
				 err);
			fid = NULL;
			goto error;
		}

		/* instantiate inode and assign the unopened fid to dentry */
		inode = v9fs_get_new_inode_from_fid(v9ses, fid, dir->i_sb);
		if (IS_ERR(inode)) {
			err = PTR_ERR(inode);
			p9_debug(P9_DEBUG_VFS, "inode creation failed %d\n",
				 err);
			goto error;
		}
		v9fs_fid_add(dentry, fid);
		d_instantiate(dentry, inode);
		fid = NULL;
		err = 0;
	} else {
		/* Not in cached mode. No need to populate inode with stat */
		inode = v9fs_get_inode(dir->i_sb, S_IFLNK, 0);
		if (IS_ERR(inode)) {
			err = PTR_ERR(inode);
			goto error;
		}
		d_instantiate(dentry, inode);
	}

error:
	if (fid)
		p9_client_clunk(fid);

	return err;
}


v9fs_umount_begin(struct super_block *sb)
{
	struct v9fs_session_info *v9ses;

	v9ses = sb->s_fs_info;
	v9fs_session_begin_cancel(v9ses);
}


v9fs_vfs_mknod_dotl(struct inode *dir, struct dentry *dentry, umode_t omode,
		dev_t rdev)
{
	int err;
	kgid_t gid;
	const unsigned char *name;
	umode_t mode;
	struct v9fs_session_info *v9ses;
	struct p9_fid *fid = NULL, *dfid = NULL;
	struct inode *inode;
	struct p9_qid qid;
	struct posix_acl *dacl = NULL, *pacl = NULL;

	p9_debug(P9_DEBUG_VFS, " %lu,%pd mode: %hx MAJOR: %u MINOR: %u\n",
		 dir->i_ino, dentry, omode,
		 MAJOR(rdev), MINOR(rdev));

	v9ses = v9fs_inode2v9ses(dir);
	dfid = v9fs_parent_fid(dentry);
	if (IS_ERR(dfid)) {
		err = PTR_ERR(dfid);
		p9_debug(P9_DEBUG_VFS, "fid lookup failed %d\n", err);
		dfid = NULL;
		goto error;
	}

	gid = v9fs_get_fsgid_for_create(dir);
	mode = omode;
	/* Update mode based on ACL value */
	err = v9fs_acl_mode(dir, &mode, &dacl, &pacl);
	if (err) {
		p9_debug(P9_DEBUG_VFS, "Failed to get acl values in mknod %d\n",
			 err);
		goto error;
	}
	name = dentry->d_name.name;

	err = p9_client_mknod_dotl(dfid, name, mode, rdev, gid, &qid);
	if (err < 0)
		goto error;

	v9fs_invalidate_inode_attr(dir);
	fid = p9_client_walk(dfid, 1, &name, 1);
	if (IS_ERR(fid)) {
		err = PTR_ERR(fid);
		p9_debug(P9_DEBUG_VFS, "p9_client_walk failed %d\n",
			 err);
		fid = NULL;
		goto error;
	}

	/* instantiate inode and assign the unopened fid to the dentry */
	if (v9ses->cache == CACHE_LOOSE || v9ses->cache == CACHE_FSCACHE) {
		inode = v9fs_get_new_inode_from_fid(v9ses, fid, dir->i_sb);
		if (IS_ERR(inode)) {
			err = PTR_ERR(inode);
			p9_debug(P9_DEBUG_VFS, "inode creation failed %d\n",
				 err);
			goto error;
		}
		v9fs_set_create_acl(inode, fid, dacl, pacl);
		v9fs_fid_add(dentry, fid);
		d_instantiate(dentry, inode);
		fid = NULL;
		err = 0;
	} else {
		/*
		 * Not in cached mode. No need to populate inode with stat.
		 * socket syscall returns a fd, so we need instantiate
		 */
		inode = v9fs_get_inode(dir->i_sb, mode, rdev);
		if (IS_ERR(inode)) {
			err = PTR_ERR(inode);
			goto error;
		}
		v9fs_set_create_acl(inode, fid, dacl, pacl);
		d_instantiate(dentry, inode);
	}
error:
	if (fid)
		p9_client_clunk(fid);
	v9fs_put_acl(dacl, pacl);
	return err;
}


static void v9fs_mmap_vm_close(struct vm_area_struct *vma)
{
	struct inode *inode;

	struct writeback_control wbc = {
		.nr_to_write = LONG_MAX,
		.sync_mode = WB_SYNC_ALL,
		.range_start = vma->vm_pgoff * PAGE_SIZE,
		 /* absolute end, byte at end included */
		.range_end = vma->vm_pgoff * PAGE_SIZE +
			(vma->vm_end - vma->vm_start - 1),
	};

	p9_debug(P9_DEBUG_VFS, "9p VMA close, %p, flushing", vma);

	inode = file_inode(vma->vm_file);

	if (!mapping_cap_writeback_dirty(inode->i_mapping))
		wbc.nr_to_write = 0;

	might_sleep();
	sync_inode(inode, &wbc);
}


int v9fs_refresh_inode(struct p9_fid *fid, struct inode *inode)
{
	int umode;
	dev_t rdev;
	struct p9_wstat *st;
	struct v9fs_session_info *v9ses;
	unsigned int flags;

	v9ses = v9fs_inode2v9ses(inode);
	st = p9_client_stat(fid);
	if (IS_ERR(st))
		return PTR_ERR(st);
	/*
	 * Don't update inode if the file type is different
	 */
	umode = p9mode2unixmode(v9ses, st, &rdev);
	if ((inode->i_mode & S_IFMT) != (umode & S_IFMT))
		goto out;

	/*
	 * We don't want to refresh inode->i_size,
	 * because we may have cached data
	 */
	flags = (v9ses->cache == CACHE_LOOSE || v9ses->cache == CACHE_FSCACHE) ?
		V9FS_STAT2INODE_KEEP_ISIZE : 0;
	v9fs_stat2inode(st, inode, inode->i_sb, flags);
out:
	p9stat_free(st);
	kfree(st);
	return 0;
}


static int v9fs_vfs_setattr(struct dentry *dentry, struct iattr *iattr)
{
	int retval;
	struct v9fs_session_info *v9ses;
	struct p9_fid *fid;
	struct p9_wstat wstat;

	p9_debug(P9_DEBUG_VFS, "\n");
	retval = setattr_prepare(dentry, iattr);
	if (retval)
		return retval;

	retval = -EPERM;
	v9ses = v9fs_dentry2v9ses(dentry);
	fid = v9fs_fid_lookup(dentry);
	if(IS_ERR(fid))
		return PTR_ERR(fid);

	v9fs_blank_wstat(&wstat);
	if (iattr->ia_valid & ATTR_MODE)
		wstat.mode = unixmode2p9mode(v9ses, iattr->ia_mode);

	if (iattr->ia_valid & ATTR_MTIME)
		wstat.mtime = iattr->ia_mtime.tv_sec;

	if (iattr->ia_valid & ATTR_ATIME)
		wstat.atime = iattr->ia_atime.tv_sec;

	if (iattr->ia_valid & ATTR_SIZE)
		wstat.length = iattr->ia_size;

	if (v9fs_proto_dotu(v9ses)) {
		if (iattr->ia_valid & ATTR_UID)
			wstat.n_uid = iattr->ia_uid;

		if (iattr->ia_valid & ATTR_GID)
			wstat.n_gid = iattr->ia_gid;
	}

	/* Write all dirty data */
	if (d_is_reg(dentry))
		filemap_write_and_wait(d_inode(dentry)->i_mapping);

	retval = p9_client_wstat(fid, &wstat);
	if (retval < 0)
		return retval;

	if ((iattr->ia_valid & ATTR_SIZE) &&
	    iattr->ia_size != i_size_read(d_inode(dentry)))
		truncate_setsize(d_inode(dentry), iattr->ia_size);

	v9fs_invalidate_inode_attr(d_inode(dentry));

	setattr_copy(d_inode(dentry), iattr);
	mark_inode_dirty(d_inode(dentry));
	return 0;
}


struct inode *v9fs_alloc_inode(struct super_block *sb)
{
	struct v9fs_inode *v9inode;
	v9inode = (struct v9fs_inode *)kmem_cache_alloc(v9fs_inode_cache,
							GFP_KERNEL);
	if (!v9inode)
		return NULL;
#ifdef CONFIG_9P_FSCACHE
	v9inode->fscache = NULL;
	mutex_init(&v9inode->fscache_lock);
#endif
	v9inode->writeback_fid = NULL;
	v9inode->cache_validity = 0;
	mutex_init(&v9inode->v_mutex);
	return &v9inode->vfs_inode;
}


Sfdouble_t sh_arith(Shell_t *shp, const char *str) { return sh_strnum(shp, str, NULL, 1); }


static_fn Namval_t *scope(Namval_t *np, struct lval *lvalue, int assign) {
    int flag = lvalue->flag;
    char *sub = 0, *cp = (char *)np;
    Namval_t *mp;
    Shell_t *shp = lvalue->shp;
    int c = 0, nosub = lvalue->nosub;
    Dt_t *sdict = (shp->st.real_fun ? shp->st.real_fun->sdict : 0);
    Dt_t *nsdict = (shp->namespace ? nv_dict(shp->namespace) : 0);
    Dt_t *root = shp->var_tree;

    nvflag_t nvflags = assign ? NV_ASSIGN : 0;
    lvalue->nosub = 0;
    if (nosub < 0 && lvalue->ovalue) return (Namval_t *)lvalue->ovalue;
    lvalue->ovalue = NULL;
    if (cp >= lvalue->expr && cp < lvalue->expr + lvalue->elen) {
        int offset;
        // Do binding to node now.
        int d = cp[flag];
        cp[flag] = 0;
        np = nv_open(cp, root, nvflags | NV_VARNAME | NV_NOADD | NV_NOFAIL);
        if ((!np || nv_isnull(np)) && sh_macfun(shp, cp, offset = stktell(shp->stk))) {
            Fun = sh_arith(shp, sub = stkptr(shp->stk, offset));
            STORE_VT(FunNode.nvalue, sfdoublep, &Fun);
            FunNode.nvshell = shp;
            nv_onattr(&FunNode, NV_NOFREE | NV_LDOUBLE | NV_RDONLY);
            cp[flag] = d;
            return &FunNode;
        }
        if (!np && assign) {
            np = nv_open(cp, root, nvflags | NV_VARNAME);
        }
        cp[flag] = d;
        if (!np) return 0;
        root = shp->last_root;
        if (cp[flag + 1] == '[') {
            flag++;
        } else {
            flag = 0;
        }
    }

    if ((lvalue->emode & ARITH_COMP) && dtvnext(root)) {
        mp = nv_search_namval(np, sdict ? sdict : root, NV_NOSCOPE);
        if (!mp && nsdict) mp = nv_search_namval(np, nsdict, 0);
        if (mp) np = mp;
    }

    while (nv_isref(np)) {
        sub = nv_refsub(np);
        np = nv_refnode(np);
        if (sub) nv_putsub(np, sub, 0, assign ? ARRAY_ADD : 0);
    }

    if (!nosub && flag) {
        int hasdot = 0;
        cp = (char *)&lvalue->expr[flag];
        if (sub) goto skip;
        sub = cp;
        while (1) {
            Namarr_t *ap;
            Namval_t *nq;
            cp = nv_endsubscript(np, cp, 0, shp);
            if (c || *cp == '.') {
                c = '.';
                while (*cp == '.') {
                    hasdot = 1;
                    cp++;
                    while (c = mb1char(&cp), isaname(c)) {
                        ;  // empty body
                    }
                }
                if (c == '[') continue;
            }
            flag = *cp;
            *cp = 0;
            if (c || hasdot) {
                sfprintf(shp->strbuf, "%s%s%c", nv_name(np), sub, 0);
                sub = sfstruse(shp->strbuf);
            }
            if (strchr(sub, '$')) sub = sh_mactrim(shp, sub, 0);
            *cp = flag;
            if (c || hasdot) {
                np = nv_open(sub, shp->var_tree, NV_VARNAME | nvflags);
                return np;
            }
            cp = nv_endsubscript(np, sub, (assign ? NV_ADD : 0) | NV_SUBQUOTE, np->nvshell);
            if (*cp != '[') break;
        skip:
            nq = nv_opensub(np);
            if (nq) {
                np = nq;
            } else {
                ap = nv_arrayptr(np);
                if (ap && !ap->table) {
                    ap->table = dtopen(&_Nvdisc, Dtoset);
                    dtuserdata(ap->table, shp, 1);
                }
                if (ap && ap->table && (nq = nv_search(nv_getsub(np), ap->table, NV_ADD))) {
                    nq->nvenv = np;
                }
                if (nq && nv_isnull(nq)) np = nv_arraychild(np, nq, 0);
            }
            sub = cp;
        }
    } else if (nosub > 0) {
        nv_putsub(np, NULL, nosub - 1, 0);
    }
    return np;
}


static void try_rgrp_unlink(struct gfs2_rgrpd *rgd, u64 *last_unlinked, u64 skip)
{
	u64 block;
	struct gfs2_sbd *sdp = rgd->rd_sbd;
	struct gfs2_glock *gl;
	struct gfs2_inode *ip;
	int error;
	int found = 0;
	struct gfs2_rbm rbm = { .rgd = rgd, .bii = 0, .offset = 0 };

	while (1) {
		down_write(&sdp->sd_log_flush_lock);
		error = gfs2_rbm_find(&rbm, GFS2_BLKST_UNLINKED, NULL, NULL,
				      true);
		up_write(&sdp->sd_log_flush_lock);
		if (error == -ENOSPC)
			break;
		if (WARN_ON_ONCE(error))
			break;

		block = gfs2_rbm_to_block(&rbm);
		if (gfs2_rbm_from_block(&rbm, block + 1))
			break;
		if (*last_unlinked != NO_BLOCK && block <= *last_unlinked)
			continue;
		if (block == skip)
			continue;
		*last_unlinked = block;

		error = gfs2_glock_get(sdp, block, &gfs2_iopen_glops, CREATE, &gl);
		if (error)
			continue;

		/* If the inode is already in cache, we can ignore it here
		 * because the existing inode disposal code will deal with
		 * it when all refs have gone away. Accessing gl_object like
		 * this is not safe in general. Here it is ok because we do
		 * not dereference the pointer, and we only need an approx
		 * answer to whether it is NULL or not.
		 */
		ip = gl->gl_object;

		if (ip || queue_work(gfs2_delete_workqueue, &gl->gl_delete) == 0)
			gfs2_glock_put(gl);
		else
			found++;

		/* Limit reclaim to sensible number of tasks */
		if (found > NR_CPUS)
			return;
	}

	rgd->rd_flags &= ~GFS2_RDF_CHECK;
	return;
}


static bool gfs2_unaligned_extlen(struct gfs2_rbm *rbm, u32 n_unaligned, u32 *len)
{
	u32 n;
	u8 res;

	for (n = 0; n < n_unaligned; n++) {
		res = gfs2_testbit(rbm);
		if (res != GFS2_BLKST_FREE)
			return true;
		(*len)--;
		if (*len == 0)
			return true;
		if (gfs2_rbm_incr(rbm))
			return true;
	}

	return false;
}


static bool gfs2_rgrp_used_recently(const struct gfs2_blkreserv *rs,
				    u64 msecs)
{
	u64 tdiff;

	tdiff = ktime_to_ns(ktime_sub(ktime_get_real(),
                            rs->rs_rbm.rgd->rd_gl->gl_dstamp));

	return tdiff > (msecs * 1000 * 1000);
}


static u32 gfs2_orlov_skip(const struct gfs2_inode *ip)
{
	const struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
	u32 skip;

	get_random_bytes(&skip, sizeof(skip));
	return skip % sdp->sd_rgrps;
}


static inline int rs_cmp(u64 blk, u32 len, struct gfs2_blkreserv *rs)
{
	u64 startblk = gfs2_rbm_to_block(&rs->rs_rbm);

	if (blk >= startblk + rs->rs_free)
		return 1;
	if (blk + len - 1 < startblk)
		return -1;
	return 0;
}


static void rs_insert(struct gfs2_inode *ip)
{
	struct rb_node **newn, *parent = NULL;
	int rc;
	struct gfs2_blkreserv *rs = &ip->i_res;
	struct gfs2_rgrpd *rgd = rs->rs_rbm.rgd;
	u64 fsblock = gfs2_rbm_to_block(&rs->rs_rbm);

	BUG_ON(gfs2_rs_active(rs));

	spin_lock(&rgd->rd_rsspin);
	newn = &rgd->rd_rstree.rb_node;
	while (*newn) {
		struct gfs2_blkreserv *cur =
			rb_entry(*newn, struct gfs2_blkreserv, rs_node);

		parent = *newn;
		rc = rs_cmp(fsblock, rs->rs_free, cur);
		if (rc > 0)
			newn = &((*newn)->rb_right);
		else if (rc < 0)
			newn = &((*newn)->rb_left);
		else {
			spin_unlock(&rgd->rd_rsspin);
			WARN_ON(1);
			return;
		}
	}

	rb_link_node(&rs->rs_node, parent, newn);
	rb_insert_color(&rs->rs_node, &rgd->rd_rstree);

	/* Do our rgrp accounting for the reservation */
	rgd->rd_reserved += rs->rs_free; /* blocks reserved */
	spin_unlock(&rgd->rd_rsspin);
	trace_gfs2_rs(rs, TRACE_RS_INSERT);
}


static bool gfs2_select_rgrp(struct gfs2_rgrpd **pos, const struct gfs2_rgrpd *begin)
{
	struct gfs2_rgrpd *rgd = *pos;
	struct gfs2_sbd *sdp = rgd->rd_sbd;

	rgd = gfs2_rgrpd_get_next(rgd);
	if (rgd == NULL)
		rgd = gfs2_rgrpd_get_first(sdp);
	*pos = rgd;
	if (rgd != begin) /* If we didn't wrap */
		return true;
	return false;
}


static void gfs2_rindex_print(const struct gfs2_rgrpd *rgd)
{
	pr_info("ri_addr = %llu\n", (unsigned long long)rgd->rd_addr);
	pr_info("ri_length = %u\n", rgd->rd_length);
	pr_info("ri_data0 = %llu\n", (unsigned long long)rgd->rd_data0);
	pr_info("ri_data = %u\n", rgd->rd_data);
	pr_info("ri_bitbytes = %u\n", rgd->rd_bitbytes);
}


void rds_sock_addref(struct rds_sock *rs)
{
	sock_hold(rds_rs_to_sk(rs));
}


static int aac_get_hba_info(struct aac_dev *dev, void __user *arg)
{
	struct aac_hba_info hbainfo;

	memset(&hbainfo, 0, sizeof(hbainfo));
	hbainfo.adapter_number		= (u8) dev->id;
	hbainfo.system_io_bus_number	= dev->pdev->bus->number;
	hbainfo.device_number		= (dev->pdev->devfn >> 3);
	hbainfo.function_number		= (dev->pdev->devfn & 0x0007);

	hbainfo.vendor_id		= dev->pdev->vendor;
	hbainfo.device_id		= dev->pdev->device;
	hbainfo.sub_vendor_id		= dev->pdev->subsystem_vendor;
	hbainfo.sub_system_id		= dev->pdev->subsystem_device;

	if (copy_to_user(arg, &hbainfo, sizeof(struct aac_hba_info))) {
		dprintk((KERN_DEBUG "aacraid: Could not copy hba info\n"));
		return -EFAULT;
	}

	return 0;
}


struct i2c_client *i2c_setup_smbus_alert(struct i2c_adapter *adapter,
					 struct i2c_smbus_alert_setup *setup)
{
	struct i2c_board_info ara_board_info = {
		I2C_BOARD_INFO("smbus_alert", 0x0c),
		.platform_data = setup,
	};

	return i2c_new_device(adapter, &ara_board_info);
}


static ssize_t ad5755_read_powerdown(struct iio_dev *indio_dev, uintptr_t priv,
	const struct iio_chan_spec *chan, char *buf)
{
	struct ad5755_state *st = iio_priv(indio_dev);

	return sprintf(buf, "%d\n",
		       (bool)(st->pwr_down & (1 << chan->channel)));
}


static int ad5755_write_ctrl(struct iio_dev *indio_dev, unsigned int channel,
	unsigned int reg, unsigned int val)
{
	int ret;

	mutex_lock(&indio_dev->mlock);
	ret = ad5755_write_ctrl_unlocked(indio_dev, channel, reg, val);
	mutex_unlock(&indio_dev->mlock);

	return ret;
}


int arc_emac_remove(struct net_device *ndev)
{
	struct arc_emac_priv *priv = netdev_priv(ndev);

	phy_disconnect(priv->phy_dev);
	priv->phy_dev = NULL;
	arc_mdio_remove(priv);
	unregister_netdev(ndev);
	netif_napi_del(&priv->napi);

	if (!IS_ERR(priv->clk)) {
		clk_disable_unprepare(priv->clk);
	}

	return 0;
}


MagickExport void ImageToCustomStream(const ImageInfo *image_info,Image *image,
  ExceptionInfo *exception)
{
  const MagickInfo
    *magick_info;

  ImageInfo
    *clone_info;

  MagickBooleanType
    blob_support,
    status;

  assert(image_info != (const ImageInfo *) NULL);
  assert(image_info->signature == MagickCoreSignature);
  if (image_info->debug != MagickFalse)
    (void) LogMagickEvent(TraceEvent,GetMagickModule(),"%s",
      image_info->filename);
  assert(image != (Image *) NULL);
  assert(image->signature == MagickCoreSignature);
  assert(image_info->custom_stream != (CustomStreamInfo *) NULL);
  assert(image_info->custom_stream->signature == MagickCoreSignature);
  assert(image_info->custom_stream->writer != (CustomStreamHandler) NULL);
  assert(exception != (ExceptionInfo *) NULL);
  clone_info=CloneImageInfo(image_info);
  clone_info->adjoin=MagickFalse;
  (void) SetImageInfo(clone_info,1,exception);
  if (*clone_info->magick != '\0')
    (void) CopyMagickString(image->magick,clone_info->magick,MagickPathExtent);
  magick_info=GetMagickInfo(image->magick,exception);
  if (magick_info == (const MagickInfo *) NULL)
    {
      (void) ThrowMagickException(exception,GetMagickModule(),
        MissingDelegateError,"NoEncodeDelegateForThisImageFormat","`%s'",
        image->magick);
      clone_info=DestroyImageInfo(clone_info);
      return;
    }
  (void) CopyMagickString(clone_info->magick,image->magick,MagickPathExtent);
  blob_support=GetMagickBlobSupport(magick_info);
  if ((blob_support != MagickFalse) &&
      (GetMagickEncoderSeekableStream(magick_info) != MagickFalse))
    {
      if ((clone_info->custom_stream->seeker == (CustomStreamSeeker) NULL) ||
          (clone_info->custom_stream->teller == (CustomStreamTeller) NULL))
        blob_support=MagickFalse;
    }
  if (blob_support != MagickFalse)
    {
      /*
        Native blob support for this image format.
      */
      (void) CloseBlob(image);
      *image->filename='\0';
      (void) WriteImage(clone_info,image,exception);
      (void) CloseBlob(image);
    }
  else
    {
      char
        unique[MagickPathExtent];

      int
        file;

      unsigned char
        *blob;

      /*
        Write file to disk in blob image format.
      */
      clone_info->custom_stream=(CustomStreamInfo *) NULL;
      blob=(unsigned char *) AcquireQuantumMemory(MagickMaxBufferExtent,
        sizeof(*blob));
      if (blob == (unsigned char *) NULL)
        {
          ThrowFileException(exception,BlobError,"UnableToWriteBlob",
            image_info->filename);
          clone_info=DestroyImageInfo(clone_info);
          return;
        }
      file=AcquireUniqueFileResource(unique);
      if (file == -1)
        {
          ThrowFileException(exception,BlobError,"UnableToWriteBlob",
            image_info->filename);
          blob=(unsigned char *) RelinquishMagickMemory(blob);
          clone_info=DestroyImageInfo(clone_info);
          return;
        }
      clone_info->file=fdopen(file,"wb+");
      if (clone_info->file != (FILE *) NULL)
        {
          ssize_t
            count;

          (void) FormatLocaleString(image->filename,MagickPathExtent,
            "%s:%s",image->magick,unique);
          status=WriteImage(clone_info,image,exception);
          (void) CloseBlob(image);
          if (status != MagickFalse)
            {
              (void) fseek(clone_info->file,0,SEEK_SET);
              count=(ssize_t) MagickMaxBufferExtent;
              while (count == (ssize_t) MagickMaxBufferExtent)
              {
                count=(ssize_t) fread(blob,sizeof(*blob),MagickMaxBufferExtent,
                  clone_info->file);
                (void) image_info->custom_stream->writer(blob,(size_t) count,
                  image_info->custom_stream->data);
              }
            }
          (void) fclose(clone_info->file);
        }
      blob=(unsigned char *) RelinquishMagickMemory(blob);
      (void) RelinquishUniqueFileResource(unique);
    }
  clone_info=DestroyImageInfo(clone_info);
}