Repository: ipfs/boxo

Issue:
gateway: limit the number of requests in flight
> Filling issue based on triage discussion today, cc @gammazero  if below prior art context is useful

Currently, we have no "requests in flight" cap in `boxo/gateway` and require reverse proxy in front of go binary to enforce the limit.

## Proposed improvement

There should be a built-in mechanism to limit the number of ongoing HTTP gateway requests in flight, and return HTTP 503 Service Unavailable with [Retry-After](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After) when limit is reached.

Make it a config option, `boxo/gateway` should ship with some implicit default, protecting people who expose it blindly to the internet 

### Prior art 

- At public gateway, 
  - the current ceiling exist at nginx and for our rainbow prod seems to be really high: [`worker_connections`](https://nginx.org/en/docs/ngx_core_module.html#worker_connections) is set to `10240` (per each instance). For reference, nginx default is 512, raising to 1024 seems to be a sensible place to start as default, protecting from traffic spikes.
  - there is a timeout of 30 seconds (nginx's `proxy_read_timeout`)  – we dont have built-in config yet, but its tracked in https://github.com/ipfs/boxo/issues/679
  



Hints from issue comments:

Solution patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 029f3520..c2ee0d72 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,10 +16,19 @@ The following emojis are used to highlight certain changes:
 
 ### Added
 
+- `gateway`: Added configurable limits for gateway resource protection:
+  - `Config.RetrievalTimeout`: Maximum duration between writes of non-empty data to HTTP response body (default: 30s). Returns 504 Gateway Timeout when gateway cannot retrieve content within this period.
+  - `Config.MaxConcurrentRequests`: Limits concurrent HTTP requests (default: 4096, suitable for most deployments). Returns 429 Too Many Requests with 60s Retry-After header when exceeded. To restore previous unlimited behavior set both `RetrievalTimeout` and `MaxConcurrentRequests` to `0`.
+    > [!IMPORTANT]
+    > If your gateway returns many HTTP 429 responses while having available resources (CPU, memory), increase `MaxConcurrentRequests`. If experiencing high load or resource exhaustion, decrease it. See the [`MaxConcurrentRequests` godoc](https://pkg.go.dev/github.com/ipfs/boxo/gateway#Config) for detailed tuning guidance.
+  - `Config.MetricsRegistry`: Optional Prometheus registry for metrics isolation. When nil, uses the default global registry. Useful for testing and deployments with multiple gateway instances.
+  - New middleware with Prometheus metrics:
+    - `ipfs_http_gw_concurrent_requests`: Gauge tracking number of concurrent requests
+    - `ipfs_http_gw_responses_total{code}`: Counter for all HTTP responses by status code
+    - `ipfs_http_gw_retrieval_timeouts_total{code,truncated}`: Counter for retrieval timeout events with details on truncation
 - `namesys/IPNSPublisher`: option to `PublishOptions` that allows for setting a custom sequence number for the IPNS record with proper validation to prevent unintentional replay attacks. [#962](https://github.com/ipfs/boxo/pull/962)
 
 ### Changed
-
 - `bitswap/network`: The connection event manager now has a `SetListeners` method. Both `bsnet` and `httpnet` now have options to provide the `ConnectionEventManager` during `New(...)`. This allows sharing the connection event manager when using both. The connection manager SHOULD be shared when using both networks with the `network.Router` utility.
 - `provider`: Distribute the responsability of providing new blocks to the places that play a role in the different providing strategies [#976](https://github.com/ipfs/boxo/pull/976). Refactor the logic to perform Provides, when the component has been given a provider:
   - Remove `providing.Exchange`
diff --git a/gateway/backend_blocks.go b/gateway/backend_blocks.go
index f5927c2a..97b10656 100644
--- a/gateway/backend_blocks.go
+++ b/gateway/backend_blocks.go
@@ -27,13 +27,12 @@ import (
 	"github.com/ipfs/go-unixfsnode/data"
 	"github.com/ipld/go-car/v2"
 	"github.com/ipld/go-car/v2/storage"
-	dagpb "github.com/ipld/go-codec-dagpb"
+	dagpb "github.com/ipld/go-codec-dagpb" // Ensure basic codecs are registered.
 	"github.com/ipld/go-ipld-prime"
-	// Ensure basic codecs are registered.
-	_ "github.com/ipld/go-ipld-prime/codec/cbor"
-	_ "github.com/ipld/go-ipld-prime/codec/dagcbor"
-	_ "github.com/ipld/go-ipld-prime/codec/dagjson"
-	_ "github.com/ipld/go-ipld-prime/codec/json"
+	_ "github.com/ipld/go-ipld-prime/codec/cbor"    // Ensure basic codecs are registered.
+	_ "github.com/ipld/go-ipld-prime/codec/dagcbor" // Ensure basic codecs are registered.
+	_ "github.com/ipld/go-ipld-prime/codec/dagjson" // Ensure basic codecs are registered.
+	_ "github.com/ipld/go-ipld-prime/codec/json"    // Ensure basic codecs are registered.
 	"github.com/ipld/go-ipld-prime/datamodel"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
 	"github.com/ipld/go-ipld-prime/node/basicnode"
@@ -113,11 +112,6 @@ func NewRemoteBlocksBackend(gatewayURL []string, httpClient *http.Client, opts .
 }
 
 func (bb *BlocksBackend) Get(ctx context.Context, path path.ImmutablePath, ranges ...ByteRange) (ContentPathMetadata, *GetResponse, error) {
-	md, nd, err := bb.getNode(ctx, path)
-	if err != nil {
-		return md, nil, err
-	}
-
 	// Only a single range is supported in responses to HTTP Range Requests.
 	// When more than one is passed in the Range header, this library will
 	// return a response for the first one and ignores remaining ones.
@@ -126,6 +120,45 @@ func (bb *BlocksBackend) Get(ctx context.Context, path path.ImmutablePath, range
 		ra = &ranges[0]
 	}
 
+	// For range requests on UnixFS files, we can optimize by only fetching
+	// the root block first to determine if it's a file, then create a
+	// UnixfsFile that will lazily fetch only the blocks needed for the range
+	if ra != nil {
+		// Get just the root block first
+		roots, lastSeg, remainder, err := bb.getPathRoots(ctx, path)
+		if err != nil {
+			return ContentPathMetadata{}, nil, err
+		}
+
+		md := ContentPathMetadata{
+			PathSegmentRoots:     roots,
+			LastSegment:          lastSeg,
+			LastSegmentRemainder: remainder,
+		}
+
+		lastRoot := lastSeg.RootCid()
+
+		// Fetch only the root block to check if it's a UnixFS file
+		rootBlock, err := bb.blockService.GetBlock(ctx, lastRoot)
+		if err != nil {
+			return md, nil, err
+		}
+
+		// Try to optimize for dag-pb blocks that might be UnixFS files
+		if rootCodec := lastRoot.Prefix().GetCodec(); rootCodec == uint64(mc.DagPb) {
+			if unixfsFileResponse := loadUnixFSFileWithLazyBlocks(ctx, path, rootBlock, &md, ra, bb.dagService); unixfsFileResponse != nil {
+				return md, unixfsFileResponse, nil
+			}
+		}
+		// Fall back to regular path for non-files or if optimization fails
+	}
+
+	// Regular path for non-range requests or when optimization doesn't apply
+	md, nd, err := bb.getNode(ctx, path)
+	if err != nil {
+		return md, nil, err
+	}
+
 	rootCodec := nd.Cid().Prefix().GetCodec()
 
 	// This covers both Raw blocks and terminal IPLD codecs like dag-cbor and dag-json
@@ -195,6 +228,70 @@ func (bb *BlocksBackend) Get(ctx context.Context, path path.ImmutablePath, range
 	return ContentPathMetadata{}, nil, fmt.Errorf("data was not a valid file or directory: %w", ErrInternalServerError) // TODO: should there be a gateway invalid content type to abstract over the various IPLD error types?
 }
 
+// loadUnixFSFileWithLazyBlocks attempts to load a UnixFS file in a way that only
+// fetches the blocks necessary for the requested byte range.
+//
+// This optimization is only possible for dag-pb files. It decodes the root block,
+// creates a UnixFS file reader, and seeks to the range start. This avoids fetching
+// unnecessary blocks when serving range requests for large files.
+//
+// Returns nil if the content is not a UnixFS file or if any step fails.
+func loadUnixFSFileWithLazyBlocks(ctx context.Context, path path.ImmutablePath, rootBlock blocks.Block, md *ContentPathMetadata, ra *ByteRange, dagService format.DAGService) *GetResponse {
+	// Try to decode as protobuf
+	nd, err := merkledag.DecodeProtobuf(rootBlock.RawData())
+	if err != nil {
+		log.Debugw("failed to decode protobuf for range request optimization",
+			"path", path,
+			"error", err)
+		return nil
+	}
+
+	// Try to create a UnixFS file - this will only fetch blocks as needed
+	f, err := ufile.NewUnixfsFile(ctx, dagService, nd)
+	if err != nil {
+		log.Debugw("failed to create UnixFS file for range request optimization",
+			"path", path,
+			"error", err)
+		return nil
+	}
+
+	// Check if it's actually a file
+	file, ok := f.(files.File)
+	if !ok {
+		// Not a file, can't optimize
+		return nil
+	}
+
+	// Get file size for range calculations
+	fileSize, err := f.Size()
+	if err != nil {
+		log.Debugw("failed to get file size for range request optimization",
+			"path", path,
+			"error", err)
+		return nil
+	}
+
+	// Set modification time if available
+	if mtime := f.ModTime(); !mtime.IsZero() {
+		md.ModTime = mtime
+	}
+
+	// Seek to the start of the requested range
+	if err := seekToRangeStart(file, ra, fileSize); err != nil {
+		log.Debugw("failed to seek to range start",
+			"path", path,
+			"error", err)
+		return nil
+	}
+
+	// Handle symlinks specially
+	if s, ok := f.(*files.Symlink); ok {
+		return NewGetResponseFromSymlink(s, fileSize)
+	}
+
+	return NewGetResponseFromReader(file, fileSize)
+}
+
 func (bb *BlocksBackend) GetAll(ctx context.Context, path path.ImmutablePath) (ContentPathMetadata, files.Node, error) {
 	md, nd, err := bb.getNode(ctx, path)
 	if err != nil {
diff --git a/gateway/doc.go b/gateway/doc.go
new file mode 100644
index 00000000..dde7c91f
--- /dev/null
+++ b/gateway/doc.go
@@ -0,0 +1,79 @@
+// Package gateway provides an HTTP gateway for serving IPFS content over HTTP.
+//
+// The gateway implements the [IPFS HTTP Gateway] specifications, including:
+//
+//   - [Path Gateway]: Serves IPFS content from URL paths like /ipfs/ and /ipns/
+//   - [Subdomain Gateway]: Serves IPFS content from subdomains for origin isolation
+//   - [Trustless Gateway]: Provides verifiable responses for light clients ([application/vnd.ipld.raw], [application/vnd.ipld.car], and [application/vnd.ipfs.ipns-record])
+//   - [DNSLink Gateway]: Resolves DNSLink names to IPFS content
+//   - [Web Redirects File]: Supports _redirects file for URL redirection (when deserialized responses are enabled)
+//
+// It supports both trustless and deserialized response formats with features for:
+//
+//   - Serving UnixFS files and directories
+//   - Streaming UnixFS directories as TAR archives (when deserialized responses are enabled)
+//   - Content type detection and handling
+//   - Raw block responses ([application/vnd.ipld.raw])
+//   - CAR streaming ([application/vnd.ipld.car])
+//   - IPNS record responses ([application/vnd.ipfs.ipns-record])
+//   - DNSLink resolution
+//   - IPNS (InterPlanetary Name System) resolution
+//   - Rate limiting and resource protection
+//   - Prometheus metrics
+//
+// # Basic Usage
+//
+// To create a gateway handler, you need an [IPFSBackend] implementation and a [Config]:
+//
+//	backend := ... // Your IPFSBackend implementation
+//	config := gateway.Config{
+//	    DeserializedResponses: true,
+//	    MaxConcurrentRequests: 4096,
+//	    RetrievalTimeout:      30 * time.Second,
+//	}
+//	handler := gateway.NewHandler(config, backend)
+//
+// The handler can then be used with any standard Go HTTP server:
+//
+//	http.ListenAndServe(":8080", handler)
+//
+// # Configuration
+//
+// The gateway behavior can be customized through the [Config] structure:
+//
+//   - [Config.DeserializedResponses]: Enable/disable deserialized responses (HTML, images, etc.)
+//   - [Config.MaxConcurrentRequests]: Limit concurrent HTTP requests (default: 4096)
+//   - [Config.RetrievalTimeout]: Maximum time between data writes (default: 30s)
+//   - [Config.PublicGateways]: Configure behavior for specific hostnames
+//   - [Config.NoDNSLink]: Disable DNSLink resolution
+//
+// # Resource Protection
+//
+// The gateway includes built-in resource protection mechanisms:
+//
+//   - Concurrent request limiting to prevent overload
+//   - Retrieval timeouts to prevent stuck requests
+//   - Prometheus metrics for monitoring
+//
+// When limits are exceeded, the gateway returns appropriate HTTP status codes:
+//   - 429 Too Many Requests: When concurrent request limit is reached
+//   - 504 Gateway Timeout: When retrieval timeout is exceeded
+//
+// # Metrics
+//
+// The gateway exposes Prometheus metrics for monitoring:
+//
+//   - ipfs_http_gw_concurrent_requests: Current number of concurrent requests
+//   - ipfs_http_gw_responses_total: Total responses by status code
+//   - ipfs_http_gw_retrieval_timeouts_total: Retrieval timeout events
+//
+// [IPFS HTTP Gateway]: https://specs.ipfs.tech/http-gateways/
+// [Path Gateway]: https://specs.ipfs.tech/http-gateways/path-gateway/
+// [Subdomain Gateway]: https://specs.ipfs.tech/http-gateways/subdomain-gateway/
+// [Trustless Gateway]: https://specs.ipfs.tech/http-gateways/trustless-gateway/
+// [DNSLink Gateway]: https://specs.ipfs.tech/http-gateways/dnslink-gateway/
+// [Web Redirects File]: https://specs.ipfs.tech/http-gateways/web-redirects-file/
+// [application/vnd.ipld.raw]: https://www.iana.org/assignments/media-types/application/vnd.ipld.raw
+// [application/vnd.ipld.car]: https://www.iana.org/assignments/media-types/application/vnd.ipld.car
+// [application/vnd.ipfs.ipns-record]: https://www.iana.org/assignments/media-types/application/vnd.ipfs.ipns-record
+package gateway
diff --git a/gateway/errors.go b/gateway/errors.go
index 5ea15ff1..62b5c873 100644
--- a/gateway/errors.go
+++ b/gateway/errors.go
@@ -24,6 +24,16 @@ var (
 	ErrBadGateway          = NewErrorStatusCodeFromStatus(http.StatusBadGateway)
 	ErrServiceUnavailable  = NewErrorStatusCodeFromStatus(http.StatusServiceUnavailable)
 	ErrTooManyRequests     = NewErrorStatusCodeFromStatus(http.StatusTooManyRequests)
+
+	// Errors for user input validation (prevent user input in logs)
+	errUnsupportedFormat        = errors.New("unsupported response format requested")
+	errConversionNotSupported   = errors.New("converting to requested format is not supported")
+	errInvalidURIQueryParameter = errors.New("failed to parse uri query parameter")
+	errInvalidURIScheme         = errors.New("uri query parameter scheme must be ipfs or ipns")
+	errUnsupportedCarScope      = errors.New("unsupported application/vnd.ipld.car scope parameter")
+	errUnsupportedCarOrder      = errors.New("unsupported application/vnd.ipld.car order parameter")
+	errUnsupportedCarDups       = errors.New("unsupported application/vnd.ipld.car dups parameter")
+	errIndexNotReadable         = errors.New("index.html could not be read: not a file")
 )
 
 // ErrorRetryAfter wraps any error with "retry after" hint. When an error of this type
@@ -166,6 +176,35 @@ func (epr ErrPartialResponse) Error() string {
 	return "received a partial CAR response from the backend"
 }
 
+// writeErrorResponse writes an error response with the given status code and message.
+// It returns HTML or plain text based on the Accept header and DisableHTMLErrors config.
+func writeErrorResponse(w http.ResponseWriter, r *http.Request, c *Config, statusCode int, message string) {
+	// Check if HTML response is appropriate
+	acceptsHTML := false
+	if c != nil && !c.DisableHTMLErrors {
+		acceptsHTML = strings.Contains(r.Header.Get("Accept"), "text/html")
+	}
+
+	if acceptsHTML {
+		w.Header().Set("Content-Type", "text/html")
+		w.WriteHeader(statusCode)
+		err := assets.ErrorTemplate.Execute(w, assets.ErrorTemplateData{
+			GlobalData: assets.GlobalData{
+				Menu: c.Menu,
+			},
+			StatusCode: statusCode,
+			StatusText: http.StatusText(statusCode),
+			Error:      message,
+		})
+		if err != nil {
+			fmt.Fprintf(w, "error during body generation: %v", err)
+		}
+	} else {
+		// plain text response
+		http.Error(w, message, statusCode)
+	}
+}
+
 func webError(w http.ResponseWriter, r *http.Request, c *Config, err error, defaultCode int) {
 	code := defaultCode
 
@@ -200,24 +239,13 @@ func webError(w http.ResponseWriter, r *http.Request, c *Config, err error, defa
 		code = gwErr.StatusCode
 	}
 
-	acceptsHTML := !c.DisableHTMLErrors && strings.Contains(r.Header.Get("Accept"), "text/html")
-	if acceptsHTML {
-		w.Header().Set("Content-Type", "text/html")
-		w.WriteHeader(code)
-		err = assets.ErrorTemplate.Execute(w, assets.ErrorTemplateData{
-			GlobalData: assets.GlobalData{
-				Menu: c.Menu,
-			},
-			StatusCode: code,
-			StatusText: http.StatusText(code),
-			Error:      err.Error(),
-		})
-		if err != nil {
-			_, _ = w.Write([]byte(fmt.Sprintf("error during body generation: %v", err)))
-		}
-	} else {
-		http.Error(w, err.Error(), code)
-	}
+	log.Debugw("serving error response",
+		"path", r.URL.Path,
+		"method", r.Method,
+		"error", err,
+		"code", code)
+
+	writeErrorResponse(w, r, c, code, err.Error())
 }
 
 // isErrNotFound returns true for IPLD errors that should return 4xx errors (e.g. the path doesn't exist, the data is
diff --git a/gateway/gateway.go b/gateway/gateway.go
index 6cf78f5a..26fa55ee 100644
--- a/gateway/gateway.go
+++ b/gateway/gateway.go
@@ -3,7 +3,6 @@ package gateway
 import (
 	"context"
 	"errors"
-	"fmt"
 	"io"
 	"strconv"
 	"strings"
@@ -14,6 +13,25 @@ import (
 	"github.com/ipfs/boxo/ipld/unixfs"
 	"github.com/ipfs/boxo/path"
 	"github.com/ipfs/go-cid"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Default values for gateway configuration limits
+const (
+	// DefaultRetrievalTimeout is the default maximum duration for initial content retrieval
+	// (time to first byte) and subsequent writes to the HTTP response body.
+	DefaultRetrievalTimeout = 30 * time.Second
+
+	// DefaultMaxConcurrentRequests is the default maximum number of concurrent HTTP requests
+	// that the gateway will process. The default value of 4096 is suitable for most deployments
+	// and aligns with common reverse proxy configurations (e.g., nginx with 8 workers × 1024 connections).
+	//
+	// When to adjust this value:
+	//  - Too many HTTP 429 responses with available resources: increase the value
+	//  - High memory usage or resource exhaustion: decrease the value
+	//
+	// See the MaxConcurrentRequests field documentation for detailed tuning guidance.
+	DefaultMaxConcurrentRequests = 4096
 )
 
 // Config is the configuration used when creating a new gateway handler.
@@ -48,6 +66,58 @@ type Config struct {
 	// directory listings, DAG previews and errors. These will be displayed to the
 	// right of "About IPFS" and "Install IPFS".
 	Menu []assets.MenuItem
+
+	// RetrievalTimeout enforces a maximum duration for content retrieval:
+	// - Time to first byte: If the gateway cannot start writing the response within
+	//   this duration (e.g., stuck searching for providers), a 504 Gateway Timeout
+	//   is returned.
+	// - Time between writes: After the first byte, the timeout resets each time new
+	//   bytes are written to the client. If the gateway cannot write additional data
+	//   within this duration after the last successful write, the response is
+	//   terminated (with 504 for headers not sent, or truncation for headers already sent).
+	// This helps free resources when the gateway gets stuck looking for providers
+	// or cannot retrieve parts of the requested content.
+	// A value of 0 disables this timeout.
+	RetrievalTimeout time.Duration
+
+	// MaxConcurrentRequests sets the maximum number of concurrent HTTP requests
+	// that the gateway will process. This limit protects against resource exhaustion
+	// by constraining memory usage, file descriptors, and concurrent goroutines.
+	//
+	// The default value of 4096 works well for most deployments. Adjust only if you
+	// experience specific symptoms:
+	//
+	// Symptoms and solutions:
+	//  - Many HTTP 429 responses with low resource usage: increase this value
+	//  - High memory usage or resource exhaustion: decrease this value
+	//  - Poor throughput despite available resources: increase this value
+	//
+	// How to tune this value:
+	//
+	//  1. Calculate your reverse proxy's capacity:
+	//     - nginx: worker_processes × worker_connections ÷ 2
+	//     - Example: 8 workers × 1024 connections = 4096 max proxied requests
+	//     - Check: nginx -T | grep -E 'worker_processes|worker_connections'
+	//     - Apache: Check MaxRequestWorkers in httpd.conf
+	//     - HAProxy: Check global maxconn and frontend/backend limits
+	//     - Caddy/Traefik: Check system limits (ulimit -n)
+	//
+	//  2. Monitor your gateway metrics:
+	//     - Count of 429 responses (indicates limit is too low)
+	//     - Active connection count (should stay below limit)
+	//     - Resource utilization (CPU, memory, file descriptors)
+	//
+	//  3. Set MaxConcurrentRequests to match your deployment:
+	//     - Start with proxy capacity (e.g., 4096 for nginx example above)
+	//     - Reduce by 5-10% if you want safety margin
+	//     - Increase if your proxy has higher limits or if using CDN
+	//
+	// A value of 0 disables the limit entirely (use with caution).
+	MaxConcurrentRequests int
+
+	// MetricsRegistry is the Prometheus registry to use for metrics.
+	// If nil, prometheus.DefaultRegisterer will be used.
+	MetricsRegistry prometheus.Registerer
 }
 
 // PublicGateway is the specification of an IPFS Public Gateway.
@@ -186,7 +256,7 @@ func NewDuplicateBlocksPolicy(dupsValue string) (DuplicateBlocksPolicy, error) {
 	case "":
 		return DuplicateBlocksUnspecified, nil
 	}
-	return 0, fmt.Errorf("unsupported application/vnd.ipld.car content type dups parameter: %q", dupsValue)
+	return 0, errUnsupportedCarDups
 }
 
 func (d DuplicateBlocksPolicy) Bool() bool {
diff --git a/gateway/handler.go b/gateway/handler.go
index 3281788f..03ff2a2c 100644
--- a/gateway/handler.go
+++ b/gateway/handler.go
@@ -68,7 +68,34 @@ type handler struct {
 //
 // [IPFS HTTP Gateway]: https://specs.ipfs.tech/http-gateways/
 func NewHandler(c Config, backend IPFSBackend) http.Handler {
-	return newHandlerWithMetrics(&c, backend)
+	// Get registry from config or use default
+	reg := c.MetricsRegistry
+	if reg == nil {
+		reg = prometheus.DefaultRegisterer
+	}
+
+	// Initialize middleware metrics with the registry
+	metrics := newMiddlewareMetrics(reg)
+
+	h := newHandlerWithMetrics(&c, backend, reg)
+
+	// Apply middleware in order (innermost to outermost)
+	var handler http.Handler = h
+
+	// Retrieval timeout middleware (innermost after main handler)
+	if c.RetrievalTimeout > 0 {
+		handler = withRetrievalTimeout(handler, c.RetrievalTimeout, &c, metrics)
+	}
+
+	// Concurrent request limiter middleware
+	if c.MaxConcurrentRequests > 0 {
+		handler = withConcurrentRequestLimiter(handler, c.MaxConcurrentRequests, &c, metrics)
+	}
+
+	// Response metrics wrapper (outermost - records ALL responses including middleware responses)
+	handler = withResponseMetrics(handler, metrics)
+
+	return handler
 }
 
 // serveContent replies to the request using the content in the provided Reader
@@ -319,8 +346,7 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) {
 		logger.Debugw("serving codec", "path", contentPath)
 		success = i.serveCodec(r.Context(), w, r, rq)
 	default: // catch-all for unsuported application/vnd.*
-		err := fmt.Errorf("unsupported format %q", responseFormat)
-		i.webError(w, r, err, http.StatusBadRequest)
+		i.webError(w, r, errUnsupportedFormat, http.StatusBadRequest)
 	}
 }
 
@@ -873,21 +899,48 @@ func (i *handler) handleWebRequestErrors(w http.ResponseWriter, r *http.Request,
 // Detect 'Cache-Control: only-if-cached' in request and return data if it is already in the local datastore.
 // https://github.com/ipfs/specs/blob/main/http-gateways/PATH_GATEWAY.md#cache-control-request-header
 func (i *handler) handleOnlyIfCached(w http.ResponseWriter, r *http.Request, contentPath path.Path) bool {
-	if r.Header.Get("Cache-Control") == "only-if-cached" {
-		if !i.backend.IsCached(r.Context(), contentPath) {
-			if r.Method == http.MethodHead {
-				w.WriteHeader(http.StatusPreconditionFailed)
-				return true
-			}
+	if r.Header.Get("Cache-Control") != "only-if-cached" {
+		return false
+	}
+
+	// If RetrievalTimeout is configured, use a timeout for the IsCached check to avoid
+	// returning 504 Gateway Timeout when the remote backend is slow.
+	// If the content is not immediately available, we should return 412 Precondition Failed.
+	// Use 80% of RetrievalTimeout for cache checks. This ensures that if the backend
+	// is slow or unresponsive, we timeout and return 412 before the overall request
+	// would hit RetrievalTimeout and return 504. This distinction is important for
+	// correct HTTP semantics: 412 means "content not in cache" while 504 means
+	// "gateway timeout", and clients may handle these differently.
+	ctx := r.Context()
+	if i.config.RetrievalTimeout != 0 {
+		cacheCheckTimeout := i.config.RetrievalTimeout * 80 / 100
+		checkCtx, cancel := context.WithTimeout(ctx, cacheCheckTimeout)
+		defer cancel()
+		ctx = checkCtx
+	}
+
+	isCached := i.backend.IsCached(ctx, contentPath)
+
+	// If the context timed out, treat as not cached
+	if ctx.Err() != nil {
+		isCached = false
+	}
+
+	if isCached && r.Method == http.MethodHead {
+		w.WriteHeader(http.StatusOK)
+		return true
+	}
+
+	if !isCached {
+		if r.Method == http.MethodHead {
+			w.WriteHeader(http.StatusPreconditionFailed)
+		} else {
 			errMsg := fmt.Sprintf("%q not in local datastore", contentPath.String())
 			http.Error(w, errMsg, http.StatusPreconditionFailed)
-			return true
-		}
-		if r.Method == http.MethodHead {
-			w.WriteHeader(http.StatusOK)
-			return true
 		}
+		return true
 	}
+
 	return false
 }
 
@@ -899,11 +952,11 @@ func handleProtocolHandlerRedirect(w http.ResponseWriter, r *http.Request, c *Co
 	if uriParam := r.URL.Query().Get("uri"); uriParam != "" {
 		u, err := url.Parse(uriParam)
 		if err != nil {
-			webError(w, r, c, fmt.Errorf("failed to parse uri query parameter: %w", err), http.StatusBadRequest)
+			webError(w, r, c, errInvalidURIQueryParameter, http.StatusBadRequest)
 			return true
 		}
 		if u.Scheme != "ipfs" && u.Scheme != "ipns" {
-			webError(w, r, c, fmt.Errorf("uri query parameter scheme must be ipfs or ipns: %w", err), http.StatusBadRequest)
+			webError(w, r, c, errInvalidURIScheme, http.StatusBadRequest)
 			return true
 		}
 
diff --git a/gateway/handler_car.go b/gateway/handler_car.go
index 221db544..dcf1b00c 100644
--- a/gateway/handler_car.go
+++ b/gateway/handler_car.go
@@ -136,8 +136,7 @@ func buildCarParams(r *http.Request, contentTypeParams map[string]string) (CarPa
 		case DagScopeEntity, DagScopeAll, DagScopeBlock:
 			params.Scope = s
 		default:
-			err := fmt.Errorf("unsupported application/vnd.ipld.car dag-scope URL parameter: %q", scopeStr)
-			return CarParams{}, err
+			return CarParams{}, errUnsupportedCarScope
 		}
 	} else {
 		params.Scope = DagScopeAll
@@ -174,7 +173,7 @@ func buildCarParams(r *http.Request, contentTypeParams map[string]string) (CarPa
 		case DagOrderUnknown, DagOrderDFS:
 			params.Order = order
 		default:
-			return CarParams{}, fmt.Errorf("unsupported application/vnd.ipld.car content type order parameter: %q", order)
+			return CarParams{}, errUnsupportedCarOrder
 		}
 	} else {
 		// when order is not specified, we use DFS as the implicit default
diff --git a/gateway/handler_codec.go b/gateway/handler_codec.go
index 4332471c..49b2e995 100644
--- a/gateway/handler_codec.go
+++ b/gateway/handler_codec.go
@@ -14,11 +14,10 @@ import (
 	"github.com/ipfs/boxo/gateway/assets"
 	"github.com/ipfs/boxo/path"
 	"github.com/ipfs/go-cid"
-	// Ensure basic codecs are registered.
-	_ "github.com/ipld/go-ipld-prime/codec/cbor"
-	_ "github.com/ipld/go-ipld-prime/codec/dagcbor"
-	_ "github.com/ipld/go-ipld-prime/codec/dagjson"
-	_ "github.com/ipld/go-ipld-prime/codec/json"
+	_ "github.com/ipld/go-ipld-prime/codec/cbor"    // Ensure basic codecs are registered.
+	_ "github.com/ipld/go-ipld-prime/codec/dagcbor" // Ensure basic codecs are registered.
+	_ "github.com/ipld/go-ipld-prime/codec/dagjson" // Ensure basic codecs are registered.
+	_ "github.com/ipld/go-ipld-prime/codec/json"    // Ensure basic codecs are registered.
 	"github.com/ipld/go-ipld-prime/multicodec"
 	"github.com/ipld/go-ipld-prime/node/basicnode"
 	mc "github.com/multiformats/go-multicodec"
@@ -145,8 +144,7 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt
 	// Let's first get the codecs that can be used with this content type.
 	toCodec, ok := contentTypeToCodec[rq.responseFormat]
 	if !ok {
-		err := fmt.Errorf("converting from %q to %q is not supported", cidCodec.String(), rq.responseFormat)
-		i.webError(w, r, err, http.StatusBadRequest)
+		i.webError(w, r, errConversionNotSupported, http.StatusBadRequest)
 		return false
 	}
 
@@ -207,7 +205,7 @@ func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r *
 		Node:       parseNode(blockCid, blockData),
 	})
 	if err != nil {
-		_, _ = w.Write([]byte(fmt.Sprintf("error during body generation: %v", err)))
+		_, _ = fmt.Fprintf(w, "error during body generation: %v", err)
 	}
 
 	return err == nil
@@ -298,6 +296,9 @@ func (i *handler) serveCodecConverted(ctx context.Context, w http.ResponseWriter
 		return true
 	}
 
+	log.Debugw("failed to write codec response",
+		"path", contentPath,
+		"error", err)
 	return false
 }
 
diff --git a/gateway/handler_ipns_record.go b/gateway/handler_ipns_record.go
index 93d1621f..56a5a91e 100644
--- a/gateway/handler_ipns_record.go
+++ b/gateway/handler_ipns_record.go
@@ -93,5 +93,8 @@ func (i *handler) serveIpnsRecord(ctx context.Context, w http.ResponseWriter, r
 		return true
 	}
 
+	log.Debugw("failed to write IPNS record response",
+		"path", rq.contentPath,
+		"error", err)
 	return false
 }
diff --git a/gateway/handler_tar.go b/gateway/handler_tar.go
index 0b263480..04d6a23a 100644
--- a/gateway/handler_tar.go
+++ b/gateway/handler_tar.go
@@ -72,7 +72,7 @@ func (i *handler) serveTAR(ctx context.Context, w http.ResponseWriter, r *http.R
 		// To improve UX/DX, we finish response stream with error message, allowing client to
 		// (1) detect error by having corrupted TAR
 		// (2) be able to reason what went wrong by instecting the tail of TAR stream
-		_, _ = w.Write([]byte(err.Error()))
+		_, _ = fmt.Fprint(w, err.Error())
 		return false
 	}
 
diff --git a/gateway/handler_unixfs_dir.go b/gateway/handler_unixfs_dir.go
index 39d7fe4a..2a029f1f 100644
--- a/gateway/handler_unixfs_dir.go
+++ b/gateway/handler_unixfs_dir.go
@@ -11,7 +11,6 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
-	"github.com/ipfs/boxo/files"
 	"github.com/ipfs/boxo/gateway/assets"
 	"github.com/ipfs/boxo/path"
 	cid "github.com/ipfs/go-cid"
@@ -92,7 +91,7 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *
 		if err == nil {
 			defer idxHeadResp.Close()
 			if !idxHeadResp.isFile {
-				i.webError(w, r, fmt.Errorf("%q could not be read: %w", imIndexPath, files.ErrNotReader), http.StatusUnprocessableEntity)
+				i.webError(w, r, errIndexNotReadable, http.StatusUnprocessableEntity)
 				return false
 			}
 			returnRangeStartsAtZero = true
@@ -105,7 +104,7 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *
 		if err == nil {
 			defer idxGetResp.Close()
 			if idxGetResp.bytes == nil {
-				i.webError(w, r, fmt.Errorf("%q could not be read: %w", imIndexPath, files.ErrNotReader), http.StatusUnprocessableEntity)
+				i.webError(w, r, errIndexNotReadable, http.StatusUnprocessableEntity)
 				return false
 			}
 			if len(ranges) > 0 {
@@ -221,7 +220,7 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *
 	rq.logger.Debugw("request processed", "tplDataDNSLink", globalData.DNSLink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash)
 
 	if err := assets.DirectoryTemplate.Execute(w, tplData); err != nil {
-		_, _ = w.Write([]byte(fmt.Sprintf("error during body generation: %v", err)))
+		_, _ = fmt.Fprintf(w, "error during body generation: %v", err)
 		return false
 	}
 
diff --git a/gateway/metrics.go b/gateway/metrics.go
index 3bc5ea65..580fef04 100644
--- a/gateway/metrics.go
+++ b/gateway/metrics.go
@@ -24,7 +24,7 @@ type ipfsBackendWithMetrics struct {
 	backendCallMetric *prometheus.HistogramVec
 }
 
-func newIPFSBackendWithMetrics(backend IPFSBackend) *ipfsBackendWithMetrics {
+func newIPFSBackendWithMetrics(backend IPFSBackend, reg prometheus.Registerer) *ipfsBackendWithMetrics {
 	// We can add buckets as a parameter in the future, but for now using static defaults
 	// suggested in https://github.com/ipfs/kubo/issues/8441
 
@@ -39,7 +39,7 @@ func newIPFSBackendWithMetrics(backend IPFSBackend) *ipfsBackendWithMetrics {
 		[]string{"name", "result"},
 	)
 
-	if err := prometheus.Register(backendCallMetric); err != nil {
+	if err := reg.Register(backendCallMetric); err != nil {
 		if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
 			backendCallMetric = are.ExistingCollector.(*prometheus.HistogramVec)
 		} else {
@@ -190,76 +190,88 @@ func (b *ipfsBackendWithMetrics) WrapContextForRequest(ctx context.Context) cont
 	return ctx
 }
 
-func newHandlerWithMetrics(c *Config, backend IPFSBackend) *handler {
+func newHandlerWithMetrics(c *Config, backend IPFSBackend, reg prometheus.Registerer) *handler {
 	i := &handler{
 		config:  c,
-		backend: newIPFSBackendWithMetrics(backend),
+		backend: newIPFSBackendWithMetrics(backend, reg),
 
 		// Response-type specific metrics
 		// ----------------------------
 		requestTypeMetric: newRequestTypeMetric(
 			"gw_request_types",
 			"The number of requests per implicit or explicit request type.",
+			reg,
 		),
 		// Generic: time it takes to execute a successful gateway request (all request types)
 		getMetric: newHistogramMetric(
 			"gw_get_duration_seconds",
 			"The time to GET a successful response to a request (all content types).",
+			reg,
 		),
 		// UnixFS: time it takes to return a file
 		unixfsFileGetMetric: newHistogramMetric(
 			"gw_unixfs_file_get_duration_seconds",
 			"The time to serve an entire UnixFS file from the gateway.",
+			reg,
 		),
 		// UnixFS: time it takes to find and serve an index.html file on behalf of a directory.
 		unixfsDirIndexGetMetric: newHistogramMetric(
 			"gw_unixfs_dir_indexhtml_get_duration_seconds",
 			"The time to serve an index.html file on behalf of a directory from the gateway. This is a subset of gw_unixfs_file_get_duration_seconds.",
+			reg,
 		),
 		// UnixFS: time it takes to generate static HTML with directory listing
 		unixfsGenDirListingGetMetric: newHistogramMetric(
 			"gw_unixfs_gen_dir_listing_get_duration_seconds",
 			"The time to serve a generated UnixFS HTML directory listing from the gateway.",
+			reg,
 		),
 		// CAR: time it takes to return requested CAR stream
 		carStreamGetMetric: newHistogramMetric(
 			"gw_car_stream_get_duration_seconds",
 			"The time to GET an entire CAR stream from the gateway.",
+			reg,
 		),
 		carStreamFailMetric: newHistogramMetric(
 			"gw_car_stream_fail_duration_seconds",
 			"How long a CAR was streamed before failing mid-stream.",
+			reg,
 		),
 		// Block: time it takes to return requested Block
 		rawBlockGetMetric: newHistogramMetric(
 			"gw_raw_block_get_duration_seconds",
 			"The time to GET an entire raw Block from the gateway.",
+			reg,
 		),
 		// TAR: time it takes to return requested TAR stream
 		tarStreamGetMetric: newHistogramMetric(
 			"gw_tar_stream_get_duration_seconds",
 			"The time to GET an entire TAR stream from the gateway.",
+			reg,
 		),
 		// TAR: time it takes to return requested TAR stream
 		tarStreamFailMetric: newHistogramMetric(
 			"gw_tar_stream_fail_duration_seconds",
 			"How long a TAR was streamed before failing mid-stream.",
+			reg,
 		),
 		// JSON/CBOR: time it takes to return requested DAG-JSON/-CBOR document
 		jsoncborDocumentGetMetric: newHistogramMetric(
 			"gw_jsoncbor_get_duration_seconds",
 			"The time to GET an entire DAG-JSON/CBOR block from the gateway.",
+			reg,
 		),
 		// IPNS Record: time it takes to return IPNS record
 		ipnsRecordGetMetric: newHistogramMetric(
 			"gw_ipns_record_get_duration_seconds",
 			"The time to GET an entire IPNS Record from the gateway.",
+			reg,
 		),
 	}
 	return i
 }
 
-func newRequestTypeMetric(name string, help string) *prometheus.CounterVec {
+func newRequestTypeMetric(name string, help string, reg prometheus.Registerer) *prometheus.CounterVec {
 	// We can add buckets as a parameter in the future, but for now using static defaults
 	// suggested in https://github.com/ipfs/kubo/issues/8441
 	metric := prometheus.NewCounterVec(
@@ -271,7 +283,7 @@ func newRequestTypeMetric(name string, help string) *prometheus.CounterVec {
 		},
 		[]string{"gateway", "type"},
 	)
-	if err := prometheus.Register(metric); err != nil {
+	if err := reg.Register(metric); err != nil {
 		if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
 			metric = are.ExistingCollector.(*prometheus.CounterVec)
 		} else {
@@ -281,7 +293,7 @@ func newRequestTypeMetric(name string, help string) *prometheus.CounterVec {
 	return metric
 }
 
-func newHistogramMetric(name string, help string) *prometheus.HistogramVec {
+func newHistogramMetric(name string, help string, reg prometheus.Registerer) *prometheus.HistogramVec {
 	// We can add buckets as a parameter in the future, but for now using static defaults
 	// suggested in https://github.com/ipfs/kubo/issues/8441
 	histogramMetric := prometheus.NewHistogramVec(
@@ -294,7 +306,7 @@ func newHistogramMetric(name string, help string) *prometheus.HistogramVec {
 		},
 		[]string{"gateway"},
 	)
-	if err := prometheus.Register(histogramMetric); err != nil {
+	if err := reg.Register(histogramMetric); err != nil {
 		if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
 			histogramMetric = are.ExistingCollector.(*prometheus.HistogramVec)
 		} else {
diff --git a/gateway/middleware_metrics.go b/gateway/middleware_metrics.go
new file mode 100644
index 00000000..18febafb
--- /dev/null
+++ b/gateway/middleware_metrics.go
@@ -0,0 +1,197 @@
+package gateway
+
+import (
+	"bufio"
+	"errors"
+	"net"
+	"net/http"
+	"strconv"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Use the main gateway logger for consistency
+// var log is defined in handler.go
+
+// middlewareMetrics holds metrics used by gateway middleware
+type middlewareMetrics struct {
+	// General HTTP response counter
+	httpResponsesTotal *prometheus.CounterVec
+
+	// Middleware-specific metrics
+	retrievalTimeouts  *prometheus.CounterVec
+	concurrentRequests prometheus.Gauge
+}
+
+// recordResponse records an HTTP response with the given status code
+func (m *middlewareMetrics) recordResponse(statusCode int) {
+	if m != nil && m.httpResponsesTotal != nil {
+		code := strconv.Itoa(statusCode)
+		m.httpResponsesTotal.With(prometheus.Labels{"code": code}).Inc()
+	}
+}
+
+// recordTimeout records both HTTP response and timeout-specific metrics
+func (m *middlewareMetrics) recordTimeout(statusCode int, truncated bool) {
+	if m == nil {
+		return
+	}
+	m.recordResponse(statusCode)
+	if m.retrievalTimeouts != nil {
+		code := strconv.Itoa(statusCode)
+		m.retrievalTimeouts.With(prometheus.Labels{
+			"code":      code,
+			"truncated": strconv.FormatBool(truncated),
+		}).Inc()
+	}
+}
+
+// incConcurrentRequests increments the concurrent requests gauge
+func (m *middlewareMetrics) incConcurrentRequests() {
+	if m != nil && m.concurrentRequests != nil {
+		m.concurrentRequests.Inc()
+	}
+}
+
+// decConcurrentRequests decrements the concurrent requests gauge
+func (m *middlewareMetrics) decConcurrentRequests() {
+	if m != nil && m.concurrentRequests != nil {
+		m.concurrentRequests.Dec()
+	}
+}
+
+// newMiddlewareMetrics creates and initializes metrics used by middleware.
+//
+// Metrics Design:
+//
+// All counter metrics include a "code" label with the HTTP status code for consistency.
+//
+// gw_responses_total{code="XXX"}
+//   - Comprehensive counter of ALL HTTP responses by status code
+//   - Includes responses from handler AND middleware
+//   - 429 responses only come from rate limiting middleware
+//
+// gw_retrieval_timeouts_total{code="XXX",truncated="true|false"}
+//   - code: The HTTP status code (504 for clean timeout, original code for truncation)
+//   - truncated: Whether response was truncated mid-stream
+//
+// gw_concurrent_requests
+//   - Gauge of currently processing requests (no code label as it's a gauge)
+//
+// Example Prometheus queries:
+//   - Rate limited requests: sum(rate(gw_responses_total{code="429"}[5m]))
+//   - Truncated 200 responses: sum(rate(gw_retrieval_timeouts_total{code="200",truncated="true"}[5m]))
+func newMiddlewareMetrics(reg prometheus.Registerer) *middlewareMetrics {
+	metrics := &middlewareMetrics{}
+
+	// Initialize the HTTP responses counter
+	metrics.httpResponsesTotal = registerOrGetMetric(
+		prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: "ipfs",
+				Subsystem: "http",
+				Name:      "gw_responses_total",
+				Help:      "Total number of HTTP responses sent by the gateway, labeled by status code.",
+			},
+			[]string{"code"},
+		),
+		"gw_responses_total",
+		reg,
+	).(*prometheus.CounterVec)
+
+	// Initialize retrieval timeout metrics
+	metrics.retrievalTimeouts = registerOrGetMetric(
+		prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: "ipfs",
+				Subsystem: "http",
+				Name:      "gw_retrieval_timeouts_total",
+				Help:      "Total number of requests that timed out due to slow content retrieval.",
+			},
+			[]string{"code", "truncated"}, // code: HTTP status, truncated: "true"/"false"
+		),
+		"gw_retrieval_timeouts_total",
+		reg,
+	).(*prometheus.CounterVec)
+
+	// Initialize the concurrent requests gauge
+	metrics.concurrentRequests = registerOrGetMetric(
+		prometheus.NewGauge(
+			prometheus.GaugeOpts{
+				Namespace: "ipfs",
+				Subsystem: "http",
+				Name:      "gw_concurrent_requests",
+				Help:      "Number of HTTP requests currently being processed by the gateway.",
+			},
+		),
+		"gw_concurrent_requests",
+		reg,
+	).(prometheus.Gauge)
+
+	return metrics
+}
+
+// withResponseMetrics wraps an http.Handler to record response status codes
+func withResponseMetrics(handler http.Handler, metrics *middlewareMetrics) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Wrap the ResponseWriter to capture status code
+		mrw := &metricsResponseWriter{
+			ResponseWriter: w,
+			statusCode:     http.StatusOK, // Default to StatusOK if WriteHeader not called
+		}
+
+		// Serve the request
+		handler.ServeHTTP(mrw, r)
+
+		// Record the response metric
+		metrics.recordResponse(mrw.statusCode)
+	})
+}
+
+// metricsResponseWriter wraps http.ResponseWriter to capture status code
+type metricsResponseWriter struct {
+	http.ResponseWriter
+	statusCode  int
+	wroteHeader bool
+}
+
+func (w *metricsResponseWriter) WriteHeader(code int) {
+	if !w.wroteHeader {
+		w.statusCode = code
+		w.wroteHeader = true
+	}
+	w.ResponseWriter.WriteHeader(code)
+}
+
+func (w *metricsResponseWriter) Write(b []byte) (int, error) {
+	if !w.wroteHeader {
+		w.WriteHeader(http.StatusOK)
+	}
+	return w.ResponseWriter.Write(b)
+}
+
+// Hijack implements http.Hijacker if the underlying ResponseWriter supports it
+func (w *metricsResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+	if hijacker, ok := w.ResponseWriter.(http.Hijacker); ok {
+		return hijacker.Hijack()
+	}
+	return nil, nil, errors.New("ResponseWriter does not support hijacking")
+}
+
+// Flush implements http.Flusher if the underlying ResponseWriter supports it
+func (w *metricsResponseWriter) Flush() {
+	if flusher, ok := w.ResponseWriter.(http.Flusher); ok {
+		flusher.Flush()
+	}
+}
+
+// registerOrGetMetric is a helper that registers a Prometheus metric and handles AlreadyRegisteredError
+func registerOrGetMetric(metric prometheus.Collector, name string, reg prometheus.Registerer) prometheus.Collector {
+	if err := reg.Register(metric); err != nil {
+		if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
+			return are.ExistingCollector
+		}
+		log.Errorf("failed to register %s: %v", name, err)
+	}
+	return metric
+}
diff --git a/gateway/middleware_ratelimit.go b/gateway/middleware_ratelimit.go
new file mode 100644
index 00000000..fed41e5a
--- /dev/null
+++ b/gateway/middleware_ratelimit.go
@@ -0,0 +1,71 @@
+package gateway
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+)
+
+const retryAfterSeconds = 60 // seconds - industry standard minimum
+
+// withConcurrentRequestLimiter limits concurrent requests using a semaphore.
+// This protects the backend from overload caused by expensive operations.
+//
+// Why semaphore instead of rate limiting:
+//   - IPFS request costs vary wildly (simple fetch vs large DAG traversal)
+//   - Semaphore limits concurrent load regardless of request cost
+//   - More appropriate than requests/second for resource protection
+//
+// Returns 429 (Too Many Requests) when at capacity:
+//   - 429 indicates client-side issue (too many requests)
+//   - 503 would incorrectly indicate server-side issue
+//   - Industry standard for rate limiting scenarios
+//
+// Retry-After is fixed at 60 seconds:
+//   - Industry standard minimum retry period
+//   - Simple and predictable for clients
+//   - Avoids complexity of dynamic calculations
+//
+// Parameters:
+//   - handler: The HTTP handler to wrap with concurrent request limiting
+//   - limit: Maximum number of concurrent requests allowed (0 disables limiting)
+//   - c: Optional configuration for controlling error page rendering (can be nil)
+func withConcurrentRequestLimiter(handler http.Handler, limit int, c *Config, metrics *middlewareMetrics) http.Handler {
+	if limit <= 0 {
+		return handler
+	}
+
+	// Create semaphore (buffered channel)
+	// Uses inverted pattern: starts empty, acquire by sending, release by receiving
+	semaphore := make(chan struct{}, limit)
+
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		select {
+		case semaphore <- struct{}{}:
+			// Acquired a slot
+			metrics.incConcurrentRequests()
+			defer func() {
+				<-semaphore // Release slot
+				metrics.decConcurrentRequests()
+			}()
+			handler.ServeHTTP(w, r)
+
+		default:
+			// At capacity - reject with 429
+			metrics.recordResponse(http.StatusTooManyRequests)
+			// No need for separate rate limits metric - 429 responses ONLY come from this middleware
+			log.Debugw("request rejected - at capacity",
+				"path", r.URL.Path,
+				"method", r.Method,
+				"remoteAddr", r.RemoteAddr,
+				"code", http.StatusTooManyRequests)
+
+			// Prevent caching of rate limit responses
+			w.Header().Set("Cache-Control", "no-store")
+			w.Header().Set("Retry-After", strconv.Itoa(retryAfterSeconds))
+
+			message := fmt.Sprintf("Too many requests. Please retry after %d seconds.", retryAfterSeconds)
+			writeErrorResponse(w, r, c, http.StatusTooManyRequests, message)
+		}
+	})
+}
diff --git a/gateway/middleware_retrieval_timeout.go b/gateway/middleware_retrieval_timeout.go
new file mode 100644
index 00000000..5ce391bd
--- /dev/null
+++ b/gateway/middleware_retrieval_timeout.go
@@ -0,0 +1,243 @@
+package gateway
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+)
+
+const truncationMessage = "\n\n[Gateway Error: Response truncated - unable to retrieve remaining data within timeout period]"
+
+// ErrRetrievalTimeout is returned on ResponseWriter Write calls after a retrieval timeout
+var ErrRetrievalTimeout = errors.New("gateway: retrieval timeout")
+
+// withRetrievalTimeout wraps an http.Handler with a timeout that enforces:
+// 1. Maximum time to first byte (initial retrieval)
+// 2. Maximum time between subsequent non-empty writes (timeout resets on each write)
+// If no data is written within the specified duration, the connection is
+// terminated with a 504 Gateway Timeout.
+//
+// Parameters:
+//   - handler: The HTTP handler to wrap with retrieval timeout
+//   - timeout: Maximum duration between writes (0 disables timeout)
+//   - c: Optional configuration for controlling error page rendering (can be nil)
+func withRetrievalTimeout(handler http.Handler, timeout time.Duration, c *Config, metrics *middlewareMetrics) http.Handler {
+	if timeout <= 0 {
+		return handler
+	}
+
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Create channels for coordination
+		handlerDone := make(chan struct{})
+		timeoutChan := make(chan struct{})
+
+		// Create a custom response writer that tracks writes
+		tw := &timeoutWriter{
+			ResponseWriter: w,
+			headers:        make(http.Header),
+			timeout:        timeout,
+			timer:          time.NewTimer(timeout),
+			request:        r,
+			config:         c,
+		}
+
+		// Start the timeout monitoring goroutine
+		go func() {
+			select {
+			case <-tw.timer.C:
+				tw.mu.Lock()
+				defer tw.mu.Unlock()
+
+				if !tw.timedOut && !tw.handlerComplete {
+					tw.timedOut = true
+					log.Debugw("retrieval timeout triggered",
+						"path", r.URL.Path,
+						"headerSent", tw.wroteHeader,
+						"bytesWritten", tw.bytesWritten)
+
+					if !tw.wroteHeader {
+						// Headers not sent yet, we can send 504
+						// Mark headers as written to prevent the handler from modifying headers
+						tw.wroteHeader = true
+						tw.headerCode = http.StatusGatewayTimeout
+
+						metrics.recordTimeout(http.StatusGatewayTimeout, false)
+						message := "Unable to retrieve content within timeout period"
+						log.Debugw("sending 504 gateway timeout",
+							"path", tw.request.URL.Path,
+							"method", tw.request.Method,
+							"remoteAddr", tw.request.RemoteAddr)
+
+						// Write error response directly to ResponseWriter (safe because handler uses tw.headers)
+						writeErrorResponse(tw.ResponseWriter, tw.request, tw.config, http.StatusGatewayTimeout, message)
+					} else {
+						// Headers already sent, response is being truncated
+						statusCode := tw.headerCode
+						if statusCode == 0 {
+							statusCode = http.StatusOK
+						}
+						metrics.recordTimeout(statusCode, true)
+
+						// Try to write truncation message (best effort)
+						fmt.Fprint(tw.ResponseWriter, truncationMessage)
+
+						// Try to hijack and force connection reset
+						if hijacker, ok := tw.ResponseWriter.(http.Hijacker); ok {
+							conn, _, err := hijacker.Hijack()
+							if err == nil {
+								tw.hijacked = true
+								// Force TCP RST instead of graceful close
+								if tcpConn, ok := conn.(*net.TCPConn); ok {
+									tcpConn.SetLinger(0)
+								}
+								conn.Close()
+								log.Debugw("response truncated due to timeout",
+									"path", tw.request.URL.Path,
+									"method", tw.request.Method,
+									"remoteAddr", tw.request.RemoteAddr,
+									"status", statusCode,
+									"bytesWritten", tw.bytesWritten)
+							} else {
+								log.Warnw("failed to hijack connection for timeout reset",
+									"path", tw.request.URL.Path,
+									"error", err)
+							}
+						}
+					}
+
+					// Signal timeout to potentially waiting handler
+					close(timeoutChan)
+				}
+
+			case <-handlerDone:
+				// Handler completed, stop timer and exit
+				tw.timer.Stop()
+			}
+		}()
+
+		// Run handler in a goroutine so we can interrupt it on timeout
+		go func() {
+			defer close(handlerDone) // Always signal completion, even on panic
+			handler.ServeHTTP(tw, r)
+			tw.mu.Lock()
+			tw.handlerComplete = true
+			tw.mu.Unlock()
+		}()
+
+		// Wait for either handler completion or timeout
+		select {
+		case <-handlerDone:
+			// Handler completed normally
+		case <-timeoutChan:
+			// Timeout occurred, response already sent by timeout goroutine
+		}
+	})
+}
+
+// timeoutWriter wraps http.ResponseWriter to track write activity
+type timeoutWriter struct {
+	http.ResponseWriter
+	headers         http.Header // Separate header map for handler to use
+	timeout         time.Duration
+	timer           *time.Timer
+	mu              sync.Mutex
+	timedOut        bool
+	wroteHeader     bool
+	headerCode      int
+	hijacked        bool
+	handlerComplete bool
+	bytesWritten    int64
+	request         *http.Request
+	config          *Config
+}
+
+// copyHeaders safely copies headers from our isolated map to the ResponseWriter.
+// This prevents race conditions by ensuring the handler only modifies tw.headers
+// while the timeout goroutine can safely write to the underlying ResponseWriter.
+func (tw *timeoutWriter) copyHeaders() {
+	if tw.wroteHeader {
+		return
+	}
+
+	// Copy headers from our map to the underlying ResponseWriter
+	dst := tw.ResponseWriter.Header()
+	for k, vv := range tw.headers {
+		// Create a new slice to avoid sharing the underlying array.
+		// dst[k][:0] reuses the destination's capacity if it exists,
+		// or creates a new slice if dst[k] is nil, then appends all values.
+		// This ensures we don't share the underlying array between maps.
+		dst[k] = append(dst[k][:0], vv...)
+	}
+
+	tw.wroteHeader = true
+}
+
+func (tw *timeoutWriter) Write(p []byte) (int, error) {
+	tw.mu.Lock()
+	defer tw.mu.Unlock()
+
+	if tw.timedOut {
+		return 0, ErrRetrievalTimeout
+	}
+
+	// Reset timer on non-empty write
+	if len(p) > 0 {
+		tw.timer.Reset(tw.timeout)
+		tw.bytesWritten += int64(len(p))
+	}
+
+	// Ensure headers are written before first body write
+	if !tw.wroteHeader {
+		tw.copyHeaders()
+		if tw.headerCode == 0 {
+			tw.headerCode = http.StatusOK
+		}
+	}
+
+	n, err := tw.ResponseWriter.Write(p)
+	return n, err
+}
+
+func (tw *timeoutWriter) WriteHeader(statusCode int) {
+	tw.mu.Lock()
+	defer tw.mu.Unlock()
+
+	if tw.timedOut || tw.wroteHeader {
+		return
+	}
+
+	tw.copyHeaders()
+	tw.headerCode = statusCode
+	tw.ResponseWriter.WriteHeader(statusCode)
+}
+
+// Header returns the header map that will be sent by WriteHeader.
+// This returns tw.headers (not the underlying ResponseWriter's headers)
+// to prevent concurrent map access between handler and timeout goroutines.
+func (tw *timeoutWriter) Header() http.Header {
+	tw.mu.Lock()
+	defer tw.mu.Unlock()
+
+	if tw.timedOut || tw.wroteHeader {
+		return make(http.Header)
+	}
+
+	return tw.headers
+}
+
+// Flush implements http.Flusher
+func (tw *timeoutWriter) Flush() {
+	tw.mu.Lock()
+	defer tw.mu.Unlock()
+
+	if tw.timedOut {
+		return
+	}
+
+	if f, ok := tw.ResponseWriter.(http.Flusher); ok {
+		f.Flush()
+	}
+}
diff --git a/go.mod b/go.mod
index 8b838458..d1248c1f 100644
--- a/go.mod
+++ b/go.mod
@@ -53,6 +53,7 @@ require (
 	github.com/multiformats/go-multistream v0.6.1
 	github.com/polydawn/refmt v0.89.0
 	github.com/prometheus/client_golang v1.23.0
+	github.com/prometheus/client_model v0.6.2
 	github.com/slok/go-http-metrics v0.13.0
 	github.com/spaolacci/murmur3 v1.1.0
 	github.com/stretchr/testify v1.10.0
@@ -140,7 +141,6 @@ require (
 	github.com/pion/turn/v4 v4.0.2 // indirect
 	github.com/pion/webrtc/v4 v4.1.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.65.0 // indirect
 	github.com/prometheus/procfs v0.17.0 // indirect
 	github.com/quic-go/qpack v0.5.1 // indirect



Test patch:
diff --git a/gateway/backend_car_test.go b/gateway/backend_car_test.go
index 3958f1e2..cccda5af 100644
--- a/gateway/backend_car_test.go
+++ b/gateway/backend_car_test.go
@@ -24,6 +24,7 @@ import (
 	carv2 "github.com/ipld/go-car/v2"
 	carbs "github.com/ipld/go-car/v2/blockstore"
 	"github.com/ipld/go-car/v2/storage"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -480,7 +481,7 @@ func TestCarBackendGetFile(t *testing.T) {
 	backend, err := NewCarBackend(fetcher)
 	require.NoError(t, err)
 
-	trustedGatewayServer := httptest.NewServer(NewHandler(Config{DeserializedResponses: true}, backend))
+	trustedGatewayServer := httptest.NewServer(NewHandler(Config{DeserializedResponses: true, MetricsRegistry: prometheus.NewRegistry()}, backend))
 	defer trustedGatewayServer.Close()
 
 	resp, err := http.Get(trustedGatewayServer.URL + "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/exampleA")
@@ -584,7 +585,7 @@ func TestCarBackendGetFileRangeRequest(t *testing.T) {
 	backend, err := NewCarBackend(fetcher)
 	require.NoError(t, err)
 
-	trustedGatewayServer := httptest.NewServer(NewHandler(Config{DeserializedResponses: true}, backend))
+	trustedGatewayServer := httptest.NewServer(NewHandler(Config{DeserializedResponses: true, MetricsRegistry: prometheus.NewRegistry()}, backend))
 	defer trustedGatewayServer.Close()
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, trustedGatewayServer.URL+"/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa", nil)
@@ -694,7 +695,7 @@ func TestCarBackendGetFileWithBadBlockReturned(t *testing.T) {
 	backend, err := NewCarBackend(fetcher)
 	require.NoError(t, err)
 
-	trustedGatewayServer := httptest.NewServer(NewHandler(Config{DeserializedResponses: true}, backend))
+	trustedGatewayServer := httptest.NewServer(NewHandler(Config{DeserializedResponses: true, MetricsRegistry: prometheus.NewRegistry()}, backend))
 	defer trustedGatewayServer.Close()
 
 	resp, err := http.Get(trustedGatewayServer.URL + "/ipfs/bafybeigcisqd7m5nf3qmuvjdbakl5bdnh4ocrmacaqkpuh77qjvggmt2sa")
@@ -799,7 +800,7 @@ func TestCarBackendGetHAMTDirectory(t *testing.T) {
 	backend, err := NewCarBackend(fetcher)
 	require.NoError(t, err)
 
-	trustedGatewayServer := httptest.NewServer(NewHandler(Config{DeserializedResponses: true}, backend))
+	trustedGatewayServer := httptest.NewServer(NewHandler(Config{DeserializedResponses: true, MetricsRegistry: prometheus.NewRegistry()}, backend))
 	defer trustedGatewayServer.Close()
 
 	resp, err := http.Get(trustedGatewayServer.URL + "/ipfs/bafybeid3fd2xxdcd3dbj7trb433h2aqssn6xovjbwnkargjv7fuog4xjdi/hamtDir/")
diff --git a/gateway/handler_car_test.go b/gateway/handler_car_test.go
index 2e79fa01..6f6d25e1 100644
--- a/gateway/handler_car_test.go
+++ b/gateway/handler_car_test.go
@@ -1,6 +1,7 @@
 package gateway
 
 import (
+	"errors"
 	"net/http"
 	"net/url"
 	"testing"
@@ -120,22 +121,25 @@ func TestCarParams(t *testing.T) {
 
 		// below ensure the implicit default (DFS and no duplicates) is correctly inferred
 		// from the value read from Accept header
-		tests := []string{
-			"application/vnd.ipld.car; dups=invalid",
-			"application/vnd.ipld.car; order=invalid",
-			"application/vnd.ipld.car; order=dfs; dups=invalid",
-			"application/vnd.ipld.car; order=invalid; dups=y",
+		tests := []struct {
+			acceptHeader string
+			expectedErr  error
+		}{
+			{"application/vnd.ipld.car; dups=invalid", errUnsupportedCarDups},
+			{"application/vnd.ipld.car; order=invalid", errUnsupportedCarOrder},
+			{"application/vnd.ipld.car; order=dfs; dups=invalid", errUnsupportedCarDups},
+			{"application/vnd.ipld.car; order=invalid; dups=y", errUnsupportedCarOrder},
 		}
 		for _, test := range tests {
 			r := mustNewRequest(t, http.MethodGet, "http://example.com/", nil)
-			r.Header.Set("Accept", test)
+			r.Header.Set("Accept", test.acceptHeader)
 
 			mediaType, formatParams, err := customResponseFormat(r)
 			assert.NoError(t, err)
 			assert.Equal(t, carResponseFormat, mediaType)
 
 			_, err = buildCarParams(r, formatParams)
-			assert.ErrorContains(t, err, "unsupported application/vnd.ipld.car content type")
+			assert.True(t, errors.Is(err, test.expectedErr), "expected error %v, got %v", test.expectedErr, err)
 		}
 	})
 }
diff --git a/gateway/handler_only_if_cached_test.go b/gateway/handler_only_if_cached_test.go
new file mode 100644
index 00000000..71c423e8
--- /dev/null
+++ b/gateway/handler_only_if_cached_test.go
@@ -0,0 +1,130 @@
+package gateway
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/ipfs/boxo/path"
+	"github.com/stretchr/testify/require"
+)
+
+// Mock backend that simulates a slow remote blockstore
+type slowBackend struct {
+	IPFSBackend
+	delay time.Duration
+}
+
+func (sb *slowBackend) IsCached(ctx context.Context, p path.Path) bool {
+	select {
+	case <-time.After(sb.delay):
+		return false
+	case <-ctx.Done():
+		// Context cancelled/timed out
+		return false
+	}
+}
+
+func TestOnlyIfCachedTimeoutBehavior(t *testing.T) {
+	testPath, _ := path.NewPath("/ipfs/QmTest")
+
+	testCases := []struct {
+		name             string
+		backendDelay     time.Duration
+		retrievalTimeout time.Duration
+		expectedMinTime  time.Duration
+		expectedMaxTime  time.Duration
+		description      string
+	}{
+		{
+			name:             "With RetrievalTimeout set",
+			backendDelay:     60 * time.Second,
+			retrievalTimeout: 5 * time.Second,
+			expectedMinTime:  3 * time.Second, // Should take at least 3s (60% of 80% of 5s = 2.4s, rounded up)
+			expectedMaxTime:  5 * time.Second, // Should not exceed the full timeout
+			description:      "Should timeout at 80% of RetrievalTimeout",
+		},
+		{
+			name:             "With RetrievalTimeout disabled",
+			backendDelay:     1 * time.Second,
+			retrievalTimeout: 0,                      // Disabled
+			expectedMinTime:  900 * time.Millisecond, // Should wait for backend (with some margin)
+			expectedMaxTime:  2 * time.Second,        // Should complete shortly after backend responds
+			description:      "Should wait for backend without timeout",
+		},
+		{
+			name:             "Quick backend response",
+			backendDelay:     10 * time.Millisecond,
+			retrievalTimeout: 5 * time.Second,
+			expectedMinTime:  5 * time.Millisecond,
+			expectedMaxTime:  100 * time.Millisecond,
+			description:      "Should return quickly when backend responds fast",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			// Setup
+			backend := &slowBackend{delay: tc.backendDelay}
+			h := &handler{
+				backend: backend,
+				config:  &Config{RetrievalTimeout: tc.retrievalTimeout},
+			}
+
+			req := httptest.NewRequest(http.MethodHead, "/ipfs/QmTest", nil)
+			req.Header.Set("Cache-Control", "only-if-cached")
+			rec := httptest.NewRecorder()
+
+			// Execute
+			start := time.Now()
+			handled := h.handleOnlyIfCached(rec, req, testPath)
+			elapsed := time.Since(start)
+
+			// Verify
+			require.True(t, handled, "Request should be handled")
+			require.Equal(t, http.StatusPreconditionFailed, rec.Code, "Should return 412")
+			require.Greater(t, elapsed, tc.expectedMinTime, tc.description)
+			require.Less(t, elapsed, tc.expectedMaxTime, tc.description)
+		})
+	}
+}
+
+// Test that only-if-cached correctly handles different HTTP methods
+func TestOnlyIfCachedMethods(t *testing.T) {
+	// Use a valid CID that is not present in the cache
+	testPath, _ := path.NewPath("/ipfs/bafkreicm2cerwpdtah2rd7rxg5jcaqsj52blfuaprkurromr5y6p3a5zlu")
+
+	// Create a mock backend that returns cached=false
+	backend := &slowBackend{delay: 10 * time.Millisecond}
+	h := &handler{
+		backend: backend,
+		config:  &Config{RetrievalTimeout: 1 * time.Second},
+	}
+
+	t.Run("HEAD method returns 412 when not cached", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodHead, "/ipfs/QmTest", nil)
+		req.Header.Set("Cache-Control", "only-if-cached")
+		rec := httptest.NewRecorder()
+
+		handled := h.handleOnlyIfCached(rec, req, testPath)
+
+		require.True(t, handled)
+		require.Equal(t, http.StatusPreconditionFailed, rec.Code)
+		require.Empty(t, rec.Body.String(), "HEAD response should have no body")
+	})
+
+	t.Run("GET method returns 412 with error message when not cached", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodGet, "/ipfs/QmTest", nil)
+		req.Header.Set("Cache-Control", "only-if-cached")
+		rec := httptest.NewRecorder()
+
+		handled := h.handleOnlyIfCached(rec, req, testPath)
+
+		require.True(t, handled)
+		require.Equal(t, http.StatusPreconditionFailed, rec.Code)
+		require.Contains(t, rec.Body.String(), "not in local datastore")
+	})
+}
diff --git a/gateway/main_test.go b/gateway/main_test.go
new file mode 100644
index 00000000..1b8fd18d
--- /dev/null
+++ b/gateway/main_test.go
@@ -0,0 +1,11 @@
+package gateway
+
+import (
+	"os"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	// Run tests
+	os.Exit(m.Run())
+}
diff --git a/gateway/middleware_metrics_test.go b/gateway/middleware_metrics_test.go
new file mode 100644
index 00000000..ef01f05a
--- /dev/null
+++ b/gateway/middleware_metrics_test.go
@@ -0,0 +1,236 @@
+package gateway
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+)
+
+// newTestMetrics creates a new middlewareMetrics instance for testing
+func newTestMetrics() *middlewareMetrics {
+	return newMiddlewareMetrics(prometheus.NewRegistry())
+}
+
+func TestWithResponseMetrics(t *testing.T) {
+	t.Run("records all response codes", func(t *testing.T) {
+		metrics := newTestMetrics()
+
+		handler := withResponseMetrics(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			switch r.URL.Path {
+			case "/success":
+				w.WriteHeader(http.StatusOK)
+				w.Write([]byte("ok"))
+			case "/notfound":
+				w.WriteHeader(http.StatusNotFound)
+				w.Write([]byte("not found"))
+			case "/error":
+				w.WriteHeader(http.StatusInternalServerError)
+				w.Write([]byte("error"))
+			}
+		}), metrics)
+
+		// Make requests with different status codes
+		testCases := []struct {
+			path         string
+			expectedCode int
+		}{
+			{"/success", http.StatusOK},
+			{"/notfound", http.StatusNotFound},
+			{"/error", http.StatusInternalServerError},
+		}
+
+		for _, tc := range testCases {
+			req := httptest.NewRequest(http.MethodGet, tc.path, nil)
+			rec := httptest.NewRecorder()
+			handler.ServeHTTP(rec, req)
+
+			if rec.Code != tc.expectedCode {
+				t.Errorf("expected status %d for %s, got %d", tc.expectedCode, tc.path, rec.Code)
+			}
+		}
+
+		// Verify metrics were recorded
+		if metrics.httpResponsesTotal != nil {
+			// Get metric values
+			ch := make(chan prometheus.Metric, 10)
+			metrics.httpResponsesTotal.Collect(ch)
+			close(ch)
+
+			foundCodes := make(map[string]bool)
+			for metric := range ch {
+				dto := &dto.Metric{}
+				metric.Write(dto)
+				if len(dto.Label) > 0 {
+					code := *dto.Label[0].Value
+					foundCodes[code] = true
+				}
+			}
+
+			// Check that our status codes were recorded
+			expectedCodes := []string{"200", "404", "500"}
+			for _, code := range expectedCodes {
+				if !foundCodes[code] {
+					t.Errorf("expected to find metric for status code %s", code)
+				}
+			}
+		}
+	})
+
+	t.Run("default status code is 200", func(t *testing.T) {
+		metrics := newTestMetrics()
+
+		handler := withResponseMetrics(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// Write without calling WriteHeader
+			w.Write([]byte("implicit 200"))
+		}), metrics)
+
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec := httptest.NewRecorder()
+		handler.ServeHTTP(rec, req)
+
+		if rec.Code != http.StatusOK {
+			t.Errorf("expected default status 200, got %d", rec.Code)
+		}
+	})
+
+	t.Run("metrics wrapper preserves flusher interface", func(t *testing.T) {
+		metrics := newTestMetrics()
+
+		handler := withResponseMetrics(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if flusher, ok := w.(http.Flusher); ok {
+				w.Write([]byte("data"))
+				flusher.Flush()
+			} else {
+				t.Error("ResponseWriter should implement Flusher")
+			}
+		}), metrics)
+
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec := httptest.NewRecorder()
+		handler.ServeHTTP(rec, req)
+	})
+}
+
+func TestMiddlewareMetricsIntegration(t *testing.T) {
+	t.Run("separate metrics for middleware events", func(t *testing.T) {
+		// Create a handler that holds the request
+		var handler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// Hold the request for 100ms
+			time.Sleep(100 * time.Millisecond)
+			w.Write([]byte("ok"))
+		})
+
+		metrics := newTestMetrics()
+
+		// Apply middleware in the same order as NewHandler
+		handler = withConcurrentRequestLimiter(handler, 1, nil, metrics)
+		handler = withResponseMetrics(handler, metrics)
+
+		// First request should succeed
+		req1 := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec1 := httptest.NewRecorder()
+		done := make(chan bool)
+		go func() {
+			handler.ServeHTTP(rec1, req1)
+			done <- true
+		}()
+
+		// Give first request time to start
+		time.Sleep(10 * time.Millisecond)
+
+		// Second request should be rate limited
+		req2 := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec2 := httptest.NewRecorder()
+		handler.ServeHTTP(rec2, req2)
+
+		if rec2.Code != http.StatusTooManyRequests {
+			t.Errorf("expected status 429 for rate limited request, got %d", rec2.Code)
+		}
+
+		// Wait for first request to complete
+		<-done
+
+		// Verify HTTP response metrics were recorded
+		if metrics.httpResponsesTotal != nil {
+			ch := make(chan prometheus.Metric, 10)
+			metrics.httpResponsesTotal.Collect(ch)
+			close(ch)
+
+			found429 := false
+			for metric := range ch {
+				dto := &dto.Metric{}
+				metric.Write(dto)
+				if len(dto.Label) > 0 {
+					if *dto.Label[0].Value == "429" {
+						found429 = true
+					}
+				}
+			}
+			if !found429 {
+				t.Error("expected to find 429 status code in general response metrics")
+			}
+		}
+	})
+
+	t.Run("timeout metrics include status code", func(t *testing.T) {
+		// Create a handler that times out
+		var handler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// Sleep longer than timeout
+			time.Sleep(100 * time.Millisecond)
+		})
+
+		metrics := newTestMetrics()
+
+		// Apply timeout middleware
+		handler = withRetrievalTimeout(handler, 50*time.Millisecond, nil, metrics)
+		handler = withResponseMetrics(handler, metrics)
+
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec := httptest.NewRecorder()
+
+		// Run in goroutine to allow timeout to trigger
+		done := make(chan bool)
+		go func() {
+			handler.ServeHTTP(rec, req)
+			done <- true
+		}()
+
+		// Wait for timeout
+		select {
+		case <-done:
+		case <-time.After(200 * time.Millisecond):
+		}
+
+		// Verify timeout metric has code label
+		if metrics.retrievalTimeouts != nil {
+			ch := make(chan prometheus.Metric, 10)
+			metrics.retrievalTimeouts.Collect(ch)
+			close(ch)
+
+			found504 := false
+			for metric := range ch {
+				dto := &dto.Metric{}
+				metric.Write(dto)
+				if len(dto.Label) >= 1 {
+					// First label should be code
+					if *dto.Label[0].Name == "code" && *dto.Label[0].Value == "504" {
+						found504 = true
+						// Verify other labels
+						for _, label := range dto.Label {
+							if *label.Name == "truncated" && *label.Value != "false" {
+								t.Error("expected truncated to be 'false' for clean timeout")
+							}
+						}
+					}
+				}
+			}
+			if !found504 {
+				t.Error("expected to find timeout metric with code '504'")
+			}
+		}
+	})
+}
diff --git a/gateway/middleware_ratelimit_test.go b/gateway/middleware_ratelimit_test.go
new file mode 100644
index 00000000..97b15763
--- /dev/null
+++ b/gateway/middleware_ratelimit_test.go
@@ -0,0 +1,288 @@
+package gateway
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestWithConcurrentRequestLimiter(t *testing.T) {
+	t.Run("returns HTML error when Accept header includes text/html", func(t *testing.T) {
+		config := &Config{DisableHTMLErrors: false}
+		blockChan := make(chan struct{})
+		defer close(blockChan)
+		started := make(chan struct{})
+
+		handler := withConcurrentRequestLimiter(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			select {
+			case <-started:
+				// Already closed, this is a subsequent request
+			default:
+				close(started) // Signal first request has started
+			}
+			// Block to simulate a busy handler
+			<-blockChan
+		}), 1, config, newTestMetrics())
+
+		// First request occupies the slot
+		req1 := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec1 := httptest.NewRecorder()
+		go handler.ServeHTTP(rec1, req1)
+
+		// Wait for first request to start and acquire the semaphore
+		<-started
+
+		// Second request with HTML accept should get HTML error
+		req2 := httptest.NewRequest(http.MethodGet, "/", nil)
+		req2.Header.Set("Accept", "text/html,application/xhtml+xml")
+		rec2 := httptest.NewRecorder()
+		handler.ServeHTTP(rec2, req2)
+
+		if rec2.Code != http.StatusTooManyRequests {
+			t.Errorf("expected status 429, got %d", rec2.Code)
+		}
+
+		contentType := rec2.Header().Get("Content-Type")
+		if contentType != "text/html" {
+			t.Errorf("expected Content-Type text/html, got %s", contentType)
+		}
+
+		body := rec2.Body.String()
+		if !bytes.Contains([]byte(body), []byte("<html")) {
+			t.Error("expected HTML response body")
+		}
+		if !bytes.Contains([]byte(body), []byte("429")) {
+			t.Error("expected 429 in HTML body")
+		}
+	})
+	t.Run("limiter disabled with zero value", func(t *testing.T) {
+		var requestCount atomic.Int32
+		handler := withConcurrentRequestLimiter(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			requestCount.Add(1)
+			w.Write([]byte("success"))
+		}), 0, nil, newTestMetrics())
+
+		// Send multiple concurrent requests
+		var wg sync.WaitGroup
+		for i := 0; i < 10; i++ {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				req := httptest.NewRequest(http.MethodGet, "/", nil)
+				rec := httptest.NewRecorder()
+				handler.ServeHTTP(rec, req)
+			}()
+		}
+		wg.Wait()
+
+		if requestCount.Load() != 10 {
+			t.Errorf("expected 10 requests to be processed, got %d", requestCount.Load())
+		}
+	})
+
+	t.Run("limits concurrent requests", func(t *testing.T) {
+		const limit = 2
+		const numRequests = 5
+		var concurrent atomic.Int32
+		var maxConcurrent atomic.Int32
+
+		// Barrier to ensure all handlers run concurrently
+		startBarrier := make(chan struct{})
+		var ready sync.WaitGroup
+		ready.Add(numRequests)
+
+		handler := withConcurrentRequestLimiter(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			cur := concurrent.Add(1)
+			defer concurrent.Add(-1)
+
+			// Track max concurrent requests
+			for {
+				max := maxConcurrent.Load()
+				if cur <= max || maxConcurrent.CompareAndSwap(max, cur) {
+					break
+				}
+			}
+
+			ready.Done()   // Signal this handler is ready
+			<-startBarrier // Wait for all handlers to be ready
+			w.Write([]byte("success"))
+		}), limit, nil, newTestMetrics())
+
+		// Send more requests than the limit
+		var wg sync.WaitGroup
+		results := make([]int, numRequests)
+		for i := 0; i < numRequests; i++ {
+			wg.Add(1)
+			go func(idx int) {
+				defer wg.Done()
+				req := httptest.NewRequest(http.MethodGet, "/", nil)
+				rec := httptest.NewRecorder()
+				handler.ServeHTTP(rec, req)
+				results[idx] = rec.Code
+				// If this request was rejected, it won't call ready.Done()
+				if rec.Code == http.StatusTooManyRequests {
+					ready.Done()
+				}
+			}(i)
+		}
+
+		// Wait for all handlers to be ready, then release them
+		ready.Wait()
+		close(startBarrier)
+		wg.Wait()
+
+		// Check that max concurrent never exceeded limit
+		if maxConcurrent.Load() > int32(limit) {
+			t.Errorf("max concurrent requests %d exceeded limit %d", maxConcurrent.Load(), limit)
+		}
+
+		// Count successful and rejected requests
+		successCount := 0
+		rejectCount := 0
+		for _, code := range results {
+			switch code {
+			case http.StatusOK:
+				successCount++
+			case http.StatusTooManyRequests:
+				rejectCount++
+			default:
+				t.Errorf("unexpected http status: %d", code)
+			}
+		}
+
+		if successCount == 0 {
+			t.Error("expected at least some successful requests")
+		}
+		if rejectCount == 0 {
+			t.Error("expected at least some rejected requests when limit exceeded")
+		}
+	})
+
+	t.Run("returns 429 with Retry-After and Cache-Control headers", func(t *testing.T) {
+		blockChan := make(chan struct{})
+		defer close(blockChan)
+		started := make(chan struct{})
+
+		handler := withConcurrentRequestLimiter(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			select {
+			case <-started:
+				// Already closed
+			default:
+				close(started) // Signal first request has started
+			}
+			<-blockChan
+			w.Write([]byte("success"))
+		}), 1, nil, newTestMetrics())
+
+		// First request should succeed
+		req1 := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec1 := httptest.NewRecorder()
+		go handler.ServeHTTP(rec1, req1)
+
+		// Wait for first request to start
+		<-started
+
+		// Second request should be rejected
+		req2 := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec2 := httptest.NewRecorder()
+		handler.ServeHTTP(rec2, req2)
+
+		if rec2.Code != http.StatusTooManyRequests {
+			t.Errorf("expected status 429, got %d", rec2.Code)
+		}
+
+		retryAfter := rec2.Header().Get("Retry-After")
+		if retryAfter == "" {
+			t.Error("expected Retry-After header to be set")
+		}
+
+		retrySeconds, err := strconv.Atoi(retryAfter)
+		if err != nil {
+			t.Errorf("Retry-After should be a number, got %s", retryAfter)
+		}
+
+		// Retry should be 60 seconds (industry standard minimum)
+		if retrySeconds != 60 {
+			t.Errorf("expected Retry-After to be 60, got %d", retrySeconds)
+		}
+
+		// Check Cache-Control header to prevent caching
+		cacheControl := rec2.Header().Get("Cache-Control")
+		if cacheControl != "no-store" {
+			t.Errorf("expected Cache-Control: no-store, got %s", cacheControl)
+		}
+	})
+
+	t.Run("Retry-After is static", func(t *testing.T) {
+		done := make(chan struct{})
+		defer close(done) // Ensure goroutine cleanup
+		started := make(chan struct{})
+
+		handler := withConcurrentRequestLimiter(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			close(started) // Signal that handler has started
+			// Hold the request until test cleanup
+			<-done
+		}), 1, nil, newTestMetrics())
+
+		// Fill the single slot
+		go func() {
+			req := httptest.NewRequest(http.MethodGet, "/", nil)
+			rec := httptest.NewRecorder()
+			handler.ServeHTTP(rec, req)
+		}()
+
+		<-started // Wait for handler to start
+
+		// Generate many rejections and verify Retry-After stays constant
+		for i := 0; i < 20; i++ {
+			req := httptest.NewRequest(http.MethodGet, "/", nil)
+			rec := httptest.NewRecorder()
+			handler.ServeHTTP(rec, req)
+
+			retryAfter := rec.Header().Get("Retry-After")
+			if retryAfter != "60" {
+				t.Errorf("expected Retry-After to be static at 60, got %s", retryAfter)
+			}
+		}
+	})
+}
+
+func TestMiddlewareIntegration(t *testing.T) {
+	t.Run("both middlewares work together", func(t *testing.T) {
+		var handler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// Simulate processing that writes data periodically
+			for i := 0; i < 3; i++ {
+				w.Write([]byte(fmt.Sprintf("data%d", i)))
+				w.(http.Flusher).Flush()
+				time.Sleep(30 * time.Millisecond)
+			}
+		})
+
+		// Apply both middlewares
+		metrics := newTestMetrics()
+		handler = withRetrievalTimeout(handler, 50*time.Millisecond, nil, metrics)
+		handler = withConcurrentRequestLimiter(handler, 2, nil, metrics)
+
+		// Test that requests work normally within limits
+		var wg sync.WaitGroup
+		for i := 0; i < 2; i++ {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				req := httptest.NewRequest(http.MethodGet, "/", nil)
+				rec := httptest.NewRecorder()
+				handler.ServeHTTP(rec, req)
+				if rec.Code != http.StatusOK {
+					t.Errorf("expected status 200, got %d", rec.Code)
+				}
+			}()
+		}
+		wg.Wait()
+	})
+}
diff --git a/gateway/middleware_retrieval_timeout_test.go b/gateway/middleware_retrieval_timeout_test.go
new file mode 100644
index 00000000..61da2052
--- /dev/null
+++ b/gateway/middleware_retrieval_timeout_test.go
@@ -0,0 +1,403 @@
+package gateway
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func TestWithRetrievalTimeout(t *testing.T) {
+	t.Run("returns HTML error when Accept header includes text/html", func(t *testing.T) {
+		config := &Config{DisableHTMLErrors: false}
+		handler := withRetrievalTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// Don't write anything - simulate stuck retrieval
+			time.Sleep(100 * time.Millisecond)
+		}), 50*time.Millisecond, config, newTestMetrics())
+
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		req.Header.Set("Accept", "text/html,application/xhtml+xml")
+		rec := httptest.NewRecorder()
+
+		done := make(chan bool)
+		go func() {
+			handler.ServeHTTP(rec, req)
+			done <- true
+		}()
+
+		select {
+		case <-done:
+		case <-time.After(200 * time.Millisecond):
+		}
+
+		if rec.Code != http.StatusGatewayTimeout {
+			t.Errorf("expected status 504, got %d", rec.Code)
+		}
+
+		contentType := rec.Header().Get("Content-Type")
+		if contentType != "text/html" {
+			t.Errorf("expected Content-Type text/html, got %s", contentType)
+		}
+
+		body := rec.Body.String()
+		if !bytes.Contains([]byte(body), []byte("<html")) {
+			t.Error("expected HTML response body")
+		}
+		if !bytes.Contains([]byte(body), []byte("504")) {
+			t.Error("expected 504 in HTML body")
+		}
+	})
+	t.Run("timeout on initial retrieval block", func(t *testing.T) {
+		blockChan := make(chan struct{})
+		defer close(blockChan) // Ensure goroutine cleanup
+
+		// Simulate a handler that blocks indefinitely on initial retrieval
+		// (e.g., searching for providers that don't exist)
+		handler := withRetrievalTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// Block until test cleanup - simulating stuck provider search
+			<-blockChan
+		}), 50*time.Millisecond, nil, newTestMetrics())
+
+		req := httptest.NewRequest(http.MethodGet, "/ipfs/bafkreif6lrhgz3fpiwypdk65qrqiey7svgpggruhbylrgv32l3izkqpsc4", nil)
+		rec := httptest.NewRecorder()
+
+		done := make(chan bool)
+		go func() {
+			handler.ServeHTTP(rec, req)
+			done <- true
+		}()
+
+		// Wait for timeout to trigger
+		select {
+		case <-done:
+			// Good, handler completed (due to timeout)
+		case <-time.After(200 * time.Millisecond):
+			t.Fatal("handler did not complete within expected time")
+		}
+
+		if rec.Code != http.StatusGatewayTimeout {
+			t.Errorf("expected status 504, got %d", rec.Code)
+		}
+		body := rec.Body.String()
+		if !bytes.Contains([]byte(body), []byte("Unable to retrieve content within timeout period")) {
+			t.Errorf("expected timeout message in body, got %s", body)
+		}
+	})
+
+	t.Run("timeout on slow initial retrieval", func(t *testing.T) {
+		// Simulate a handler that takes too long to retrieve initial data
+		handler := withRetrievalTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// Simulate slow provider discovery/block retrieval
+			time.Sleep(100 * time.Millisecond)
+			// This write should never happen due to timeout
+			w.Write([]byte("should not appear"))
+		}), 50*time.Millisecond, nil, newTestMetrics())
+
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec := httptest.NewRecorder()
+
+		done := make(chan bool)
+		go func() {
+			handler.ServeHTTP(rec, req)
+			done <- true
+		}()
+
+		select {
+		case <-done:
+		case <-time.After(200 * time.Millisecond):
+			t.Fatal("handler did not complete within expected time")
+		}
+
+		if rec.Code != http.StatusGatewayTimeout {
+			t.Errorf("expected status 504, got %d", rec.Code)
+		}
+		body := rec.Body.String()
+		if bytes.Contains([]byte(body), []byte("should not appear")) {
+			t.Errorf("body should not contain 'should not appear', got %s", body)
+		}
+	})
+	t.Run("timeout disabled with zero value", func(t *testing.T) {
+		handler := withRetrievalTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			time.Sleep(100 * time.Millisecond)
+			w.Write([]byte("success"))
+		}), 0, nil, newTestMetrics())
+
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec := httptest.NewRecorder()
+		handler.ServeHTTP(rec, req)
+
+		if rec.Code != http.StatusOK {
+			t.Errorf("expected status 200, got %d", rec.Code)
+		}
+		if rec.Body.String() != "success" {
+			t.Errorf("expected body 'success', got %s", rec.Body.String())
+		}
+	})
+
+	t.Run("timeout triggers after no data", func(t *testing.T) {
+		handler := withRetrievalTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// Don't write anything for longer than timeout
+			time.Sleep(150 * time.Millisecond)
+		}), 50*time.Millisecond, nil, newTestMetrics())
+
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec := httptest.NewRecorder()
+
+		// Run in goroutine to allow timeout to trigger
+		done := make(chan bool)
+		go func() {
+			handler.ServeHTTP(rec, req)
+			done <- true
+		}()
+
+		// Wait for handler to complete (including timeout)
+		select {
+		case <-done:
+		case <-time.After(200 * time.Millisecond):
+		}
+
+		if rec.Code != http.StatusGatewayTimeout {
+			t.Errorf("expected status 504, got %d", rec.Code)
+		}
+	})
+
+	t.Run("timeout resets on data write", func(t *testing.T) {
+		handler := withRetrievalTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// Write data in chunks, each write should reset the timeout
+			for i := 0; i < 3; i++ {
+				w.Write([]byte(fmt.Sprintf("chunk%d", i)))
+				w.(http.Flusher).Flush()
+				time.Sleep(75 * time.Millisecond) // Less than timeout
+			}
+		}), 100*time.Millisecond, nil, newTestMetrics())
+
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec := httptest.NewRecorder()
+		handler.ServeHTTP(rec, req)
+
+		if rec.Code != http.StatusOK {
+			t.Errorf("expected status 200, got %d", rec.Code)
+		}
+		expected := "chunk0chunk1chunk2"
+		if rec.Body.String() != expected {
+			t.Errorf("expected body '%s', got %s", expected, rec.Body.String())
+		}
+	})
+
+	t.Run("empty writes don't reset timeout", func(t *testing.T) {
+		handler := withRetrievalTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Write([]byte("initial"))
+			// Empty writes shouldn't reset timeout
+			for i := 0; i < 5; i++ {
+				w.Write([]byte{})
+				time.Sleep(30 * time.Millisecond)
+			}
+		}), 100*time.Millisecond, nil, newTestMetrics())
+
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec := httptest.NewRecorder()
+		handler.ServeHTTP(rec, req)
+
+		// Wait for timeout
+		time.Sleep(200 * time.Millisecond)
+
+		// The initial write should succeed, but timeout should trigger after
+		body := rec.Body.String()
+		if !bytes.Contains([]byte(body), []byte("initial")) {
+			t.Errorf("expected body to contain 'initial', got %s", body)
+		}
+	})
+
+	t.Run("truncation message on mid-stream timeout", func(t *testing.T) {
+		handler := withRetrievalTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte("partial data"))
+			w.(http.Flusher).Flush()
+			// Now sleep longer than timeout
+			time.Sleep(150 * time.Millisecond)
+			// This write should fail
+			w.Write([]byte("should not appear"))
+		}), 50*time.Millisecond, nil, newTestMetrics())
+
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec := httptest.NewRecorder()
+
+		done := make(chan bool)
+		go func() {
+			handler.ServeHTTP(rec, req)
+			done <- true
+		}()
+
+		select {
+		case <-done:
+		case <-time.After(300 * time.Millisecond):
+		}
+
+		body := rec.Body.String()
+		// Should contain the initial data
+		if !bytes.Contains([]byte(body), []byte("partial data")) {
+			t.Errorf("expected body to contain 'partial data', got %s", body)
+		}
+		// Should contain truncation message
+		if !bytes.Contains([]byte(body), []byte(truncationMessage)) {
+			t.Errorf("expected body to contain truncation message, got %s", body)
+		}
+		// Should not contain data after timeout
+		if bytes.Contains([]byte(body), []byte("should not appear")) {
+			t.Errorf("body should not contain 'should not appear', got %s", body)
+		}
+	})
+
+	t.Run("tracks status code for truncated responses", func(t *testing.T) {
+		handler := withRetrievalTimeout(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusPartialContent) // 206
+			w.Write([]byte("partial"))
+			w.(http.Flusher).Flush()
+			time.Sleep(150 * time.Millisecond)
+		}), 50*time.Millisecond, nil, newTestMetrics())
+
+		req := httptest.NewRequest(http.MethodGet, "/", nil)
+		rec := httptest.NewRecorder()
+
+		done := make(chan bool)
+		go func() {
+			handler.ServeHTTP(rec, req)
+			done <- true
+		}()
+
+		select {
+		case <-done:
+		case <-time.After(300 * time.Millisecond):
+		}
+
+		// The recorder should have captured the original status
+		if rec.Code != http.StatusPartialContent {
+			t.Errorf("expected status 206, got %d", rec.Code)
+		}
+	})
+}
+
+// TestConcurrentHeaderAccessRace verifies that no "concurrent map writes" panic
+// occurs when the timeout handler writes an error response while the request
+// handler is still trying to modify headers.
+//
+// The race condition occurs when both the timeout goroutine and the handler
+// goroutine try to access the same underlying http.Header map. Without proper
+// isolation, this causes a "concurrent map writes" panic.
+//
+// This test is designed to be run with the race detector:
+//
+//	go test -race -run TestConcurrentHeaderAccessRace ./gateway
+//
+// For more thorough testing with multiple iterations:
+//
+//	go test -shuffle=on -count=10 -failfast -race -parallel 8 -run TestConcurrentHeaderAccessRace ./gateway -v
+//
+// The race is triggered by having the handler continuously modify headers
+// while a timeout occurs, causing both goroutines to access the same header
+// map if not properly isolated.
+func TestConcurrentHeaderAccessRace(t *testing.T) {
+	t.Run("race before headers sent (504 response)", func(t *testing.T) {
+		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// Continuously modify headers to create race
+			for i := 0; i < 100; i++ {
+				w.Header().Set("X-Test", fmt.Sprintf("value-%d", i))
+				time.Sleep(time.Microsecond)
+			}
+		})
+
+		config := &Config{DisableHTMLErrors: true}
+		timeoutHandler := withRetrievalTimeout(handler, 5*time.Millisecond, config, newTestMetrics())
+
+		// Run with race detector
+		for i := 0; i < 5; i++ {
+			rec := httptest.NewRecorder()
+			req := httptest.NewRequest(http.MethodGet, "/test", nil)
+
+			done := make(chan bool)
+			go func() {
+				defer func() {
+					if r := recover(); r != nil {
+						// Check if it's the concurrent map writes panic
+						if panicStr, ok := r.(string); ok && panicStr == "concurrent map writes" {
+							t.Logf("Reproduced concurrent map writes panic!")
+						}
+					}
+					done <- true
+				}()
+				timeoutHandler.ServeHTTP(rec, req)
+			}()
+
+			select {
+			case <-done:
+			case <-time.After(100 * time.Millisecond):
+			}
+		}
+	})
+
+	t.Run("race after headers sent (200 response truncation)", func(t *testing.T) {
+		// Test the race condition when timeout occurs after headers are already sent.
+		// This tests the truncation path where we write truncation message and hijack connection.
+
+		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// Write headers and some data first
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte("initial data"))
+			if f, ok := w.(http.Flusher); ok {
+				f.Flush()
+			}
+
+			// Then continue modifying headers (which should be no-op after WriteHeader)
+			// and writing more data slowly to trigger timeout during response
+			for i := 0; i < 100; i++ {
+				// Try to set headers after WriteHeader (should be ignored but could race)
+				w.Header().Set("X-Late-Header", fmt.Sprintf("value-%d", i))
+
+				// Write small chunks of data
+				if i%10 == 0 {
+					w.Write([]byte(fmt.Sprintf("chunk-%d", i)))
+					if f, ok := w.(http.Flusher); ok {
+						f.Flush()
+					}
+				}
+				time.Sleep(time.Millisecond)
+			}
+		})
+
+		config := &Config{DisableHTMLErrors: true}
+		timeoutHandler := withRetrievalTimeout(handler, 20*time.Millisecond, config, newTestMetrics())
+
+		// Run with race detector
+		for i := 0; i < 5; i++ {
+			rec := httptest.NewRecorder()
+			req := httptest.NewRequest(http.MethodGet, "/test", nil)
+
+			done := make(chan bool)
+			go func() {
+				defer func() {
+					if r := recover(); r != nil {
+						// Check if it's the concurrent map writes panic
+						if panicStr, ok := r.(string); ok && panicStr == "concurrent map writes" {
+							t.Errorf("Reproduced concurrent map writes panic in truncation path!")
+						}
+					}
+					done <- true
+				}()
+				timeoutHandler.ServeHTTP(rec, req)
+			}()
+
+			select {
+			case <-done:
+			case <-time.After(150 * time.Millisecond):
+			}
+
+			// Verify we got initial data before timeout
+			body := rec.Body.String()
+			if !bytes.Contains([]byte(body), []byte("initial data")) {
+				t.Errorf("Expected initial data in response, got: %s", body)
+			}
+		}
+	})
+}
diff --git a/gateway/utilities_test.go b/gateway/utilities_test.go
index 22f5750f..c186455e 100644
--- a/gateway/utilities_test.go
+++ b/gateway/utilities_test.go
@@ -22,6 +22,7 @@ import (
 	carblockstore "github.com/ipld/go-car/v2/blockstore"
 	"github.com/libp2p/go-libp2p/core/crypto"
 	"github.com/libp2p/go-libp2p/core/routing"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -233,6 +234,7 @@ func newTestServerAndNode(t *testing.T, fixturesFile string) (*httptest.Server,
 func newTestServer(t *testing.T, backend IPFSBackend) *httptest.Server {
 	return newTestServerWithConfig(t, backend, Config{
 		DeserializedResponses: true,
+		MetricsRegistry:       prometheus.NewRegistry(),
 	})
 }
 
@@ -241,6 +243,10 @@ func newTestServerWithConfig(t *testing.T, backend IPFSBackend, config Config) *
 }
 
 func newTestServerWithConfigAndHeaders(t *testing.T, backend IPFSBackend, config Config, headers map[string][]string) *httptest.Server {
+	// Ensure each test has its own isolated metrics registry
+	if config.MetricsRegistry == nil {
+		config.MetricsRegistry = prometheus.NewRegistry()
+	}
 	handler := NewHandler(config, backend)
 	mux := http.NewServeMux()
 	mux.Handle("/ipfs/", handler)