Repository: ipfs/boxo

Issue:
gateway: error when host is IP and UseSubdomains=true
It is not possible to have subdomains on IP – there should be validation that errors when someone tries to run subdomain gateway on IP:

```json
"PublicGateways": {
      "127.0.0.1:3334": {
        "DeserializedResponses": true,
        "UseSubdomains": true
      },
```




Discussion:
@gammazero: Closed by #903

Solution patch:
diff --git a/.github/workflows/generated-pr.yml b/.github/workflows/generated-pr.yml
new file mode 100644
index 00000000..b8c5cc63
--- /dev/null
+++ b/.github/workflows/generated-pr.yml
@@ -0,0 +1,14 @@
+name: Close Generated PRs
+
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+
+permissions:
+  issues: write
+  pull-requests: write
+
+jobs:
+  stale:
+    uses: ipdxco/unified-github-workflows/.github/workflows/reusable-generated-pr.yml@v1
diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml
index 16d65d72..7c955c41 100644
--- a/.github/workflows/stale.yml
+++ b/.github/workflows/stale.yml
@@ -1,8 +1,9 @@
-name: Close and mark stale issue
+name: Close Stale Issues
 
 on:
   schedule:
     - cron: '0 0 * * *'
+  workflow_dispatch:
 
 permissions:
   issues: write
@@ -10,4 +11,4 @@ permissions:
 
 jobs:
   stale:
-    uses: pl-strflt/.github/.github/workflows/reusable-stale-issue.yml@v0.3
+    uses: ipdxco/unified-github-workflows/.github/workflows/reusable-stale-issue.yml@v1
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4c901905..a72562e9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,9 +16,13 @@ The following emojis are used to highlight certain changes:
 
 ### Added
 
+- `gateway`: generated HTML with UnixFS directory listings now include a button for copying CIDs of child entities [#899](https://github.com/ipfs/boxo/pull/899)
+
 ### Changed
 
 - upgrade to `go-libp2p` [v0.41.1](https://github.com/libp2p/go-libp2p/releases/tag/v0.41.1)
+- `bitswap/network`: Add a new `requests_in_flight` metric gauge that measures how many bitswap streams are being written or read at a given time.
+- improve speed of data onboarding by batching/bufering provider queue writes [#888](https://github.com/ipfs/boxo/pull/888)
 
 ### Removed
 
@@ -26,6 +30,7 @@ The following emojis are used to highlight certain changes:
 
 - `gateway`: query parameters are now supported and preserved in redirects triggered by a [`_redirects`](https://specs.ipfs.tech/http-gateways/web-redirects-file/) file [#886](https://github.com/ipfs/boxo/pull/886)
 - `provider`: adjusted first reprovide timing after node reboot [#890](https://github.com/ipfs/boxo/pull/890)
+- `gateway`: validate configuration and warn when `UseSubdomains=true` is used with IP-based hostnames [#903](https://github.com/ipfs/boxo/pull/903)
 
 ### Security
 
@@ -75,7 +80,6 @@ The following emojis are used to highlight certain changes:
 ### Added
 
 - `bitswap/client`: Add `DontHaveTimeoutConfig` type alias and `func DontHaveTimeoutConfig()` to expose config defined in internal package.
-
 ### Changed
 
 - move `ipld/unixfs` from gogo protobuf [#841](https://github.com/ipfs/boxo/pull/841)
diff --git a/bitswap/network/bsnet/ipfs_impl.go b/bitswap/network/bsnet/ipfs_impl.go
index 427f3ff0..3c3a6a1f 100644
--- a/bitswap/network/bsnet/ipfs_impl.go
+++ b/bitswap/network/bsnet/ipfs_impl.go
@@ -293,6 +293,9 @@ func (bsnet *impl) msgToStream(ctx context.Context, s network.Stream, msg bsmsg.
 		log.Warnf("error setting deadline: %s", err)
 	}
 
+	bsnet.metrics.RequestsInFlight.Inc()
+	defer bsnet.metrics.RequestsInFlight.Dec()
+
 	// Older Bitswap versions use a slightly different wire format so we need
 	// to convert the message to the appropriate format depending on the remote
 	// peer's Bitswap version.
@@ -423,6 +426,12 @@ func (bsnet *impl) DisconnectFrom(ctx context.Context, p peer.ID) error {
 func (bsnet *impl) handleNewStream(s network.Stream) {
 	defer s.Close()
 
+	// In HTTPnet this metric measures sending the request and reading the
+	// response.
+	// In bitswap these are de-coupled, but we can measure them separately.
+	bsnet.metrics.RequestsInFlight.Inc()
+	defer bsnet.metrics.RequestsInFlight.Dec()
+
 	if len(bsnet.receivers) == 0 {
 		_ = s.Reset()
 		return
diff --git a/bitswap/network/bsnet/metrics.go b/bitswap/network/bsnet/metrics.go
index 442bc2b2..ec83e407 100644
--- a/bitswap/network/bsnet/metrics.go
+++ b/bitswap/network/bsnet/metrics.go
@@ -10,6 +10,10 @@ var durationHistogramBuckets = []float64{0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30,
 
 var blockSizesHistogramBuckets = []float64{1, 128 << 10, 256 << 10, 512 << 10, 1024 << 10, 2048 << 10, 4092 << 10}
 
+func requestsInFlight(ctx context.Context) imetrics.Gauge {
+	return imetrics.NewCtx(ctx, "requests_in_flight", "Current number of in-flight requests").Gauge()
+}
+
 func responseSizes(ctx context.Context) imetrics.Histogram {
 	return imetrics.NewCtx(ctx, "response_bytes", "Histogram of bitswap response sizes").Histogram(blockSizesHistogramBuckets)
 }
@@ -27,6 +31,7 @@ func wantlistsSeconds(ctx context.Context) imetrics.Histogram {
 }
 
 type metrics struct {
+	RequestsInFlight    imetrics.Gauge
 	WantlistsTotal      imetrics.Counter
 	WantlistsItemsTotal imetrics.Counter
 	WantlistsSeconds    imetrics.Histogram
@@ -37,6 +42,7 @@ func newMetrics() *metrics {
 	ctx := imetrics.CtxScope(context.Background(), "exchange_bitswap")
 
 	return &metrics{
+		RequestsInFlight:    requestsInFlight(ctx),
 		WantlistsTotal:      wantlistsTotal(ctx),
 		WantlistsItemsTotal: wantlistsItemsTotal(ctx),
 		WantlistsSeconds:    wantlistsSeconds(ctx),
diff --git a/gateway/assets/directory.html b/gateway/assets/directory.html
index 888ecabf..e1bbc4fd 100644
--- a/gateway/assets/directory.html
+++ b/gateway/assets/directory.html
@@ -25,6 +25,9 @@
         {{ if .Hash }}
         <div class="ipfs-hash" translate="no">
           {{- .Hash -}}
+          <div class="type-icon copy-button" title="Copy CID">
+            <div class="ipfs-copy" onclick="navigator.clipboard.writeText('{{ .Hash }}')">&nbsp;</div>
+          </div>
         </div>
         {{ end }}
       </div>
@@ -56,9 +59,14 @@
           </div>
           <div class="nowrap">
             {{ if .Hash }}
-            <a class="ipfs-hash" translate="no" href={{ if $root.DNSLink }}"https://cid.ipfs.tech/#{{ .Hash | urlEscape}}" target="_blank" rel="noreferrer noopener"{{ else }}"{{ $root.GatewayURL }}/ipfs/{{ .Hash | urlEscape}}?filename={{ .Name | urlEscape }}"{{ end }}>
+            <div>
+              <a class="ipfs-hash" translate="no" href={{ if $root.DNSLink }}"https://cid.ipfs.tech/#{{ .Hash | urlEscape}}" target="_blank" rel="noreferrer noopener"{{ else }}"{{ $root.GatewayURL }}/ipfs/{{ .Hash | urlEscape}}?filename={{ .Name | urlEscape }}"{{ end }}>
               {{- .ShortHash -}}
-            </a>
+              </a>
+              <div class="type-icon copy-button" title="Copy CID">
+                <div class="ipfs-copy" onclick="navigator.clipboard.writeText('{{ .Hash }}')">&nbsp;</div>
+              </div>
+            </div>
             {{ end }}
           </div>
           <div class="nowrap" title="Cumulative size of IPFS DAG (data + metadata)">{{ .Size }}</div>
diff --git a/gateway/assets/icons.css b/gateway/assets/icons.css
index 6f4dbc8e..22fb30e6 100644
--- a/gateway/assets/icons.css
+++ b/gateway/assets/icons.css
@@ -42,6 +42,12 @@
 	background-size: contain;
 }
 
+.ipfs-copy {
+	background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'%3E%3Cpath d='M67.36 30.73H29.1a3.92 3.92 0 00-3.91 3.91V72.9a3.92 3.92 0 003.91 3.91h38.26a3.92 3.92 0 003.91-3.91V34.64a3.91 3.91 0 00-3.91-3.91zm1.31 42.17a1.31 1.31 0 01-1.31 1.31H29.1a1.31 1.31 0 01-1.31-1.31V34.64a1.31 1.31 0 011.31-1.31h38.26a1.31 1.31 0 011.31 1.31z'/%3E%3Cpath d='M72.75 23.19h-32.3a1.3 1.3 0 000 2.6h32.3A4.3 4.3 0 0177 30.08v32.3a1.3 1.3 0 002.6 0v-32.3a6.9 6.9 0 00-6.85-6.89zM35.37 45.68H47.9a.75.75 0 000-1.5H35.37a.75.75 0 000 1.5zM61.1 53H35.37a.75.75 0 000 1.5H61.1a.75.75 0 000-1.5zm0 8.86H35.37a.75.75 0 100 1.5H61.1a.75.75 0 100-1.5z'/%3E%3C/svg%3E");
+	background-repeat: no-repeat;
+	background-size: contain;
+}
+
 .ipfs-cpp {
 	background-image: url("data:image/svg+xml,%0A%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 72 100'%3E%3ClinearGradient id='a' gradientUnits='userSpaceOnUse' x1='36.108' y1='3.003' x2='36.108' y2='101.001' gradientTransform='matrix(1 0 0 -1 0 102)'%3E%3Cstop offset='0' stop-color='%23c8d4db'/%3E%3Cstop offset='.139' stop-color='%23d8e1e6'/%3E%3Cstop offset='.359' stop-color='%23ebf0f3'/%3E%3Cstop offset='.617' stop-color='%23f9fafb'/%3E%3Cstop offset='1' stop-color='%23fff'/%3E%3C/linearGradient%3E%3Cpath d='M45.1 1L72 27.7V99H.2V1h44.9z' fill='url(%23a)'/%3E%3Cpath d='M45.1 1L72 27.7V99H.2V1h44.9z' stroke='%237191a1' stroke-width='2' fill='none'/%3E%3Cpath d='M22.1 83.8L26 85c-.6 2.2-1.6 3.8-3 4.8-1.4 1.1-3.2 1.6-5.3 1.6-2.7 0-4.8-.9-6.5-2.7s-2.6-4.3-2.6-7.4c0-3.3.9-5.9 2.6-7.7s4-2.7 6.8-2.7c2.4 0 4.4.7 6 2.2.9.8 1.6 2.1 2 3.7l-4 .9c-.2-1-.7-1.8-1.5-2.4-.7-.6-1.7-.9-2.7-.9-1.5 0-2.7.5-3.6 1.6s-1.4 2.8-1.4 5.1c0 2.5.5 4.3 1.4 5.4s2.1 1.6 3.5 1.6c1.1 0 2-.3 2.8-1s1.3-1.9 1.6-3.3zm7.4 7.3V71.2H36c2.5 0 4.1.1 4.8.3 1.1.3 2.1.9 2.9 1.9.8 1 1.2 2.3 1.2 3.9 0 1.2-.2 2.2-.7 3.1-.4.8-1 1.5-1.7 2s-1.4.8-2.1.9c-1 .2-2.4.3-4.2.3h-2.6v7.5h-4.1zm4-16.5v5.6h2.2c1.6 0 2.7-.1 3.2-.3.5-.2 1-.5 1.3-1 .3-.4.5-1 .5-1.5 0-.7-.2-1.3-.6-1.8-.4-.5-1-.8-1.6-.9-.5-.1-1.4-.1-2.9-.1h-2.1zm14.6 16.5V71.2h6.5c2.5 0 4.1.1 4.8.3 1.1.3 2.1.9 2.9 1.9.8 1 1.2 2.3 1.2 3.9 0 1.2-.2 2.2-.7 3.1-.4.8-1 1.5-1.7 2s-1.4.8-2.1.9c-1 .2-2.4.3-4.2.3h-2.6v7.5h-4.1zm4-16.5v5.6h2.2c1.6 0 2.7-.1 3.2-.3.5-.2 1-.5 1.3-1 .3-.4.5-1 .5-1.5 0-.7-.2-1.3-.6-1.8-.4-.5-1-.8-1.6-.9-.5-.1-1.4-.1-2.9-.1h-2.1z' fill='%234c6c7b'/%3E%3ClinearGradient id='b' gradientUnits='userSpaceOnUse' x1='13.15' y1='22' x2='54.15' y2='22' gradientTransform='translate(0 2)'%3E%3Cstop offset='0' stop-color='%237291a1'/%3E%3Cstop offset='1' stop-color='%23cad5db'/%3E%3C/linearGradient%3E%3Cpath d='M13.1 22h41v4h-41v-4z' fill='url(%23b)'/%3E%3ClinearGradient id='c' gradientUnits='userSpaceOnUse' x1='19.15' y1='33.75' x2='60.15' y2='33.75' gradientTransform='translate(0 2)'%3E%3Cstop offset='0' stop-color='%237291a1'/%3E%3Cstop offset='1' stop-color='%23cad5db'/%3E%3C/linearGradient%3E%3Cpath d='M19.1 33.7h41v4.1h-41v-4.1z' fill='url(%23c)'/%3E%3ClinearGradient id='d' gradientUnits='userSpaceOnUse' x1='13.15' y1='45.75' x2='54.15' y2='45.75' gradientTransform='translate(0 2)'%3E%3Cstop offset='0' stop-color='%237291a1'/%3E%3Cstop offset='1' stop-color='%23cad5db'/%3E%3C/linearGradient%3E%3Cpath d='M13.1 45.7h41v4.1h-41v-4.1z' fill='url(%23d)'/%3E%3ClinearGradient id='e' gradientUnits='userSpaceOnUse' x1='19.15' y1='58' x2='60.15' y2='58' gradientTransform='translate(0 2)'%3E%3Cstop offset='0' stop-color='%237291a1'/%3E%3Cstop offset='1' stop-color='%23cad5db'/%3E%3C/linearGradient%3E%3Cpath d='M19.1 58h41v4h-41v-4z' fill='url(%23e)'/%3E%3ClinearGradient id='f' gradientUnits='userSpaceOnUse' x1='45.122' y1='74.229' x2='58.575' y2='87.683' gradientTransform='matrix(1 0 0 -1 0 102)'%3E%3Cstop offset='0' stop-color='%23fff'/%3E%3Cstop offset='.35' stop-color='%23fafbfb'/%3E%3Cstop offset='.532' stop-color='%23edf1f4'/%3E%3Cstop offset='.675' stop-color='%23dde5e9'/%3E%3Cstop offset='.799' stop-color='%23c7d3da'/%3E%3Cstop offset='.908' stop-color='%23adbdc7'/%3E%3Cstop offset='1' stop-color='%2392a5b0'/%3E%3C/linearGradient%3E%3Cpath d='M45.1 1L72 27.7H45.1V1z' fill='url(%23f)'/%3E%3Cpath d='M45.1 1L72 27.7H45.1V1z' stroke='%237191a1' stroke-width='2' stroke-linejoin='bevel' fill='none'/%3E%3C/svg%3E");
 	background-repeat: no-repeat;
diff --git a/gateway/assets/style.css b/gateway/assets/style.css
index a8dfdaf7..4585d4b4 100644
--- a/gateway/assets/style.css
+++ b/gateway/assets/style.css
@@ -66,6 +66,19 @@ a:visited {
 	font-family: var(--monospace);
 }
 
+.copy-button {
+  cursor: pointer;
+  display: inline-flex;
+  align-items: center;
+  justify-content: center;
+  height: 1.15em;
+  transition: background-color 0.3s ease;
+}
+
+.copy-button:active {
+  opacity: 0.25;
+}
+
 #header {
 	align-items: center;
 	background: var(--navy);
diff --git a/gateway/hostname.go b/gateway/hostname.go
index 4c078cec..ede21d8e 100644
--- a/gateway/hostname.go
+++ b/gateway/hostname.go
@@ -554,6 +554,14 @@ func prepareHostnameGateways(gateways map[string]*PublicGateway) *hostnameGatewa
 	}
 
 	for hostname, gw := range gateways {
+		// Validate that UseSubdomains is not enabled for IP addresses
+		if gw.UseSubdomains {
+			hostWithoutPort := stripPort(hostname)
+			if net.ParseIP(hostWithoutPort) != nil {
+				log.Warn("invalid gateway configuration: UseSubdomains cannot be enabled for IP address %s", hostname)
+				continue
+			}
+		}
 		if strings.Contains(hostname, "*") {
 			// from *.domain.tld, construct a regexp that match any direct subdomain
 			// of .domain.tld.
diff --git a/provider/internal/queue/queue.go b/provider/internal/queue/queue.go
index 97def8e8..7144141c 100644
--- a/provider/internal/queue/queue.go
+++ b/provider/internal/queue/queue.go
@@ -2,10 +2,13 @@ package queue
 
 import (
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"sync"
+	"time"
 
+	"github.com/gammazero/deque"
 	cid "github.com/ipfs/go-cid"
 	datastore "github.com/ipfs/go-datastore"
 	namespace "github.com/ipfs/go-datastore/namespace"
@@ -15,136 +18,251 @@ import (
 
 var log = logging.Logger("provider.queue")
 
-// Queue provides a best-effort durability, FIFO interface to the datastore for storing cids
+const (
+	// batchSize is the limit on number of CIDs kept in memory at which ther
+	// are all written to the datastore.
+	batchSize = 16 * 1024
+	// idleWriteTime is the amout of time to check if the queue has been idle
+	// (no input or output). If the queue has been idle since the last check,
+	// then write all buffered CIDs to the datastore.
+	idleWriteTime = time.Minute
+	// shutdownTimeout is the duration that Close waits to finish writing CIDs
+	// to the datastore.
+	shutdownTimeout = 10 * time.Second
+)
+
+// Queue provides a FIFO interface to the datastore for storing cids.
+//
+// CIDs in the process of being provided when a crash or shutdown occurs may be
+// in the queue when the node is brought back online depending on whether they
+// were fully written to the underlying datastore.
 //
-// Best-effort durability just means that cids in the process of being provided when a
-// crash or shutdown occurs may be in the queue when the node is brought back online
-// depending on whether the underlying datastore has synchronous or asynchronous writes.
+// Input to the queue is buffered in memory. The contents of the buffer are
+// written to the datastore when the input buffer contains batchSize items, or
+// when idleWriteTime has elapsed since the previous batch write or dequeue. CIDs to
+// dequeue are read, in order, from the input buffer if there are none in the
+// datastore. Otherwise they are read from the datastore.
+//
+// If queued items are read from the input buffer before it reaches its limit,
+// then queued items can remain in memory. When the queue is closed, any
+// remaining items in memory are written to the datastore.
 type Queue struct {
-	// used to differentiate queues in datastore
-	// e.g. provider vs reprovider
-	ctx     context.Context
-	ds      datastore.Datastore // Must be threadsafe
-	dequeue chan cid.Cid
-	enqueue chan cid.Cid
-	close   context.CancelFunc
-	closed  sync.WaitGroup
-
-	counter uint64
+	close     context.CancelFunc
+	closed    chan error
+	closeOnce sync.Once
+	dequeue   chan cid.Cid
+	ds        datastore.Batching
+	enqueue   chan cid.Cid
 }
 
-// NewQueue creates a queue for cids
-func NewQueue(ds datastore.Datastore) *Queue {
-	namespaced := namespace.Wrap(ds, datastore.NewKey("/queue"))
-	cancelCtx, cancel := context.WithCancel(context.Background())
+// New creates a queue for cids.
+func New(ds datastore.Batching) *Queue {
+	ctx, cancel := context.WithCancel(context.Background())
+
 	q := &Queue{
-		ctx:     cancelCtx,
-		ds:      namespaced,
+		close:   cancel,
+		closed:  make(chan error, 1),
 		dequeue: make(chan cid.Cid),
+		ds:      namespace.Wrap(ds, datastore.NewKey("/queue")),
 		enqueue: make(chan cid.Cid),
-		close:   cancel,
 	}
-	q.closed.Add(1)
-	go q.worker()
+
+	go q.worker(ctx)
+
 	return q
 }
 
-// Close stops the queue
+// Close stops the queue.
 func (q *Queue) Close() error {
-	q.close()
-	q.closed.Wait()
-	// We don't close dequeue because the provider which consume this get caught in
-	// an infinite loop dequeing cid.Undef if we do that.
-	// The provider has it's own select on top of dequeue and will handle this by itself.
-	return nil
+	var err error
+	q.closeOnce.Do(func() {
+		// Close input queue and wait for worker to finish reading it.
+		close(q.enqueue)
+		select {
+		case <-q.closed:
+		case <-time.After(shutdownTimeout):
+			q.close() // force immediate shutdown
+			err = <-q.closed
+		}
+		close(q.dequeue) // no more output from this queue
+	})
+	return err
 }
 
-// Enqueue puts a cid in the queue
-func (q *Queue) Enqueue(cid cid.Cid) error {
-	select {
-	case q.enqueue <- cid:
-		return nil
-	case <-q.ctx.Done():
-		return errors.New("failed to enqueue CID: shutting down")
+// Enqueue puts a cid in the queue.
+func (q *Queue) Enqueue(c cid.Cid) (err error) {
+	if c == cid.Undef {
+		return
 	}
+	defer func() {
+		if r := recover(); r != nil {
+			err = errors.New("failed to enqueue CID: shutting down")
+		}
+	}()
+	q.enqueue <- c
+	return
 }
 
-// Dequeue returns a channel that if listened to will remove entries from the queue
+// Dequeue returns a channel that for reading entries from the queue,
 func (q *Queue) Dequeue() <-chan cid.Cid {
 	return q.dequeue
 }
 
+func makeCidString(c cid.Cid) string {
+	data := c.Bytes()
+	if len(data) > 4 {
+		data = data[len(data)-4:]
+	}
+	return base64.RawURLEncoding.EncodeToString(data)
+}
+
+func makeKey(c cid.Cid, counter uint64) datastore.Key {
+	return datastore.NewKey(fmt.Sprintf("%020d/%s", counter, makeCidString(c)))
+}
+
 // worker run dequeues and enqueues when available.
-func (q *Queue) worker() {
-	var k datastore.Key = datastore.Key{}
-	var c cid.Cid = cid.Undef
+func (q *Queue) worker(ctx context.Context) {
+	defer close(q.closed)
+
+	var (
+		c       cid.Cid
+		counter uint64
+		k       datastore.Key = datastore.Key{}
+		inBuf   deque.Deque[cid.Cid]
+	)
+
+	const baseCap = 1024
+	inBuf.SetBaseCap(baseCap)
+
+	defer func() {
+		if c != cid.Undef {
+			if err := q.ds.Put(ctx, k, c.Bytes()); err != nil {
+				log.Errorw("Failed to write cid to datastore", "err", err)
+			}
+			counter++
+		}
+		if inBuf.Len() != 0 {
+			err := q.commitInput(ctx, counter, &inBuf)
+			if err != nil && !errors.Is(err, context.Canceled) {
+				log.Error(err)
+				if inBuf.Len() != 0 {
+					q.closed <- fmt.Errorf("provider queue: %d cids not written to datastore", inBuf.Len())
+				}
+			}
+		}
+	}()
+
+	var (
+		commit  bool
+		dsEmpty bool
+		err     error
+		idle    bool
+	)
+
+	readInBuf := q.enqueue
 
-	defer q.closed.Done()
-	defer q.close()
+	batchTimer := time.NewTimer(idleWriteTime)
+	defer batchTimer.Stop()
 
 	for {
 		if c == cid.Undef {
-			head, err := q.getQueueHead()
-
-			switch {
-			case err != nil:
-				log.Errorf("error querying for head of queue: %s, stopping provider", err)
-				return
-			case head != nil:
-				k = datastore.NewKey(head.Key)
-				c, err = cid.Parse(head.Value)
+			if !dsEmpty {
+				head, err := q.getQueueHead(ctx)
 				if err != nil {
-					log.Warnf("error parsing queue entry cid with key (%s), removing it from queue: %s", head.Key, err)
-					err = q.ds.Delete(q.ctx, k)
-					if err != nil {
-						log.Errorf("error deleting queue entry with key (%s), due to error (%s), stopping provider", head.Key, err)
+					log.Errorw("Error querying for head of queue, stopping provider", "err", err)
+					return
+				}
+				if head != nil {
+					k = datastore.NewKey(head.Key)
+					if err = q.ds.Delete(ctx, k); err != nil {
+						log.Errorw("Error deleting queue entry, stopping provider", "err", err, "key", head.Key)
 						return
 					}
-					continue
+					c, err = cid.Parse(head.Value)
+					if err != nil {
+						log.Warnw("Error parsing queue entry cid, removing it from queue", "err", err, "key", head.Key)
+						continue
+					}
+				} else {
+					dsEmpty = true
 				}
-			default:
-				c = cid.Undef
+			}
+			if dsEmpty && inBuf.Len() != 0 {
+				// There were no queued CIDs in the datastore, so read one from
+				// the input buffer.
+				c = inBuf.PopFront()
+				k = makeKey(c, counter)
 			}
 		}
 
-		// If c != cid.Undef set dequeue and attempt write, otherwise wait for enqueue
+		// If c != cid.Undef set dequeue and attempt write.
 		var dequeue chan cid.Cid
 		if c != cid.Undef {
 			dequeue = q.dequeue
 		}
 
 		select {
-		case toQueue := <-q.enqueue:
-			keyPath := fmt.Sprintf("%020d/%s", q.counter, c.String())
-			q.counter++
-			nextKey := datastore.NewKey(keyPath)
+		case toQueue, ok := <-readInBuf:
+			if !ok {
+				return
+			}
+			idle = false
 
 			if c == cid.Undef {
-				// fast path, skip rereading the datastore if we don't have anything in hand yet
+				// Use this CID as the next output since there was nothing in
+				// the datastore or buffer previously.
 				c = toQueue
-				k = nextKey
+				k = makeKey(c, counter)
+				continue
 			}
 
-			if err := q.ds.Put(q.ctx, nextKey, toQueue.Bytes()); err != nil {
-				log.Errorf("Failed to enqueue cid: %s", err)
-				continue
+			inBuf.PushBack(toQueue)
+			if inBuf.Len() >= batchSize {
+				commit = true
 			}
 		case dequeue <- c:
-			err := q.ds.Delete(q.ctx, k)
-			if err != nil {
-				log.Errorf("Failed to delete queued cid %s with key %s: %s", c, k, err)
-				continue
-			}
 			c = cid.Undef
-		case <-q.ctx.Done():
+			idle = false
+		case <-batchTimer.C:
+			if idle {
+				if inBuf.Len() != 0 {
+					commit = true
+				} else {
+					if inBuf.Cap() > baseCap {
+						inBuf = deque.Deque[cid.Cid]{}
+						inBuf.SetBaseCap(baseCap)
+					}
+				}
+			}
+			idle = true
+			batchTimer.Reset(idleWriteTime)
+
+		case <-ctx.Done():
 			return
 		}
+
+		if commit {
+			commit = false
+			n := inBuf.Len()
+			err = q.commitInput(ctx, counter, &inBuf)
+			if err != nil {
+				if !errors.Is(err, context.Canceled) {
+					log.Errorw("Error writing CIDs to datastore, stopping provider", "err", err)
+				}
+				return
+			}
+			counter += uint64(n)
+			dsEmpty = false
+		}
 	}
 }
 
-func (q *Queue) getQueueHead() (*query.Entry, error) {
-	qry := query.Query{Orders: []query.Order{query.OrderByKey{}}, Limit: 1}
-	results, err := q.ds.Query(q.ctx, qry)
+func (q *Queue) getQueueHead(ctx context.Context) (*query.Entry, error) {
+	qry := query.Query{
+		Orders: []query.Order{query.OrderByKey{}},
+		Limit:  1,
+	}
+	results, err := q.ds.Query(ctx, qry)
 	if err != nil {
 		return nil, err
 	}
@@ -156,3 +274,29 @@ func (q *Queue) getQueueHead() (*query.Entry, error) {
 
 	return &r.Entry, r.Error
 }
+
+func (q *Queue) commitInput(ctx context.Context, counter uint64, cids *deque.Deque[cid.Cid]) error {
+	b, err := q.ds.Batch(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to create batch: %w", err)
+	}
+
+	cstr := makeCidString(cids.Front())
+	n := cids.Len()
+	for i := 0; i < n; i++ {
+		c := cids.At(i)
+		key := datastore.NewKey(fmt.Sprintf("%020d/%s", counter, cstr))
+		if err = b.Put(ctx, key, c.Bytes()); err != nil {
+			log.Errorw("Failed to add cid to batch", "err", err)
+			continue
+		}
+		counter++
+	}
+	cids.Clear()
+
+	if err = b.Commit(ctx); err != nil {
+		return fmt.Errorf("failed to commit batch to datastore: %w", err)
+	}
+
+	return nil
+}
diff --git a/provider/reprovider.go b/provider/reprovider.go
index 94e077da..002168e9 100644
--- a/provider/reprovider.go
+++ b/provider/reprovider.go
@@ -138,7 +138,7 @@ func New(ds datastore.Batching, opts ...Option) (System, error) {
 	}
 
 	s.ds = namespace.Wrap(ds, s.keyPrefix)
-	s.q = queue.NewQueue(s.ds)
+	s.q = queue.New(s.ds)
 
 	// This is after the options processing so we do not have to worry about leaking a context if there is an
 	// initialization error processing the options
@@ -423,10 +423,12 @@ func (s *reprovider) run() {
 		// after the first reprovide, schedule periodical reprovides
 		nextReprovideTicker := time.NewTicker(s.reprovideInterval)
 
-		for s.ctx.Err() == nil {
+		for {
 			err := s.Reprovide(context.Background())
-
-			if s.ctx.Err() == nil && err != nil {
+			if err != nil {
+				if s.ctx.Err() != nil {
+					return
+				}
 				log.Errorf("failed to reprovide: %s", err)
 			}
 			select {



Test patch:
diff --git a/gateway/hostname_test.go b/gateway/hostname_test.go
index b6ad366a..1af729b1 100644
--- a/gateway/hostname_test.go
+++ b/gateway/hostname_test.go
@@ -407,3 +407,84 @@ func TestHasDNSLinkRecordWithLocalIP(t *testing.T) {
 		require.True(t, result, "example.com should have a valid DNSLink record")
 	})
 }
+
+func TestValidateSubdomainsForIP(t *testing.T) {
+	t.Parallel()
+
+	// Test cases
+	tests := []struct {
+		name          string
+		hostname      string
+		useSubdomains bool
+		shouldSkip    bool
+	}{
+		{
+			name:          "IP with subdomains enabled",
+			hostname:      "127.0.0.1:8080",
+			useSubdomains: true,
+			shouldSkip:    true,
+		},
+		{
+			name:          "IPv6 with subdomains enabled",
+			hostname:      "[::1]:8080",
+			useSubdomains: true,
+			shouldSkip:    true,
+		},
+		{
+			name:          "IP without port with subdomains enabled",
+			hostname:      "192.168.1.1",
+			useSubdomains: true,
+			shouldSkip:    true,
+		},
+		{
+			name:          "Domain with subdomains enabled",
+			hostname:      "example.com:8080",
+			useSubdomains: true,
+			shouldSkip:    false,
+		},
+		{
+			name:          "IP with subdomains disabled",
+			hostname:      "10.0.0.1:8080",
+			useSubdomains: false,
+			shouldSkip:    false,
+		},
+		{
+			name:          "IPv6 without port with subdomains enabled",
+			hostname:      "::1",
+			useSubdomains: true,
+			shouldSkip:    true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Create test gateway config
+			gw := &PublicGateway{
+				UseSubdomains: tt.useSubdomains,
+			}
+
+			// Prepare gateways map
+			gateways := make(map[string]*PublicGateway)
+			gateways[tt.hostname] = gw
+
+			// Run the validation
+			validatedGateways := prepareHostnameGateways(gateways)
+
+			// Check if gateway was skipped by looking in both exact and wildcard maps
+			_, existsExact := validatedGateways.exact[tt.hostname]
+			existsWildcard := false
+			for re := range validatedGateways.wildcard {
+				if re.MatchString(tt.hostname) {
+					existsWildcard = true
+					break
+				}
+			}
+			exists := existsExact || existsWildcard
+			if tt.shouldSkip {
+				assert.False(t, exists, "Gateway with UseSubdomains=true should be skipped for IP address")
+			} else {
+				assert.True(t, exists, "Gateway should not be skipped")
+			}
+		})
+	}
+}
diff --git a/provider/internal/queue/queue_test.go b/provider/internal/queue/queue_test.go
index a9a49cc6..d0250e99 100644
--- a/provider/internal/queue/queue_test.go
+++ b/provider/internal/queue/queue_test.go
@@ -5,73 +5,65 @@ import (
 	"testing"
 	"time"
 
-	"github.com/ipfs/boxo/internal/test"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
 	"github.com/ipfs/go-datastore/sync"
 	"github.com/ipfs/go-test/random"
 )
 
-const blockSize = 4
-
-func makeCids(n int) []cid.Cid {
-	blks := random.BlocksOfSize(n, blockSize)
-	cids := make([]cid.Cid, n)
-	for i := 0; i < n; i++ {
-		cids[i] = blks[i].Cid()
-	}
-	return cids
-}
-
 func assertOrdered(cids []cid.Cid, q *Queue, t *testing.T) {
-	for _, c := range cids {
+	t.Helper()
+
+	for i, c := range cids {
 		select {
-		case dequeued := <-q.dequeue:
+		case dequeued, ok := <-q.Dequeue():
+			if !ok {
+				t.Fatal("queue closed")
+			}
 			if c != dequeued {
-				t.Fatalf("Error in ordering of CIDs retrieved from queue. Expected: %s, got: %s", c, dequeued)
+				t.Fatalf("Error in ordering of CID %d retrieved from queue. Expected: %s, got: %s", i, c, dequeued)
 			}
 
-		case <-time.After(time.Second * 1):
+		case <-time.After(time.Second):
 			t.Fatal("Timeout waiting for cids to be provided.")
 		}
 	}
 }
 
 func TestBasicOperation(t *testing.T) {
-	test.Flaky(t)
-	ctx := context.Background()
-	defer ctx.Done()
-
 	ds := sync.MutexWrap(datastore.NewMapDatastore())
-	queue := NewQueue(ds)
+	queue := New(ds)
 	defer queue.Close()
 
-	cids := makeCids(10)
-
+	cids := random.Cids(10)
 	for _, c := range cids {
 		queue.Enqueue(c)
 	}
 
 	assertOrdered(cids, queue, t)
+
+	err := queue.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err = queue.Close(); err != nil {
+		t.Fatal(err)
+	}
 }
 
 func TestMangledData(t *testing.T) {
-	test.Flaky(t)
-	ctx := context.Background()
-	defer ctx.Done()
-
 	ds := sync.MutexWrap(datastore.NewMapDatastore())
-	queue := NewQueue(ds)
+	queue := New(ds)
 	defer queue.Close()
 
-	cids := makeCids(10)
+	cids := random.Cids(10)
 	for _, c := range cids {
 		queue.Enqueue(c)
 	}
 
 	// put bad data in the queue
 	queueKey := datastore.NewKey("/test/0")
-	err := queue.ds.Put(ctx, queueKey, []byte("borked"))
+	err := queue.ds.Put(context.Background(), queueKey, []byte("borked"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -82,45 +74,46 @@ func TestMangledData(t *testing.T) {
 }
 
 func TestInitialization(t *testing.T) {
-	test.Flaky(t)
-	ctx := context.Background()
-	defer ctx.Done()
-
 	ds := sync.MutexWrap(datastore.NewMapDatastore())
-	queue := NewQueue(ds)
+	queue := New(ds)
 	defer queue.Close()
 
-	cids := makeCids(10)
+	cids := random.Cids(10)
 	for _, c := range cids {
 		queue.Enqueue(c)
 	}
 
 	assertOrdered(cids[:5], queue, t)
 
+	err := queue.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	// make a new queue, same data
-	queue = NewQueue(ds)
+	queue = New(ds)
 	defer queue.Close()
 
 	assertOrdered(cids[5:], queue, t)
 }
 
 func TestInitializationWithManyCids(t *testing.T) {
-	test.Flaky(t)
-	ctx := context.Background()
-	defer ctx.Done()
-
 	ds := sync.MutexWrap(datastore.NewMapDatastore())
-	queue := NewQueue(ds)
+	queue := New(ds)
+	defer queue.Close()
 
-	cids := makeCids(25)
+	cids := random.Cids(25)
 	for _, c := range cids {
 		queue.Enqueue(c)
 	}
 
-	queue.Close()
+	err := queue.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	// make a new queue, same data
-	queue = NewQueue(ds)
+	queue = New(ds)
 	defer queue.Close()
 
 	assertOrdered(cids, queue, t)