Repository: ipfs/kubo

Issue:
Kubo stopped listing pin after some times
### Checklist

- [X] This is a bug report, not a question. Ask questions on [discuss.ipfs.tech](https://discuss.ipfs.tech/c/help/13).
- [X] I have searched on the [issue tracker](https://github.com/ipfs/kubo/issues?q=is%3Aissue) for my bug.
- [X] I am running the latest [kubo version](https://dist.ipfs.tech/#kubo) or have an issue updating.

### Installation method

built from source

### Version

```Text
Kubo version: 0.32.1
Repo version: 16
System version: amd64/linux
Golang version: go1.23.3
```


### Config

```json
{
  "API": {
    "HTTPHeaders": {}
  },
  "Addresses": {
    "API": "/ip4/127.0.0.1/tcp/5001",
    "Announce": [],
    "AppendAnnounce": [],
    "Gateway": "/ip4/127.0.0.1/tcp/8080",
    "NoAnnounce": [
      "/ip4/10.0.0.0/ipcidr/8",
      "/ip4/100.64.0.0/ipcidr/10",
      "/ip4/169.254.0.0/ipcidr/16",
      "/ip4/172.16.0.0/ipcidr/12",
      "/ip4/192.0.0.0/ipcidr/24",
      "/ip4/192.0.2.0/ipcidr/24",
      "/ip4/192.168.0.0/ipcidr/16",
      "/ip4/198.18.0.0/ipcidr/15",
      "/ip4/198.51.100.0/ipcidr/24",
      "/ip4/203.0.113.0/ipcidr/24",
      "/ip4/240.0.0.0/ipcidr/4",
      "/ip6/100::/ipcidr/64",
      "/ip6/2001:2::/ipcidr/48",
      "/ip6/2001:db8::/ipcidr/32",
      "/ip6/fc00::/ipcidr/7",
      "/ip6/fe80::/ipcidr/10"
    ],
    "Swarm": [
      "/ip4/0.0.0.0/tcp/4001",
      "/ip6/::/tcp/4001",
      "/ip4/0.0.0.0/udp/4001/quic-v1",
      "/ip4/0.0.0.0/udp/4001/quic-v1/webtransport",
      "/ip4/0.0.0.0/udp/4001/webrtc-direct",
      "/ip6/::/udp/4001/quic-v1",
      "/ip6/::/udp/4001/quic-v1/webtransport",
      "/ip6/::/udp/4001/webrtc-direct"
    ]
  },
  "AutoNAT": {},
  "AutoTLS": {},
  "Bootstrap": [
    "/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN",
    "/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa",
    "/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb",
    "/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt",
    "/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
    "/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ"
  ],
  "DNS": {
    "Resolvers": {}
  },
  "Datastore": {
    "BloomFilterSize": 0,
    "GCPeriod": "1h",
    "HashOnRead": false,
    "Spec": {
      "mounts": [
        {
          "child": {
            "path": "blocks",
            "shardFunc": "/repo/flatfs/shard/v1/next-to-last/3",
            "sync": false,
            "type": "flatfs"
          },
          "mountpoint": "/blocks",
          "prefix": "flatfs.datastore",
          "type": "measure"
        },
        {
          "child": {
            "compression": "none",
            "path": "datastore",
            "type": "levelds"
          },
          "mountpoint": "/",
          "prefix": "leveldb.datastore",
          "type": "measure"
        }
      ],
      "type": "mount"
    },
    "StorageGCWatermark": 90,
    "StorageMax": "31TB"
  },
  "Discovery": {
    "MDNS": {
      "Enabled": false
    }
  },
  "Experimental": {
    "FilestoreEnabled": false,
    "GraphsyncEnabled": false,
    "Libp2pStreamMounting": false,
    "OptimisticProvide": false,
    "OptimisticProvideJobsPoolSize": 0,
    "P2pHttpProxy": false,
    "StrategicProviding": false,
    "UrlstoreEnabled": false
  },
  "Gateway": {
    "APICommands": [],
    "DeserializedResponses": null,
    "DisableHTMLErrors": null,
    "ExposeRoutingAPI": null,
    "HTTPHeaders": {},
    "NoDNSLink": false,
    "NoFetch": false,
    "PathPrefixes": [],
    "PublicGateways": {
      "cfg": {
        "DeserializedResponses": null,
        "InlineDNSLink": null,
        "NoDNSLink": false,
        "Paths": null,
        "UseSubdomains": false
      },
      "localhost": {
        "DeserializedResponses": null,
        "InlineDNSLink": null,
        "NoDNSLink": false,
        "Paths": [
          "/ipfs"
        ],
        "UseSubdomains": false
      }
    },
    "RootRedirect": ""
  },
  "Identity": {
    "PeerID": "12D3KooWAJJJwXsB5b68cbq69KpXiKqQAgTKssg76heHkg6mo2qB"
  },
  "Import": {
    "CidVersion": null,
    "HashFunction": null,
    "UnixFSChunker": null,
    "UnixFSRawLeaves": null
  },
  "Internal": {
    "Bitswap": {
      "EngineBlockstoreWorkerCount": 128,
      "EngineTaskWorkerCount": 8,
      "MaxOutstandingBytesPerPeer": null,
      "ProviderSearchDelay": null,
      "TaskWorkerCount": 8
    }
  },
  "Ipns": {
    "RecordLifetime": "",
    "RepublishPeriod": "",
    "ResolveCacheSize": 128
  },
  "Migration": {
    "DownloadSources": [],
    "Keep": ""
  },
  "Mounts": {
    "FuseAllowOther": false,
    "IPFS": "/ipfs",
    "IPNS": "/ipns"
  },
  "Peering": {
    "Peers": "removed for brevity"
  },
  "Pinning": {},
  "Plugins": {
    "Plugins": null
  },
  "Provider": {
    "Strategy": ""
  },
  "Pubsub": {
    "DisableSigning": false,
    "Router": ""
  },
  "Reprovider": {
    "Interval": "24h0m0s",
    "Strategy": "all"
  },
  "Routing": {
    "AcceleratedDHTClient": true,
    "Methods": null,
    "Routers": null,
    "Type": "auto"
  },
  "Swarm": {
    "AddrFilters": [
      "/ip4/10.0.0.0/ipcidr/8",
      "/ip4/100.64.0.0/ipcidr/10",
      "/ip4/169.254.0.0/ipcidr/16",
      "/ip4/172.16.0.0/ipcidr/12",
      "/ip4/192.0.0.0/ipcidr/24",
      "/ip4/192.0.2.0/ipcidr/24",
      "/ip4/192.168.0.0/ipcidr/16",
      "/ip4/198.18.0.0/ipcidr/15",
      "/ip4/198.51.100.0/ipcidr/24",
      "/ip4/203.0.113.0/ipcidr/24",
      "/ip4/240.0.0.0/ipcidr/4",
      "/ip6/100::/ipcidr/64",
      "/ip6/2001:2::/ipcidr/48",
      "/ip6/2001:db8::/ipcidr/32",
      "/ip6/fc00::/ipcidr/7",
      "/ip6/fe80::/ipcidr/10"
    ],
    "ConnMgr": {
      "GracePeriod": "30s",
      "HighWater": 4096,
      "LowWater": 1024
    },
    "DisableBandwidthMetrics": false,
    "DisableNatPortMap": true,
    "RelayClient": {
      "Enabled": false
    },
    "RelayService": {
      "Enabled": false
    },
    "ResourceMgr": {
      "Limits": {},
      "MaxMemory": "24GB"
    },
    "Transports": {
      "Multiplexers": {},
      "Network": {},
      "Security": {}
    }
  },
  "Version": {}
}
```


### Description

Hello,

We started to experience an issue with Kubo in our 2-node cluster where Kubo don't list pin anymore.

We have 2 nodes that both pin all the pinset we keep track of, which is around 16.39 million pins right now.

Last weeks (while we were still using 0.29), Kubo stopped responding to the `/pin/ls` queries sent by the cluster, those requests were hanging "indefinitely" (as in, when using curl I stopped the command after ~16h without response). Our `ipfs-cluster` process is returning the following in the log when this happens:
```
Nov 24 22:16:34 ipfs-01 ipfs-cluster-service[3875697]: 2024-11-24T22:16:34.328Z        ERROR        pintracker        stateless/stateless.go:540        Post "http://127.0.0.1:5001/api/v0/pin/ls?stream=true&arg=QmcPpbdw8k8fyhas77WjzPMqSAgVYPonbVve3xTPtxL8ab&type=recursive": context canceled
Nov 24 22:16:34 ipfs-01 ipfs-cluster-service[3875697]: 2024-11-24T22:16:34.328Z        ERROR        cluster        ipfs-cluster/cluster.go:2022        12D3KooTheOtherClusterNodeHash: error in broadcast response from 12D3KooTheOtherClusterNodeHash: context canceled
Nov 24 22:16:34 ipfs-01 ipfs-cluster-service[3875697]: 2024-11-24T22:16:34.447Z        ERROR        ipfshttp        ipfshttp/ipfshttp.go:1276        error posting to IPFS:Post "http://127.0.0.1:5001/api/v0/pin/ls?stream=true&arg=QmRSGNoqrbTsKCgHbi8xCeQEn4sQXFDjfNEUgXWG5wAg6U&type=recursive": context canceled
```

This started out of the blue, there was no change on the server. The issue remained after upgrading to 0.32.1.

At that time, we had the bloom filter activated, deactivating it did improve the situation for a while (maybe 24h), and then the issue started to show up again. In retrospect, I think it may not be related to the bloom filter at all).

This is the typical metrics reported by `ipfs-cluster` which show when Kubo stop responding to `/pin/ls`:

<img width="1385" alt="Screenshot 2024-11-25 at 10 31 12" src="https://github.com/user-attachments/assets/799d8968-716e-41f7-b599-26dd7e58bdc9">

The graph on top is the number of pins the cluster is keeping track of, and on the one on the bottom is the number of pins reported by Kubo. When restarting Kubo it generally jumps to the expected amount, and after a while it drops to 0. At that point any attempt to list pin from Kubo fails.

We only have the metrics reported by ipfs-cluster because of [this Kubo bug](https://github.com/ipfs/kubo/issues/9891).

The server CPU, RAM, and disk utilization is fairly low when this issue show up, so it doesn't look like it a performance issue. The only metric that started to go out of bound is the number of open file descriptors which grow and reached the 128k limit set. I bumped it to 1.28 million, but it still reaches it (with or without the bloom filter):
<img width="1385" alt="Screenshot 2024-11-25 at 10 31 40" src="https://github.com/user-attachments/assets/64f8c695-fbf0-442f-ab1c-35b40d8b7467">

The FDs limit is set both at the systemd unit level, and via `IPFS_FD_MAX`.

Restarting Kubo make it work again most of the time, but sometimes it doesn't change anything and it instantly starts to fail.

Here is some profiling data from one of our nodes:
- when it's starting and respond to pin listing: https://drive.google.com/file/d/1-HagskynT6D-WoZ_JxIvHiu8kZGS1EUd/view?usp=sharing
- when it's in this fail state: https://drive.google.com/file/d/1fexP-ZBLxkvcCVtZTHPcIUNalzJsfemh/view?usp=sharing

More info about the system:
- Nixos, current version of Nixpkgs being the PR that updated Kubo to 0.32.1
- AMD Ryzen 5 Pro 3600 - 6c/12t - 3.6 GHz/4.2 GHz
- 128 GB ECC 2666 MHz
- 2×512 GB SSD NVMe
  - one with the system
  - the other is split and is used as `logs` and `cache` for ZFS
- one ZFS ZRAID-0 pool with 4×6 TB HDD SATA

Kubo also emit a lot of:
```
INFO        net/identify        identify/id.go:457        failed negotiate identify protocol with peer        {"peer": "12D3KooWMyK8arvRjtC33rxTRZfKDcyQgZTC9yWpfMFHRbp
ngXwK", "error": "Application error 0x1 (remote): conn-31160279: system: cannot reserve inbound connection: resource limit exceeded"}
```
But `ipfs swarm resources` doesn't return anything above 5-15%, so I think this error is actually on the remote node side and not related to our issue, right?

Anything else we could gather to help solve this issue?

Right now I'm out of ideas to get our cluster back into a working state (beside restarting Kubo every 2h but that's not a solution since it will prevent us from reproving the pins to the rest of the network)

Edit with additional info:
- Kubo is launched without the `--enable-gc` flag, as prescribed by ipfs-cluster doc.

Discussion:
@11qu1d: I can confirm we are seeing the same behaviour while testing `ipfs/kubo:v0.32.1` docker image in k8s paired with ipfs-cluster (smaller test cluster ~17k pins).

Logs from ipfs-cluster:
```
2024-11-25T15:01:17.201Z	ERROR	ipfshttp	ipfshttp/ipfshttp.go:1276	error posting to IPFS:Post "http://localhost:5001/api/v0/pin/ls?stream=true&type=recursive": context deadline exceeded
2024-11-25T15:01:17.201Z	ERROR	pintracker	stateless/stateless.go:364	could not get pinset from IPFS: Post "http://localhost:5001/api/v0/pin/ls?stream=true&type=recursive": context deadline exceeded
2024-11-25T15:01:17.201Z	ERROR	pintracker	stateless/stateless.go:570	could not get pinset from IPFS: Post "http://localhost:5001/api/v0/pin/ls?stream=true&type=recursive": context deadline exceeded
```
No errors or other logs observed in Kubo.

Also, calling the API endpoint or running `ipfs pin ls --stream` in Kubo just hangs until the pod is restarted. After a while it stops working again.@hsanjuan: Hello, per the details you provide, I just think leveldb exploded in some way. How many files are there in the `.ipfs/leveldb` folder? Chances are, it is compacting... or was compacting and it was stopped.

Do you have monitoring for disk-reads? Is it trying to read/write a lot from disk even when nothing is being added?

I would recommend to switch leveldb to pebble (so, flatfs + pebble). You will loose the pinset (not the blocks, just the list of things you have pinned) but cluster will add the pins again, in time. @Mayeu: > How many files are there in the .ipfs/leveldb folder?

    $ ls -l /ipfs-data/node/datastore/*.ldb | wc -l
    1917
    
> Do you have monitoring for disk-reads? Is it trying to read/write a lot from disk even when nothing is being added?

We are running Netdata on the node so if you know of a particular metric that could be interesting I can take a look specifically at those.

In the meantime, this is the reported number of pins for the Kubo node between the 5th and 9th of December:
<img width="977" alt="Screenshot 2024-12-12 at 12 27 28" src="https://github.com/user-attachments/assets/3f5d829e-b60f-4794-ae90-c43a909d1076" />

Disk bandwidth at that time:
<img width="977" alt="Screenshot 2024-12-12 at 12 25 01" src="https://github.com/user-attachments/assets/a35f2128-af71-470f-be5f-7ae8a7304f85" />

and Disk utilization per systemd service:
<img width="977" alt="Screenshot 2024-12-12 at 12 26 20" src="https://github.com/user-attachments/assets/51c2408f-6ff0-4843-881c-3b3b656204f5" />

So there is less data read when Kubo is stuck, but the total number of reads & writes operations don't seem to be impacted.

> I would recommend to switch leveldb to pebble (so, flatfs + pebble). You will loose the pinset (not the blocks, just the list of things you have pinned) but cluster will add the pins again, in time.

I was thinking of that since I saw that pebble was added to Kubo in a previous version.

Just to confirm the process, that would go like so:

1. Stop Kubo
2. Update the datastore configuration to replace the levelds store type by a pebble store
3. delete the previous datastore data
4. Restart the node
5. `ipfs-cluster` will then ask Kubo to repin everything, which won't download anything since the data is still on disk.@hsanjuan: Yes, when switching you will need to edit `datastore_spec`. You can also set pebble to live in a different folder (not the `datastore` folder used by leveldb. I would keep that folder around just in case you need to go back.

Do you use MFS for anything?

Also, regarding the ipfs-pins graph, it goes to 0 because of https://github.com/ipfs-cluster/ipfs-cluster/pull/2122. From now on it will stay at the last reported amount when `pin ls` worked..

Even if it won't need to download data, it will need to add 16M items to the datastore, and pinning will make it traverse everything it has.@Mayeu: > Yes, when switching you will need to edit datastore_spec. You can also set pebble to live in a different folder (not the datastore folder used by leveldb. I would keep that folder around just in case you need to go back.

Thank you, I'm going to try that today on one of our node.

> Do you use MFS for anything?

No we don't use MFS, we only add new pin via the cluster API, and when needed we access our data via Kubo's gateway using the CIDs. As far as I understand this doesn't involve the MFS subsystems.

> [...] the ipfs-pins graph, it goes to 0 because of https://github.com/ipfs-cluster/ipfs-cluster/pull/2122. [...]

Good to know, thank you 👍@Mayeu: I have switch one of our node to using the pebble datastore, right now it is slowly adding back the whole pinset to pebble.@ehsan6sha: > pebble

hi
did changing it to pebble solved the issue?@Mayeu: @ehsan6sha still too soon to tell. Our node is still adding the data back into the pebble store. It has only catched up 50% of the previous pins right now.@github-actions: Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.@lidel: @Mayeu how does it look like now? (assuming it finished re-pinning)@github-actions: Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.@Mayeu: Hello, I'm just getting back to work after my end of the year vacation, so first, happy new year to you all :)

On the matter, it seems that something unexpected happened during my break:

<img width="1446" alt="Screenshot 2025-01-06 at 13 46 16" src="https://github.com/user-attachments/assets/2d024c66-5e34-4d78-9499-fa799bb525a3" />

Around the 22nd of December, the pinning stopped at ~14.18M (on 16.51M at that point) and then dropped to 0 until yesterday when it spiked back to the previous point and continue to pin. Right now there is still 0.7M pin that needs to be added to the DB.

I'm currently going through our logs and I'll report back with what I can find there.@Mayeu: Sadly, I can't find anything because the amount log produce meant that the log before the spike up was purged 🤦🏻‍♀️

I'm updating our log retention config for that machine to keep much (much) more logs, and "hope" to see that drop again.

On the bright side, this node is now fully caught up with the cluster state, so we'll see if this issue shows up again. But since the 4th of December, our first node (which is still using LevelDB) didn't experience that issue.
@hsanjuan: As mentioned, cannot trust the graph much due to the bug I pointed above... you are better tracking the "pending" items (queued, pinning, error) and comparing that to total items in cluster pinset, rather than using this metric right now.@hsanjuan: (and happy new year!)@gammazero: @Mayeu We will wait for another week, and assume the issue is resolved if we do not hear from you.@Mayeu: > As mentioned, cannot trust the graph much due to the bug I pointed above... you are better tracking the "pending" items (queued, pinning, error) and comparing that to total items in cluster pinset, rather than using this metric right now.

@hsanjuan right, I forgot about that. We do gather  those as well.

Here they are for the past 60 days:

Pin queued:

<img width="1556" alt="Screenshot 2025-01-08 at 11 58 56" src="https://github.com/user-attachments/assets/86ef621b-1030-4c9e-8601-195a9f083d5e" />

Error:
 
<img width="1556" alt="Screenshot 2025-01-08 at 11 48 59" src="https://github.com/user-attachments/assets/8a45f980-0e0b-4131-a5d7-b55f990e0ac2" />

Pinning:
    
<img width="1556" alt="Screenshot 2025-01-08 at 11 49 09" src="https://github.com/user-attachments/assets/d0bbe825-5451-4dbb-b7f6-c13ccd85aa88" />

Just a reminder of the timeline:
- We realized that we were encountering the issue around the 15th of November, and started to change the node configurations and thus restart the nodes regularly (one of those being switching the number of concurrent pin from 100 to 50).
- I switched our second node to using the pebble datastore on the 13th of December.

For comparison, here are the graphs for our first node (still using LevelDBs), which didn't experience as many issues (it still encountered that listing issue, but for some reason it stabilized pretty quickly after we realized there was an issue):

Error and Pinning:
  
<img width="1556" alt="Screenshot 2025-01-08 at 11 29 45" src="https://github.com/user-attachments/assets/a8374368-fade-4e53-bc63-43a6e21a79f4" />

Queued:

<img width="1556" alt="Screenshot 2025-01-08 at 11 30 14" src="https://github.com/user-attachments/assets/f5ce691b-b443-437a-8554-8f9b2893e02f" />
@hsanjuan: So my understanding is that the new node still hangs on `pin ls` sometimes?@Mayeu: @hsanjuan I can't go back in the logs before the 5th of January so I'm not sure why the new node was stuck between the 22nd of December and the 5th of January.

Between the 5th and today there are 4184 `pin/ls` error in the logs, the last ones being yesterday morning. So it is still happening yes :/@hsanjuan: and the errors are `context.Cancelled`? How long does `pin ls` take when you call it manually ? @hsanjuan: Also, assume you call it manually and it is not streaming anything at all for a few minutes... can you `ipfs diag profile` then and post the result? Or look yourself which goroutine is hanging for that amount of time and on which lock it is waiting.@Mayeu: > and the errors are context.Cancelled?

Yes, the errors are the `context.Cancelled` one.

> How long does `pin ls` take when you call it manually ?

Previously when the issue arose with LevelDB, it "never" finished ("never" as in I stopped curl after 24h). But I don't yet have been able to catch it myself with Pebble. I don't think it will "never" finish anymore since it seems to happen regularly and then resolve itself as this number of error per day shows:

```
date              _count
--------------------------------
2025-01-07        3848
2025-01-08        463
2025-01-09        195
```

I'll script something to triggers a diagnostic if a curl doesn't finish after a few minutes, because I'm not sure I can react myself to those events.

FYI (and mine in 6 months), I'm getting those numbers with [angle-grinder](https://github.com/rcoh/angle-grinder):
```
journalctl -u ipfs-cluster  --grep "ERROR.*ipfshttp.*/pin/ls.*context canceled" | agrind '* | parse "* * * * *[*]: *T*Z*" as month, day, time, server, process, pid, date, time, msg | count by date | sort by date'
```@github-actions: Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.@hsanjuan: Hello, @Mayeu .

There logs should print info messages of the sort: "Full pinset listing finished" . How long does it take to list the pinset when it works?

The ipfs-cluster config has a `ipfshttp.ipfs_request_timeout` option. If your errors come from cluster logs, it may be that pin ls request are timing out. Given the amount of pins that you have, it will likely take more than the default of 5 minutes.

If it is not that, and you can reproduce this by calling `ipfs pin ls --type recursive` directly, then we need the kubo profiles to see why it is deadlocked.@Mayeu: Sorry for the delay here, I finally got to check the data that was gathered last week.

> The ipfs-cluster config has a ipfshttp.ipfs_request_timeout option. [...]

We have set this timeout to 20 minutes in our case.

Here is one profiling that was triggered while a `pin/ls` manual call was hanging for ~1min 50 after the error triggered in the log (so after the 20 min timeout): https://drive.google.com/file/d/1eFb1rHmPKDHvdP-1HEjyUvdweN-JVIvt/view?usp=sharing

At the longest some requests took 14 minutes (after the log entry) to respond. But at that point partition was full of profiles so I don't have a profile matching those request. I'll try again if you want.

> There logs should print info messages of the sort: "Full pinset listing finished" . How long does it take to list the pinset when it works?

There isn't any log at the start of the listing process, right? The earliest I can find is when the process is around 500k pins.

On our node using pebble it seems to take around 1m:
```
Jan 22 00:48:04 ipfs-02 ipfs-cluster-service[2716182]: 2025-01-22T00:48:04.502Z        INFO        dsstate        dsstate/datastore.go:196        Full pinset listing in progress: 500000 pins so far
Jan 22 00:48:06 ipfs-02 ipfs-cluster-service[2716182]: 2025-01-22T00:48:06.554Z        INFO        dsstate        dsstate/datastore.go:196        Full pinset listing in progress: 1000000 pins so far
[...]    
Jan 22 00:49:01 ipfs-02 ipfs-cluster-service[2716182]: 2025-01-22T00:49:01.603Z        INFO        dsstate        dsstate/datastore.go:201        Full pinset listing finished: 16630186 pins
```

On our node still using leveldb, this takes around 45s:
```
Jan 23 10:20:42 ipfs-cluster-service[3941117]: 2025-01-23T10:20:42.921Z        INFO        dsstate        dsstate/datastore.go:196        Full pinset listing in progress: 500000 pins so far
Jan 23 10:20:44 ipfs-01 ipfs-cluster-service[3941117]: 2025-01-23T10:20:44.117Z        INFO        dsstate        dsstate/datastore.go:196        Full pinset listing in progress: 1000000 pins so far [...]
Jan 23 10:21:24 ipfs-01 ipfs-cluster-service[3941117]: 2025-01-23T10:21:24.420Z        INFO        dsstate        dsstate/datastore.go:201        Full pinset listing finished: 16633864 pins
 ```@hsanjuan: I requested access to the profile. If you manage to get another manual `ipfs pin ls --stream true --type=recursive` hanging for like 10 minutes, that would be good too.@Mayeu: Ha sorry I thought that I shared a public link, I updated the share right to be for anyone with the link.

I'll start the script again to try to get a profile for a request that hangs for longer.@hsanjuan: @Mayeu the `goroutine.stacks` file shows 218 goroutines calling PinLs(cid?), they have all been waiting for 2 minutes for read access.

There is a goroutine listing pins at the same time.

And there are 5 waiting for a write Lock to pin something, which they cannot get while the listing is ongoing.

I don't see anything weird at the pebble level, it seems to be reading as requested by pinLs. Do you think the ongoing pin/ls requests is returning any data, or just hanging? logs showing on cluster?@github-actions: Oops, seems like we needed more information for this issue, please comment with more details or this issue will be closed in 7 days.@Mayeu: > they have all been waiting for 2 minutes for read access.

That surprising, when a log entry triggers the cluster has already been waiting for its 20 min timeout. Or maybe I'm missing something ?

> Do you think the ongoing pin/ls requests is returning any data, or just hanging?

The few times I witnessed the issue directly no data was returned for a while it was just hanging waiting for a response from Kubo.

"a while" spanning between 30s to a few hour with the Pebble store, and more than 24h when we initially had the issue with LevelDB.

> logs showing on cluster?

At time when this happens, the cluster only shows the "context cancelled" error:
```
error posting to IPFS:Post "http://127.0.0.1:5001/api/v0/pin/ls?stream=true&arg=QmRSGNoqrbTsKCgHbi8xCeQEn4sQXFDjfNEUgXWG5wAg6U&type=recursive": context canceled
```

I restarted the script to get a profile when this error shows up in the log. I improved it a bit because multiple profiles could have been run at the same time previously, and maybe that could have influenced the running time of `pin/ls`?. I'll check tomorrow and report back.

Sidenote: both nodes have been upgraded to `0.33.0`, I'm waiting for `0.33.1` to hit nixpkgs to upgrade to that version.@hsanjuan: Any news @Mayeu ?@Mayeu: Hello,

For the past few days the lock up were shorter than during my previous data gathering tentative, the longest I got was [this profile](https://drive.google.com/file/d/1-IBIEaNqb3HxLPoJLX5z044dBaEzwGrF/view?usp=sharing), during which a manual call to `pin/ls` was hanging for a bit more than 9 mins based on the output of the script.@hsanjuan: I think I have something... The `reprovider` is running, and apparently very slowly since it is waiting to send pins for 284 minutes.

It is holding a read lock on the dspinner. However, there are also some writers waiting to write on the same lock (doPinRecursive). This prevents more read-locks to be granted (judging by what it says here https://stackoverflow.com/questions/46786900/the-implementions-of-rwmutex-lock-in-golang, which matches what the stacks show).

As a result, everything hangs. I will study how we can address this, given that you already have accelerateddht providing enabled (I assume you need dht providing for every block, otherwise best to disable or change strategy to "roots"). At your pinset size, dht providing IS a bottleneck.@hsanjuan: Also explained here: https://pkg.go.dev/sync#RWMutex@Mayeu: Thank you for spending your time on this 🙏

> I assume you need dht providing for every block, otherwise best to disable or change strategy to "roots"

Good insight, I don't think we wish to do the reproviding at the block level.

For CIDs that points to a file, we definitely don't care about reproviding at the block level, the roots will be enough.

But how would "roots" behave for CIDs that point to a folder? If somebody requests only one file in that folder, will our server still answer that request or will it only answer requests to get the whole folder?@hsanjuan: > But how would "roots" behave for CIDs that point to a folder? If somebody requests only one file in that folder, will our server still answer that request or will it only answer requests to get the whole folder?

It could be that, if you are only announcing the folder's CID and people request a file by file's CID, that no routing to that file is discovered. If people request `<folder_cid>/filename.jpg` things would likely work.

If the reporovider does not announce a CID in particular because it is set to `roots`, then things may still work if discovery happens via bitswap. For that to happen, the clients looking for the content need to be connected to the server in advance so that they ask it directly for content.@hsanjuan: There's a fix at #10746 and it would be good if someone can confirm improvements. I am testing myself as well on our clusters.@hsanjuan: Fix now on master@Mayeu: @hsanjuan Hello, sorry I was on vacation last week and I'm only checking this issue right now. I'll try the master branch on our second server and report back any improvement in the number of reported errors :+1:@Mayeu: Hello, I was only able to update our kubo recently, but I can confirm this change improved the situation:

```
2025-03-29        1108
2025-03-30        1453
2025-03-31        1667
2025-04-01        5184
2025-04-02        6258
2025-04-03        1870
2025-04-04        2856
2025-04-05        727 <-- This is the day I updated our node
2025-04-06        434
2025-04-07        739
2025-04-08        1346
2025-04-09        17
2025-04-10        0
2025-04-11        461
2025-04-12        432
2025-04-13        376
2025-04-14        413
```

Those errors still happen, but for the past week it's consistently lower than before.
Also
- We never experienced the complete lockup issue since I opened this ticket. 
- I didn't yet changed the reproviding to root only, so I'm expecting further improvement after that

So I think we can keep this ticket closed 👍 

Solution patch:
diff --git a/core/node/provider.go b/core/node/provider.go
index d6ab26b0c..78f0761cc 100644
--- a/core/node/provider.go
+++ b/core/node/provider.go
@@ -167,16 +167,23 @@ func newProvidingStrategy(onlyPinned, onlyRoots bool) interface{} {
 		IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
 	}
 	return func(in input) provider.KeyChanFunc {
+		// Pinner-related CIDs will be buffered in memory to avoid
+		// deadlocking the pinner when the providing process is slow.
+
 		if onlyRoots {
-			return provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher)
+			return provider.NewBufferedProvider(
+				provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher),
+			)
 		}
 
 		if onlyPinned {
-			return provider.NewPinnedProvider(false, in.Pinner, in.IPLDFetcher)
+			return provider.NewBufferedProvider(
+				provider.NewPinnedProvider(false, in.Pinner, in.IPLDFetcher),
+			)
 		}
 
 		return provider.NewPrioritizedProvider(
-			provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher),
+			provider.NewBufferedProvider(provider.NewPinnedProvider(true, in.Pinner, in.IPLDFetcher)),
 			provider.NewBlockstoreProvider(in.Blockstore),
 		)
 	}
diff --git a/docs/changelogs/v0.34.md b/docs/changelogs/v0.34.md
index b11ec3699..8c1c5753e 100644
--- a/docs/changelogs/v0.34.md
+++ b/docs/changelogs/v0.34.md
@@ -14,6 +14,7 @@
   - [Badger datastore update](#badger-datastore-update)
   - [Datastore Implementation Updates](#datastore-implementation-updates)
   - [One Multi-error Package](#one-multi-error-package)
+  - [Fix hanging pinset operations during reprovides](#fix-hanging-pinset-operations-during-reprovides)
   - [📦️ Important dependency updates](#-important-dependency-updates)
 - [👨‍👩‍👧‍👦 Contributors](#-contributors)
 
@@ -74,6 +75,10 @@ The go-ds-xxx datastore implementations have been updated to support the updated
 
 Kubo previously depended on multiple multi-error packages, `github.com/hashicorp/go-multierror` and `go.uber.org/multierr`. These have nearly identical functionality so there was no need to use both. Therefore, `go.uber.org/multierr` was selected as the package to depend on. Any future code needing multi-error functionality should use `go.uber.org/multierr` to avoid introducing unneeded dependencies.
 
+#### Fix hanging pinset operations during reprovides
+
+The reprovide process can be quite slow. In default settings, the reprovide process will start reading CIDs that belong to the pinset. During this operation, starvation can occur for other operations that need pinset access (see https://github.com/ipfs/kubo/issue/10596). We have now switch to buffering pinset-related cids that are going to be reprovided in memory, so that we can free pinset mutexes as soon as possible so that pinset-writes and subsequent read operations can proceed. The downside is larger pinsets will need some extra memory, with an estimation of ~1GiB of RAM memory-use per 20 million items to be reprovided.
+
 #### 📦️ Important dependency updates
 
 - update `go-libp2p` to [v0.41.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.41.0) (incl. [v0.40.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.40.0))
diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod
index 8dffe9c14..8ca4390ed 100644
--- a/docs/examples/kubo-as-a-library/go.mod
+++ b/docs/examples/kubo-as-a-library/go.mod
@@ -7,7 +7,7 @@ go 1.24
 replace github.com/ipfs/kubo => ./../../..
 
 require (
-	github.com/ipfs/boxo v0.29.0
+	github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0
 	github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
 	github.com/libp2p/go-libp2p v0.41.0
 	github.com/multiformats/go-multiaddr v0.15.0
@@ -52,7 +52,7 @@ require (
 	github.com/francoispqt/gojay v1.2.13 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.6 // indirect
-	github.com/gammazero/chanqueue v1.0.0 // indirect
+	github.com/gammazero/chanqueue v1.1.0 // indirect
 	github.com/gammazero/deque v1.0.0 // indirect
 	github.com/getsentry/sentry-go v0.27.0 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.5 // indirect
diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum
index 9a5fa3be1..6207a1d8a 100644
--- a/docs/examples/kubo-as-a-library/go.sum
+++ b/docs/examples/kubo-as-a-library/go.sum
@@ -166,8 +166,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc=
 github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc=
-github.com/gammazero/chanqueue v1.0.0 h1:FER/sMailGFA3DDvFooEkipAMU+3c9Bg3bheloPSz6o=
-github.com/gammazero/chanqueue v1.0.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc=
+github.com/gammazero/chanqueue v1.1.0 h1:yiwtloc1azhgGLFo2gMloJtQvkYD936Ai7tBfa+rYJw=
+github.com/gammazero/chanqueue v1.1.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc=
 github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34=
 github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo=
 github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
@@ -298,8 +298,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd
 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.29.0 h1:clzd7PglUcE+Ufq1KucS3aKID7pzGVaSgcdRsW395t4=
-github.com/ipfs/boxo v0.29.0/go.mod h1:c3R52nMlgMsN1tADffYcogKoVRsX1RJE1TMYSpJ4uVs=
+github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0 h1:wkFvXf7SqRiAu5ezj6lsieE2PwDk3nUkFOs9/+5lWdo=
+github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0/go.mod h1:MkDJStXiJS9U99cbAijHdcmwNfVn5DKYBmQCOgjY2NU=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=
diff --git a/go.mod b/go.mod
index 0bcc1869f..0bcfbc99d 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
 	github.com/hashicorp/go-version v1.7.0
 	github.com/ipfs-shipyard/nopfs v0.0.14
 	github.com/ipfs-shipyard/nopfs/ipfs v0.25.0
-	github.com/ipfs/boxo v0.29.0
+	github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0
 	github.com/ipfs/go-block-format v0.2.0
 	github.com/ipfs/go-cid v0.5.0
 	github.com/ipfs/go-cidutil v0.1.0
@@ -125,7 +125,7 @@ require (
 	github.com/flynn/noise v1.1.0 // indirect
 	github.com/francoispqt/gojay v1.2.13 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.6 // indirect
-	github.com/gammazero/chanqueue v1.0.0 // indirect
+	github.com/gammazero/chanqueue v1.1.0 // indirect
 	github.com/gammazero/deque v1.0.0 // indirect
 	github.com/getsentry/sentry-go v0.27.0 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.5 // indirect
diff --git a/go.sum b/go.sum
index e29d9352f..7e2b6e63a 100644
--- a/go.sum
+++ b/go.sum
@@ -200,8 +200,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc=
 github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc=
-github.com/gammazero/chanqueue v1.0.0 h1:FER/sMailGFA3DDvFooEkipAMU+3c9Bg3bheloPSz6o=
-github.com/gammazero/chanqueue v1.0.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc=
+github.com/gammazero/chanqueue v1.1.0 h1:yiwtloc1azhgGLFo2gMloJtQvkYD936Ai7tBfa+rYJw=
+github.com/gammazero/chanqueue v1.1.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc=
 github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34=
 github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo=
 github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
@@ -362,8 +362,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.25.0 h1:OqNqsGZPX8zh3eFMO8Lf8EHRRnSGBMqcd
 github.com/ipfs-shipyard/nopfs/ipfs v0.25.0/go.mod h1:BxhUdtBgOXg1B+gAPEplkg/GpyTZY+kCMSfsJvvydqU=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.29.0 h1:clzd7PglUcE+Ufq1KucS3aKID7pzGVaSgcdRsW395t4=
-github.com/ipfs/boxo v0.29.0/go.mod h1:c3R52nMlgMsN1tADffYcogKoVRsX1RJE1TMYSpJ4uVs=
+github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0 h1:wkFvXf7SqRiAu5ezj6lsieE2PwDk3nUkFOs9/+5lWdo=
+github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0/go.mod h1:MkDJStXiJS9U99cbAijHdcmwNfVn5DKYBmQCOgjY2NU=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=



Test patch:
diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod
index 6fca5043c..6fb22bf3d 100644
--- a/test/dependencies/go.mod
+++ b/test/dependencies/go.mod
@@ -116,7 +116,7 @@ require (
 	github.com/huin/goupnp v1.3.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/ipfs/bbloom v0.0.4 // indirect
-	github.com/ipfs/boxo v0.29.0 // indirect
+	github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0 // indirect
 	github.com/ipfs/go-block-format v0.2.0 // indirect
 	github.com/ipfs/go-cid v0.5.0 // indirect
 	github.com/ipfs/go-datastore v0.8.2 // indirect
diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum
index 5ec6fbf57..1fa188603 100644
--- a/test/dependencies/go.sum
+++ b/test/dependencies/go.sum
@@ -156,8 +156,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo=
 github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA=
-github.com/gammazero/chanqueue v1.0.0 h1:FER/sMailGFA3DDvFooEkipAMU+3c9Bg3bheloPSz6o=
-github.com/gammazero/chanqueue v1.0.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc=
+github.com/gammazero/chanqueue v1.1.0 h1:yiwtloc1azhgGLFo2gMloJtQvkYD936Ai7tBfa+rYJw=
+github.com/gammazero/chanqueue v1.1.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc=
 github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34=
 github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -294,8 +294,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.29.0 h1:clzd7PglUcE+Ufq1KucS3aKID7pzGVaSgcdRsW395t4=
-github.com/ipfs/boxo v0.29.0/go.mod h1:c3R52nMlgMsN1tADffYcogKoVRsX1RJE1TMYSpJ4uVs=
+github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0 h1:wkFvXf7SqRiAu5ezj6lsieE2PwDk3nUkFOs9/+5lWdo=
+github.com/ipfs/boxo v0.29.1-0.20250307073615-93ea580a3bb0/go.mod h1:MkDJStXiJS9U99cbAijHdcmwNfVn5DKYBmQCOgjY2NU=
 github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs=
 github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM=
 github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg=