Repository: hirosystems/bitcoin-indexer
Pull request #531: upgrade rocksdb version
Author: login: ASuciuX
Created: 2025-05-09T11:03:40Z | Merged: 2025-05-20T16:13:49Z

- fixes https://github.com/hirosystems/bitcoin-indexer/issues/527

Tested and confirmed to work as expected on the mainnet bitcoin-indexer running on Linux.

Extra dependencies might be required (from os to os), as specified in the `README.md` file.


Diff:
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index c780ec36..203b323c 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -186,6 +186,17 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Install LLVM and Clang
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y llvm clang libclang-dev
+
+      - name: Install Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: '1.85'
+          components: clippy
+
       - name: Run clippy
         id: clippy
         run: |
@@ -251,8 +262,15 @@ jobs:
           docker compose -f ../../dockerfiles/docker-compose.dev.postgres.yml up -d
           docker compose -f ../../dockerfiles/docker-compose.dev.postgres.yml logs -t -f --no-color &> docker-compose-logs.txt &
 
+      - name: Install LLVM and CLang
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y llvm clang libclang-dev
+
       - name: Install Rust
         uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: '1.85'
 
       - name: Run doc tests
         run: |
diff --git a/Cargo.lock b/Cargo.lock
index 30265c45..54c0243f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -302,15 +302,16 @@ checksum = "383d29d513d8764dcdc42ea295d979eb99c3c9f00607b3692cf68a431f7dca72"
 
 [[package]]
 name = "bindgen"
-version = "0.65.1"
+version = "0.68.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5"
+checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078"
 dependencies = [
- "bitflags 1.3.2",
+ "bitflags 2.9.0",
  "cexpr",
  "clang-sys",
  "lazy_static",
  "lazycell",
+ "log",
  "peeking_take_while",
  "prettyplease",
  "proc-macro2",
@@ -319,29 +320,27 @@ dependencies = [
  "rustc-hash 1.1.0",
  "shlex",
  "syn 2.0.100",
+ "which",
 ]
 
 [[package]]
 name = "bindgen"
-version = "0.68.1"
+version = "0.69.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "726e4313eb6ec35d2730258ad4e15b547ee75d6afaa1361a922e78e59b7d8078"
+checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
 dependencies = [
  "bitflags 2.9.0",
  "cexpr",
  "clang-sys",
+ "itertools",
  "lazy_static",
  "lazycell",
- "log",
- "peeking_take_while",
- "prettyplease",
  "proc-macro2",
  "quote",
  "regex",
  "rustc-hash 1.1.0",
  "shlex",
  "syn 2.0.100",
- "which",
 ]
 
 [[package]]
@@ -1977,6 +1976,15 @@ version = "1.70.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
 
+[[package]]
+name = "itertools"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.15"
@@ -2064,14 +2072,13 @@ dependencies = [
 
 [[package]]
 name = "librocksdb-sys"
-version = "0.11.0+8.1.1"
+version = "0.17.1+9.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3386f101bcb4bd252d8e9d2fb41ec3b0862a15a62b478c355b2982efa469e3e"
+checksum = "2b7869a512ae9982f4d46ba482c2a304f1efd80c6412a3d4bf57bb79a619679f"
 dependencies = [
- "bindgen 0.65.1",
+ "bindgen 0.69.5",
  "bzip2-sys",
  "cc",
- "glob",
  "libc",
  "libz-sys",
 ]
@@ -2619,9 +2626,9 @@ dependencies = [
 
 [[package]]
 name = "prettyplease"
-version = "0.2.31"
+version = "0.2.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb"
+checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6"
 dependencies = [
  "proc-macro2",
  "syn 2.0.100",
@@ -3184,9 +3191,9 @@ dependencies = [
 
 [[package]]
 name = "rocksdb"
-version = "0.21.0"
+version = "0.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb6f170a4041d50a0ce04b0d2e14916d6ca863ea2e422689a5b694395d299ffe"
+checksum = "26ec73b20525cb235bad420f911473b69f9fe27cc856c5461bccd7e4af037f43"
 dependencies = [
  "libc",
  "librocksdb-sys",
diff --git a/README.md b/README.md
index f5db4d52..148ae548 100644
--- a/README.md
+++ b/README.md
@@ -35,6 +35,8 @@ $ cargo bitcoin-indexer-install
 
 Docker images are also available at https://hub.docker.com/r/hirosystems/bitcoin-indexer
 
+Note: You may need to install additional LLVM and Clang dependencies if they are not already available on your system.
+
 ## Running the Indexer
 
 The following command will start indexing Ordinals activity and will continue to stream new blocks
@@ -50,6 +52,38 @@ $ bitcoin-indexer runes service start --config-path <path>
 
 A fully synced Bitcoin node is required for indexing to start.
 
+### Running Tests
+
+The test suite can be run in a Docker environment that includes all necessary dependencies (PostgreSQL, Rust toolchain, etc.). This ensures consistent test execution across all development environments.
+
+To run the tests:
+
+```bash
+./scripts/run-tests.sh
+```
+
+This script will:
+1. Start a PostgreSQL container
+2. Build the Bitcoin Indexer Docker image
+3. Run all tests in the Docker environment
+4. Clean up containers after completion
+
+For more granular test control, you can run specific test suites using cargo:
+
+```bash
+# Start docker postgres
+docker compose -f dockerfiles/docker-compose.dev.postgres.yml up -d
+
+# Run all tests
+cargo test --workspace
+
+# Run tests for a specific component
+cargo test -p components/bitcoind
+
+# Stop and remove docker postgres
+docker compose -f dockerfiles/docker-compose.dev.postgres.yml down -v -t 0
+```
+
 ## Running an API
 
 Once the index starts advancing, you can deploy the Ordinals API or Runes API to read the same data
diff --git a/components/ordinals/Cargo.toml b/components/ordinals/Cargo.toml
index 913524f8..21777193 100644
--- a/components/ordinals/Cargo.toml
+++ b/components/ordinals/Cargo.toml
@@ -34,8 +34,8 @@ fxhash = "0.2.1"
 anyhow = { version = "1.0.56", features = ["backtrace"] }
 progressing = '3'
 futures = "0.3.28"
-rocksdb = { version = "0.21.0", default-features = false, features = [
-    "snappy",
+rocksdb = { version = "0.23.0", default-features = false, features = [
+    "snappy", "bindgen-runtime"
 ] }
 pprof = { version = "0.14.0", features = ["flamegraph"], optional = true }
 hyper = { version = "=0.14.27" }
diff --git a/dockerfiles/components/bitcoin-indexer.dockerfile b/dockerfiles/components/bitcoin-indexer.dockerfile
index 499f7fb0..f36ca7cb 100644
--- a/dockerfiles/components/bitcoin-indexer.dockerfile
+++ b/dockerfiles/components/bitcoin-indexer.dockerfile
@@ -2,8 +2,28 @@ FROM rust:bullseye AS build
 
 WORKDIR /src
 
-RUN apt-get update && apt-get install -y ca-certificates pkg-config libssl-dev libclang-11-dev libunwind-dev libunwind8 curl gnupg
-RUN rustup update 1.81 && rustup default 1.81
+RUN apt-get update && \
+    apt-get install -y \
+    wget && \
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+    echo "deb http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-18 main" >> /etc/apt/sources.list.d/llvm.list && \
+    apt-get update && \
+    apt-get install -y \
+    pkg-config \
+    libssl-dev \
+    libunwind-dev \
+    libunwind8 \
+    curl \
+    libsnappy-dev \
+    libgflags-dev \
+    zlib1g-dev \
+    libbz2-dev \
+    liblz4-dev \
+    libzstd-dev \
+    clang-18 \
+    libclang-18-dev \
+    llvm-18-dev
+RUN rustup update 1.85 && rustup default 1.85
 
 RUN mkdir /out
 COPY ./Cargo.toml /src/Cargo.toml
@@ -16,7 +36,30 @@ RUN cp /src/target/release/bitcoin-indexer /out
 
 FROM debian:bullseye-slim
 
-RUN apt-get update && apt-get install -y ca-certificates libssl-dev libclang-11-dev libunwind-dev libunwind8 sqlite3
+# Install runtime dependencies for LLVM/Clang 18 and other necessary libs
+RUN apt-get update && \
+    apt-get install -y \
+    gnupg \
+    ca-certificates \
+    wget && \
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+    echo "deb http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-18 main" >> /etc/apt/sources.list.d/llvm.list && \
+    apt-get update && \
+    apt-get install -y \
+    pkg-config \
+    libssl-dev \
+    libunwind-dev \
+    libunwind8 \
+    libsnappy-dev \
+    libgflags-dev \
+    zlib1g-dev \
+    libbz2-dev \
+    liblz4-dev \
+    libzstd-dev \
+    clang-18 \
+    libclang-18-dev \
+    llvm-18-dev
+
 COPY --from=build /out/bitcoin-indexer /bin/bitcoin-indexer
 
 WORKDIR /workspace
diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh
new file mode 100755
index 00000000..71887005
--- /dev/null
+++ b/scripts/run-tests.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+set -e
+
+# Start PostgreSQL container
+echo "Starting PostgreSQL environment..."
+docker compose -f dockerfiles/docker-compose.dev.postgres.yml up -d
+echo "PostgreSQL containers started"
+
+# Build the image targeting the build stage
+echo "Building Bitcoin Indexer image..."
+docker build --no-cache --target build -t test-bitcoin-indexer -f dockerfiles/components/bitcoin-indexer.dockerfile .
+
+# Run the tests
+echo "Running tests..."
+docker run -it --rm \
+  -v $(pwd):/src \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  --network host \
+  test-bitcoin-indexer \
+  bash -c "cd /src && RUST_BACKTRACE=1 cargo test --workspace --color=always --no-fail-fast -- --nocapture --show-output"
+
+# Clean up
+echo "Cleaning up containers..."
+docker compose -f dockerfiles/docker-compose.dev.postgres.yml down -v -t 0
+echo "Test environment cleanup complete"


Reviews:
user: login: ASuciuX
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-19T15:09:38Z
user: login: ASuciuX
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-19T15:10:02Z
user: login: rafaelcr
state: CHANGES_REQUESTED
author_association: COLLABORATOR
submitted_at: 2025-05-19T15:24:56Z
user: login: ASuciuX
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-19T18:10:31Z
user: login: ASuciuX
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-19T18:10:43Z
user: login: rafaelcr
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-19T20:44:10Z
user: login: ASuciuX
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-19T21:01:23Z
user: login: rafaelcr
body: LGTM! Just remove the remaining cargo config.

Also don't forget to include `chore(deps):` as the stashed commit prefix in this case
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-05-19T21:03:03Z
user: login: ASuciuX
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-19T21:04:10Z

Review comments:
path: test-script.sh
user: login: ASuciuX
body: This has been created to allow users to run the suite and test their contributions without needing to install dependencies on their local machine. It only impacts the Docker image used, ensuring a clean and isolated environment.
created_at: 2025-05-19T15:09:37Z
author_association: CONTRIBUTOR
path: test-script.sh
user: login: ASuciuX
body: Is there a more suitable place to move this to? Also, more appropriate name
created_at: 2025-05-19T15:10:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -251,8 +262,15 @@ jobs:
           docker compose -f ../../dockerfiles/docker-compose.dev.postgres.yml up -d
           docker compose -f ../../dockerfiles/docker-compose.dev.postgres.yml logs -t -f --no-color &> docker-compose-logs.txt &
 
+      - name: Install LLVM and CLang
+        run: |
+         sudo apt-get update
path: .github/workflows/ci.yaml
user: login: rafaelcr
body: ```suggestion
          sudo apt-get update
```
created_at: 2025-05-19T15:18:39Z
author_association: COLLABORATOR
path: test-script.sh
user: login: rafaelcr
body: How about `scripts/run-tests.sh`? Also, it would be neat to have a command configured in `.cargo/config.toml` to run this script
created_at: 2025-05-19T15:24:52Z
author_association: COLLABORATOR
path: test-script.sh
user: login: ASuciuX
body: Thanks! I’ve added an alias as well. Since Cargo aliases don’t natively support running shell scripts directly (they're limited to Cargo commands), I created a separate cargo-tasks project to handle this, and to support any future tasks that might come up too.

I've added it outside the components folder, that requires to also copy that to the bitcoin-indexer image to run it.
created_at: 2025-05-19T18:10:31Z
author_association: CONTRIBUTOR
diff_hunk: @@ -251,8 +262,15 @@ jobs:
           docker compose -f ../../dockerfiles/docker-compose.dev.postgres.yml up -d
           docker compose -f ../../dockerfiles/docker-compose.dev.postgres.yml logs -t -f --no-color &> docker-compose-logs.txt &
 
+      - name: Install LLVM and CLang
+        run: |
+         sudo apt-get update
path: .github/workflows/ci.yaml
user: login: ASuciuX
body: Updated
created_at: 2025-05-19T18:10:43Z
author_association: CONTRIBUTOR
path: test-script.sh
user: login: rafaelcr
body: > Cargo aliases don’t natively support running shell scripts directly

Ohh hmm I didn't know this. I think we might as well just leave it as a shell script (without the alias and the new component) and adding a note to the README on how to run tests. Sorry about the change!
created_at: 2025-05-19T20:44:10Z
author_association: COLLABORATOR
path: test-script.sh
user: login: ASuciuX
body: No worries, I've removed that and add explanations to the README on how to run tests with this shell, and, also without it.
created_at: 2025-05-19T21:01:23Z
author_association: CONTRIBUTOR
diff_hunk: @@ -3,6 +3,7 @@ bitcoin-indexer-install = "install --path components/cli --locked --force"
 bitcoin-indexer-fmt = "fmt -- --config group_imports=StdExternalCrate,imports_granularity=Crate"
 bitcoin-indexer-clippy = "clippy --tests --all-features --all-targets -- -A clippy::too_many_arguments -A clippy::needless_return -A clippy::type_complexity -A clippy::ptr_arg"
 bitcoin-indexer-clippy-cli = "clippy --tests --all-features --all-targets --message-format=short -- -A clippy::too_many_arguments -A clippy::needless_return -A clippy::type_complexity -A clippy::ptr_arg -D warnings"
+bitcoin-indexer-test = "run --package cargo-tasks"
path: .cargo/config.toml
user: login: rafaelcr
body: Remove
created_at: 2025-05-19T21:02:26Z
author_association: COLLABORATOR
diff_hunk: @@ -3,6 +3,7 @@ bitcoin-indexer-install = "install --path components/cli --locked --force"
 bitcoin-indexer-fmt = "fmt -- --config group_imports=StdExternalCrate,imports_granularity=Crate"
 bitcoin-indexer-clippy = "clippy --tests --all-features --all-targets -- -A clippy::too_many_arguments -A clippy::needless_return -A clippy::type_complexity -A clippy::ptr_arg"
 bitcoin-indexer-clippy-cli = "clippy --tests --all-features --all-targets --message-format=short -- -A clippy::too_many_arguments -A clippy::needless_return -A clippy::type_complexity -A clippy::ptr_arg -D warnings"
+bitcoin-indexer-test = "run --package cargo-tasks"
path: .cargo/config.toml
user: login: ASuciuX
body: Thanks for pointing it, done.
created_at: 2025-05-19T21:04:10Z
author_association: CONTRIBUTOR

PR comments:
user: login: github-actions[bot]
created_at: 2025-05-09T11:04:45Z
body: Vercel deployment URL: https://bitcoin-indexer-3wmo7exb0-hirosystems.vercel.app :rocket:
<!-- thollander/actions-comment-pull-request "vercel" -->
author_association: NONE
user: login: codecov[bot]
created_at: 2025-05-09T11:04:48Z
body: ## [Codecov](https://app.codecov.io/gh/hirosystems/bitcoin-indexer/pull/531?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=hirosystems) Report
All modified and coverable lines are covered by tests :white_check_mark:

:loudspeaker: Thoughts on this report? [Let us know!](https://github.com/codecov/feedback/issues/255)
author_association: NONE