Repository: informalsystems/tendermint-rs (language: rust)

Issue:
Can not generate tendermint-proto from source with buf dependencies
### Description

Originally reported as https://github.com/cometbft/cometbft/issues/565

tendermint-proto-compiler needs to be able to process buf dependencies in the `buf.lock` file in order to fetch third-party
protobuf modules used by CometBFT proto module source in releases starting from 0.37.0.
For #1193 I have scrounged by generating off the `v0.37.0-alpha.1` release and checking that the proto source has not changed between then and the eventual 0.37.x releases, but to adapt to any future changes we need to adapt to the buf way of managing protobuf dependencies.

### Definition of "done"

The tool that's used to generate code for tendermint-proto (either t-p-compiler or, if you feel ambitious, switch completely to buf.build) can deal with the buf dependencies in CometBFT releases since `v0.37.0`.


Discussion / hints:
We're dealing with a similar issue on cosmos-rust: https://github.com/cosmos/cosmos-rust/issues/318

So far we modified our `proto-build` to shell out to `buf`, however `protoc-gen-prost` looks promising: https://github.com/neoeinstein/protoc-gen-prost
I'm implementing it as a shell-out as well, to have a tendermint-proto-compiler that can work with older v0.34.x checkouts as well as v0.37.x and later revisions in the CometBFT repository.

We will switch to full-on `buf` for generation and transition to using [versioned cometbft protos](https://github.com/cometbft/cometbft/pull/495) as a source at a later time.

Fix patch:
diff --git a/.changelog/unreleased/improvements/1293-buf-proto-deps.md b/.changelog/unreleased/improvements/1293-buf-proto-deps.md
new file mode 100644
index 000000000..91c3a31c7
--- /dev/null
+++ b/.changelog/unreleased/improvements/1293-buf-proto-deps.md
@@ -0,0 +1,3 @@
+- [`tools/proto-compiler`] Parse and fetch proto dependencies as listed in the
+  `buf.lock` file
+  ([\#1293](https://github.com/informalsystems/tendermint-rs/pull/1293)).
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 9fee19a47..7342584ae 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -73,6 +73,11 @@ jobs:
           unzip /tmp/protoc.zip -d ${HOME}/.local
           echo "PROTOC=${HOME}/.local/bin/protoc" >> $GITHUB_ENV
           export PATH="${PATH}:${HOME}/.local/bin"
+      - name: Install buf
+        run: |
+          curl -sSL https://github.com/bufbuild/buf/releases/download/v1.15.1/buf-Linux-x86_64 \
+            -o /usr/local/bin/buf
+          chmod +x /usr/local/bin/buf
       - name: Regenerate proto definitions
         working-directory: ./tools/proto-compiler/
         run: cargo run
diff --git a/proto/src/prost/v0_34/tendermint.abci.rs b/proto/src/prost/v0_34/tendermint.abci.rs
index 9d20c30f1..ae59b8286 100644
--- a/proto/src/prost/v0_34/tendermint.abci.rs
+++ b/proto/src/prost/v0_34/tendermint.abci.rs
@@ -336,8 +336,8 @@ pub struct ResponseCheckTx {
     pub sender: ::prost::alloc::string::String,
     #[prost(int64, tag = "10")]
     pub priority: i64,
-    /// mempool_error is set by Tendermint.
-    /// ABCI applications creating a ResponseCheckTX should not set mempool_error.
+    /// mempool_error is set by CometBFT.
+    /// ABCI applictions creating a ResponseCheckTX should not set mempool_error.
     #[prost(string, tag = "11")]
     pub mempool_error: ::prost::alloc::string::String,
 }
@@ -438,6 +438,18 @@ pub mod response_offer_snapshot {
                 Result::RejectSender => "REJECT_SENDER",
             }
         }
+        /// Creates an enum from field names used in the ProtoBuf definition.
+        pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+            match value {
+                "UNKNOWN" => Some(Self::Unknown),
+                "ACCEPT" => Some(Self::Accept),
+                "ABORT" => Some(Self::Abort),
+                "REJECT" => Some(Self::Reject),
+                "REJECT_FORMAT" => Some(Self::RejectFormat),
+                "REJECT_SENDER" => Some(Self::RejectSender),
+                _ => None,
+            }
+        }
     }
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
@@ -501,6 +513,18 @@ pub mod response_apply_snapshot_chunk {
                 Result::RejectSnapshot => "REJECT_SNAPSHOT",
             }
         }
+        /// Creates an enum from field names used in the ProtoBuf definition.
+        pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+            match value {
+                "UNKNOWN" => Some(Self::Unknown),
+                "ACCEPT" => Some(Self::Accept),
+                "ABORT" => Some(Self::Abort),
+                "RETRY" => Some(Self::Retry),
+                "RETRY_SNAPSHOT" => Some(Self::RetrySnapshot),
+                "REJECT_SNAPSHOT" => Some(Self::RejectSnapshot),
+                _ => None,
+            }
+        }
     }
 }
 /// ConsensusParams contains all consensus-relevant parameters
@@ -661,6 +685,14 @@ impl CheckTxType {
             CheckTxType::Recheck => "RECHECK",
         }
     }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "NEW" => Some(Self::New),
+            "RECHECK" => Some(Self::Recheck),
+            _ => None,
+        }
+    }
 }
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
 #[repr(i32)]
@@ -681,4 +713,13 @@ impl EvidenceType {
             EvidenceType::LightClientAttack => "LIGHT_CLIENT_ATTACK",
         }
     }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "UNKNOWN" => Some(Self::Unknown),
+            "DUPLICATE_VOTE" => Some(Self::DuplicateVote),
+            "LIGHT_CLIENT_ATTACK" => Some(Self::LightClientAttack),
+            _ => None,
+        }
+    }
 }
diff --git a/proto/src/prost/v0_34/tendermint.crypto.rs b/proto/src/prost/v0_34/tendermint.crypto.rs
index fed646029..a6273013b 100644
--- a/proto/src/prost/v0_34/tendermint.crypto.rs
+++ b/proto/src/prost/v0_34/tendermint.crypto.rs
@@ -55,7 +55,7 @@ pub struct ProofOps {
     #[prost(message, repeated, tag = "1")]
     pub ops: ::prost::alloc::vec::Vec<ProofOp>,
 }
-/// PublicKey defines the keys available for use with Tendermint Validators
+/// PublicKey defines the keys available for use with Validators
 #[derive(::serde::Deserialize, ::serde::Serialize)]
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
diff --git a/proto/src/prost/v0_34/tendermint.privval.rs b/proto/src/prost/v0_34/tendermint.privval.rs
index 1fae1cbee..8485c3816 100644
--- a/proto/src/prost/v0_34/tendermint.privval.rs
+++ b/proto/src/prost/v0_34/tendermint.privval.rs
@@ -120,4 +120,16 @@ impl Errors {
             Errors::WriteTimeout => "ERRORS_WRITE_TIMEOUT",
         }
     }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "ERRORS_UNKNOWN" => Some(Self::Unknown),
+            "ERRORS_UNEXPECTED_RESPONSE" => Some(Self::UnexpectedResponse),
+            "ERRORS_NO_CONNECTION" => Some(Self::NoConnection),
+            "ERRORS_CONNECTION_TIMEOUT" => Some(Self::ConnectionTimeout),
+            "ERRORS_READ_TIMEOUT" => Some(Self::ReadTimeout),
+            "ERRORS_WRITE_TIMEOUT" => Some(Self::WriteTimeout),
+            _ => None,
+        }
+    }
 }
diff --git a/proto/src/prost/v0_34/tendermint.types.rs b/proto/src/prost/v0_34/tendermint.types.rs
index 66c081611..229dec5d9 100644
--- a/proto/src/prost/v0_34/tendermint.types.rs
+++ b/proto/src/prost/v0_34/tendermint.types.rs
@@ -67,7 +67,7 @@ pub struct BlockId {
     #[serde(alias = "parts")]
     pub part_set_header: ::core::option::Option<PartSetHeader>,
 }
-/// Header defines the structure of a Tendermint block header.
+/// Header defines the structure of a block header.
 #[derive(::serde::Deserialize, ::serde::Serialize)]
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
@@ -290,6 +290,16 @@ impl BlockIdFlag {
             BlockIdFlag::Nil => "BLOCK_ID_FLAG_NIL",
         }
     }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "BLOCK_ID_FLAG_UNKNOWN" => Some(Self::Unknown),
+            "BLOCK_ID_FLAG_ABSENT" => Some(Self::Absent),
+            "BLOCK_ID_FLAG_COMMIT" => Some(Self::Commit),
+            "BLOCK_ID_FLAG_NIL" => Some(Self::Nil),
+            _ => None,
+        }
+    }
 }
 /// SignedMsgType is a type of signed message in the consensus.
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
@@ -315,6 +325,16 @@ impl SignedMsgType {
             SignedMsgType::Proposal => "SIGNED_MSG_TYPE_PROPOSAL",
         }
     }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "SIGNED_MSG_TYPE_UNKNOWN" => Some(Self::Unknown),
+            "SIGNED_MSG_TYPE_PREVOTE" => Some(Self::Prevote),
+            "SIGNED_MSG_TYPE_PRECOMMIT" => Some(Self::Precommit),
+            "SIGNED_MSG_TYPE_PROPOSAL" => Some(Self::Proposal),
+            _ => None,
+        }
+    }
 }
 /// ConsensusParams contains consensus critical parameters that determine the
 /// validity of blocks.
diff --git a/proto/src/prost/v0_37/tendermint.abci.rs b/proto/src/prost/v0_37/tendermint.abci.rs
index acccf14ad..9a200e06f 100644
--- a/proto/src/prost/v0_37/tendermint.abci.rs
+++ b/proto/src/prost/v0_37/tendermint.abci.rs
@@ -367,7 +367,7 @@ pub struct ResponseCheckTx {
     pub sender: ::prost::alloc::string::String,
     #[prost(int64, tag = "10")]
     pub priority: i64,
-    /// mempool_error is set by Tendermint.
+    /// mempool_error is set by CometBFT.
     /// ABCI applictions creating a ResponseCheckTX should not set mempool_error.
     #[prost(string, tag = "11")]
     pub mempool_error: ::prost::alloc::string::String,
@@ -469,6 +469,18 @@ pub mod response_offer_snapshot {
                 Result::RejectSender => "REJECT_SENDER",
             }
         }
+        /// Creates an enum from field names used in the ProtoBuf definition.
+        pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+            match value {
+                "UNKNOWN" => Some(Self::Unknown),
+                "ACCEPT" => Some(Self::Accept),
+                "ABORT" => Some(Self::Abort),
+                "REJECT" => Some(Self::Reject),
+                "REJECT_FORMAT" => Some(Self::RejectFormat),
+                "REJECT_SENDER" => Some(Self::RejectSender),
+                _ => None,
+            }
+        }
     }
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
@@ -532,6 +544,18 @@ pub mod response_apply_snapshot_chunk {
                 Result::RejectSnapshot => "REJECT_SNAPSHOT",
             }
         }
+        /// Creates an enum from field names used in the ProtoBuf definition.
+        pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+            match value {
+                "UNKNOWN" => Some(Self::Unknown),
+                "ACCEPT" => Some(Self::Accept),
+                "ABORT" => Some(Self::Abort),
+                "RETRY" => Some(Self::Retry),
+                "RETRY_SNAPSHOT" => Some(Self::RetrySnapshot),
+                "REJECT_SNAPSHOT" => Some(Self::RejectSnapshot),
+                _ => None,
+            }
+        }
     }
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
@@ -577,6 +601,15 @@ pub mod response_process_proposal {
                 ProposalStatus::Reject => "REJECT",
             }
         }
+        /// Creates an enum from field names used in the ProtoBuf definition.
+        pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+            match value {
+                "UNKNOWN" => Some(Self::Unknown),
+                "ACCEPT" => Some(Self::Accept),
+                "REJECT" => Some(Self::Reject),
+                _ => None,
+            }
+        }
     }
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
@@ -734,6 +767,14 @@ impl CheckTxType {
             CheckTxType::Recheck => "RECHECK",
         }
     }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "NEW" => Some(Self::New),
+            "RECHECK" => Some(Self::Recheck),
+            _ => None,
+        }
+    }
 }
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
 #[repr(i32)]
@@ -754,4 +795,13 @@ impl MisbehaviorType {
             MisbehaviorType::LightClientAttack => "LIGHT_CLIENT_ATTACK",
         }
     }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "UNKNOWN" => Some(Self::Unknown),
+            "DUPLICATE_VOTE" => Some(Self::DuplicateVote),
+            "LIGHT_CLIENT_ATTACK" => Some(Self::LightClientAttack),
+            _ => None,
+        }
+    }
 }
diff --git a/proto/src/prost/v0_37/tendermint.crypto.rs b/proto/src/prost/v0_37/tendermint.crypto.rs
index ac0853255..a6273013b 100644
--- a/proto/src/prost/v0_37/tendermint.crypto.rs
+++ b/proto/src/prost/v0_37/tendermint.crypto.rs
@@ -36,7 +36,7 @@ pub struct DominoOp {
     pub output: ::prost::alloc::string::String,
 }
 /// ProofOp defines an operation used for calculating Merkle root
-/// The data could be arbitrary format, providing necessary data
+/// The data could be arbitrary format, providing nessecary data
 /// for example neighbouring node hash
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
@@ -55,7 +55,7 @@ pub struct ProofOps {
     #[prost(message, repeated, tag = "1")]
     pub ops: ::prost::alloc::vec::Vec<ProofOp>,
 }
-/// PublicKey defines the keys available for use with Tendermint Validators
+/// PublicKey defines the keys available for use with Validators
 #[derive(::serde::Deserialize, ::serde::Serialize)]
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
diff --git a/proto/src/prost/v0_37/tendermint.privval.rs b/proto/src/prost/v0_37/tendermint.privval.rs
index 1fae1cbee..8485c3816 100644
--- a/proto/src/prost/v0_37/tendermint.privval.rs
+++ b/proto/src/prost/v0_37/tendermint.privval.rs
@@ -120,4 +120,16 @@ impl Errors {
             Errors::WriteTimeout => "ERRORS_WRITE_TIMEOUT",
         }
     }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "ERRORS_UNKNOWN" => Some(Self::Unknown),
+            "ERRORS_UNEXPECTED_RESPONSE" => Some(Self::UnexpectedResponse),
+            "ERRORS_NO_CONNECTION" => Some(Self::NoConnection),
+            "ERRORS_CONNECTION_TIMEOUT" => Some(Self::ConnectionTimeout),
+            "ERRORS_READ_TIMEOUT" => Some(Self::ReadTimeout),
+            "ERRORS_WRITE_TIMEOUT" => Some(Self::WriteTimeout),
+            _ => None,
+        }
+    }
 }
diff --git a/proto/src/prost/v0_37/tendermint.types.rs b/proto/src/prost/v0_37/tendermint.types.rs
index 3b93c0932..3ccc14ee6 100644
--- a/proto/src/prost/v0_37/tendermint.types.rs
+++ b/proto/src/prost/v0_37/tendermint.types.rs
@@ -67,7 +67,7 @@ pub struct BlockId {
     #[serde(alias = "parts")]
     pub part_set_header: ::core::option::Option<PartSetHeader>,
 }
-/// Header defines the structure of a Tendermint block header.
+/// Header defines the structure of a block header.
 #[derive(::serde::Deserialize, ::serde::Serialize)]
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
@@ -290,6 +290,16 @@ impl BlockIdFlag {
             BlockIdFlag::Nil => "BLOCK_ID_FLAG_NIL",
         }
     }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "BLOCK_ID_FLAG_UNKNOWN" => Some(Self::Unknown),
+            "BLOCK_ID_FLAG_ABSENT" => Some(Self::Absent),
+            "BLOCK_ID_FLAG_COMMIT" => Some(Self::Commit),
+            "BLOCK_ID_FLAG_NIL" => Some(Self::Nil),
+            _ => None,
+        }
+    }
 }
 /// SignedMsgType is a type of signed message in the consensus.
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
@@ -315,6 +325,16 @@ impl SignedMsgType {
             SignedMsgType::Proposal => "SIGNED_MSG_TYPE_PROPOSAL",
         }
     }
+    /// Creates an enum from field names used in the ProtoBuf definition.
+    pub fn from_str_name(value: &str) -> ::core::option::Option<Self> {
+        match value {
+            "SIGNED_MSG_TYPE_UNKNOWN" => Some(Self::Unknown),
+            "SIGNED_MSG_TYPE_PREVOTE" => Some(Self::Prevote),
+            "SIGNED_MSG_TYPE_PRECOMMIT" => Some(Self::Precommit),
+            "SIGNED_MSG_TYPE_PROPOSAL" => Some(Self::Proposal),
+            _ => None,
+        }
+    }
 }
 /// ConsensusParams contains consensus critical parameters that determine the
 /// validity of blocks.
diff --git a/proto/src/tendermint/v0_34.rs b/proto/src/tendermint/v0_34.rs
index 2457ed879..561b12534 100644
--- a/proto/src/tendermint/v0_34.rs
+++ b/proto/src/tendermint/v0_34.rs
@@ -61,6 +61,6 @@ pub mod version {
 }
 
 pub mod meta {
-    pub const REPOSITORY: &str = "https://github.com/tendermint/tendermint";
-    pub const COMMITISH: &str = "v0.34.24";
+    pub const REPOSITORY: &str = "https://github.com/cometbft/cometbft";
+    pub const COMMITISH: &str = "v0.34.27";
 }
diff --git a/proto/src/tendermint/v0_37.rs b/proto/src/tendermint/v0_37.rs
index 292f1a337..6f987b3e6 100644
--- a/proto/src/tendermint/v0_37.rs
+++ b/proto/src/tendermint/v0_37.rs
@@ -61,6 +61,6 @@ pub mod version {
 }
 
 pub mod meta {
-    pub const REPOSITORY: &str = "https://github.com/tendermint/tendermint";
-    pub const COMMITISH: &str = "v0.37.0-alpha.1";
+    pub const REPOSITORY: &str = "https://github.com/cometbft/cometbft";
+    pub const COMMITISH: &str = "v0.37.0";
 }
diff --git a/tools/proto-compiler/Cargo.toml b/tools/proto-compiler/Cargo.toml
index c844577f1..c26d62e5a 100644
--- a/tools/proto-compiler/Cargo.toml
+++ b/tools/proto-compiler/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "tendermint-proto-compiler"
-version = "0.2.0"
+version = "0.3.0"
 authors = ["Informal Systems <hello@informal.systems>"]
 edition = "2021"
 publish = false
@@ -9,5 +9,8 @@ publish = false
 walkdir         = { version = "2.3" }
 prost-build     = { version = "0.11.4" }
 git2            = { version = "0.16" }
-tempfile        = { version = "3.2.0" }
+tempfile        = { version = "3.5.0" }
 subtle-encoding = { version = "0.5" }
+serde           = { version = "1.0", features = ["derive"] }
+serde_yaml      = { version = "0.9" }
+thiserror       = { version = "1.0" }
diff --git a/tools/proto-compiler/src/buf_build.rs b/tools/proto-compiler/src/buf_build.rs
new file mode 100644
index 000000000..d7f4e4917
--- /dev/null
+++ b/tools/proto-compiler/src/buf_build.rs
@@ -0,0 +1,74 @@
+//! Support for [buf](https://buf.build) build tools.
+
+use serde::Deserialize;
+
+use std::fs::File;
+use std::io::{self, BufReader};
+use std::path::Path;
+use std::process::{Command, ExitStatus};
+
+/// Errors that can occur when working with buf files.
+#[derive(Debug, thiserror::Error)]
+pub enum FileError {
+    #[error(transparent)]
+    Io(#[from] io::Error),
+    #[error(transparent)]
+    Yaml(#[from] serde_yaml::Error),
+    #[error("unsupported lock file version {}", .encountered)]
+    UnsupportedVersion { encountered: String },
+}
+
+/// Serialization object for the `buf.lock` file.
+#[derive(Debug, Deserialize)]
+struct LockFile {
+    pub version: String,
+    pub deps: Vec<LockedDep>,
+}
+
+/// Serialization object for an entry under the `deps` key in a `buf.lock` file.
+#[derive(Debug, Deserialize)]
+pub struct LockedDep {
+    pub remote: String,
+    pub owner: String,
+    pub repository: String,
+    pub commit: String,
+}
+
+pub fn read_locked_deps(lockfile_path: impl AsRef<Path>) -> Result<Vec<LockedDep>, FileError> {
+    let file = File::open(lockfile_path)?;
+    let lock_config: LockFile = serde_yaml::from_reader(BufReader::new(file))?;
+    if lock_config.version != "v1" {
+        return Err(FileError::UnsupportedVersion {
+            encountered: lock_config.version,
+        });
+    }
+    Ok(lock_config.deps)
+}
+
+/// Errors that can occur from invocation of the buf CLI tool.
+#[derive(Debug, thiserror::Error)]
+pub enum ToolError {
+    #[error("failed to execute buf: {}", .0)]
+    Spawn(#[from] io::Error),
+    #[error("buf exited with {}", .0)]
+    Failure(ExitStatus),
+}
+
+pub fn export_dep_module(dep: &LockedDep, out_dir: &Path) -> Result<(), ToolError> {
+    let module_ref = format!(
+        "{}/{}/{}:{}",
+        dep.remote, dep.owner, dep.repository, dep.commit
+    );
+    println!("Exporting `{}` into `{}`", module_ref, out_dir.display());
+    let status = Command::new("buf")
+        .arg("export")
+        .arg(module_ref)
+        .arg("-o")
+        .arg(out_dir)
+        .status()?;
+    if status.success() {
+        Ok(())
+    } else {
+        Err(ToolError::Failure(status))
+    }
+}
diff --git a/tools/proto-compiler/src/constants.rs b/tools/proto-compiler/src/constants.rs
index 556f9d8b4..0a8413844 100644
--- a/tools/proto-compiler/src/constants.rs
+++ b/tools/proto-compiler/src/constants.rs
@@ -1,10 +1,9 @@
 /// Tendermint protobuf version
 
-/// Tendermint repository URL.
-pub const TENDERMINT_REPO: &str = "https://github.com/tendermint/tendermint";
-
 /// Information on a Tendermint snapshot to generate prost structures from.
 pub struct TendermintVersion {
+    /// Repository URL.
+    pub repo: &'static str,
     /// Identifier to use in module names.
     pub ident: &'static str,
     /// A commitish reference in the tendermint git repository, for example:
@@ -18,12 +17,14 @@ pub struct TendermintVersion {
 /// All Tendermint versions to generate code for
 pub const TENDERMINT_VERSIONS: &[TendermintVersion] = &[
     TendermintVersion {
+        repo: "https://github.com/cometbft/cometbft",
         ident: "v0_34",
-        commitish: "v0.34.24",
+        commitish: "v0.34.27",
     },
     TendermintVersion {
+        repo: "https://github.com/cometbft/cometbft",
         ident: "v0_37",
-        commitish: "v0.37.0-alpha.1",
+        commitish: "v0.37.0",
     },
 ];
 
diff --git a/tools/proto-compiler/src/functions.rs b/tools/proto-compiler/src/functions.rs
index e38edb2ca..32873b704 100644
--- a/tools/proto-compiler/src/functions.rs
+++ b/tools/proto-compiler/src/functions.rs
@@ -190,23 +190,21 @@ pub fn copy_files(src_dir: &Path, target_dir: &Path) {
     }
 }
 
-/// Walk through the list of directories and gather all *.proto files
-pub fn find_proto_files(proto_paths: Vec<PathBuf>) -> Vec<PathBuf> {
+/// Walk through the directory recursively and gather all *.proto files
+pub fn find_proto_files(proto_path: &Path) -> Vec<PathBuf> {
     let mut protos: Vec<PathBuf> = vec![];
-    for proto_path in &proto_paths {
-        protos.append(
-            &mut WalkDir::new(proto_path)
-                .into_iter()
-                .filter_map(|e| e.ok())
-                .filter(|e| {
-                    e.file_type().is_file()
-                        && e.path().extension().is_some()
-                        && e.path().extension().unwrap() == "proto"
-                })
-                .map(|e| e.into_path())
-                .collect(),
-        );
-    }
+    protos.append(
+        &mut WalkDir::new(proto_path)
+            .into_iter()
+            .filter_map(|e| e.ok())
+            .filter(|e| {
+                e.file_type().is_file()
+                    && e.path().extension().is_some()
+                    && e.path().extension().unwrap() == "proto"
+            })
+            .map(|e| e.into_path())
+            .collect(),
+    );
     protos
 }
 
@@ -266,9 +264,9 @@ pub fn generate_tendermint_mod(prost_dir: &Path, version: &TendermintVersion, ta
         "{}\npub mod meta {{\n{}pub const REPOSITORY: &str = \"{}\";\n{}pub const COMMITISH: &str = \"{}\";\n}}\n",
         content,
         tab,
-        crate::constants::TENDERMINT_REPO,
+        version.repo,
         tab,
-        &version.commitish,
+        version.commitish,
     );
 
     let tendermint_mod_target = target_dir.join(format!("{}.rs", version.ident));
diff --git a/tools/proto-compiler/src/main.rs b/tools/proto-compiler/src/main.rs
index 0ff9c2eb5..90d9f3df9 100644
--- a/tools/proto-compiler/src/main.rs
+++ b/tools/proto-compiler/src/main.rs
@@ -1,20 +1,22 @@
 use std::{
     env::var,
+    fs,
     path::{Path, PathBuf},
     process,
 };
 
 use tempfile::tempdir;
 
+mod buf_build;
+use crate::buf_build::{export_dep_module, read_locked_deps};
+
 mod functions;
 use functions::{
     copy_files, find_proto_files, generate_tendermint_lib, generate_tendermint_mod, get_commitish,
 };
 
 mod constants;
-use constants::{
-    CUSTOM_FIELD_ATTRIBUTES, CUSTOM_TYPE_ATTRIBUTES, TENDERMINT_REPO, TENDERMINT_VERSIONS,
-};
+use constants::{CUSTOM_FIELD_ATTRIBUTES, CUSTOM_TYPE_ATTRIBUTES, TENDERMINT_VERSIONS};
 
 fn main() {
     let root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
@@ -30,18 +32,48 @@ fn main() {
 
     for version in TENDERMINT_VERSIONS {
         println!(
-            "[info] => Fetching {TENDERMINT_REPO} at {} into {tendermint_dir:?}",
-            &version.commitish,
+            "[info] => Fetching {} at {} into {tendermint_dir:?}",
+            version.repo, version.commitish,
         );
-        get_commitish(&tendermint_dir, TENDERMINT_REPO, &version.commitish); // This panics if it fails.
+        get_commitish(&tendermint_dir, &version.repo, &version.commitish); // This panics if it fails.
+
+        let proto_path = tendermint_dir.join("proto");
+
+        let mut proto_includes_paths = vec![tendermint_dir.join("proto")];
+
+        let buf_lock_path = proto_path.join("buf.lock");
+        let _temp_dirs = if fs::metadata(&buf_lock_path).is_ok() {
+            // A new-style proto module with buf dependencies.
+            // Fetch the dependencies and add them to include paths.
+            match read_locked_deps(&buf_lock_path) {
+                Ok(deps) => deps
+                    .iter()
+                    .map(|dep| {
+                        let mod_dir = tempdir().unwrap();
+                        if let Err(e) = export_dep_module(dep, mod_dir.path()) {
+                            eprintln!(
+                                "Failed to export module {}/{}/{}: {}",
+                                dep.remote, dep.owner, dep.repository, e,
+                            );
+                            process::exit(1);
+                        }
+                        proto_includes_paths.push(mod_dir.path().to_owned());
+                        mod_dir
+                    })
+                    .collect::<Vec<_>>(),
+                Err(e) => {
+                    eprintln!("Failed to read {}: {}", buf_lock_path.display(), e);
+                    process::exit(1);
+                },
+            }
+        } else {
+            // Old school, assume the dependency protos are bundled in the tree.
+            proto_includes_paths.push(tendermint_dir.join("third_party").join("proto"));
+            vec![]
+        };
 
-        let proto_paths = vec![tendermint_dir.join("proto")];
-        let proto_includes_paths = vec![
-            tendermint_dir.join("proto"),
-            tendermint_dir.join("third_party").join("proto"),
-        ];
         // List available proto files
-        let protos = find_proto_files(proto_paths);
+        let protos = find_proto_files(&proto_path);
 
         let ver_target_dir = target_dir.join("prost").join(&version.ident);
         let ver_module_dir = target_dir.join("tendermint");


Test patch: