Repository: tokio-rs/prost (language: rust)

Issue:
Support simpler generate of large dependency trees
This has outstanding PRs to implement in https://github.com/danburkert/prost/pull/123 and https://github.com/danburkert/prost/pull/333, but since these seem to have not gotten feedback from maintainers for a while I thought it made sense to open an issue to track the use case, requirements, current status, etc.

Basically, today its quite challenging to handle complex dependencies. For my example use case, I am trying to build a project that has 500 `.proto`s with many inter-dependencies. Currently, there does not seem to be an easy way to do this.

It would be great if either, or both, of (1) some improvements to the codegen to simplify this use case, and (2) documentation on how to handle these large dependency trees.


Discussion / hints:

Fix patch:
diff --git a/Cargo.toml b/Cargo.toml
index d176bea22..5441d03e1 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -24,7 +24,9 @@ members = [
   "tests",
   "tests-2015",
   "tests-no-std",
+  "tests/single-include",
 ]
+
 exclude = [
   # The fuzz crate can't be compiled or tested without the 'cargo fuzz' command,
   # so exclude it from normal builds.
diff --git a/prost-build/Cargo.toml b/prost-build/Cargo.toml
index 201849e1a..c34b0226a 100644
--- a/prost-build/Cargo.toml
+++ b/prost-build/Cargo.toml
@@ -28,3 +28,4 @@ which = { version = "4", default-features = false }
 
 [dev-dependencies]
 env_logger = { version = "0.8", default-features = false }
+
diff --git a/prost-build/src/lib.rs b/prost-build/src/lib.rs
index 8bb0cb429..3a6074ee4 100644
--- a/prost-build/src/lib.rs
+++ b/prost-build/src/lib.rs
@@ -122,7 +122,7 @@ use std::env;
 use std::ffi::{OsStr, OsString};
 use std::fmt;
 use std::fs;
-use std::io::{Error, ErrorKind, Result};
+use std::io::{Error, ErrorKind, Result, Write};
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
@@ -233,6 +233,7 @@ pub struct Config {
     default_package_filename: String,
     protoc_args: Vec<OsString>,
     disable_comments: PathMap<()>,
+    include_file: Option<PathBuf>,
 }
 
 impl Config {
@@ -690,6 +691,39 @@ impl Config {
         self
     }
 
+    /// Configures the optional module filename for easy inclusion of all generated Rust files
+    ///
+    /// If set, generates a file (inside the `OUT_DIR` or `out_dir()` as appropriate) which contains
+    /// a set of `pub mod XXX` statements combining to load all Rust files generated.  This can allow
+    /// for a shortcut where multiple related proto files have been compiled together resulting in
+    /// a semi-complex set of includes.
+    ///
+    /// Turning a need for:
+    ///
+    /// ```rust,no_run,ignore
+    /// pub mod Foo {
+    ///     pub mod Bar {
+    ///         include!(concat!(env!("OUT_DIR"), "/foo.bar.rs"));
+    ///     }
+    ///     pub mod Baz {
+    ///         include!(concat!(env!("OUT_DIR"), "/foo.baz.rs"));
+    ///     }
+    /// }
+    /// ```
+    ///
+    /// Into the simpler:
+    ///
+    /// ```rust,no_run,ignore
+    /// include!(concat!(env!("OUT_DIR"), "/_includes.rs"));
+    /// ```
+    pub fn include_file<P>(&mut self, path: P) -> &mut Self
+    where
+        P: Into<PathBuf>,
+    {
+        self.include_file = Some(path.into());
+        self
+    }
+
     /// Compile `.proto` files into Rust files during a Cargo build with additional code generator
     /// configuration options.
     ///
@@ -713,12 +747,16 @@ impl Config {
         protos: &[impl AsRef<Path>],
         includes: &[impl AsRef<Path>],
     ) -> Result<()> {
+        let mut target_is_env = false;
         let target: PathBuf = self.out_dir.clone().map(Ok).unwrap_or_else(|| {
             env::var_os("OUT_DIR")
                 .ok_or_else(|| {
                     Error::new(ErrorKind::Other, "OUT_DIR environment variable is not set")
                 })
-                .map(Into::into)
+                .map(|val| {
+                    target_is_env = true;
+                    Into::into(val)
+                })
         })?;
 
         // TODO: This should probably emit 'rerun-if-changed=PATH' directives for cargo, however
@@ -781,7 +819,7 @@ impl Config {
         })?;
 
         let modules = self.generate(file_descriptor_set.file)?;
-        for (module, content) in modules {
+        for (module, content) in &modules {
             let mut filename = if module.is_empty() {
                 self.default_package_filename.clone()
             } else {
@@ -805,9 +843,83 @@ impl Config {
             }
         }
 
+        if let Some(ref include_file) = self.include_file {
+            trace!("Writing include file: {:?}", target.join(include_file));
+            let mut file = fs::File::create(target.join(include_file))?;
+            self.write_includes(
+                modules.keys().collect(),
+                &mut file,
+                0,
+                if target_is_env { None } else { Some(&target) },
+            )?;
+            file.flush()?;
+        }
+
         Ok(())
     }
 
+    fn write_includes(
+        &self,
+        mut entries: Vec<&Module>,
+        outfile: &mut fs::File,
+        depth: usize,
+        basepath: Option<&PathBuf>,
+    ) -> Result<usize> {
+        let mut written = 0;
+        while !entries.is_empty() {
+            let modident = &entries[0][depth];
+            let matching: Vec<&Module> = entries
+                .iter()
+                .filter(|&v| &v[depth] == modident)
+                .copied()
+                .collect();
+            {
+                // Will NLL sort this mess out?
+                let _temp = entries
+                    .drain(..)
+                    .filter(|&v| &v[depth] != modident)
+                    .collect();
+                entries = _temp;
+            }
+            self.write_line(outfile, depth, &format!("pub mod {} {{", modident))?;
+            let subwritten = self.write_includes(
+                matching
+                    .iter()
+                    .filter(|v| v.len() > depth + 1)
+                    .copied()
+                    .collect(),
+                outfile,
+                depth + 1,
+                basepath,
+            )?;
+            written += subwritten;
+            if subwritten != matching.len() {
+                let modname = matching[0][..=depth].join(".");
+                if let Some(buf) = basepath {
+                    self.write_line(
+                        outfile,
+                        depth + 1,
+                        &format!("include!(\"{}/{}.rs\");", buf.display(), modname),
+                    )?;
+                } else {
+                    self.write_line(
+                        outfile,
+                        depth + 1,
+                        &format!("include!(concat!(env!(\"OUT_DIR\"), \"/{}.rs\"));", modname),
+                    )?;
+                }
+                written += 1;
+            }
+
+            self.write_line(outfile, depth, "}")?;
+        }
+        Ok(written)
+    }
+
+    fn write_line(&self, outfile: &mut fs::File, depth: usize, line: &str) -> Result<()> {
+        outfile.write_all(format!("{}{}\n", ("    ").to_owned().repeat(depth), line).as_bytes())
+    }
+
     fn generate(&mut self, files: Vec<FileDescriptorProto>) -> Result<HashMap<Module, String>> {
         let mut modules = HashMap::new();
         let mut packages = HashMap::new();
@@ -864,6 +976,7 @@ impl default::Default for Config {
             default_package_filename: "_".to_string(),
             protoc_args: Vec::new(),
             disable_comments: PathMap::default(),
+            include_file: None,
         }
     }
 }
@@ -1037,6 +1150,7 @@ mod tests {
 
         Config::new()
             .service_generator(Box::new(gen))
+            .include_file("_protos.rs")
             .compile_protos(&["src/hello.proto", "src/goodbye.proto"], &["src"])
             .unwrap();
 


Test patch:
diff --git a/tests/single-include/Cargo.toml b/tests/single-include/Cargo.toml
new file mode 100644
index 000000000..1d8eebebd
--- /dev/null
+++ b/tests/single-include/Cargo.toml
@@ -0,0 +1,13 @@
+[package]
+name = "single_include"
+version = "0.1.0"
+authors = ["Cameron Dart <cdart2@illinois.edu>"]
+edition = "2018"
+publish = false
+license = "MIT"
+
+[dependencies]
+prost = { path = "../../../prost" }
+
+[build-dependencies]
+prost-build = { path = "../../prost-build" }
diff --git a/tests/single-include/build.rs b/tests/single-include/build.rs
new file mode 100644
index 000000000..5ab6a9316
--- /dev/null
+++ b/tests/single-include/build.rs
@@ -0,0 +1,8 @@
+use prost_build::Config;
+
+fn main() {
+    Config::new()
+        .include_file("lib.rs")
+        .compile_protos(&["protos/search.proto"], &["protos"])
+        .unwrap();
+}
diff --git a/tests/single-include/protos/search.proto b/tests/single-include/protos/search.proto
new file mode 100644
index 000000000..1d3c29e3d
--- /dev/null
+++ b/tests/single-include/protos/search.proto
@@ -0,0 +1,9 @@
+syntax = "proto3";
+
+package search;
+
+message SearchRequest {
+  string query = 1;
+  int32 page_number = 2;
+  int32 result_per_page = 3;
+}
diff --git a/tests/single-include/src/lib.rs b/tests/single-include/src/lib.rs
new file mode 100644
index 000000000..f7072f67a
--- /dev/null
+++ b/tests/single-include/src/lib.rs
@@ -0,0 +1,1 @@
+include!(concat!(env!("OUT_DIR"), "/lib.rs"));
diff --git a/tests/single-include/src/main.rs b/tests/single-include/src/main.rs
new file mode 100644
index 000000000..c36342382
--- /dev/null
+++ b/tests/single-include/src/main.rs
@@ -0,0 +1,3 @@
+fn main() {
+    let _search_req = single_include::search::SearchRequest::default();
+}