Change:
When building from git source, the build system makes a deep clone of git submodules, even those, like or , that had been disabled with configure options. ( and respectively) Not downloading a couple hundred megabytes of data spread over tens of thousands of small files is especially important on flash devices (and slow connections).
At the very least it should probably do a shallower clone of those repos. As best I can tell, you should be able to just pass to just fine.
Yep, that would be a great start. However the gist of this report is to completely forego downloading the source trees that are never going to be used.
I've just tested your patch, having done a shallow clone first and is not cloned at all but still is. Still, a huge improvement; thanks! I also paid attention to the amount of data downloaded by fully cloning other modules, like , , etc. and it seems bootstrapping from a source snapshot is still the fastest option, requiring a 50% smaller download.

Discussion:
use std::collections::HashMap; use std::env; use std::fs::{self, File}; <del> use std::path::{PathBuf, Path}; </del> <ins> use std::path::{Component, PathBuf, Path}; </ins> use std::process::Command; use build_helper::{run_silent, output};

/// This will detect if any submodules are out of date an run the necessary /// commands to sync them all with upstream. fn update_submodules(&self) { <ins> struct Submodule<'a> { path: &'a Path, state: State, } enum State { // The submodule may have staged/unstaged changes MaybeDirty, // Or could be initialized but never updated NotInitialized, // The submodule, itself, has extra commits but those changes haven't been commited to // the (outer) git repository OutOfSync, } </ins> if !self.config.submodules { return } if fs::metadata(self.src.join(".git")).is_err() { return } <ins> let git = || { let mut cmd = Command::new("git"); cmd.current_dir(&self.src); return cmd }; </ins> let git_submodule = || { let mut cmd = Command::new("git"); cmd.current_dir(&self.src).arg("submodule");

//        of detecting whether we need to run all the submodule commands //        below. let out = output(git_submodule().arg("status")); <del> if !out.lines().any(|l| l.starts_with("+") || l.starts_with("-")) { return </del> <ins> let mut submodules = vec![]; for line in out.lines() { // NOTE `git submodule status` output looks like this: // // -5066b7dcab7e700844b0e2ba71b8af9dc627a59b src/liblibc // +b37ef24aa82d2be3a3cc0fe89bf82292f4ca181c src/compiler-rt (remotes/origin/..) //  e058ca661692a8d01f8cf9d35939dfe3105ce968 src/jemalloc (3.6.0-533-ge058ca6) // // The first character can be '-', '+' or ' ' and denotes the `State` of the submodule // Right next to this character is the SHA-1 of the submodule HEAD // And after that comes the path to the submodule let path = Path::new(line[1..].split(' ').skip(1).next().unwrap()); let state = if line.starts_with('-') { State::NotInitialized } else if line.starts_with('*') { State::OutOfSync } else if line.starts_with(' ') { State::MaybeDirty } else { panic!("unexpected git submodule state: {:?}", line.chars().next()); }; submodules.push(Submodule { path: path, state: state }) </ins> } self.run(git_submodule().arg("sync")); <del> self.run(git_submodule().arg("init")); self.run(git_submodule().arg("update")); self.run(git_submodule().arg("update").arg("--recursive")); self.run(git_submodule().arg("status").arg("--recursive")); self.run(git_submodule().arg("foreach").arg("--recursive") .arg("git").arg("clean").arg("-fdx")); self.run(git_submodule().arg("foreach").arg("--recursive") .arg("git").arg("checkout").arg(".")); </del> <ins> for submodule in submodules { // If using llvm-root then don't touch the llvm submodule. if submodule.path.components().any(|c| c == Component::Normal("llvm".as_ref())) && self.config.target_config.get(&self.config.build) .and_then(|c| c.llvm_config.as_ref()).is_some() { continue } if submodule.path.components().any(|c| c == Component::Normal("jemalloc".as_ref())) && !self.config.use_jemalloc { continue } match submodule.state { State::MaybeDirty => { // drop staged changes self.run(git().arg("-C").arg(submodule.path).args(&["reset", "--hard"])); // drops unstaged changes self.run(git().arg("-C").arg(submodule.path).args(&["clean", "-fdx"])); }, State::NotInitialized => { self.run(git_submodule().arg("init").arg(submodule.path)); self.run(git_submodule().arg("update").arg(submodule.path)); }, State::OutOfSync => { // drops submodule commits that weren't reported to the (outer) git repository self.run(git_submodule().arg("update").arg(submodule.path)); self.run(git().arg("-C").arg(submodule.path).args(&["reset", "--hard"])); self.run(git().arg("-C").arg(submodule.path).args(&["clean", "-fdx"])); }, } } </ins> } /// Clear out `dir` if `input` is newer.