/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use anyhow::Result;
use async_trait::async_trait;
use clap::Args;
use derived_data_service_if::types as thrift;

#[derive(Clone, Debug)]
pub struct RemoteDerivationOptions {
    pub derive_remotely: bool,
    pub address: Address,
}

#[derive(Clone, Debug)]
pub enum Address {
    SmcTier(String),
    HostPort(String),
    Empty,
}

/// Command line arguments for controlling remote derivation
#[derive(Args, Debug)]
pub struct RemoteDerivationArgs {
    /// Derive data remotely using the derived data service
    #[clap(long)]
    pub derive_remotely: bool,

    /// Specify SMC tier for the derived data service
    #[clap(long, value_name = "SMC", group = "Address")]
    pub derive_remotely_tier: Option<String>,

    /// Specify Host:Port pair to connect to derived data service
    #[clap(long, value_name = "HOST:PORT", group = "Address")]
    pub derive_remotely_hostport: Option<String>,
}

impl From<RemoteDerivationArgs> for RemoteDerivationOptions {
    fn from(args: RemoteDerivationArgs) -> Self {
        let address = match (args.derive_remotely_tier, args.derive_remotely_hostport) {
            (Some(tier), _) => Address::SmcTier(tier),
            (_, Some(host_port)) => Address::HostPort(host_port),
            (_, _) => Address::Empty,
        };
        RemoteDerivationOptions {
            derive_remotely: args.derive_remotely,
            address,
        }
    }
}

#[async_trait]
pub trait DerivationClient: Send + Sync {
    async fn derive_remotely(
        &self,
        request: &thrift::DeriveRequest,
    ) -> Result<thrift::DeriveResponse>;

    async fn poll(&self, request: &thrift::DeriveRequest) -> Result<thrift::DeriveResponse>;
}


// @generated by autocargo
use std::env;
use std::fs;
use std::path::Path;

use thrift_compiler::Config;
use thrift_compiler::GenContext;

#[rustfmt::skip]
fn main() {
    // Rerun if this gets rewritten.
    println!("cargo:rerun-if-changed=thrift_build.rs");

    let out_dir = env::var_os("OUT_DIR").expect("OUT_DIR env not provided");
    let out_dir: &Path = out_dir.as_ref();
    fs::write(
        out_dir.join("cratemap"),
        "changeset_info_thrift derived_data_thrift
derived_data_service crate
fb303_core fb303_core
filenodes filenodes_if
git_types_thrift git_types_thrift
mercurial_thrift mercurial_thrift
mononoke_types_thrift mononoke_types_thrift",
    ).expect("Failed to write cratemap");

    let conf = {
        let mut conf = Config::from_env(GenContext::Lib).expect("Failed to instantiate thrift_compiler::Config");

        let path_from_manifest_to_base: &Path = "../../../../..".as_ref();
        let cargo_manifest_dir =
            env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not provided");
        let cargo_manifest_dir: &Path = cargo_manifest_dir.as_ref();
        let base_path = cargo_manifest_dir
            .join(path_from_manifest_to_base)
            .canonicalize()
            .expect("Failed to canonicalize base_path");
        // TODO: replace canonicalize() with std::path::absolute() when
        // https://github.com/rust-lang/rust/pull/91673 is available (~Rust 1.60)
        // and remove this block.
        #[cfg(windows)]
        let base_path = Path::new(
            base_path
                .as_path()
                .to_string_lossy()
                .trim_start_matches(r"\\?\"),
            )
            .to_path_buf();

        conf.base_path(base_path);

        let options = "types_crate=derived_data_service_if__types";
        if !options.is_empty() {
            conf.options(options);
        }

        let include_srcs = vec![

        ];
        conf.include_srcs(include_srcs);

        conf
    };

    conf
        .run(&[
            "derived_data_service.thrift"
        ])
        .expect("Failed while running thrift compilation");
}


// @generated by autocargo
::codegen_includer_proc_macro::include!();


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::BTreeMap;
use std::collections::HashMap;

use anyhow::Context;
use anyhow::Error;
use blobstore::Loadable;
use borrowed::borrowed;
use context::CoreContext;
use derived_data::batch::split_batch_in_linear_stacks;
use derived_data::batch::FileConflicts;
use derived_data::batch::StackItem;
use derived_data_manager::BonsaiDerivable;
use derived_data_manager::DerivationContext;
use futures::stream::FuturesOrdered;
use futures::stream::TryStreamExt;
use itertools::Itertools;
use mononoke_types::ChangesetId;
use stats::prelude::*;

use crate::derive::derive_skeleton_manifest_stack;
use crate::RootSkeletonManifestId;
use crate::SkeletonManifestId;

define_stats! {
    prefix = "mononoke.derived_data.skeleton_manifest";
    new_parallel: timeseries(Rate, Sum),
}

/// Derive a batch of skeleton manifests, potentially doing it faster than
/// deriving skeleton manifests sequentially.  The primary purpose of this is
/// to be used while backfilling skeleton manifests for a large repository.
///
/// This is the same mechanism as fsnodes, see `derive_fsnode_in_batch` for
/// more details.
pub async fn derive_skeleton_manifests_in_batch(
    ctx: &CoreContext,
    derivation_ctx: &DerivationContext,
    batch: Vec<ChangesetId>,
    gap_size: Option<usize>,
) -> Result<HashMap<ChangesetId, RootSkeletonManifestId>, Error> {
    let linear_stacks = split_batch_in_linear_stacks(
        ctx,
        derivation_ctx.blobstore(),
        batch,
        FileConflicts::ChangeDelete.into(),
    )
    .await?;
    let mut res: HashMap<ChangesetId, RootSkeletonManifestId> = HashMap::new();
    for linear_stack in linear_stacks {
        // Fetch the parent skeleton manifests, either from a previous
        // iteration of this loop (which will have stored the mapping in
        // `res`, or from the main mapping, where they should already be
        // derived.
        let parent_skeleton_manifests = linear_stack
            .parents
            .into_iter()
            .map(|p| {
                borrowed!(res);
                async move {
                    anyhow::Result::<_>::Ok(
                        match res.get(&p) {
                            Some(sk_mf_id) => sk_mf_id.clone(),
                            None => derivation_ctx.fetch_dependency(ctx, p).await?,
                        }
                        .into_skeleton_manifest_id(),
                    )
                }
            })
            .collect::<FuturesOrdered<_>>()
            .try_collect::<Vec<_>>()
            .await?;
        STATS::new_parallel.add_value(1);
        new_batch_derivation(
            ctx,
            derivation_ctx,
            parent_skeleton_manifests,
            gap_size,
            linear_stack.stack_items,
            &mut res,
        )
        .await?;
    }

    Ok(res)
}

pub async fn new_batch_derivation(
    ctx: &CoreContext,
    derivation_ctx: &DerivationContext,
    parent_skeleton_manifests: Vec<SkeletonManifestId>,
    gap_size: Option<usize>,
    file_changes: Vec<StackItem>,
    already_derived: &mut HashMap<ChangesetId, RootSkeletonManifestId>,
) -> Result<(), Error> {
    if parent_skeleton_manifests.len() > 1 {
        // we can't derive stack for a merge commit,
        // so let's derive it without batching
        for item in file_changes {
            let bonsai = item.cs_id.load(ctx, derivation_ctx.blobstore()).await?;
            let parents = derivation_ctx
                .fetch_unknown_parents(ctx, Some(already_derived), &bonsai)
                .await?;
            let derived =
                RootSkeletonManifestId::derive_single(ctx, derivation_ctx, bonsai, parents).await?;
            already_derived.insert(item.cs_id, derived);
        }
    } else {
        let first = file_changes.first().map(|item| item.cs_id);
        let last = file_changes.last().map(|item| item.cs_id);

        let file_changes: Vec<_> = match gap_size {
            Some(gap_size) => file_changes
                .into_iter()
                .chunks(gap_size)
                .into_iter()
                .filter_map(|chunk| {
                    // We are deriving with gaps - that means we are deriving
                    // just for the top commit out of `gap_size` linear stack
                    // of commits. To do that we need to combine all file changes
                    // for a given linear stack together
                    let mut combined_file_changes = BTreeMap::new();
                    let mut last_cs_id = None;
                    for item in chunk {
                        combined_file_changes.extend(item.per_commit_file_changes);
                        last_cs_id = Some(item.cs_id);
                    }
                    Some((last_cs_id?, combined_file_changes))
                })
                .collect(),
            None => file_changes
                .into_iter()
                .map(|item| (item.cs_id, item.per_commit_file_changes))
                .collect(),
        };

        let derived = derive_skeleton_manifest_stack(
            ctx,
            derivation_ctx,
            file_changes,
            parent_skeleton_manifests.get(0).copied(),
        )
        .await
        .with_context(|| format!("failed deriving stack of {:?} to {:?}", first, last,))?;

        already_derived.extend(
            derived
                .into_iter()
                .map(|(csid, mf_id)| (csid, RootSkeletonManifestId(mf_id))),
        );
    }

    Ok(())
}

#[cfg(test)]
mod test {
    use bonsai_hg_mapping::BonsaiHgMapping;
    use bookmarks::Bookmarks;
    use changeset_fetcher::ChangesetFetcher;
    use changeset_fetcher::ChangesetFetcherArc;
    use changesets::Changesets;
    use derived_data_manager::BatchDeriveOptions;
    use fbinit::FacebookInit;
    use filestore::FilestoreConfig;
    use fixtures::Linear;
    use fixtures::TestRepoFixture;
    use futures::compat::Stream01CompatExt;
    use repo_blobstore::RepoBlobstore;
    use repo_derived_data::RepoDerivedData;
    use repo_derived_data::RepoDerivedDataRef;
    use revset::AncestorsNodeStream;
    use test_repo_factory::TestRepoFactory;
    use tests_utils::bookmark;
    use tests_utils::drawdag::create_from_dag;
    use tests_utils::resolve_cs_id;

    use super::*;

    #[facet::container]
    #[derive(Clone)]
    struct TestRepo {
        #[facet]
        bonsai_hg_mapping: dyn BonsaiHgMapping,
        #[facet]
        bookmarks: dyn Bookmarks,
        #[facet]
        changesets: dyn Changesets,
        #[facet]
        changeset_fetcher: dyn ChangesetFetcher,
        #[facet]
        repo_derived_data: RepoDerivedData,
        #[facet]
        filestore_config: FilestoreConfig,
        #[facet]
        repo_blobstore: RepoBlobstore,
    }

    #[fbinit::test]
    async fn batch_derive(fb: FacebookInit) -> Result<(), Error> {
        let ctx = CoreContext::test_mock(fb);

        let new_batch = {
            let repo = Linear::getrepo(fb).await;
            let master_cs_id = resolve_cs_id(&ctx, &repo, "master").await?;

            let mut cs_ids =
                AncestorsNodeStream::new(ctx.clone(), &repo.changeset_fetcher_arc(), master_cs_id)
                    .compat()
                    .try_collect::<Vec<_>>()
                    .await?;
            cs_ids.reverse();
            let manager = repo.repo_derived_data().manager();

            manager
                .derive_exactly_batch::<RootSkeletonManifestId>(
                    &ctx,
                    cs_ids,
                    BatchDeriveOptions::Parallel { gap_size: None },
                    None,
                )
                .await?;
            manager
                .fetch_derived::<RootSkeletonManifestId>(&ctx, master_cs_id, None)
                .await?
                .unwrap()
                .into_skeleton_manifest_id()
        };

        let sequential = {
            let repo = Linear::getrepo(fb).await;
            let master_cs_id = resolve_cs_id(&ctx, &repo, "master").await?;
            repo.repo_derived_data()
                .manager()
                .derive::<RootSkeletonManifestId>(&ctx, master_cs_id, None)
                .await?
                .into_skeleton_manifest_id()
        };

        assert_eq!(new_batch, sequential);
        Ok(())
    }

    #[fbinit::test]
    async fn batch_derive_with_merge(fb: FacebookInit) -> Result<(), Error> {
        let ctx = CoreContext::test_mock(fb);
        let new_batch = {
            let repo = repo_with_merge(&ctx).await?;
            let master_cs_id = resolve_cs_id(&ctx, &repo, "master").await?;

            let mut cs_ids =
                AncestorsNodeStream::new(ctx.clone(), &repo.changeset_fetcher_arc(), master_cs_id)
                    .compat()
                    .try_collect::<Vec<_>>()
                    .await?;
            cs_ids.reverse();

            let manager = repo.repo_derived_data().manager();

            manager
                .derive_exactly_batch::<RootSkeletonManifestId>(
                    &ctx,
                    cs_ids,
                    BatchDeriveOptions::Parallel { gap_size: None },
                    None,
                )
                .await?;

            manager
                .fetch_derived::<RootSkeletonManifestId>(&ctx, master_cs_id, None)
                .await?
                .unwrap()
                .into_skeleton_manifest_id()
        };

        let sequential = {
            let repo = repo_with_merge(&ctx).await?;
            let master_cs_id = resolve_cs_id(&ctx, &repo, "master").await?;
            repo.repo_derived_data()
                .manager()
                .derive::<RootSkeletonManifestId>(&ctx, master_cs_id, None)
                .await?
                .into_skeleton_manifest_id()
        };

        assert_eq!(new_batch, sequential);
        Ok(())
    }

    async fn repo_with_merge(ctx: &CoreContext) -> Result<TestRepo, Error> {
        let repo: TestRepo = TestRepoFactory::new(ctx.fb)?.build().await?;

        let commit_map = create_from_dag(
            ctx,
            &repo,
            r##"
            A-M
             /
            B
            "##,
        )
        .await?;

        let m = commit_map.get(&"M".to_string()).unwrap();
        bookmark(ctx, &repo, "master").set_to(*m).await?;

        Ok(repo)
    }

    #[fbinit::test]
    async fn batch_derive_with_gaps(fb: FacebookInit) -> Result<(), Error> {
        let ctx = CoreContext::test_mock(fb);

        let repo = Linear::getrepo(fb).await;
        let master_cs_id = resolve_cs_id(&ctx, &repo, "master").await?;
        let ddm = repo.repo_derived_data().manager();
        ddm.derive::<RootSkeletonManifestId>(&ctx, master_cs_id, None)
            .await?
            .into_skeleton_manifest_id();

        for gap_size in 1..12 {
            let new_batch_with_gaps = derive_new_batch(fb, &ctx, gap_size).await?;
            for (cs_id, derived_with_gaps) in new_batch_with_gaps {
                let derived_sequential = ddm
                    .fetch_derived::<RootSkeletonManifestId>(&ctx, cs_id, None)
                    .await?;
                assert_eq!(derived_with_gaps, derived_sequential.unwrap());
            }
        }

        Ok(())
    }

    async fn derive_new_batch(
        fb: FacebookInit,
        ctx: &CoreContext,
        gap_size: usize,
    ) -> Result<HashMap<ChangesetId, RootSkeletonManifestId>, Error> {
        let repo = Linear::getrepo(fb).await;
        let master_cs_id = resolve_cs_id(ctx, &repo, "master").await?;

        let mut cs_ids =
            AncestorsNodeStream::new(ctx.clone(), &repo.changeset_fetcher_arc(), master_cs_id)
                .compat()
                .try_collect::<Vec<_>>()
                .await?;
        cs_ids.reverse();
        let manager = repo.repo_derived_data().manager();

        manager
            .derive_exactly_batch::<RootSkeletonManifestId>(
                ctx,
                cs_ids.clone(),
                BatchDeriveOptions::Parallel {
                    gap_size: Some(gap_size),
                },
                None,
            )
            .await?;

        let derived = cs_ids
            .chunks(gap_size)
            .filter_map(|chunk| chunk.last().cloned())
            .collect();

        let derived = manager
            .fetch_derived_batch::<RootSkeletonManifestId>(ctx, derived, None)
            .await?;
        for cs_id in cs_ids {
            if !derived.contains_key(&cs_id) {
                assert_eq!(
                    manager
                        .fetch_derived::<RootSkeletonManifestId>(ctx, cs_id, None)
                        .await?,
                    None
                );
            }
        }

        Ok(derived)
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::BTreeMap;
use std::collections::HashMap;
use std::collections::HashSet;
use std::sync::Arc;

use anyhow::format_err;
use anyhow::Context;
use anyhow::Error;
use anyhow::Result;
use blobstore::Blobstore;
use blobstore::Loadable;
use borrowed::borrowed;
use cloned::cloned;
use context::CoreContext;
use derived_data_manager::DerivationContext;
use futures::channel::mpsc;
use futures::future::BoxFuture;
use futures::future::FutureExt;
use futures::stream::FuturesOrdered;
use futures::stream::FuturesUnordered;
use futures::stream::TryStreamExt;
use manifest::derive_manifest_with_io_sender;
use manifest::derive_manifests_for_simple_stack_of_commits;
use manifest::Entry;
use manifest::ManifestChanges;
use manifest::TreeInfo;
use mononoke_types::skeleton_manifest::SkeletonManifest;
use mononoke_types::skeleton_manifest::SkeletonManifestDirectory;
use mononoke_types::skeleton_manifest::SkeletonManifestEntry;
use mononoke_types::skeleton_manifest::SkeletonManifestSummary;
use mononoke_types::BlobstoreKey;
use mononoke_types::BlobstoreValue;
use mononoke_types::ChangesetId;
use mononoke_types::ContentId;
use mononoke_types::FileType;
use mononoke_types::MPath;
use mononoke_types::MPathElement;
use mononoke_types::SkeletonManifestId;
use sorted_vector_map::SortedVectorMap;

use crate::SkeletonManifestDerivationError;

pub(crate) async fn derive_skeleton_manifest_stack(
    ctx: &CoreContext,
    derivation_ctx: &DerivationContext,
    file_changes: Vec<(ChangesetId, BTreeMap<MPath, Option<(ContentId, FileType)>>)>,
    parent: Option<SkeletonManifestId>,
) -> Result<HashMap<ChangesetId, SkeletonManifestId>, Error> {
    let blobstore = derivation_ctx.blobstore();

    let manifest_changes = file_changes
        .into_iter()
        .map(|(cs_id, file_changes)| ManifestChanges {
            cs_id,
            changes: file_changes.into_iter().collect(),
        })
        .collect::<Vec<_>>();

    let res = derive_manifests_for_simple_stack_of_commits(
        ctx.clone(),
        blobstore.clone(),
        parent,
        manifest_changes,
        {
            cloned!(blobstore, ctx);
            move |tree_info, _cs_id| {
                cloned!(blobstore, ctx);
                async move { create_skeleton_manifest(&ctx, &blobstore, None, tree_info).await }
            }
        },
        |_leaf_info, _cs_id| async { Ok((None, ())) },
    )
    .await?;

    Ok(res.into_iter().collect())
}

/// Derives skeleton manifests for bonsai_changeset `cs_id` given parent
/// skeleton manifests. Note that `derive_manifest()` does a lot of the heavy
/// lifting for us, and this crate has to provide only functions to create a
/// single fsnode, and check that the leaf entries are valid during merges.
pub(crate) async fn derive_skeleton_manifest(
    ctx: &CoreContext,
    derivation_ctx: &DerivationContext,
    parents: Vec<SkeletonManifestId>,
    changes: Vec<(MPath, Option<(ContentId, FileType)>)>,
) -> Result<SkeletonManifestId> {
    let blobstore = derivation_ctx.blobstore();

    // We must box and store the derivation future, otherwise lifetime
    // analysis is unable to see that the blobstore lasts long enough.
    let derive_fut =
        derive_manifest_with_io_sender(
            ctx.clone(),
            blobstore.clone(),
            parents.clone(),
            changes,
            {
                cloned!(blobstore, ctx);
                move |tree_info, sender| {
                    cloned!(blobstore, ctx);
                    async move {
                        create_skeleton_manifest(&ctx, &blobstore, Some(sender), tree_info).await
                    }
                }
            },
            |_leaf_info, _sender| async { Ok((None, ())) },
        )
        .boxed();
    let maybe_tree_id = derive_fut.await?;

    match maybe_tree_id {
        Some(tree_id) => Ok(tree_id),
        None => {
            // All files have been deleted, generate empty fsnode
            let tree_info = TreeInfo {
                path: None,
                parents,
                subentries: Default::default(),
            };
            let (_, tree_id) = create_skeleton_manifest(ctx, blobstore, None, tree_info).await?;
            Ok(tree_id)
        }
    }
}

/// Collect all the subentries for a new skeleton manifest, re-using entries
/// from the parent skeleton manifests to avoid fetching too much.
async fn collect_skeleton_subentries(
    ctx: &CoreContext,
    blobstore: &Arc<dyn Blobstore>,
    parents: &[SkeletonManifestId],
    subentries: BTreeMap<
        MPathElement,
        (
            Option<Option<SkeletonManifestSummary>>,
            Entry<SkeletonManifestId, ()>,
        ),
    >,
) -> Result<Vec<(MPathElement, SkeletonManifestEntry)>> {
    // Load the parent skeleton manifests and collect their entries into a cache
    let mut dir_cache = HashMap::new();
    let mut parent_skeletons = parents
        .iter()
        .map({
            move |skeleton_id| async move {
                skeleton_id
                    .load(ctx, blobstore)
                    .await
                    .context(SkeletonManifestDerivationError::MissingParent(*skeleton_id))
            }
        })
        .collect::<FuturesUnordered<_>>();
    while let Some(parent_skeleton) = parent_skeletons.try_next().await? {
        for (_elem, entry) in parent_skeleton.list() {
            if let SkeletonManifestEntry::Directory(dir) = entry {
                dir_cache.entry(*dir.id()).or_insert_with(|| dir.clone());
            }
        }
    }

    // Find (from the traversal or the cache) or fetch (from the blobstore)
    // the `SkeletonManifestEntry` for each of the subentries.
    borrowed!(dir_cache);
    subentries
        .into_iter()
        .map(move |(elem, (summary, entry))| {
            async move {
                match entry {
                    Entry::Tree(skeleton_id) => {
                        if let Some(Some(summary)) = summary {
                            // The subdirectory was just created. Use the
                            // summary we just calculated.
                            Ok((
                                elem.clone(),
                                SkeletonManifestEntry::Directory(SkeletonManifestDirectory::new(
                                    skeleton_id,
                                    summary,
                                )),
                            ))
                        } else if let Some(entry) = dir_cache.get(&skeleton_id) {
                            // The subdirectory was already in this
                            // directory. Use the cached entry.
                            Ok((
                                elem.clone(),
                                SkeletonManifestEntry::Directory(entry.clone()),
                            ))
                        } else {
                            // Some other directory is being used. Fetch its
                            // summary from the blobstore.
                            let skeleton_manifest =
                                skeleton_id.load(ctx, blobstore).await.with_context({
                                    || {
                                        SkeletonManifestDerivationError::MissingSubentry(
                                            String::from_utf8_lossy(elem.as_ref()).to_string(),
                                            skeleton_id,
                                        )
                                    }
                                })?;

                            let entry =
                                SkeletonManifestEntry::Directory(SkeletonManifestDirectory::new(
                                    skeleton_id,
                                    skeleton_manifest.summary().clone(),
                                ));
                            Ok((elem.clone(), entry))
                        }
                    }
                    Entry::Leaf(()) => Ok((elem.clone(), SkeletonManifestEntry::File)),
                }
            }
        })
        .collect::<FuturesOrdered<_>>()
        .try_collect()
        .await
}

/// Create a new skeleton manifest for the tree described by `tree_info`.
async fn create_skeleton_manifest(
    ctx: &CoreContext,
    blobstore: &Arc<dyn Blobstore>,
    sender: Option<mpsc::UnboundedSender<BoxFuture<'static, Result<(), Error>>>>,
    tree_info: TreeInfo<SkeletonManifestId, (), Option<SkeletonManifestSummary>>,
) -> Result<(Option<SkeletonManifestSummary>, SkeletonManifestId)> {
    let entries =
        collect_skeleton_subentries(ctx, blobstore, &tree_info.parents, tree_info.subentries)
            .await?;

    // Build a summary of the entries and store it as the new skeleton
    // manifest.
    let entries: SortedVectorMap<_, _> = entries.into_iter().collect();
    let mut summary = SkeletonManifestSummary {
        child_files_count: 0,
        descendant_files_count: 0,
        child_dirs_count: 0,
        descendant_dirs_count: 0,
        max_path_len: 0,
        max_path_wchar_len: 0,
        child_case_conflicts: false,
        descendant_case_conflicts: false,
        child_non_utf8_filenames: false,
        descendant_non_utf8_filenames: false,
        child_invalid_windows_filenames: false,
        descendant_invalid_windows_filenames: false,
    };
    let mut lower_entries = HashSet::new();
    for (elem, entry) in entries.iter() {
        let elem_len = elem.len() as u32;
        let elem_wchar_len = elem.wchar_len() as u32;
        match entry {
            SkeletonManifestEntry::Directory(dir) => {
                let sub = dir.summary();
                summary.descendant_files_count += sub.descendant_files_count;
                summary.child_dirs_count += 1;
                summary.descendant_dirs_count += sub.descendant_dirs_count + 1;
                summary.max_path_len = summary.max_path_len.max(elem_len + 1 + sub.max_path_len);
                summary.max_path_wchar_len = summary
                    .max_path_wchar_len
                    .max(elem_wchar_len + 1 + sub.max_path_wchar_len);
                summary.descendant_case_conflicts |=
                    sub.child_case_conflicts | sub.descendant_case_conflicts;
                summary.descendant_non_utf8_filenames |=
                    sub.child_non_utf8_filenames | sub.descendant_non_utf8_filenames;
                summary.descendant_invalid_windows_filenames |=
                    sub.child_invalid_windows_filenames | sub.descendant_invalid_windows_filenames;
            }
            SkeletonManifestEntry::File => {
                summary.descendant_files_count += 1;
                summary.child_files_count += 1;
                summary.max_path_len = summary.max_path_len.max(elem_len);
                summary.max_path_wchar_len = summary.max_path_wchar_len.max(elem_wchar_len);
            }
        }
        if !summary.child_case_conflicts {
            if let Some(lower) = elem.to_lowercase_utf8() {
                if !lower_entries.insert(lower) {
                    // A case conflict has been found.
                    summary.child_case_conflicts = true;
                }
            }
        }
        if !summary.child_non_utf8_filenames && !elem.is_utf8() {
            // A non-utf8 filename has been found.
            summary.child_non_utf8_filenames = true;
        }
        if !summary.child_invalid_windows_filenames && !elem.is_valid_windows_filename() {
            // An invalid windows filename has been found.
            summary.child_invalid_windows_filenames = true;
        }
    }
    let skeleton_manifest = SkeletonManifest::new(entries, summary.clone());
    let blob = skeleton_manifest.into_blob();
    let skeleton_manifest_id = *blob.id();

    // Skeleton manifests are frequently unchanged.  If any of our parents
    // have the same ID, there is no need to store this blob.
    if !tree_info.parents.contains(&skeleton_manifest_id) {
        let key = skeleton_manifest_id.blobstore_key();
        let f = {
            cloned!(ctx, blobstore);
            async move { blobstore.put(&ctx, key, blob.into()).await }
        };

        match sender {
            Some(sender) => sender
                .unbounded_send(f.boxed())
                .map_err(|err| format_err!("failed to send skeleton manifest future {}", err))?,
            None => f.await?,
        };
    }
    Ok((Some(summary), skeleton_manifest_id))
}

#[cfg(test)]
mod test {
    use anyhow::anyhow;
    use bonsai_hg_mapping::BonsaiHgMapping;
    use bookmarks::Bookmarks;
    use changeset_fetcher::ChangesetFetcher;
    use changesets::Changesets;
    use fbinit::FacebookInit;
    use filestore::FilestoreConfig;
    use mononoke_types::ChangesetId;
    use pretty_assertions::assert_eq;
    use repo_blobstore::RepoBlobstore;
    use repo_blobstore::RepoBlobstoreRef;
    use repo_derived_data::RepoDerivedData;
    use repo_derived_data::RepoDerivedDataRef;
    use tests_utils::drawdag::changes;
    use tests_utils::drawdag::create_from_dag_with_changes;
    use tests_utils::CreateCommitContext;

    use super::*;
    use crate::mapping::get_file_changes;

    #[facet::container]
    struct TestRepo {
        #[facet]
        bonsai_hg_mapping: dyn BonsaiHgMapping,
        #[facet]
        bookmarks: dyn Bookmarks,
        #[facet]
        changesets: dyn Changesets,
        #[facet]
        changeset_fetcher: dyn ChangesetFetcher,
        #[facet]
        repo_derived_data: RepoDerivedData,
        #[facet]
        repo_blobstore: RepoBlobstore,
        #[facet]
        filestore_config: FilestoreConfig,
    }

    const B_FILES: &[&str] = &[
        "dir1/subdir1/subsubdir1/file1",
        "dir1/subdir1/subsubdir2/file2",
        "dir1/subdir1/subsubdir3/file3",
        "dir1/subdir2/subsubdir1/file1",
        "dir1/subdir2/subsubdir2/file1",
        "dir2/subdir1/subsubdir1/file1",
    ];

    const C_FILES: &[&str] = &[
        "dir1/subdir1/subsubdir1/FILE1",
        "dir1/subdir1/SUBSUBDIR2/file2",
        "dir1/subdir1/SUBSUBDIR3/FILE3",
        "dir1/subdir2/SUBSUBDIR1/file1",
    ];

    const D_FILES: &[&str] = &[
        "dir1/subdir1/subsubdir2/file2",
        "dir1/subdir1/SUBSUBDIR3/FILE3",
    ];

    const E_FILES: &[&str] = &[
        "dir1/subdir1/subsubdir1/FILE1",
        "dir1/subdir2/subsubdir1/file1",
    ];

    const G_FILES: &[&str] = &[
        "windows/dirnamewith\u{00E9}/filenamewith\u{9854}",
        "nonwin/dir1/com7",
        "nonwin/dir2/PRN.txt",
        "nonwin/dir3/Nul.JPG",
        "nonwin/dir4/enddot.",
        "nonwin/dir5/endspace ",
        "nonwin/dir6/ctrl\tchars.txt",
    ];

    const H_FILES: &[&str] = &["dir1/subdir1/SUBSUBDIR3/File3"];

    const J_FILES: &[&str] = &["dir3/FILE1", "dir3/file1"];

    const K_FILES: &[&str] = &["dir3/file2"];

    const L_FILES: &[&str] = &["dir2/subdir1/subsubdir1/FILE1"];

    fn add_files<'a>(
        mut c: CreateCommitContext<'a, TestRepo>,
        files: &[&str],
    ) -> CreateCommitContext<'a, TestRepo> {
        for &file in files {
            c = c.add_file(file, file);
        }
        c
    }

    fn delete_files<'a>(
        mut c: CreateCommitContext<'a, TestRepo>,
        files: &[&str],
    ) -> CreateCommitContext<'a, TestRepo> {
        for &file in files {
            c = c.delete_file(file);
        }
        c
    }

    async fn init_repo(ctx: &CoreContext) -> Result<(TestRepo, BTreeMap<String, ChangesetId>)> {
        let repo: TestRepo = test_repo_factory::build_empty(ctx.fb).await.unwrap();
        let changesets = create_from_dag_with_changes(
            ctx,
            &repo,
            r##"
                    J-K
                     /
                A-B-C-D-E-F-G
                     \
                      H
                       \
                      L-M
            "##,
            changes! {
                "B" => |c| add_files(c, B_FILES),
                "C" => |c| add_files(c, C_FILES),
                "D" => |c| delete_files(c, D_FILES),
                "E" => |c| delete_files(c, E_FILES),
                "F" => |c| c.add_file(b"cp1252/euro\x80".as_ref(), "euro"),
                "G" => |c| add_files(c, G_FILES),
                "H" => |c| add_files(c, H_FILES),
                "J" => |c| add_files(c, J_FILES),
                "K" => |c| add_files(c, K_FILES),
                "L" => |c| add_files(c, L_FILES),
            },
        )
        .await?;
        Ok((repo, changesets))
    }

    fn skeleton_dir<'a>(
        skeleton: &'a SkeletonManifest,
        elem: &[u8],
    ) -> Result<&'a SkeletonManifestDirectory> {
        match skeleton.lookup(&MPathElement::new(elem.to_vec())?) {
            Some(SkeletonManifestEntry::Directory(dir)) => Ok(dir),
            entry => Err(anyhow!("Unexpected skeleton manifest entry: {:?}", entry)),
        }
    }

    #[fbinit::test]
    async fn test_skeleton_manifests(fb: FacebookInit) -> Result<()> {
        let ctx = CoreContext::test_mock(fb);
        let (repo, changesets) = init_repo(&ctx).await?;
        let derivation_ctx = repo.repo_derived_data().manager().derivation_context(None);

        let a_bcs = changesets["A"].load(&ctx, repo.repo_blobstore()).await?;
        let a_skeleton_id =
            derive_skeleton_manifest(&ctx, &derivation_ctx, vec![], get_file_changes(&a_bcs))
                .await?;
        let a_skeleton = a_skeleton_id.load(&ctx, repo.repo_blobstore()).await?;
        assert_eq!(
            a_skeleton.lookup(&MPathElement::new(b"A".to_vec())?),
            Some(&SkeletonManifestEntry::File)
        );
        assert_eq!(
            a_skeleton.summary(),
            &SkeletonManifestSummary {
                child_files_count: 1,
                descendant_files_count: 1,
                max_path_len: 1,
                max_path_wchar_len: 1,
                ..Default::default()
            }
        );

        // Changeset B introduces some subdirectories
        let b_bcs = changesets["B"].load(&ctx, repo.repo_blobstore()).await?;
        let b_skeleton_id = derive_skeleton_manifest(
            &ctx,
            &derivation_ctx,
            vec![a_skeleton_id],
            get_file_changes(&b_bcs),
        )
        .await?;
        let b_skeleton = b_skeleton_id.load(&ctx, repo.repo_blobstore()).await?;
        assert_eq!(
            b_skeleton.summary(),
            &SkeletonManifestSummary {
                child_files_count: 2,
                descendant_files_count: 8,
                child_dirs_count: 2,
                descendant_dirs_count: 11,
                max_path_len: 29,
                max_path_wchar_len: 29,
                ..Default::default()
            }
        );
        assert_eq!(
            skeleton_dir(&b_skeleton, b"dir1")?.summary(),
            &SkeletonManifestSummary {
                child_files_count: 0,
                descendant_files_count: 5,
                child_dirs_count: 2,
                descendant_dirs_count: 7,
                max_path_len: 24,
                max_path_wchar_len: 24,
                ..Default::default()
            }
        );
        assert_eq!(
            b_skeleton
                .first_case_conflict(&ctx, repo.repo_blobstore())
                .await?,
            None,
        );

        // Changeset C introduces some case conflicts
        let c_bcs = changesets["C"].load(&ctx, repo.repo_blobstore()).await?;
        let c_skeleton_id = derive_skeleton_manifest(
            &ctx,
            &derivation_ctx,
            vec![b_skeleton_id],
            get_file_changes(&c_bcs),
        )
        .await?;
        let c_skeleton = c_skeleton_id.load(&ctx, repo.repo_blobstore()).await?;
        assert_eq!(
            c_skeleton.summary(),
            &SkeletonManifestSummary {
                child_files_count: 3,
                descendant_files_count: 13,
                child_dirs_count: 2,
                descendant_dirs_count: 14,
                max_path_len: 29,
                max_path_wchar_len: 29,
                descendant_case_conflicts: true,
                ..Default::default()
            }
        );
        assert_eq!(
            c_skeleton
                .first_case_conflict(&ctx, repo.repo_blobstore())
                .await?,
            Some((
                MPath::new(b"dir1/subdir1/SUBSUBDIR2")?,
                MPath::new(b"dir1/subdir1/subsubdir2")?
            ))
        );
        assert_eq!(
            c_skeleton
                .clone()
                .first_new_case_conflict(&ctx, repo.repo_blobstore(), vec![b_skeleton])
                .await?,
            Some((
                MPath::new(b"dir1/subdir1/SUBSUBDIR2")?,
                MPath::new(b"dir1/subdir1/subsubdir2")?
            ))
        );

        let c_sk_dir1 = skeleton_dir(&c_skeleton, b"dir1")?
            .id()
            .load(&ctx, repo.repo_blobstore())
            .await?;
        assert!(!c_sk_dir1.summary().child_case_conflicts);
        assert!(c_sk_dir1.summary().descendant_case_conflicts);

        let c_sk_subdir1 = skeleton_dir(&c_sk_dir1, b"subdir1")?
            .id()
            .load(&ctx, repo.repo_blobstore())
            .await?;
        assert!(c_sk_subdir1.summary().child_case_conflicts);
        assert!(c_sk_subdir1.summary().descendant_case_conflicts);

        let c_sk_subsubdir1 = skeleton_dir(&c_sk_subdir1, b"subsubdir1")?;
        assert!(c_sk_subsubdir1.summary().child_case_conflicts);
        assert!(!c_sk_subsubdir1.summary().descendant_case_conflicts);

        let c_sk_subsubdir2 = skeleton_dir(&c_sk_subdir1, b"subsubdir2")?;
        assert!(!c_sk_subsubdir2.summary().child_case_conflicts);
        assert!(!c_sk_subsubdir2.summary().descendant_case_conflicts);

        let c_sk_subdir2 = skeleton_dir(&c_sk_dir1, b"subdir2")?;
        assert!(c_sk_subdir2.summary().child_case_conflicts);
        assert!(!c_sk_subdir2.summary().descendant_case_conflicts);

        let c_sk_dir2 = skeleton_dir(&c_skeleton, b"dir2")?;
        assert!(!c_sk_dir2.summary().child_case_conflicts);
        assert!(!c_sk_dir2.summary().descendant_case_conflicts);

        // Changeset D removes some of the conflicts
        let d_bcs = changesets["D"].load(&ctx, repo.repo_blobstore()).await?;
        let d_skeleton_id = derive_skeleton_manifest(
            &ctx,
            &derivation_ctx,
            vec![c_skeleton_id],
            get_file_changes(&d_bcs),
        )
        .await?;
        let d_skeleton = d_skeleton_id.load(&ctx, repo.repo_blobstore()).await?;
        assert!(!d_skeleton.summary().child_case_conflicts);
        assert!(d_skeleton.summary().descendant_case_conflicts);
        assert_eq!(
            d_skeleton
                .first_case_conflict(&ctx, repo.repo_blobstore())
                .await?,
            Some((
                MPath::new(b"dir1/subdir1/subsubdir1/FILE1")?,
                MPath::new(b"dir1/subdir1/subsubdir1/file1")?
            ))
        );
        assert_eq!(
            d_skeleton
                .clone()
                .first_new_case_conflict(&ctx, repo.repo_blobstore(), vec![c_skeleton.clone()])
                .await?,
            None,
        );

        let d_sk_dir1 = skeleton_dir(&d_skeleton, b"dir1")?
            .id()
            .load(&ctx, repo.repo_blobstore())
            .await?;
        assert!(!d_sk_dir1.summary().child_case_conflicts);
        assert!(d_sk_dir1.summary().descendant_case_conflicts);

        let d_sk_subdir1 = skeleton_dir(&d_sk_dir1, b"subdir1")?
            .id()
            .load(&ctx, repo.repo_blobstore())
            .await?;
        assert!(!d_sk_subdir1.summary().child_case_conflicts);
        assert!(d_sk_subdir1.summary().descendant_case_conflicts);

        // Changeset E removes them all
        let e_bcs = changesets["E"].load(&ctx, repo.repo_blobstore()).await?;
        let e_skeleton_id = derive_skeleton_manifest(
            &ctx,
            &derivation_ctx,
            vec![d_skeleton_id],
            get_file_changes(&e_bcs),
        )
        .await?;
        let e_skeleton = e_skeleton_id.load(&ctx, repo.repo_blobstore()).await?;
        assert_eq!(
            e_skeleton.summary(),
            &SkeletonManifestSummary {
                child_files_count: 5,
                descendant_files_count: 11,
                child_dirs_count: 2,
                descendant_dirs_count: 11,
                max_path_len: 29,
                max_path_wchar_len: 29,
                ..Default::default()
            }
        );

        // Changeset F adds a non-UTF-8 filename
        let f_bcs = changesets["F"].load(&ctx, repo.repo_blobstore()).await?;
        let f_skeleton_id = derive_skeleton_manifest(
            &ctx,
            &derivation_ctx,
            vec![e_skeleton_id],
            get_file_changes(&f_bcs),
        )
        .await?;
        let f_skeleton = f_skeleton_id.load(&ctx, repo.repo_blobstore()).await?;
        assert_eq!(
            f_skeleton.summary(),
            &SkeletonManifestSummary {
                child_files_count: 6,
                descendant_files_count: 13,
                child_dirs_count: 3,
                descendant_dirs_count: 12,
                max_path_len: 29,
                max_path_wchar_len: 29,
                descendant_non_utf8_filenames: true,
                ..Default::default()
            }
        );
        assert_eq!(
            skeleton_dir(&f_skeleton, b"cp1252")?.summary(),
            &SkeletonManifestSummary {
                child_files_count: 1,
                descendant_files_count: 1,
                child_dirs_count: 0,
                descendant_dirs_count: 0,
                max_path_len: 5,
                max_path_wchar_len: 5,
                child_non_utf8_filenames: true,
                ..Default::default()
            }
        );

        // Changeset G adds some files that are not valid on Windows
        let g_bcs = changesets["G"].load(&ctx, repo.repo_blobstore()).await?;
        let g_skeleton_id = derive_skeleton_manifest(
            &ctx,
            &derivation_ctx,
            vec![f_skeleton_id],
            get_file_changes(&g_bcs),
        )
        .await?;
        let g_skeleton = g_skeleton_id.load(&ctx, repo.repo_blobstore()).await?;
        assert_eq!(
            g_skeleton.summary(),
            &SkeletonManifestSummary {
                child_files_count: 7,
                descendant_files_count: 21,
                child_dirs_count: 5,
                descendant_dirs_count: 21,
                max_path_len: 37,
                max_path_wchar_len: 34,
                descendant_non_utf8_filenames: true,
                descendant_invalid_windows_filenames: true,
                ..Default::default()
            }
        );
        assert_eq!(
            skeleton_dir(&g_skeleton, b"windows")?.summary(),
            &SkeletonManifestSummary {
                child_files_count: 0,
                descendant_files_count: 1,
                child_dirs_count: 1,
                descendant_dirs_count: 1,
                max_path_len: 29,
                max_path_wchar_len: 26,
                ..Default::default()
            }
        );
        let g_sk_nonwin = skeleton_dir(&g_skeleton, b"nonwin")?
            .id()
            .load(&ctx, repo.repo_blobstore())
            .await?;
        for dir in ["dir1", "dir2", "dir3", "dir4", "dir5", "dir6"] {
            assert_eq!(
                skeleton_dir(&g_sk_nonwin, dir.as_bytes())?
                    .summary()
                    .child_invalid_windows_filenames,
                true,
                "Missing child_invalid_windows_filenames for {}",
                dir
            );
        }

        // Changeset H introduces a new case conflict on top of the ones
        // already in changeset C.
        let h_bcs = changesets["H"].load(&ctx, repo.repo_blobstore()).await?;
        let h_skeleton_id = derive_skeleton_manifest(
            &ctx,
            &derivation_ctx,
            vec![c_skeleton_id],
            get_file_changes(&h_bcs),
        )
        .await?;
        let h_skeleton = h_skeleton_id.load(&ctx, repo.repo_blobstore()).await?;
        assert_eq!(
            h_skeleton.summary(),
            &SkeletonManifestSummary {
                child_files_count: 4,
                descendant_files_count: 15,
                child_dirs_count: 2,
                descendant_dirs_count: 14,
                max_path_len: 29,
                max_path_wchar_len: 29,
                descendant_case_conflicts: true,
                ..Default::default()
            }
        );
        assert_eq!(
            h_skeleton
                .first_case_conflict(&ctx, repo.repo_blobstore())
                .await?,
            Some((
                MPath::new(b"dir1/subdir1/SUBSUBDIR2")?,
                MPath::new(b"dir1/subdir1/subsubdir2")?
            ))
        );
        assert_eq!(
            h_skeleton
                .clone()
                .first_new_case_conflict(&ctx, repo.repo_blobstore(), vec![c_skeleton])
                .await?,
            Some((
                MPath::new(b"dir1/subdir1/SUBSUBDIR3/FILE3")?,
                MPath::new(b"dir1/subdir1/SUBSUBDIR3/File3")?,
            ))
        );

        // Changeset J has an internal case conflict.
        let j_bcs = changesets["J"].load(&ctx, repo.repo_blobstore()).await?;
        let j_skeleton_id =
            derive_skeleton_manifest(&ctx, &derivation_ctx, vec![], get_file_changes(&j_bcs))
                .await?;
        let j_skeleton = j_skeleton_id.load(&ctx, repo.repo_blobstore()).await?;
        assert_eq!(
            j_skeleton
                .first_case_conflict(&ctx, repo.repo_blobstore())
                .await?,
            Some((MPath::new(b"dir3/FILE1")?, MPath::new(b"dir3/file1")?))
        );

        // Changeset K is a merge of H and J, both of which have case
        // conflicts.  It does not introduce any new case conflicts.
        let k_bcs = changesets["K"].load(&ctx, repo.repo_blobstore()).await?;
        let k_skeleton_id = derive_skeleton_manifest(
            &ctx,
            &derivation_ctx,
            vec![h_skeleton_id, j_skeleton_id],
            get_file_changes(&k_bcs),
        )
        .await?;
        let k_skeleton = k_skeleton_id.load(&ctx, repo.repo_blobstore()).await?;
        assert_eq!(
            k_skeleton
                .first_new_case_conflict(
                    &ctx,
                    repo.repo_blobstore(),
                    vec![h_skeleton.clone(), j_skeleton]
                )
                .await?,
            None,
        );

        // Changeset L has no case conflicts.
        let l_bcs = changesets["L"].load(&ctx, repo.repo_blobstore()).await?;
        let l_skeleton_id =
            derive_skeleton_manifest(&ctx, &derivation_ctx, vec![], get_file_changes(&l_bcs))
                .await?;
        let l_skeleton = l_skeleton_id.load(&ctx, repo.repo_blobstore()).await?;
        assert_eq!(
            l_skeleton
                .first_case_conflict(&ctx, repo.repo_blobstore())
                .await?,
            None,
        );

        // Changeset M introduces a case conflict by virtue of merging the
        // manifests of H (which has its own case conflicts) and L.
        let m_bcs = changesets["M"].load(&ctx, repo.repo_blobstore()).await?;
        let m_skeleton_id = derive_skeleton_manifest(
            &ctx,
            &derivation_ctx,
            vec![h_skeleton_id, l_skeleton_id],
            get_file_changes(&m_bcs),
        )
        .await?;
        let m_skeleton = m_skeleton_id.load(&ctx, repo.repo_blobstore()).await?;
        assert_eq!(
            m_skeleton
                .first_case_conflict(&ctx, repo.repo_blobstore())
                .await?,
            Some((
                MPath::new(b"dir1/subdir1/SUBSUBDIR2")?,
                MPath::new(b"dir1/subdir1/subsubdir2")?
            ))
        );
        assert_eq!(
            m_skeleton
                .first_new_case_conflict(&ctx, repo.repo_blobstore(), vec![h_skeleton, l_skeleton])
                .await?,
            Some((
                MPath::new(b"dir2/subdir1/subsubdir1/FILE1")?,
                MPath::new(b"dir2/subdir1/subsubdir1/file1")?
            )),
        );

        Ok(())
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use mononoke_types::SkeletonManifestId;
use thiserror::Error;

mod batch;
mod derive;
pub mod mapping;

pub use mapping::format_key;
pub use mapping::RootSkeletonManifestId;

#[derive(Debug, Error)]
pub enum SkeletonManifestDerivationError {
    #[error("Invalid bonsai changeset: {0}")]
    InvalidBonsai(String),
    #[error("Missing skeleton manifest parent: {0}")]
    MissingParent(SkeletonManifestId),
    #[error("Missing skeleton manifest subentry for '{0}': {1}")]
    MissingSubentry(String, SkeletonManifestId),
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;

use anyhow::anyhow;
use anyhow::Error;
use anyhow::Result;
use async_trait::async_trait;
use blobstore::Blobstore;
use blobstore::BlobstoreGetData;
use bytes::Bytes;
use context::CoreContext;
use derived_data::impl_bonsai_derived_via_manager;
use derived_data_manager::dependencies;
use derived_data_manager::BonsaiDerivable;
use derived_data_manager::DerivableType;
use derived_data_manager::DerivationContext;
use derived_data_service_if::types as thrift;
use mononoke_types::BlobstoreBytes;
use mononoke_types::BonsaiChangeset;
use mononoke_types::ChangesetId;
use mononoke_types::ContentId;
use mononoke_types::FileType;
use mononoke_types::MPath;
use mononoke_types::SkeletonManifestId;

use crate::batch::derive_skeleton_manifests_in_batch;
use crate::derive::derive_skeleton_manifest;

#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub struct RootSkeletonManifestId(pub(crate) SkeletonManifestId);

impl RootSkeletonManifestId {
    pub fn skeleton_manifest_id(&self) -> &SkeletonManifestId {
        &self.0
    }
    pub fn into_skeleton_manifest_id(self) -> SkeletonManifestId {
        self.0
    }
}

impl TryFrom<BlobstoreBytes> for RootSkeletonManifestId {
    type Error = Error;

    fn try_from(blob_bytes: BlobstoreBytes) -> Result<Self> {
        SkeletonManifestId::from_bytes(&blob_bytes.into_bytes()).map(RootSkeletonManifestId)
    }
}

impl TryFrom<BlobstoreGetData> for RootSkeletonManifestId {
    type Error = Error;

    fn try_from(blob_get_data: BlobstoreGetData) -> Result<Self> {
        blob_get_data.into_bytes().try_into()
    }
}

impl From<RootSkeletonManifestId> for BlobstoreBytes {
    fn from(root_skeleton_manifest_id: RootSkeletonManifestId) -> Self {
        BlobstoreBytes::from_bytes(Bytes::copy_from_slice(
            root_skeleton_manifest_id.0.blake2().as_ref(),
        ))
    }
}

pub fn format_key(derivation_ctx: &DerivationContext, changeset_id: ChangesetId) -> String {
    let root_prefix = "derived_root_skeletonmanifest.";
    let key_prefix = derivation_ctx.mapping_key_prefix::<RootSkeletonManifestId>();
    format!("{}{}{}", root_prefix, key_prefix, changeset_id)
}

#[async_trait]
impl BonsaiDerivable for RootSkeletonManifestId {
    const VARIANT: DerivableType = DerivableType::SkeletonManifests;

    type Dependencies = dependencies![];

    async fn derive_single(
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        bonsai: BonsaiChangeset,
        parents: Vec<Self>,
    ) -> Result<Self, Error> {
        let id = derive_skeleton_manifest(
            ctx,
            derivation_ctx,
            parents
                .into_iter()
                .map(RootSkeletonManifestId::into_skeleton_manifest_id)
                .collect(),
            get_file_changes(&bonsai),
        )
        .await?;
        Ok(RootSkeletonManifestId(id))
    }

    async fn derive_batch(
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        bonsais: Vec<BonsaiChangeset>,
        gap_size: Option<usize>,
    ) -> Result<HashMap<ChangesetId, Self>> {
        derive_skeleton_manifests_in_batch(
            ctx,
            derivation_ctx,
            bonsais.into_iter().map(|b| b.get_changeset_id()).collect(),
            gap_size,
        )
        .await
    }

    async fn store_mapping(
        self,
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        changeset_id: ChangesetId,
    ) -> Result<()> {
        let key = format_key(derivation_ctx, changeset_id);
        derivation_ctx.blobstore().put(ctx, key, self.into()).await
    }

    async fn fetch(
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        changeset_id: ChangesetId,
    ) -> Result<Option<Self>> {
        let key = format_key(derivation_ctx, changeset_id);
        Ok(derivation_ctx
            .blobstore()
            .get(ctx, &key)
            .await?
            .map(TryInto::try_into)
            .transpose()?)
    }

    fn from_thrift(data: thrift::DerivedData) -> Result<Self> {
        if let thrift::DerivedData::skeleton_manifest(
            thrift::DerivedDataSkeletonManifest::root_skeleton_manifest_id(id),
        ) = data
        {
            SkeletonManifestId::from_thrift(id).map(Self)
        } else {
            Err(anyhow!(
                "Can't convert {} from provided thrift::DerivedData",
                Self::NAME.to_string(),
            ))
        }
    }

    fn into_thrift(data: Self) -> Result<thrift::DerivedData> {
        Ok(thrift::DerivedData::skeleton_manifest(
            thrift::DerivedDataSkeletonManifest::root_skeleton_manifest_id(
                data.skeleton_manifest_id().into_thrift(),
            ),
        ))
    }
}

impl_bonsai_derived_via_manager!(RootSkeletonManifestId);

pub fn get_file_changes(bcs: &BonsaiChangeset) -> Vec<(MPath, Option<(ContentId, FileType)>)> {
    bcs.file_changes()
        .map(|(mpath, file_change)| {
            (
                mpath.clone(),
                file_change
                    .simplify()
                    .map(|bc| (bc.content_id(), bc.file_type())),
            )
        })
        .collect()
}

#[cfg(test)]
mod test {
    use blobstore::Loadable;
    use bookmarks::BookmarkKey;
    use bookmarks::BookmarksRef;
    use borrowed::borrowed;
    use changeset_fetcher::ChangesetFetcherArc;
    use derived_data_test_utils::iterate_all_manifest_entries;
    use fbinit::FacebookInit;
    use fixtures::BranchEven;
    use fixtures::BranchUneven;
    use fixtures::BranchWide;
    use fixtures::Linear;
    use fixtures::ManyDiamonds;
    use fixtures::ManyFilesDirs;
    use fixtures::MergeEven;
    use fixtures::MergeUneven;
    use fixtures::TestRepoFixture;
    use fixtures::UnsharedMergeEven;
    use fixtures::UnsharedMergeUneven;
    use futures::compat::Stream01CompatExt;
    use futures::future::Future;
    use futures::stream::Stream;
    use futures::stream::TryStreamExt;
    use futures::try_join;
    use manifest::Entry;
    use mercurial_derivation::DeriveHgChangeset;
    use mercurial_types::HgChangesetId;
    use mercurial_types::HgManifestId;
    use mononoke_types::ChangesetId;
    use repo_blobstore::RepoBlobstoreRef;
    use repo_derived_data::RepoDerivedDataRef;
    use revset::AncestorsNodeStream;
    use tokio::runtime::Runtime;

    use super::*;

    async fn fetch_manifest_by_cs_id(
        ctx: &CoreContext,
        repo: &impl RepoBlobstoreRef,
        hg_cs_id: HgChangesetId,
    ) -> Result<HgManifestId> {
        Ok(hg_cs_id
            .load(ctx, repo.repo_blobstore())
            .await?
            .manifestid())
    }

    async fn verify_skeleton_manifest(
        ctx: &CoreContext,
        repo: &(impl RepoDerivedDataRef + RepoBlobstoreRef + Send + Sync),
        bcs_id: ChangesetId,
        hg_cs_id: HgChangesetId,
    ) -> Result<()> {
        let manager = repo.repo_derived_data().manager();
        let root_skeleton_manifest_id = manager
            .derive::<RootSkeletonManifestId>(ctx, bcs_id, None)
            .await?
            .into_skeleton_manifest_id();

        let skeleton_manifest_entries =
            iterate_all_manifest_entries(ctx, repo, Entry::Tree(root_skeleton_manifest_id))
                .map_ok(|(path, _)| path)
                .try_collect::<Vec<_>>();

        let root_mf_id = fetch_manifest_by_cs_id(ctx, repo, hg_cs_id).await?;

        let filenode_entries = iterate_all_manifest_entries(ctx, repo, Entry::Tree(root_mf_id))
            .map_ok(|(path, _)| path)
            .try_collect::<Vec<_>>();

        let (mut skeleton_manifest_entries, mut filenode_entries) =
            try_join!(skeleton_manifest_entries, filenode_entries)?;
        skeleton_manifest_entries.sort();
        filenode_entries.sort();
        assert_eq!(skeleton_manifest_entries, filenode_entries);
        Ok(())
    }

    async fn all_commits<'a>(
        ctx: &'a CoreContext,
        repo: &'a (impl BookmarksRef + ChangesetFetcherArc + RepoDerivedDataRef + Send + Sync),
    ) -> Result<impl Stream<Item = Result<(ChangesetId, HgChangesetId)>> + 'a> {
        let master_book = BookmarkKey::new("master").unwrap();
        let bcs_id = repo
            .bookmarks()
            .get(ctx.clone(), &master_book)
            .await?
            .unwrap();

        Ok(
            AncestorsNodeStream::new(ctx.clone(), &repo.changeset_fetcher_arc(), bcs_id.clone())
                .compat()
                .and_then(move |new_bcs_id| async move {
                    let hg_cs_id = repo.derive_hg_changeset(ctx, new_bcs_id).await?;
                    Ok((new_bcs_id, hg_cs_id))
                }),
        )
    }

    fn verify_repo<F, R>(fb: FacebookInit, repo: F, runtime: &Runtime)
    where
        F: Future<Output = R>,
        R: BookmarksRef + ChangesetFetcherArc + RepoBlobstoreRef + RepoDerivedDataRef + Send + Sync,
    {
        let ctx = CoreContext::test_mock(fb);
        let repo = runtime.block_on(repo);
        borrowed!(ctx, repo);

        runtime
            .block_on(async move {
                all_commits(ctx, repo)
                    .await
                    .unwrap()
                    .try_for_each(move |(bcs_id, hg_cs_id)| async move {
                        verify_skeleton_manifest(ctx, repo, bcs_id, hg_cs_id).await
                    })
                    .await
            })
            .unwrap();
    }

    #[fbinit::test]
    fn test_derive_data(fb: FacebookInit) {
        let runtime = Runtime::new().unwrap();
        verify_repo(fb, Linear::getrepo(fb), &runtime);
        verify_repo(fb, BranchEven::getrepo(fb), &runtime);
        verify_repo(fb, BranchUneven::getrepo(fb), &runtime);
        verify_repo(fb, BranchWide::getrepo(fb), &runtime);
        verify_repo(fb, ManyDiamonds::getrepo(fb), &runtime);
        verify_repo(fb, ManyFilesDirs::getrepo(fb), &runtime);
        verify_repo(fb, MergeEven::getrepo(fb), &runtime);
        verify_repo(fb, MergeUneven::getrepo(fb), &runtime);
        verify_repo(fb, UnsharedMergeEven::getrepo(fb), &runtime);
        verify_repo(fb, UnsharedMergeUneven::getrepo(fb), &runtime);
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::cmp::Ordering as CmpOrdering;
use std::collections::BTreeMap;

use anyhow::Error;
use blobstore::Blobstore;
use blobstore::Loadable;
use context::CoreContext;
use futures::stream;
use futures::stream::StreamExt;
use futures::stream::TryStreamExt;
use itertools::Itertools;
use mononoke_types::BonsaiChangeset;
use mononoke_types::ChangesetId;
use mononoke_types::ContentId;
use mononoke_types::FileChange;
use mononoke_types::FileType;
use mononoke_types::MPath;

pub type FileToContent = BTreeMap<MPath, Option<(ContentId, FileType)>>;
pub const DEFAULT_STACK_FILE_CHANGES_LIMIT: u64 = 10000;

#[derive(Copy, Clone, Debug)]
pub enum FileConflicts {
    /// Stacks should be split if files conflict in terms of change vs delete.
    ChangeDelete,

    /// Stacks should be split on any file change conflict.
    AnyChange,
}

pub struct SplitOptions {
    pub file_conflicts: FileConflicts,
    // Commits with copy info shouldn't be stacked with any other commit
    pub copy_info: bool,
    // Limits how many file changes should be in a single stack
    pub file_changes_limit: u64,
}

impl From<FileConflicts> for SplitOptions {
    fn from(fc: FileConflicts) -> Self {
        Self {
            file_conflicts: fc,
            copy_info: false,
            // Pick a reasonable default
            file_changes_limit: DEFAULT_STACK_FILE_CHANGES_LIMIT,
        }
    }
}

pub async fn split_batch_in_linear_stacks(
    ctx: &CoreContext,
    blobstore: &impl Blobstore,
    batch: Vec<ChangesetId>,
    split_opts: SplitOptions,
) -> Result<Vec<LinearStack>, Error> {
    let bonsais = stream::iter(
        batch
            .into_iter()
            .map(|bcs_id| async move { bcs_id.load(ctx, blobstore).await }),
    )
    .buffered(100)
    .try_collect::<Vec<_>>()
    .await?;
    split_bonsais_in_linear_stacks(&bonsais, split_opts)
}

/// We follow a few rules when splitting a batch in the stacks:
/// 1) Merges go to a separate batch
/// 2) If two commits have two files where one is a prefix of another, then they
///    go to a separate stacks (because of the way bonsai interprets these files)
/// 3) If there are file conflicts (see SplitOptions for details) then a commit go to a
///    separate stack.
/// 4) We check any other requirements set by the SplitOptions.
pub fn split_bonsais_in_linear_stacks(
    bonsais: &[BonsaiChangeset],
    split_opts: SplitOptions,
) -> Result<Vec<LinearStack>, Error> {
    let start_bcs = match bonsais.first() {
        Some(val) => val,
        None => {
            return Ok(vec![]);
        }
    };

    let mut linear_stacks = Vec::new();
    let mut cur_linear_stack = LinearStack::new(start_bcs.parents().collect::<Vec<_>>());
    cur_linear_stack.push(start_bcs);

    for (prev_bcs, bcs) in bonsais.iter().tuple_windows() {
        if !cur_linear_stack.can_be_in_same_linear_stack(prev_bcs, bcs, &split_opts) {
            linear_stacks.push(cur_linear_stack);
            cur_linear_stack = LinearStack::new(bcs.parents().collect::<Vec<_>>());
        }

        cur_linear_stack.push(bcs);
    }

    linear_stacks.push(cur_linear_stack);

    Ok(linear_stacks)
}

// Stores a linear stack.
#[derive(Clone, Debug)]
pub struct LinearStack {
    pub parents: Vec<ChangesetId>,
    pub stack_items: Vec<StackItem>,
    total_file_changes_len: u64,
}

#[derive(Clone, Debug)]
pub struct StackItem {
    pub cs_id: ChangesetId,
    pub combined_file_changes: FileToContent,
    pub per_commit_file_changes: FileToContent,
}

impl LinearStack {
    fn new(parents: Vec<ChangesetId>) -> Self {
        Self {
            parents,
            stack_items: vec![],
            total_file_changes_len: 0,
        }
    }

    fn push(&mut self, bcs: &BonsaiChangeset) {
        let cs_id = bcs.get_changeset_id();
        let file_changes = bcs
            .file_changes()
            .map(|(path, fc)| {
                (
                    path.clone(),
                    fc.simplify().map(|bc| (bc.content_id(), bc.file_type())),
                )
            })
            .collect::<BTreeMap<_, _>>();

        self.total_file_changes_len += file_changes.len() as u64;
        let mut combined_file_changes = self.get_last_file_changes().cloned().unwrap_or_default();
        combined_file_changes.extend(file_changes.clone());
        self.stack_items.push(StackItem {
            cs_id,
            combined_file_changes,
            per_commit_file_changes: file_changes,
        });
    }

    fn can_be_in_same_linear_stack(
        &self,
        prev: &BonsaiChangeset,
        next: &BonsaiChangeset,
        split_opts: &SplitOptions,
    ) -> bool {
        // Each merge should go in a separate stack
        if prev.is_merge() || next.is_merge() {
            return false;
        }

        if split_opts.copy_info {
            if has_copy_info(prev) || has_copy_info(next) {
                return false;
            }
        }

        // The next commit should be stacked on top of the previous one
        let prev_cs_id = prev.get_changeset_id();
        if !next.parents().any(|p| p == prev_cs_id) {
            return false;
        }

        // Check if stack will get too big
        let next_len = next.file_changes().len() as u64;
        if self.total_file_changes_len + next_len > split_opts.file_changes_limit {
            return false;
        }

        // There must be no file conflicts when adding the new changes.
        if let Some(cur_file_changes) = self.get_last_file_changes() {
            let new_file_changes = next.file_changes().collect();
            if has_file_conflict(
                cur_file_changes,
                new_file_changes,
                split_opts.file_conflicts,
            ) {
                return false;
            }
        }

        true
    }

    fn get_last_file_changes(&self) -> Option<&FileToContent> {
        self.stack_items
            .last()
            .map(|item| &item.combined_file_changes)
    }
}

fn has_copy_info(cs: &BonsaiChangeset) -> bool {
    for (_, fc) in cs.file_changes() {
        use FileChange::*;
        match fc {
            Change(fc) => {
                if fc.copy_from().is_some() {
                    return true;
                }
            }
            UntrackedChange(_) | Deletion | UntrackedDeletion => {}
        }
    }

    false
}

/// Returns true if:
/// 1) any of the files in `left` are prefix of any file in `right` and vice-versa
/// 2) File with the same name was deleted in `left` and added in `right` or vice-versa
fn has_file_conflict(
    left: &FileToContent,
    right: BTreeMap<&MPath, &FileChange>,
    file_conflicts: FileConflicts,
) -> bool {
    let mut left = left.iter().peekable();
    let mut right = right.into_iter().peekable();
    let mut state = (left.next(), right.next());
    loop {
        state = match state {
            (Some((l_path, l_content)), Some((r_path, r_file_change))) => {
                match l_path.cmp(r_path) {
                    CmpOrdering::Equal => {
                        match file_conflicts {
                            FileConflicts::ChangeDelete => {
                                if l_content.is_some() != r_file_change.is_changed() {
                                    // File is deleted and modified in two
                                    // different commits - this is a conflict,
                                    // they can't be in the same stack
                                    return true;
                                }
                            }
                            FileConflicts::AnyChange => {
                                return true;
                            }
                        }

                        // It's possible for a single conflict to have a path
                        // conflict (usually it happens when a file is replaced
                        // with a directory). The code below checks if we have
                        // a conflict like that and exists early if we do.
                        if let Some((next_l_path, _)) = left.peek() {
                            if r_path.is_prefix_of(*next_l_path) {
                                return true;
                            }
                        } else if let Some((next_r_path, _)) = right.peek() {
                            if l_path.is_prefix_of(*next_r_path) {
                                return true;
                            }
                        }
                        (left.next(), right.next())
                    }
                    CmpOrdering::Less => {
                        if l_path.is_prefix_of(r_path) {
                            return true;
                        }
                        (left.next(), Some((r_path, r_file_change)))
                    }
                    CmpOrdering::Greater => {
                        if r_path.is_prefix_of(l_path) {
                            return true;
                        }
                        (Some((l_path, l_content)), right.next())
                    }
                }
            }
            _ => break,
        };
    }

    false
}

#[cfg(test)]
mod test {
    use bonsai_hg_mapping::BonsaiHgMapping;
    use bookmarks::Bookmarks;
    use changeset_fetcher::ChangesetFetcher;
    use changesets::Changesets;
    use fbinit::FacebookInit;
    use filestore::FilestoreConfig;
    use maplit::btreemap;
    use mononoke_types::FileType;
    use repo_blobstore::RepoBlobstore;
    use repo_blobstore::RepoBlobstoreRef;
    use repo_derived_data::RepoDerivedData;
    use tests_utils::CreateCommitContext;

    use super::*;

    #[facet::container]
    struct TestRepo {
        #[facet]
        bonsai_hg_mapping: dyn BonsaiHgMapping,
        #[facet]
        bookmarks: dyn Bookmarks,
        #[facet]
        repo_blobstore: RepoBlobstore,
        #[facet]
        repo_derived_data: RepoDerivedData,
        #[facet]
        filestore_config: FilestoreConfig,
        #[facet]
        changeset_fetcher: dyn ChangesetFetcher,
        #[facet]
        changesets: dyn Changesets,
    }

    #[fbinit::test]
    async fn test_split_batch_in_linear_stacks_simple(fb: FacebookInit) -> Result<(), Error> {
        let ctx = CoreContext::test_mock(fb);
        let repo: TestRepo = test_repo_factory::build_empty(fb).await?;

        let file1 = "file1";
        let root = CreateCommitContext::new_root(&ctx, &repo)
            .add_file(file1, "content1")
            .commit()
            .await?;
        let file2 = "file2";
        let second = CreateCommitContext::new(&ctx, &repo, vec![root])
            .add_file(file2, "content2")
            .commit()
            .await?;

        let linear_stacks = split_batch_in_linear_stacks(
            &ctx,
            repo.repo_blobstore(),
            vec![root],
            FileConflicts::ChangeDelete.into(),
        )
        .await?;
        assert_linear_stacks(
            &ctx,
            &repo,
            linear_stacks,
            vec![(
                vec![],
                vec![btreemap! {file1 => Some(("content1".to_string(), FileType::Regular))}],
            )],
        )
        .await?;

        let linear_stacks = split_batch_in_linear_stacks(
            &ctx,
            repo.repo_blobstore(),
            vec![second],
            FileConflicts::ChangeDelete.into(),
        )
        .await?;
        assert_linear_stacks(
            &ctx,
            &repo,
            linear_stacks,
            vec![(
                vec![root],
                vec![btreemap! {file2 => Some(("content2".to_string(), FileType::Regular))}],
            )],
        )
        .await?;

        let linear_stacks = split_batch_in_linear_stacks(
            &ctx,
            repo.repo_blobstore(),
            vec![root, second],
            FileConflicts::ChangeDelete.into(),
        )
        .await?;
        assert_linear_stacks(
            &ctx,
            &repo,
            linear_stacks.clone(),
            vec![(
                vec![],
                vec![
                    btreemap! {file1 => Some(("content1".to_string(), FileType::Regular))},
                    btreemap! {
                        file1 => Some(("content1".to_string(), FileType::Regular)),
                        file2 => Some(("content2".to_string(), FileType::Regular)),
                    },
                ],
            )],
        )
        .await?;

        // Check that per_commit_file_changes are correct
        assert_eq!(
            linear_stacks[0].stack_items[0]
                .per_commit_file_changes
                .keys()
                .collect::<Vec<_>>(),
            vec![&MPath::new(file1)?]
        );

        assert_eq!(
            linear_stacks[0].stack_items[1]
                .per_commit_file_changes
                .keys()
                .collect::<Vec<_>>(),
            vec![&MPath::new(file2)?]
        );

        Ok(())
    }

    #[fbinit::test]
    async fn test_split_batch_in_linear_stacks_with_limit(fb: FacebookInit) -> Result<(), Error> {
        let ctx = CoreContext::test_mock(fb);
        let repo: TestRepo = test_repo_factory::build_empty(fb).await?;

        let file1 = "file1";
        let root = CreateCommitContext::new_root(&ctx, &repo)
            .add_file(file1, "content1")
            .commit()
            .await?;
        let file2 = "file2";
        let second = CreateCommitContext::new(&ctx, &repo, vec![root])
            .add_file(file2, "content2")
            .commit()
            .await?;

        let linear_stacks = split_batch_in_linear_stacks(
            &ctx,
            repo.repo_blobstore(),
            vec![root, second],
            SplitOptions {
                file_conflicts: FileConflicts::ChangeDelete,
                copy_info: false,
                file_changes_limit: 1,
            },
        )
        .await?;
        assert_linear_stacks(
            &ctx,
            &repo,
            linear_stacks.clone(),
            vec![
                (
                    vec![],
                    vec![btreemap! {file1 => Some(("content1".to_string(), FileType::Regular))}],
                ),
                (
                    vec![root],
                    vec![btreemap! {
                        file2 => Some(("content2".to_string(), FileType::Regular)),
                    }],
                ),
            ],
        )
        .await?;

        // Check that per_commit_file_changes are correct
        assert_eq!(
            linear_stacks[0].stack_items[0]
                .per_commit_file_changes
                .keys()
                .collect::<Vec<_>>(),
            vec![&MPath::new(file1)?]
        );

        assert_eq!(
            linear_stacks[1].stack_items[0]
                .per_commit_file_changes
                .keys()
                .collect::<Vec<_>>(),
            vec![&MPath::new(file2)?]
        );

        Ok(())
    }
    #[fbinit::test]
    async fn test_split_batch_in_linear_stacks_merge(fb: FacebookInit) -> Result<(), Error> {
        let repo: TestRepo = test_repo_factory::build_empty(fb).await?;
        let ctx = CoreContext::test_mock(fb);

        let file1 = "file1";
        let p1 = CreateCommitContext::new_root(&ctx, &repo)
            .add_file(file1, "content1")
            .commit()
            .await?;
        let file2 = "file2";
        let p2 = CreateCommitContext::new_root(&ctx, &repo)
            .add_file(file2, "content2")
            .commit()
            .await?;
        let merge_file = "merge";
        let merge = CreateCommitContext::new(&ctx, &repo, vec![p1, p2])
            .add_file(merge_file, "merge")
            .commit()
            .await?;

        let linear_stacks = split_batch_in_linear_stacks(
            &ctx,
            repo.repo_blobstore(),
            vec![p1, merge],
            FileConflicts::ChangeDelete.into(),
        )
        .await?;
        assert_linear_stacks(
            &ctx,
            &repo,
            linear_stacks,
            vec![
                (
                    vec![],
                    vec![btreemap! {file1 => Some(("content1".to_string(), FileType::Regular))}],
                ),
                (
                    vec![p1, p2],
                    vec![btreemap! {merge_file => Some(("merge".to_string(), FileType::Regular))}],
                ),
            ],
        )
        .await?;

        let linear_stacks = split_batch_in_linear_stacks(
            &ctx,
            repo.repo_blobstore(),
            vec![p1, p2],
            FileConflicts::ChangeDelete.into(),
        )
        .await?;
        assert_linear_stacks(
            &ctx,
            &repo,
            linear_stacks,
            vec![
                (
                    vec![],
                    vec![btreemap! {file1 => Some(("content1".to_string(), FileType::Regular))}],
                ),
                (
                    vec![],
                    vec![btreemap! {file2 => Some(("content2".to_string(), FileType::Regular))}],
                ),
            ],
        )
        .await?;

        Ok(())
    }

    #[fbinit::test]
    async fn test_split_batch_in_linear_stacks_replace_dir_with_file(
        fb: FacebookInit,
    ) -> Result<(), Error> {
        let repo: TestRepo = test_repo_factory::build_empty(fb).await?;
        let ctx = CoreContext::test_mock(fb);

        let dir = "dir";
        let file1 = format!("{}/file1", dir);
        let root = CreateCommitContext::new_root(&ctx, &repo)
            .add_file(file1.as_str(), "content1")
            .commit()
            .await?;
        let child = CreateCommitContext::new(&ctx, &repo, vec![root])
            .add_file(dir, "replace_dir_with_file")
            .commit()
            .await?;

        let linear_stacks = split_batch_in_linear_stacks(
            &ctx,
            repo.repo_blobstore(),
            vec![root, child],
            FileConflicts::ChangeDelete.into(),
        )
        .await?;
        assert_linear_stacks(
            &ctx,
            &repo,
            linear_stacks,
            vec![
                (
                    vec![],
                    vec![btreemap! {file1.as_str() => Some(("content1".to_string(), FileType::Regular))}],
                ),
                (
                    vec![root],
                    vec![btreemap! {dir => Some(("replace_dir_with_file".to_string(), FileType::Regular))}],
                ),
            ],
        )
        .await?;

        Ok(())
    }

    #[fbinit::test]
    async fn test_split_batch_in_linear_stacks_delete_file(fb: FacebookInit) -> Result<(), Error> {
        let repo: TestRepo = test_repo_factory::build_empty(fb).await?;
        let ctx = CoreContext::test_mock(fb);

        let file1 = "file1";
        let root = CreateCommitContext::new_root(&ctx, &repo)
            .add_file(file1, "content1")
            .commit()
            .await?;
        let child = CreateCommitContext::new(&ctx, &repo, vec![root])
            .delete_file(file1)
            .commit()
            .await?;

        let linear_stacks = split_batch_in_linear_stacks(
            &ctx,
            repo.repo_blobstore(),
            vec![root, child],
            FileConflicts::ChangeDelete.into(),
        )
        .await?;
        assert_linear_stacks(
            &ctx,
            &repo,
            linear_stacks,
            vec![
                (
                    vec![],
                    vec![btreemap! {file1 => Some(("content1".to_string(), FileType::Regular))}],
                ),
                (vec![root], vec![btreemap! {file1 => None}]),
            ],
        )
        .await?;

        Ok(())
    }

    #[fbinit::test]
    async fn test_split_batch_in_linear_stacks_add_same_file(
        fb: FacebookInit,
    ) -> Result<(), Error> {
        let repo: TestRepo = test_repo_factory::build_empty(fb).await?;
        let ctx = CoreContext::test_mock(fb);

        let file1 = "file1";
        let root = CreateCommitContext::new_root(&ctx, &repo)
            .add_file(file1, "content1")
            .commit()
            .await?;
        let child = CreateCommitContext::new(&ctx, &repo, vec![root])
            .add_file(file1, "content2")
            .commit()
            .await?;

        // With ChangeDelete, the stack is combined.
        let linear_stacks = split_batch_in_linear_stacks(
            &ctx,
            repo.repo_blobstore(),
            vec![root, child],
            FileConflicts::ChangeDelete.into(),
        )
        .await?;
        assert_linear_stacks(
            &ctx,
            &repo,
            linear_stacks,
            vec![(
                vec![],
                vec![
                    btreemap! {file1 => Some(("content1".to_string(), FileType::Regular))},
                    btreemap! {file1 => Some(("content2".to_string(), FileType::Regular))},
                ],
            )],
        )
        .await?;

        // With AnyChange, the stack is split.
        let linear_stacks = split_batch_in_linear_stacks(
            &ctx,
            repo.repo_blobstore(),
            vec![root, child],
            FileConflicts::AnyChange.into(),
        )
        .await?;
        assert_linear_stacks(
            &ctx,
            &repo,
            linear_stacks,
            vec![
                (
                    vec![],
                    vec![btreemap! {file1 => Some(("content1".to_string(), FileType::Regular))}],
                ),
                (
                    vec![root],
                    vec![btreemap! {file1 => Some(("content2".to_string(), FileType::Regular))}],
                ),
            ],
        )
        .await?;
        Ok(())
    }

    #[fbinit::test]
    async fn test_split_batch_in_linear_stacks_replace_file_with_dir(
        fb: FacebookInit,
    ) -> Result<(), Error> {
        let ctx = CoreContext::test_mock(fb);
        let repo: TestRepo = test_repo_factory::build_empty(fb).await?;

        let root = CreateCommitContext::new_root(&ctx, &repo)
            .add_file("dir", "content1")
            .commit()
            .await?;
        let second = CreateCommitContext::new(&ctx, &repo, vec![root])
            .delete_file("dir")
            .add_file("dir/file", "content1")
            .commit()
            .await?;
        let third = CreateCommitContext::new(&ctx, &repo, vec![second])
            .delete_file("dir")
            .commit()
            .await?;

        let linear_stacks = split_batch_in_linear_stacks(
            &ctx,
            repo.repo_blobstore(),
            vec![root, second, third],
            FileConflicts::ChangeDelete.into(),
        )
        .await?;
        assert_linear_stacks(
            &ctx,
            &repo,
            linear_stacks,
            vec![
                (
                    vec![],
                    vec![btreemap! {"dir" => Some(("content1".to_string(), FileType::Regular))}],
                ),
                (
                    vec![root],
                    vec![btreemap! {
                        "dir" => None,
                        "dir/file" => Some(("content1".to_string(), FileType::Regular))
                    }],
                ),
                (
                    vec![second],
                    vec![btreemap! {
                        "dir" => None,
                    }],
                ),
            ],
        )
        .await?;

        Ok(())
    }

    #[fbinit::test]
    async fn test_split_batch_in_linear_stacks_copy_info(fb: FacebookInit) -> Result<(), Error> {
        let ctx = CoreContext::test_mock(fb);
        let repo: TestRepo = test_repo_factory::build_empty(fb).await?;

        let root = CreateCommitContext::new_root(&ctx, &repo)
            .add_file("dir", "content1")
            .commit()
            .await?;
        let second = CreateCommitContext::new(&ctx, &repo, vec![root])
            .add_file_with_copy_info("copiedfile", "content1", (root, "dir"))
            .commit()
            .await?;
        let third = CreateCommitContext::new(&ctx, &repo, vec![second])
            .add_file("dir2", "content2")
            .commit()
            .await?;

        let linear_stacks = split_batch_in_linear_stacks(
            &ctx,
            repo.repo_blobstore(),
            vec![root, second, third],
            SplitOptions {
                file_conflicts: FileConflicts::ChangeDelete,
                copy_info: true,
                file_changes_limit: DEFAULT_STACK_FILE_CHANGES_LIMIT,
            },
        )
        .await?;
        assert_linear_stacks(
            &ctx,
            &repo,
            linear_stacks,
            vec![
                (
                    vec![],
                    vec![btreemap! {"dir" => Some(("content1".to_string(), FileType::Regular))}],
                ),
                (
                    vec![root],
                    vec![btreemap! {
                        "copiedfile" => Some(("content1".to_string(), FileType::Regular))
                    }],
                ),
                (
                    vec![second],
                    vec![btreemap! {
                        "dir2" => Some(("content2".to_string(), FileType::Regular))
                    }],
                ),
            ],
        )
        .await?;

        Ok(())
    }

    async fn assert_linear_stacks(
        ctx: &CoreContext,
        repo: impl RepoBlobstoreRef,
        actual: Vec<LinearStack>,
        expected: Vec<(
            Vec<ChangesetId>,
            Vec<BTreeMap<&str, Option<(String, FileType)>>>,
        )>,
    ) -> Result<(), Error> {
        let mut actual_res = vec![];
        for linear_stack in actual {
            let LinearStack {
                parents,
                stack_items,
                ..
            } = linear_stack;

            let mut paths_for_the_whole_stack = vec![];
            for item in stack_items {
                let file_to_content = item.combined_file_changes;
                let mut paths = btreemap![];
                for (path, maybe_content) in file_to_content {
                    let maybe_content = match maybe_content {
                        Some((content_id, file_type)) => {
                            let content = filestore::fetch_concat(
                                &repo.repo_blobstore(),
                                ctx,
                                filestore::FetchKey::Canonical(content_id),
                            )
                            .await?;

                            let content = String::from_utf8(content.to_vec())?;
                            Some((content, file_type))
                        }
                        None => None,
                    };
                    paths.insert(path, maybe_content);
                }
                paths_for_the_whole_stack.push(paths);
            }

            actual_res.push((parents, paths_for_the_whole_stack));
        }

        let expected = expected
            .into_iter()
            .map(|(parents, linear_stack)| {
                let paths = linear_stack
                    .into_iter()
                    .map(|paths| {
                        paths
                            .into_iter()
                            .map(|(p, maybe_content)| (MPath::new(p).unwrap(), maybe_content))
                            .collect::<BTreeMap<_, _>>()
                    })
                    .collect::<Vec<_>>();

                (parents, paths)
            })
            .collect::<Vec<_>>();

        assert_eq!(actual_res, expected);

        Ok(())
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

//! # Derived Data
//!
//! This crate defines the traits that are used to implement data derivation
//! in Mononoke.
//!
//! Each type of derived data can be derived from a changeset when given:
//!   * The bonsai changeset (including its changeset ID).
//!   * The derived data of the same type for the immediate parent changesets.
//!
//! ## Traits
//!
//! ### BonsaiDerivable
//!
//! The `manager::BonsaiDerivable` trait defines how derivation occurs.  Each
//! derived data type must implement `BonsaiDerivable` to describe how to
//! derive a new value from the inputs given above.
//!
//! As a performance enhancement, a derived data type may also implement the
//! `derive_batch` method of this trait to implement a fast path for deriving
//! data from a batch of changesets.  The default implementation derives each
//! changeset in the batch sequentially.
//!
//! This trait also defines how derived data is stored and fetched via the
//! `store_mapping` and `fetch` method, which should store the data in a
//! way that can be fetched via its changeset id only.
//!
//! ### BonsaiDerived
//!
//! The `BonsaiDerived` trait ties these together by proving a default
//! mapping implementation that uses the configuration on the repository,
//! provided the derived data type is enabled for that repository.
//!
//! ## Usage
//!
//! The usual usage for deriving a particular derived data type in a
//! repository is the methods of `RepoDerivedData` or `DerivedDataManager`:
//!
//! ```ignore
//! use repo_derived_data::RepoDerivedDataRef;
//!
//! let value: DerivedDataType = repo.repo_derived_data().derive(ctx, cs_id).await?;
//! // Batch derivation
//! let manager = repo.repo_derived_data().manager();
//! manager.derive_exactly_batch::<DerivedDataType>(ctx, cs_ids.clone(), BatchDeriveOptions, None).await?;
//! let values: Vec<DerivedDataType> = manager.fetch_derived_batch(ctx, cs_ids, None).await?;
//! ```

use anyhow::Error;
use async_trait::async_trait;
use context::CoreContext;
use context::SessionClass;
use mononoke_types::ChangesetId;
use repo_derived_data::RepoDerivedDataRef;
use repo_identity::RepoIdentityRef;

pub mod batch;

pub use derived_data_manager::DerivationError as DeriveError;
pub use metaconfig_types::DerivedDataTypesConfig;

pub mod macro_export {
    pub use anyhow::Error;
    pub use async_trait::async_trait;
    pub use context::CoreContext;
    pub use derived_data_manager::BonsaiDerivable;
    pub use mononoke_types::ChangesetId;
    pub use repo_derived_data::RepoDerivedDataRef;

    pub use super::BonsaiDerived;
    pub use super::DeriveError;
}

/// Trait for accessing data that can be derived from bonsai changesets, such
/// as Mercurial or Git changesets, unodes, fsnodes, skeleton manifests and
/// other derived data.
#[async_trait]
pub trait BonsaiDerived: Sized + Send + Sync + Clone + 'static {
    const DERIVABLE_NAME: &'static str;

    /// This function is the entrypoint for changeset derivation.  It will
    /// derive an instance of the derived data type based on the bonsai
    /// changeset representation.
    ///
    /// The derived data will be saved in a mapping from the changeset id,
    /// so that subsequent derives will just fetch.
    ///
    /// This function fails immediately if this type of derived data is not
    /// enabled for this repo.
    async fn derive(
        ctx: &CoreContext,
        repo: &(impl RepoDerivedDataRef + Send + Sync),
        csid: ChangesetId,
    ) -> Result<Self, DeriveError>;

    /// Fetch the derived data in cases where we might not want to trigger
    /// derivation, e.g. when scrubbing.
    async fn fetch_derived(
        ctx: &CoreContext,
        repo: &(impl RepoDerivedDataRef + Send + Sync),
        csid: &ChangesetId,
    ) -> Result<Option<Self>, Error>;

    /// Returns `true` if derived data has already been derived for this
    /// changeset.
    async fn is_derived(
        ctx: &CoreContext,
        repo: &(impl RepoDerivedDataRef + Send + Sync),
        csid: &ChangesetId,
    ) -> Result<bool, DeriveError> {
        Ok(Self::fetch_derived(ctx, repo, csid).await?.is_some())
    }

    /// Returns the number of ancestors of `csid` that are not yet derived,
    /// or at most `limit`.
    ///
    /// This function fails immediately if derived data is not enabled for
    /// this repo.
    async fn count_underived(
        ctx: &CoreContext,
        repo: &(impl RepoDerivedDataRef + Send + Sync),
        csid: &ChangesetId,
        limit: u64,
    ) -> Result<u64, DeriveError>;
}

#[macro_export]
macro_rules! impl_bonsai_derived_via_manager {
    ($derivable:ty) => {
        #[$crate::macro_export::async_trait]
        impl $crate::macro_export::BonsaiDerived for $derivable {
            const DERIVABLE_NAME: &'static str =
                <$derivable as $crate::macro_export::BonsaiDerivable>::NAME;

            async fn derive(
                ctx: &$crate::macro_export::CoreContext,
                repo: &(impl $crate::macro_export::RepoDerivedDataRef + Send + Sync),
                csid: $crate::macro_export::ChangesetId,
            ) -> Result<Self, $crate::macro_export::DeriveError> {
                repo.repo_derived_data().derive::<Self>(ctx, csid).await
            }

            async fn fetch_derived(
                ctx: &$crate::macro_export::CoreContext,
                repo: &(impl $crate::macro_export::RepoDerivedDataRef + Send + Sync),
                csid: &$crate::macro_export::ChangesetId,
            ) -> Result<Option<Self>, $crate::macro_export::Error> {
                Ok(repo
                    .repo_derived_data()
                    .fetch_derived::<Self>(ctx, *csid)
                    .await?)
            }

            async fn count_underived(
                ctx: &$crate::macro_export::CoreContext,
                repo: &(impl $crate::macro_export::RepoDerivedDataRef + Send + Sync),
                csid: &$crate::macro_export::ChangesetId,
                limit: u64,
            ) -> Result<u64, $crate::macro_export::DeriveError> {
                repo.repo_derived_data()
                    .count_underived::<Self>(ctx, *csid, Some(limit))
                    .await
            }
        }
    };
}

pub fn override_ctx(mut ctx: CoreContext, repo: impl RepoIdentityRef) -> CoreContext {
    let use_bg_class = tunables::tunables()
        .by_repo_derived_data_use_background_session_class(repo.repo_identity().name());
    if let Some(true) = use_bg_class {
        ctx.session_mut()
            .override_session_class(SessionClass::BackgroundUnlessTooSlow);
        ctx
    } else {
        ctx
    }
}