/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::sync::Arc;
use std::sync::Mutex;
use std::time::Duration;

use anyhow::anyhow;
use anyhow::Error;
use anyhow::Result;
use async_trait::async_trait;
use blobstore::Blobstore;
use blobstore::BlobstoreBytes;
use bonsai_hg_mapping::BonsaiHgMapping;
use bookmarks::BookmarkKey;
use bookmarks::Bookmarks;
use bookmarks::BookmarksRef;
use bytes::Bytes;
use cacheblob::LeaseOps;
use changeset_fetcher::ChangesetFetcher;
use changesets::Changesets;
use changesets::ChangesetsRef;
use cloned::cloned;
use context::CoreContext;
use derived_data_manager::BonsaiDerivable;
use derived_data_manager::DerivationError;
use fbinit::FacebookInit;
use filestore::FilestoreConfig;
use fixtures::BranchEven;
use fixtures::BranchUneven;
use fixtures::BranchWide;
use fixtures::Linear;
use fixtures::ManyDiamonds;
use fixtures::ManyFilesDirs;
use fixtures::MergeEven;
use fixtures::MergeUneven;
use fixtures::TestRepoFixture;
use fixtures::UnsharedMergeEven;
use fixtures::UnsharedMergeUneven;
use futures::future::BoxFuture;
use futures_stats::TimedFutureExt;
use futures_stats::TimedTryFutureExt;
use generation_derivation::make_test_repo_factory;
use generation_derivation::DerivedGeneration;
use lock_ext::LockExt;
use maplit::hashmap;
use mononoke_types::ChangesetId;
use mononoke_types::MPath;
use mononoke_types::RepositoryId;
use repo_blobstore::RepoBlobstore;
use repo_blobstore::RepoBlobstoreRef;
use repo_derived_data::RepoDerivedData;
use repo_derived_data::RepoDerivedDataArc;
use repo_derived_data::RepoDerivedDataRef;
use repo_identity::RepoIdentity;
use repo_identity::RepoIdentityRef;
use tests_utils::CreateCommitContext;
use tunables::override_tunables;
use tunables::MononokeTunables;

#[facet::container]
#[derive(Clone)]
struct TestRepo {
    #[facet]
    bonsai_hg_mapping: dyn BonsaiHgMapping,
    #[facet]
    bookmarks: dyn Bookmarks,
    #[facet]
    changesets: dyn Changesets,
    #[facet]
    changeset_fetcher: dyn ChangesetFetcher,
    #[facet]
    filestore_config: FilestoreConfig,
    #[facet]
    repo_blobstore: RepoBlobstore,
    #[facet]
    repo_identity: RepoIdentity,
    #[facet]
    repo_derived_data: RepoDerivedData,
}

async fn derive_for_master(
    ctx: &CoreContext,
    repo: &(impl BookmarksRef + ChangesetsRef + RepoDerivedDataRef),
) -> Result<()> {
    let master = repo
        .bookmarks()
        .get(ctx.clone(), &BookmarkKey::new("master")?)
        .await?
        .expect("master should be set");
    let expected = repo
        .changesets()
        .get(ctx, master)
        .await?
        .expect("changeset should exist")
        .gen;

    let derived = repo
        .repo_derived_data()
        .derive::<DerivedGeneration>(ctx, master)
        .await?;

    assert_eq!(expected, derived.generation);

    Ok(())
}

#[fbinit::test]
async fn test_derive_fixtures(fb: FacebookInit) -> Result<()> {
    let ctx = CoreContext::test_mock(fb);
    let mut factory = make_test_repo_factory(fb);

    let repo: TestRepo = factory.with_id(RepositoryId::new(1)).build().await?;
    BranchEven::initrepo(fb, &repo).await;
    derive_for_master(&ctx, &repo).await?;

    let repo: TestRepo = factory.with_id(RepositoryId::new(2)).build().await?;
    BranchUneven::initrepo(fb, &repo).await;
    derive_for_master(&ctx, &repo).await?;

    let repo: TestRepo = factory.with_id(RepositoryId::new(3)).build().await?;
    BranchWide::initrepo(fb, &repo).await;
    derive_for_master(&ctx, &repo).await?;

    let repo: TestRepo = factory.with_id(RepositoryId::new(4)).build().await?;
    Linear::initrepo(fb, &repo).await;
    derive_for_master(&ctx, &repo).await?;

    let repo: TestRepo = factory.with_id(RepositoryId::new(5)).build().await?;
    ManyFilesDirs::initrepo(fb, &repo).await;
    derive_for_master(&ctx, &repo).await?;

    let repo: TestRepo = factory.with_id(RepositoryId::new(6)).build().await?;
    MergeEven::initrepo(fb, &repo).await;
    derive_for_master(&ctx, &repo).await?;

    let repo: TestRepo = factory.with_id(RepositoryId::new(7)).build().await?;
    MergeUneven::initrepo(fb, &repo).await;
    derive_for_master(&ctx, &repo).await?;

    let repo: TestRepo = factory.with_id(RepositoryId::new(8)).build().await?;
    UnsharedMergeEven::initrepo(fb, &repo).await;
    derive_for_master(&ctx, &repo).await?;

    let repo: TestRepo = factory.with_id(RepositoryId::new(9)).build().await?;
    UnsharedMergeUneven::initrepo(fb, &repo).await;
    derive_for_master(&ctx, &repo).await?;

    let repo: TestRepo = factory.with_id(RepositoryId::new(10)).build().await?;
    ManyDiamonds::initrepo(fb, &repo).await;
    derive_for_master(&ctx, &repo).await?;

    Ok(())
}

#[fbinit::test]
/// Test that derivation is successful even when there are gaps (i.e. some
/// derived changesets do not have their parents derived).
async fn test_gapped_derivation(fb: FacebookInit) -> Result<()> {
    let ctx = CoreContext::test_mock(fb);
    let repo: TestRepo = make_test_repo_factory(fb).build().await?;
    Linear::initrepo(fb, &repo).await;

    let master = repo
        .bookmarks()
        .get(ctx.clone(), &BookmarkKey::new("master")?)
        .await?
        .expect("master should be set");
    let master_anc1 = repo
        .changesets()
        .get(&ctx, master)
        .await?
        .expect("changeset should exist")
        .parents
        .first()
        .expect("changeset should have a parent")
        .clone();
    let master_anc2 = repo
        .changesets()
        .get(&ctx, master_anc1)
        .await?
        .expect("changeset should exist")
        .parents
        .first()
        .expect("changeset should have a parent")
        .clone();
    // Insert a derived entry for the first ancestor changeset.  We will
    // deliberately use a different value than is expected.
    repo.repo_blobstore()
        .put(
            &ctx,
            format!("repo0.test_generation.{}", master_anc1),
            BlobstoreBytes::from_bytes(Bytes::from_static(b"41")),
        )
        .await?;

    // Derivation was based on that different value.
    let derived = repo
        .repo_derived_data()
        .derive::<DerivedGeneration>(&ctx, master)
        .await?;

    assert_eq!(42, derived.generation);

    // The other ancestors were not derived.
    let derived_anc2 = repo
        .repo_derived_data()
        .fetch_derived::<DerivedGeneration>(&ctx, master_anc2)
        .await?;
    assert!(derived_anc2.is_none());

    Ok(())
}

#[fbinit::test]
async fn test_leases(fb: FacebookInit) -> Result<(), Error> {
    let ctx = CoreContext::test_mock(fb);
    let repo: TestRepo = make_test_repo_factory(fb).build().await?;
    Linear::initrepo(fb, &repo).await;

    let master = repo
        .bookmarks()
        .get(ctx.clone(), &BookmarkKey::new("master")?)
        .await?
        .expect("master should be set");

    let lease = repo.repo_derived_data().lease();
    let lease_key = format!(
        "repo{}.{}.{}",
        repo.repo_identity().id(),
        DerivedGeneration::NAME,
        master
    );

    // take lease
    assert!(lease.try_add_put_lease(&lease_key).await?);
    assert!(!lease.try_add_put_lease(&lease_key).await?);

    let output = Arc::new(Mutex::new(None));
    tokio::spawn({
        cloned!(ctx, repo, output);
        async move {
            let result = repo
                .repo_derived_data()
                .derive::<DerivedGeneration>(&ctx, master)
                .await;
            output.with(move |output| {
                let _ = output.insert(result);
            });
        }
    });

    // Let the derivation process get started, however derivation won't
    // happen yet.
    tokio::time::sleep(Duration::from_millis(300)).await;

    assert!(
        repo.repo_derived_data()
            .fetch_derived::<DerivedGeneration>(&ctx, master)
            .await?
            .is_none()
    );

    // Release the lease, allowing derivation to proceed.
    lease.release_lease(&lease_key).await;
    tokio::time::sleep(Duration::from_millis(3000)).await;

    let expected = repo
        .changesets()
        .get(&ctx, master)
        .await?
        .expect("changeset should exist")
        .gen;
    let result = output
        .with(|output| output.take())
        .expect("scheduled derivation should have completed")?;
    assert_eq!(expected, result.generation,);

    // Take the lease again.
    assert!(lease.try_add_put_lease(&lease_key).await?);

    // This time it should succeed, as the lease won't be necessary.
    let result = repo
        .repo_derived_data()
        .derive::<DerivedGeneration>(&ctx, master)
        .await?;
    assert_eq!(expected, result.generation);
    lease.release_lease(&lease_key).await;
    Ok(())
}

#[derive(Debug)]
struct FailingLease;

impl std::fmt::Display for FailingLease {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(f, "FailingLease")
    }
}

#[async_trait]
impl LeaseOps for FailingLease {
    async fn try_add_put_lease(&self, _key: &str) -> Result<bool> {
        Err(anyhow!("error"))
    }

    fn renew_lease_until(&self, _ctx: CoreContext, _key: &str, _done: BoxFuture<'static, ()>) {}

    async fn wait_for_other_leases(&self, _key: &str) {}

    async fn release_lease(&self, _key: &str) {}
}

#[fbinit::test]
async fn test_always_failing_lease(fb: FacebookInit) -> Result<(), Error> {
    let ctx = CoreContext::test_mock(fb);
    let repo: TestRepo = make_test_repo_factory(fb)
        .with_derived_data_lease(|| Arc::new(FailingLease))
        .build()
        .await?;
    Linear::initrepo(fb, &repo).await;

    let master = repo
        .bookmarks()
        .get(ctx.clone(), &BookmarkKey::new("master")?)
        .await?
        .expect("master should be set");

    let lease = repo.repo_derived_data().lease();
    let lease_key = format!(
        "repo{}.{}.{}",
        repo.repo_identity().id(),
        DerivedGeneration::NAME,
        master,
    );

    // Taking the lease should fail
    assert!(lease.try_add_put_lease(&lease_key).await.is_err());

    // Derivation should succeed even though lease always fails
    let result = repo
        .repo_derived_data()
        .derive::<DerivedGeneration>(&ctx, master)
        .await?;
    let expected = repo
        .changesets()
        .get(&ctx, master)
        .await?
        .expect("changeset should exist")
        .gen;
    assert_eq!(expected, result.generation);

    Ok(())
}

#[fbinit::test]
async fn test_parallel_derivation(fb: FacebookInit) -> Result<(), Error> {
    let ctx = CoreContext::test_mock(fb);
    let repo: TestRepo = make_test_repo_factory(fb).build().await?;

    // Create a commit with lots of parents, and make each derivation take
    // 2 seconds.  Check that derivations happen in parallel by ensuring
    // it completes in a shorter time.
    let mut parents = vec![];
    for i in 0..8 {
        let p = CreateCommitContext::new_root(&ctx, &repo)
            .add_file(MPath::new(format!("file_{}", i))?, format!("{}", i))
            .add_extra("test-derive-delay", "2")
            .commit()
            .await?;
        parents.push(p);
    }

    let merge = CreateCommitContext::new(&ctx, &repo, parents)
        .add_extra("test-derive-delay", "2")
        .commit()
        .await?;

    let (stats, _res) = repo
        .repo_derived_data()
        .derive::<DerivedGeneration>(&ctx, merge)
        .try_timed()
        .await?;

    assert!(stats.completion_time > Duration::from_secs(2));
    assert!(stats.completion_time < Duration::from_secs(10));

    Ok(())
}

async fn ensure_tunables_disable_derivation(
    ctx: &CoreContext,
    repo: &impl RepoDerivedDataArc,
    csid: ChangesetId,
    tunables: MononokeTunables,
) -> Result<(), Error> {
    let spawned_derivation = tokio::spawn({
        let ctx = ctx.clone();
        let repo_derived_data = repo.repo_derived_data_arc();
        async move {
            repo_derived_data
                .derive::<DerivedGeneration>(&ctx, csid)
                .timed()
                .await
        }
    });
    tokio::time::sleep(Duration::from_millis(1000)).await;

    override_tunables(Some(Arc::new(tunables)));

    let (stats, res) = spawned_derivation.await?;

    assert!(matches!(res, Err(DerivationError::Disabled(..))));

    eprintln!("derivation cancelled after {:?}", stats.completion_time);
    assert!(stats.completion_time < Duration::from_secs(15));

    Ok(())
}

#[fbinit::test]
/// Test that very slow derivation can be cancelled mid-flight by setting
/// the appropriate values in tunables.
async fn test_cancelling_slow_derivation(fb: FacebookInit) -> Result<(), Error> {
    let ctx = CoreContext::test_mock(fb);
    let repo: TestRepo = make_test_repo_factory(fb).build().await?;

    let create_tunables = || {
        let tunables = MononokeTunables::default();
        // Make delay smaller so that the test runs faster
        tunables.update_ints(&hashmap! {
            "derived_data_disabled_watcher_delay_secs".to_string() => 1,
        });
        tunables
    };

    let commit = CreateCommitContext::new_root(&ctx, &repo)
        .add_file(MPath::new("file")?, "content")
        .add_extra("test-derive-delay", "20")
        .commit()
        .await?;

    // Reset tunables.
    override_tunables(Some(Arc::new(create_tunables())));

    // Disable derived data for all types
    let tunables_to_disable_all = create_tunables();
    tunables_to_disable_all.update_by_repo_bools(&hashmap! {
        repo.repo_identity().name().to_string() => hashmap! {
            "all_derived_data_disabled".to_string() => true,
        },
    });
    ensure_tunables_disable_derivation(&ctx, &repo, commit, tunables_to_disable_all).await?;

    // Reset tunables.
    override_tunables(Some(Arc::new(create_tunables())));

    // Disable derived data for a single type
    let tunables_to_disable_by_type = create_tunables();
    tunables_to_disable_by_type.update_by_repo_vec_of_strings(&hashmap! {
        repo.repo_identity().name().to_string() => hashmap! {
            "derived_data_types_disabled".to_string() => vec![DerivedGeneration::NAME.to_string()],
        },
    });
    ensure_tunables_disable_derivation(&ctx, &repo, commit, tunables_to_disable_by_type).await?;

    Ok(())
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;
use std::time::Duration;

use anyhow::bail;
use anyhow::Error;
use anyhow::Result;
use async_trait::async_trait;
use blobstore::BlobstoreBytes;
use blobstore::BlobstoreGetData;
use bytes::Bytes;
use context::CoreContext;
use derived_data_manager::dependencies;
use derived_data_manager::BonsaiDerivable;
use derived_data_manager::DerivableType;
use derived_data_manager::DerivationContext;
use derived_data_service_if::types as thrift;
use fbinit::FacebookInit;
use mononoke_types::BonsaiChangeset;
use mononoke_types::ChangesetId;
use test_repo_factory::TestRepoFactory;

#[derive(Clone, Debug)]
pub struct DerivedGeneration {
    pub generation: u64,
}

impl From<DerivedGeneration> for BlobstoreBytes {
    fn from(derived: DerivedGeneration) -> BlobstoreBytes {
        let generation = derived.generation.to_string();
        let data = Bytes::copy_from_slice(generation.as_bytes());
        BlobstoreBytes::from_bytes(data)
    }
}

impl TryFrom<BlobstoreBytes> for DerivedGeneration {
    type Error = Error;

    fn try_from(blob_bytes: BlobstoreBytes) -> Result<Self> {
        let generation = std::str::from_utf8(blob_bytes.as_bytes())?.parse::<u64>()?;
        Ok(DerivedGeneration { generation })
    }
}

impl TryFrom<BlobstoreGetData> for DerivedGeneration {
    type Error = Error;

    fn try_from(data: BlobstoreGetData) -> Result<Self> {
        data.into_bytes().try_into()
    }
}

#[async_trait]
impl BonsaiDerivable for DerivedGeneration {
    const NAME: &'static str = "test_generation";
    // Using existing variant for unodes to avoid adding new for test only
    const VARIANT: DerivableType = DerivableType::Unodes;

    type Dependencies = dependencies![];

    async fn derive_single(
        _ctx: &CoreContext,
        _derivation_ctx: &DerivationContext,
        bonsai: BonsaiChangeset,
        parents: Vec<Self>,
    ) -> Result<Self> {
        if let Some(delay_str) = bonsai
            .hg_extra()
            .collect::<HashMap<_, _>>()
            .get("test-derive-delay")
        {
            let delay = std::str::from_utf8(delay_str)?.parse::<f64>()?;
            tokio::time::sleep(Duration::from_secs_f64(delay)).await;
        }
        let mut generation = 1;
        for parent in parents {
            if parent.generation >= generation {
                generation = parent.generation + 1;
            }
        }
        let derived = DerivedGeneration { generation };
        Ok(derived)
    }

    async fn store_mapping(
        self,
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        changeset_id: ChangesetId,
    ) -> Result<()> {
        derivation_ctx
            .blobstore()
            .put(
                ctx,
                format!(
                    "repo{}.test_generation.{}",
                    derivation_ctx.repo_id(),
                    changeset_id,
                ),
                self.into(),
            )
            .await?;
        Ok(())
    }

    async fn fetch(
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        changeset_id: ChangesetId,
    ) -> Result<Option<Self>> {
        match derivation_ctx
            .blobstore()
            .get(
                ctx,
                &format!(
                    "repo{}.test_generation.{}",
                    derivation_ctx.repo_id(),
                    changeset_id
                ),
            )
            .await?
        {
            Some(blob) => Ok(Some(blob.try_into()?)),
            None => Ok(None),
        }
    }

    fn from_thrift(_: thrift::DerivedData) -> Result<Self> {
        bail!("Not implemented for {}", Self::NAME);
    }

    fn into_thrift(_: Self) -> Result<thrift::DerivedData> {
        bail!("Not implemented for {}", Self::NAME);
    }
}

pub fn make_test_repo_factory(fb: FacebookInit) -> TestRepoFactory {
    let mut factory = TestRepoFactory::new(fb).unwrap();
    factory.with_config_override(|repo_config| {
        repo_config
            .derived_data_config
            .get_active_config()
            .expect("No enabled derived data types config")
            .types
            .insert(DerivedGeneration::NAME.to_string());
    });
    factory
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use anyhow::anyhow;
use anyhow::Error;
use anyhow::Result;
use blobstore::Loadable;
use bonsai_hg_mapping::BonsaiHgMappingRef;
use bounded_traversal::bounded_traversal_stream;
use context::CoreContext;
use futures::future::FutureExt;
use futures::stream;
use futures::stream::Stream;
use futures::stream::TryStreamExt;
use manifest::Entry;
use manifest::Manifest;
use mercurial_types::HgChangesetId;
use mononoke_types::BonsaiChangeset;
use mononoke_types::ChangesetId;
use mononoke_types::MPath;
use repo_blobstore::RepoBlobstoreRef;

pub async fn bonsai_changeset_from_hg(
    ctx: &CoreContext,
    repo: impl RepoBlobstoreRef + BonsaiHgMappingRef,
    s: &str,
) -> Result<(ChangesetId, BonsaiChangeset)> {
    let hg_cs_id = s.parse::<HgChangesetId>()?;
    let bcs_id = repo
        .bonsai_hg_mapping()
        .get_bonsai_from_hg(ctx, hg_cs_id)
        .await?
        .ok_or_else(|| anyhow!("Failed to find bonsai changeset id for {}", hg_cs_id))?;
    let bcs = bcs_id.load(ctx, repo.repo_blobstore()).await?;
    Ok((bcs_id, bcs))
}

pub fn iterate_all_manifest_entries<'a, MfId, LId>(
    ctx: &'a CoreContext,
    repo: &'a (impl RepoBlobstoreRef + Send + Sync),
    entry: Entry<MfId, LId>,
) -> impl Stream<Item = Result<(Option<MPath>, Entry<MfId, LId>)>> + 'a
where
    MfId: Loadable + Send + Sync + Clone + 'a,
    LId: Send + Clone + 'static,
    <MfId as Loadable>::Value: Manifest<TreeId = MfId, LeafId = LId>,
{
    bounded_traversal_stream(256, Some((None, entry)), move |(path, entry)| {
        async move {
            match entry {
                Entry::Leaf(_) => Ok((vec![(path, entry.clone())], vec![])),
                Entry::Tree(tree) => {
                    let mf = tree.load(ctx, repo.repo_blobstore()).await?;
                    let recurse = mf
                        .list()
                        .map(|(basename, new_entry)| {
                            let path = MPath::join_opt_element(path.as_ref(), &basename);
                            (Some(path), new_entry)
                        })
                        .collect();

                    Ok::<_, Error>((vec![(path, Entry::Tree(tree))], recurse))
                }
            }
        }
        .boxed()
    })
    .map_ok(|entries| stream::iter(entries.into_iter().map(Ok::<_, Error>)))
    .try_flatten()
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::BTreeMap;
use std::collections::HashMap;
use std::sync::Arc;

use anyhow::format_err;
use anyhow::Error;
use blobstore::Blobstore;
use blobstore::Loadable;
use borrowed::borrowed;
use cloned::cloned;
use context::CoreContext;
use derived_data_manager::DerivationContext;
use futures::channel::mpsc;
use futures::future::try_join_all;
use futures::future::BoxFuture;
use futures::future::FutureExt as NewFutureExt;
use futures::future::TryFutureExt;
use futures::future::{self as new_future};
use manifest::derive_manifest_with_io_sender;
use manifest::derive_manifests_for_simple_stack_of_commits;
use manifest::Entry;
use manifest::LeafInfo;
use manifest::ManifestChanges;
use manifest::TreeInfo;
use metaconfig_types::UnodeVersion;
use mononoke_types::unode::FileUnode;
use mononoke_types::unode::ManifestUnode;
use mononoke_types::unode::UnodeEntry;
use mononoke_types::BlobstoreKey;
use mononoke_types::BlobstoreValue;
use mononoke_types::ChangesetId;
use mononoke_types::ContentId;
use mononoke_types::FileType;
use mononoke_types::FileUnodeId;
use mononoke_types::MPath;
use mononoke_types::MPathElement;
use mononoke_types::MPathHash;
use mononoke_types::ManifestUnodeId;
use sorted_vector_map::SortedVectorMap;

use crate::ErrorKind;

pub(crate) async fn derive_unode_manifest_stack(
    ctx: &CoreContext,
    derivation_ctx: &DerivationContext,
    file_changes: Vec<(ChangesetId, BTreeMap<MPath, Option<(ContentId, FileType)>>)>,
    parent: Option<ManifestUnodeId>,
    unode_version: UnodeVersion,
) -> Result<HashMap<ChangesetId, ManifestUnodeId>, Error> {
    let blobstore = derivation_ctx.blobstore();

    let manifest_changes = file_changes
        .into_iter()
        .map(|(cs_id, file_changes)| ManifestChanges {
            cs_id,
            changes: file_changes.into_iter().collect(),
        })
        .collect::<Vec<_>>();

    let res = derive_manifests_for_simple_stack_of_commits(
        ctx.clone(),
        blobstore.clone(),
        parent,
        manifest_changes,
        {
            cloned!(ctx, blobstore);
            move |tree_info, cs_id| {
                create_unode_manifest(
                    ctx.clone(),
                    cs_id,
                    blobstore.clone(),
                    None,
                    tree_info,
                    unode_version,
                )
            }
        },
        {
            cloned!(ctx, blobstore);
            move |leaf_info, cs_id| {
                create_unode_file(
                    ctx.clone(),
                    cs_id,
                    blobstore.clone(),
                    None,
                    leaf_info,
                    unode_version,
                )
            }
        },
    )
    .await?;

    Ok(res.into_iter().collect())
}

/// Derives unode manifests for bonsai changeset `cs_id` given parent unode manifests.
/// Note that `derive_manifest()` does a lot of the heavy lifting for us, and this crate has to
/// provide only functions to create a single unode file or single unode tree (
/// `create_unode_manifest` and `create_unode_file`).
pub(crate) async fn derive_unode_manifest(
    ctx: &CoreContext,
    derivation_ctx: &DerivationContext,
    cs_id: ChangesetId,
    parents: Vec<ManifestUnodeId>,
    changes: Vec<(MPath, Option<(ContentId, FileType)>)>,
    unode_version: UnodeVersion,
) -> Result<ManifestUnodeId, Error> {
    let parents: Vec<_> = parents.into_iter().collect();
    let blobstore = derivation_ctx.blobstore();

    let maybe_tree_id = derive_manifest_with_io_sender(
        ctx.clone(),
        blobstore.clone(),
        parents.clone(),
        changes,
        {
            cloned!(ctx, blobstore);
            move |tree_info, sender| {
                create_unode_manifest(
                    ctx.clone(),
                    cs_id,
                    blobstore.clone(),
                    Some(sender),
                    tree_info,
                    unode_version,
                )
            }
        },
        {
            cloned!(ctx, blobstore);
            move |leaf_info, sender| {
                create_unode_file(
                    ctx.clone(),
                    cs_id,
                    blobstore.clone(),
                    Some(sender),
                    leaf_info,
                    unode_version,
                )
            }
        },
    )
    .await?;

    match maybe_tree_id {
        Some(tree_id) => Ok(tree_id),
        None => {
            // All files have been deleted, generate empty **root** manifest
            let tree_info = TreeInfo {
                path: None,
                parents,
                subentries: Default::default(),
            };
            let ((), tree_id) = create_unode_manifest(
                ctx.clone(),
                cs_id,
                blobstore.clone(),
                None,
                tree_info,
                unode_version,
            )
            .await?;
            Ok(tree_id)
        }
    }
}

// Note that in some rare cases it's possible to have unode where one parent is ancestor of another
// (that applies both to files and directories)
//
//  Example:
//       4 o <- merge commit modifies file 'A'
//        / \
//     2 o ---> changed file 'A' content to 'B'
//       |   |
//       | 3 o -> changed some other file
//       \  /
//      1 o  <- created file 'A' with content 'A'
//
// In that case unode for file 'A' in a merge commit will have two parents - from commit '2' and
// from commit '1', and unode from commit '1' is ancestor of unode from commit '2'.
// Case like that might create slight confusion, however it should be rare and we should be
// able to fix it in the ui.
async fn create_unode_manifest(
    ctx: CoreContext,
    linknode: ChangesetId,
    blobstore: Arc<dyn Blobstore>,
    sender: Option<mpsc::UnboundedSender<BoxFuture<'static, Result<(), Error>>>>,
    tree_info: TreeInfo<ManifestUnodeId, FileUnodeId, ()>,
    unode_version: UnodeVersion,
) -> Result<((), ManifestUnodeId), Error> {
    let mut subentries = SortedVectorMap::new();
    for (basename, (_context, entry)) in tree_info.subentries {
        match entry {
            Entry::Tree(mf_unode) => {
                subentries.insert(basename, UnodeEntry::Directory(mf_unode));
            }
            Entry::Leaf(file_unode) => {
                subentries.insert(basename, UnodeEntry::File(file_unode));
            }
        }
    }
    if can_reuse(unode_version) && tree_info.parents.len() > 1 {
        if let Some(mf_unode_id) =
            reuse_manifest_parent(&ctx, &blobstore, &tree_info.parents, &subentries).await?
        {
            return Ok(((), mf_unode_id));
        }
    }

    let mf_unode = ManifestUnode::new(tree_info.parents, subentries, linknode);
    let mf_unode_id = mf_unode.get_unode_id();

    let key = mf_unode_id.blobstore_key();
    let blob = mf_unode.into_blob();
    let f = async move { blobstore.put(&ctx, key, blob.into()).await };

    match sender {
        Some(sender) => sender
            .unbounded_send(f.boxed())
            .map_err(|err| format_err!("failed to send manifest future {}", err))?,
        None => f.await?,
    };
    Ok(((), mf_unode_id))
}

async fn create_unode_file(
    ctx: CoreContext,
    linknode: ChangesetId,
    blobstore: Arc<dyn Blobstore>,
    sender: Option<mpsc::UnboundedSender<BoxFuture<'static, Result<(), Error>>>>,
    leaf_info: LeafInfo<FileUnodeId, (ContentId, FileType)>,
    unode_version: UnodeVersion,
) -> Result<((), FileUnodeId), Error> {
    borrowed!(ctx, blobstore);

    async fn save_unode(
        ctx: &CoreContext,
        blobstore: &Arc<dyn Blobstore>,
        sender: Option<mpsc::UnboundedSender<BoxFuture<'static, Result<(), Error>>>>,
        parents: Vec<FileUnodeId>,
        content_id: ContentId,
        file_type: FileType,
        path_hash: MPathHash,
        linknode: ChangesetId,
        unode_version: UnodeVersion,
    ) -> Result<FileUnodeId, Error> {
        if can_reuse(unode_version) && parents.len() > 1 {
            if let Some(parent) =
                reuse_file_parent(ctx, blobstore, &parents, &content_id, file_type).await?
            {
                return Ok(parent);
            }
        };
        let file_unode = FileUnode::new(parents, content_id, file_type, path_hash, linknode);
        let file_unode_id = file_unode.get_unode_id();

        let f = {
            cloned!(ctx, blobstore);
            async move {
                blobstore
                    .put(
                        &ctx,
                        file_unode_id.blobstore_key(),
                        file_unode.into_blob().into(),
                    )
                    .await
            }
            .boxed()
        };

        match sender {
            Some(sender) => {
                sender
                    .unbounded_send(f)
                    .map_err(|err| format_err!("failed to send manifest future {}", err))?;
            }
            None => f.await?,
        };
        Ok(file_unode_id)
    }

    let LeafInfo {
        leaf,
        path,
        parents,
    } = leaf_info;

    let leaf_id = if let Some((content_id, file_type)) = leaf {
        save_unode(
            ctx,
            blobstore,
            sender,
            parents,
            content_id,
            file_type,
            path.get_path_hash(),
            linknode,
            unode_version,
        )
        .await?
    } else {
        // We can end up in this codepath if there are at least 2 parent commits have a unode with
        // this file path, and these unodes are different, but current bonsai changeset have no
        // changes for this file path.
        //
        //  Example:
        //         o <- merge commit, it doesn't modify any of the files
        //        / \
        //       o ---> changed file 'A' content to 'B'
        //       |  |
        //       |   o -> changed file 'A content to 'B' as well
        //       \  /
        //        o  <- created file 'A' with content 'A'
        //
        // In that case we need to check file content and file type.
        // if they are the same then we need to create a new file unode.
        //
        // If content or file type are different then we need to return an error

        // Note that there's a difference from how we handle this case in mercurial manifests.
        // In mercurial manifests we compare file content with copy information, while in unodes
        // copy information is ignored. It might mean that some bonsai changesets would be
        // considered valid for unode manifests, but invalid for mercurial
        if parents.len() < 2 {
            return Err(ErrorKind::InvalidBonsai(
                "no change is provided, but file unode has only one parent".to_string(),
            )
            .into());
        }
        let parent_unodes = try_join_all(
            parents
                .clone()
                .into_iter()
                .map(|id| async move { id.load(ctx, blobstore).await }),
        )
        .await?;

        match return_if_unique_filenode(&parent_unodes) {
            Some((content_id, file_type)) => {
                save_unode(
                    ctx,
                    blobstore,
                    sender,
                    parents,
                    content_id.clone(),
                    *file_type,
                    path.get_path_hash(),
                    linknode,
                    unode_version,
                )
                .await?
            }
            _ => {
                return Err(ErrorKind::InvalidBonsai(
                    "no change is provided, but content is different".to_string(),
                )
                .into());
            }
        }
    };

    Ok(((), leaf_id))
}

// reuse_manifest_parent() and reuse_file_parent() are used in unodes v2 in order to avoid
// creating unnecessary unodes that were created in unodes v1. Consider this graph with a diamond merge:
//
//   O <- merge commit, doesn't change anything
//  / \
// P1  |  <- modified "file.txt"
// |   P2    <- created "other.txt"
// \  /
//  ROOT <- created "file.txt"
//
// In unodes v1 merge commit would contain a unode for "file.txt" which points to "file.txt"
// unode from P1 and and "file.txt" unode from ROOT. This is
// actually quite unexpected - merge commit didn't touch "file.txt" at all, but "file.txt"'s history
// would contain a merge commit.
//
// reuse_manifest_parent() and reuse_file_parent() are heuristics that help fix this problem.
// They check if content of the unode in the merge commit is the same as the content of unode
// in one of the parents. If yes, then this unode is reused. Note - it actually might also lead
// to surprising results in some cases:
//
//   O <- merge commit, doesn't change anything
//  / \
// P1  |  <- modified "file.txt" to "B"
// |   P2    <- modified "file.txt" to "B"
// \  /
//  ROOT <- created "file.txt" with content "A"
//
// In that case it would be ideal to preserve that "file.txt" was modified in both commits.
// But we consider this case is rare enough to not worry about it.

fn can_reuse(unode_version: UnodeVersion) -> bool {
    unode_version == UnodeVersion::V2 || tunables::tunables().force_unode_v2().unwrap_or_default()
}

async fn reuse_manifest_parent(
    ctx: &CoreContext,
    blobstore: &Arc<dyn Blobstore>,
    parents: &[ManifestUnodeId],
    subentries: &SortedVectorMap<MPathElement, UnodeEntry>,
) -> Result<Option<ManifestUnodeId>, Error> {
    let parents = new_future::try_join_all(
        parents
            .iter()
            .map(|id| id.load(ctx, blobstore).map_err(Error::from)),
    )
    .await?;

    if let Some(mf_unode) = parents.iter().find(|p| p.subentries() == subentries) {
        Ok(Some(mf_unode.get_unode_id()))
    } else {
        Ok(None)
    }
}

async fn reuse_file_parent(
    ctx: &CoreContext,
    blobstore: &Arc<dyn Blobstore>,
    parents: &[FileUnodeId],
    content_id: &ContentId,
    file_type: FileType,
) -> Result<Option<FileUnodeId>, Error> {
    let parents = new_future::try_join_all(
        parents
            .iter()
            .map(|id| id.load(ctx, blobstore).map_err(Error::from)),
    )
    .await?;

    if let Some(file_unode) = parents
        .iter()
        .find(|p| p.content_id() == content_id && p.file_type() == &file_type)
    {
        Ok(Some(file_unode.get_unode_id()))
    } else {
        Ok(None)
    }
}

// If all elements in `unodes` are the same than this element is returned, otherwise None is returned
fn return_if_unique_filenode(unodes: &[FileUnode]) -> Option<(&ContentId, &FileType)> {
    let mut iter = unodes
        .iter()
        .map(|elem| (elem.content_id(), elem.file_type()));
    let first_elem = iter.next()?;
    if iter.all(|next_elem| next_elem == first_elem) {
        Some(first_elem)
    } else {
        None
    }
}

#[cfg(test)]
mod tests {
    use std::collections::BTreeMap;
    use std::collections::HashSet;
    use std::collections::VecDeque;

    use anyhow::Result;
    use async_trait::async_trait;
    use blobrepo::save_bonsai_changesets;
    use blobstore::Storable;
    use bytes::Bytes;
    use derived_data::BonsaiDerived;
    use derived_data_test_utils::bonsai_changeset_from_hg;
    use derived_data_test_utils::iterate_all_manifest_entries;
    use fbinit::FacebookInit;
    use filenodes_derivation::FilenodesOnlyPublic;
    use fixtures::Linear;
    use fixtures::TestRepoFixture;
    use futures::TryStreamExt;
    use manifest::ManifestOps;
    use maplit::btreemap;
    use mercurial_derivation::DeriveHgChangeset;
    use mercurial_types::blobs::HgBlobManifest;
    use mercurial_types::HgFileNodeId;
    use mercurial_types::HgManifestId;
    use mononoke_types::BlobstoreValue;
    use mononoke_types::BonsaiChangeset;
    use mononoke_types::BonsaiChangesetMut;
    use mononoke_types::DateTime;
    use mononoke_types::FileChange;
    use mononoke_types::FileContents;
    use mononoke_types::RepoPath;
    use repo_derived_data::RepoDerivedDataRef;
    use test_repo_factory::TestRepoFactory;
    use tests_utils::resolve_cs_id;
    use tests_utils::CreateCommitContext;

    use super::*;
    use crate::mapping::get_file_changes;
    use crate::mapping::RootUnodeManifestId;
    use crate::tests::TestRepo;

    #[fbinit::test]
    async fn linear_test(fb: FacebookInit) -> Result<(), Error> {
        let repo: TestRepo = Linear::get_custom_test_repo(fb).await;
        let derivation_ctx = repo.repo_derived_data().manager().derivation_context(None);
        let ctx = CoreContext::test_mock(fb);

        // Derive filenodes because they are going to be used in this test
        let master_cs_id = resolve_cs_id(&ctx, &repo, "master").await?;
        FilenodesOnlyPublic::derive(&ctx, &repo, master_cs_id).await?;

        let parent_unode_id = {
            let parent_hg_cs = "2d7d4ba9ce0a6ffd222de7785b249ead9c51c536";
            let (bcs_id, bcs) = bonsai_changeset_from_hg(&ctx, &repo, parent_hg_cs).await?;
            let unode_id = derive_unode_manifest(
                &ctx,
                &derivation_ctx,
                bcs_id,
                vec![],
                get_file_changes(&bcs),
                UnodeVersion::V2,
            )
            .await?;

            // Make sure it's saved in the blobstore
            unode_id.load(&ctx, &repo.repo_blobstore).await?;
            let all_unodes: Vec<_> =
                iterate_all_manifest_entries(&ctx, &repo, Entry::Tree(unode_id))
                    .try_collect()
                    .await?;
            let mut paths: Vec<_> = all_unodes.into_iter().map(|(path, _)| path).collect();
            paths.sort();
            assert_eq!(
                paths,
                vec![
                    None,
                    Some(MPath::new("1").unwrap()),
                    Some(MPath::new("files").unwrap())
                ]
            );
            unode_id
        };

        {
            let child_hg_cs = "3e0e761030db6e479a7fb58b12881883f9f8c63f";
            let (bcs_id, bcs) = bonsai_changeset_from_hg(&ctx, &repo, child_hg_cs).await?;

            let unode_id = derive_unode_manifest(
                &ctx,
                &derivation_ctx,
                bcs_id,
                vec![parent_unode_id.clone()],
                get_file_changes(&bcs),
                UnodeVersion::V2,
            )
            .await?;

            // Make sure it's saved in the blobstore
            let root_unode = unode_id.load(&ctx, &repo.repo_blobstore).await?;
            assert_eq!(root_unode.parents(), &vec![parent_unode_id]);

            let root_filenode_id = fetch_root_filenode_id(fb, repo.clone(), bcs_id).await?;
            assert_eq!(
                find_unode_history(fb, repo.clone(), UnodeEntry::Directory(unode_id)).await?,
                find_filenode_history(fb, repo.clone(), root_filenode_id).await?,
            );

            let all_unodes: Vec<_> =
                iterate_all_manifest_entries(&ctx, &repo, Entry::Tree(unode_id))
                    .try_collect()
                    .await?;
            let mut paths: Vec<_> = all_unodes.into_iter().map(|(path, _)| path).collect();
            paths.sort();
            assert_eq!(
                paths,
                vec![
                    None,
                    Some(MPath::new("1").unwrap()),
                    Some(MPath::new("2").unwrap()),
                    Some(MPath::new("files").unwrap())
                ]
            );
        }
        Ok(())
    }

    #[fbinit::test]
    async fn test_same_content_different_paths(fb: FacebookInit) -> Result<(), Error> {
        let repo: TestRepo = Linear::get_custom_test_repo(fb).await;
        let ctx = CoreContext::test_mock(fb);

        async fn check_unode_uniqeness(
            ctx: CoreContext,
            repo: TestRepo,
            file_changes: BTreeMap<MPath, FileChange>,
        ) -> Result<(), Error> {
            let derivation_ctx = repo.repo_derived_data().manager().derivation_context(None);
            let bcs = create_bonsai_changeset(ctx.fb, repo.clone(), file_changes).await?;
            let bcs_id = bcs.get_changeset_id();

            let unode_id = derive_unode_manifest(
                &ctx,
                &derivation_ctx,
                bcs_id,
                vec![],
                get_file_changes(&bcs),
                UnodeVersion::V2,
            )
            .await?;
            let unode_mf = unode_id.load(&ctx, &repo.repo_blobstore).await?;

            // Unodes should be unique even if content is the same. Check it
            let vals: Vec<_> = unode_mf.list().collect();
            assert_eq!(vals.len(), 2);
            assert_ne!(vals.get(0), vals.get(1));
            Ok(())
        }

        let file_changes = store_files(
            ctx.clone(),
            btreemap! {"file1" => Some(("content", FileType::Regular)), "file2" => Some(("content", FileType::Regular))},
            repo.clone(),
        ).await?;
        check_unode_uniqeness(ctx.clone(), repo.clone(), file_changes).await?;

        let file_changes = store_files(
            ctx.clone(),
            btreemap! {"dir1/file" => Some(("content", FileType::Regular)), "dir2/file" => Some(("content", FileType::Regular))},
            repo.clone(),
        ).await?;
        check_unode_uniqeness(ctx.clone(), repo.clone(), file_changes).await?;
        Ok(())
    }

    #[fbinit::test]
    async fn test_same_content_no_change(fb: FacebookInit) -> Result<(), Error> {
        let repo: TestRepo = Linear::get_custom_test_repo(fb).await;
        let ctx = CoreContext::test_mock(fb);

        build_diamond_graph(
            ctx.clone(),
            repo.clone(),
            btreemap! {"A" => Some(("A", FileType::Regular))},
            btreemap! {"A" => Some(("B", FileType::Regular))},
            btreemap! {"A" => Some(("B", FileType::Regular))},
            btreemap! {},
        )
        .await?;

        // Content is different - fail!
        assert!(
            build_diamond_graph(
                ctx.clone(),
                repo.clone(),
                btreemap! {"A" => Some(("A", FileType::Regular))},
                btreemap! {"A" => Some(("B", FileType::Regular))},
                btreemap! {"A" => Some(("C", FileType::Regular))},
                btreemap! {},
            )
            .await
            .is_err()
        );

        // Type is different - fail!
        assert!(
            build_diamond_graph(
                ctx,
                repo,
                btreemap! {"A" => Some(("A", FileType::Regular))},
                btreemap! {"A" => Some(("B", FileType::Regular))},
                btreemap! {"A" => Some(("B", FileType::Executable))},
                btreemap! {},
            )
            .await
            .is_err()
        );

        Ok(())
    }

    async fn diamond_merge_unodes_v2(fb: FacebookInit) -> Result<(), Error> {
        let ctx = CoreContext::test_mock(fb);
        let factory = TestRepoFactory::new(fb)?;
        let repo: TestRepo = factory.build().await?;
        let merged_files = "dir/file.txt";
        let root_commit = CreateCommitContext::new_root(&ctx, &repo)
            .add_file(merged_files, "a")
            .commit()
            .await?;

        let p1 = CreateCommitContext::new(&ctx, &repo, vec![root_commit])
            .add_file("p1.txt", "p1")
            .add_file(merged_files, "b")
            .commit()
            .await?;

        let p2 = CreateCommitContext::new(&ctx, &repo, vec![root_commit])
            .add_file("p2.txt", "p2")
            .commit()
            .await?;

        let merge = CreateCommitContext::new(&ctx, &repo, vec![p1, p2])
            .add_file(merged_files, "b")
            .commit()
            .await?;

        let find_unodes = {
            |ctx: CoreContext, repo: TestRepo| async move {
                let p1_root_unode_mf_id = RootUnodeManifestId::derive(&ctx, &repo, p1).await?;

                let mut p1_unodes: Vec<_> = p1_root_unode_mf_id
                    .manifest_unode_id()
                    .find_entries(
                        ctx.clone(),
                        repo.repo_blobstore.clone(),
                        vec![Some(MPath::new(merged_files)?), Some(MPath::new("dir")?)],
                        // Some(MPath::new(&merged_files)?),
                    )
                    .try_collect()
                    .await?;
                p1_unodes.sort_by_key(|(path, _)| path.clone());

                let merge_root_unode_mf_id =
                    RootUnodeManifestId::derive(&ctx, &repo, merge).await?;

                let mut merge_unodes: Vec<_> = merge_root_unode_mf_id
                    .manifest_unode_id()
                    .find_entries(
                        ctx.clone(),
                        repo.repo_blobstore.clone(),
                        vec![Some(MPath::new(merged_files)?), Some(MPath::new("dir")?)],
                    )
                    .try_collect()
                    .await?;
                merge_unodes.sort_by_key(|(path, _)| path.clone());

                let res: Result<_, Error> = Ok((p1_unodes, merge_unodes));
                res
            }
        };

        // Unodes v2 should just reuse merged filenode
        let (p1_unodes, merge_unodes) = find_unodes(ctx.clone(), repo.clone()).await?;
        assert_eq!(p1_unodes, merge_unodes);
        Ok(())
    }

    #[fbinit::test]
    async fn test_diamond_merge_unodes_v2(fb: FacebookInit) -> Result<(), Error> {
        diamond_merge_unodes_v2(fb).await
    }

    #[fbinit::test]
    async fn test_parent_order(fb: FacebookInit) -> Result<(), Error> {
        let repo: TestRepo = test_repo_factory::build_empty(fb).await.unwrap();
        let derivation_ctx = repo.repo_derived_data().manager().derivation_context(None);
        let ctx = CoreContext::test_mock(fb);

        let p1_root_unode_id = create_changeset_and_derive_unode(
            ctx.clone(),
            repo.clone(),
            btreemap! {"A" => Some(("A", FileType::Regular))},
        )
        .await?;

        let p2_root_unode_id = create_changeset_and_derive_unode(
            ctx.clone(),
            repo.clone(),
            btreemap! {"A" => Some(("B", FileType::Regular))},
        )
        .await?;

        let file_changes = store_files(
            ctx.clone(),
            btreemap! { "A" => Some(("C", FileType::Regular)) },
            repo.clone(),
        )
        .await?;
        let bcs = create_bonsai_changeset(fb, repo.clone(), file_changes).await?;
        let bcs_id = bcs.get_changeset_id();

        let root_unode = derive_unode_manifest(
            &ctx,
            &derivation_ctx,
            bcs_id,
            vec![p1_root_unode_id, p2_root_unode_id],
            get_file_changes(&bcs),
            UnodeVersion::V2,
        )
        .await?;

        // Make sure hash is the same if nothing was changed
        let same_root_unode = derive_unode_manifest(
            &ctx,
            &derivation_ctx,
            bcs_id,
            vec![p1_root_unode_id, p2_root_unode_id],
            get_file_changes(&bcs),
            UnodeVersion::V2,
        )
        .await?;
        assert_eq!(root_unode, same_root_unode);

        // Now change parent order, make sure hashes are different
        let reverse_root_unode = derive_unode_manifest(
            &ctx,
            &derivation_ctx,
            bcs_id,
            vec![p2_root_unode_id, p1_root_unode_id],
            get_file_changes(&bcs),
            UnodeVersion::V2,
        )
        .await?;

        assert_ne!(root_unode, reverse_root_unode);

        Ok(())
    }

    async fn create_changeset_and_derive_unode(
        ctx: CoreContext,
        repo: TestRepo,
        file_changes: BTreeMap<&str, Option<(&str, FileType)>>,
    ) -> Result<ManifestUnodeId, Error> {
        let file_changes = store_files(ctx.clone(), file_changes, repo.clone()).await?;
        let bcs = create_bonsai_changeset(ctx.fb, repo.clone(), file_changes).await?;

        let bcs_id = bcs.get_changeset_id();
        let derivation_ctx = repo.repo_derived_data().manager().derivation_context(None);

        derive_unode_manifest(
            &ctx,
            &derivation_ctx,
            bcs_id,
            vec![],
            get_file_changes(&bcs),
            UnodeVersion::V2,
        )
        .await
    }

    async fn build_diamond_graph(
        ctx: CoreContext,
        repo: TestRepo,
        changes_first: BTreeMap<&str, Option<(&str, FileType)>>,
        changes_merge_p1: BTreeMap<&str, Option<(&str, FileType)>>,
        changes_merge_p2: BTreeMap<&str, Option<(&str, FileType)>>,
        changes_merge: BTreeMap<&str, Option<(&str, FileType)>>,
    ) -> Result<ManifestUnodeId> {
        let derivation_ctx = repo.repo_derived_data().manager().derivation_context(None);
        let file_changes = store_files(ctx.clone(), changes_first, repo.clone()).await?;

        let bcs = create_bonsai_changeset(ctx.fb, repo.clone(), file_changes).await?;
        let first_bcs_id = bcs.get_changeset_id();

        let first_unode_id = derive_unode_manifest(
            &ctx,
            &derivation_ctx,
            first_bcs_id,
            vec![],
            get_file_changes(&bcs),
            UnodeVersion::V2,
        )
        .await?;

        let (merge_p1, merge_p1_unode_id) = {
            let file_changes = store_files(ctx.clone(), changes_merge_p1, repo.clone()).await?;
            let merge_p1 = create_bonsai_changeset_with_params(
                ctx.fb,
                repo.clone(),
                file_changes.clone(),
                "merge_p1",
                vec![first_bcs_id.clone()],
            )
            .await?;
            let merge_p1_id = merge_p1.get_changeset_id();
            let merge_p1_unode_id = derive_unode_manifest(
                &ctx,
                &derivation_ctx,
                merge_p1_id,
                vec![first_unode_id.clone()],
                get_file_changes(&merge_p1),
                UnodeVersion::V2,
            )
            .await?;
            (merge_p1, merge_p1_unode_id)
        };

        let (merge_p2, merge_p2_unode_id) = {
            let file_changes = store_files(ctx.clone(), changes_merge_p2, repo.clone()).await?;

            let merge_p2 = create_bonsai_changeset_with_params(
                ctx.fb,
                repo.clone(),
                file_changes,
                "merge_p2",
                vec![first_bcs_id.clone()],
            )
            .await?;
            let merge_p2_id = merge_p2.get_changeset_id();
            let merge_p2_unode_id = derive_unode_manifest(
                &ctx,
                &derivation_ctx,
                merge_p2_id,
                vec![first_unode_id.clone()],
                get_file_changes(&merge_p2),
                UnodeVersion::V2,
            )
            .await?;
            (merge_p2, merge_p2_unode_id)
        };

        let file_changes = store_files(ctx.clone(), changes_merge, repo.clone()).await?;
        let merge = create_bonsai_changeset_with_params(
            ctx.fb,
            repo.clone(),
            file_changes,
            "merge",
            vec![merge_p1.get_changeset_id(), merge_p2.get_changeset_id()],
        )
        .await?;
        let merge_id = merge.get_changeset_id();
        derive_unode_manifest(
            &ctx,
            &derivation_ctx,
            merge_id,
            vec![merge_p1_unode_id, merge_p2_unode_id],
            get_file_changes(&merge),
            UnodeVersion::V2,
        )
        .await
    }

    async fn create_bonsai_changeset(
        fb: FacebookInit,
        repo: TestRepo,
        file_changes: BTreeMap<MPath, FileChange>,
    ) -> Result<BonsaiChangeset, Error> {
        create_bonsai_changeset_with_params(fb, repo, file_changes, "message", vec![]).await
    }

    async fn create_bonsai_changeset_with_params(
        fb: FacebookInit,
        repo: TestRepo,
        file_changes: BTreeMap<MPath, FileChange>,
        message: &str,
        parents: Vec<ChangesetId>,
    ) -> Result<BonsaiChangeset, Error> {
        let bcs = BonsaiChangesetMut {
            parents,
            author: "author".to_string(),
            author_date: DateTime::now(),
            committer: None,
            committer_date: None,
            message: message.to_string(),
            hg_extra: Default::default(),
            git_extra_headers: None,
            git_tree_hash: None,
            file_changes: file_changes.into(),
            is_snapshot: false,
            git_annotated_tag: None,
        }
        .freeze()
        .unwrap();

        save_bonsai_changesets(vec![bcs.clone()], CoreContext::test_mock(fb), &repo).await?;
        Ok(bcs)
    }

    async fn store_files(
        ctx: CoreContext,
        files: BTreeMap<&str, Option<(&str, FileType)>>,
        repo: TestRepo,
    ) -> Result<BTreeMap<MPath, FileChange>, Error> {
        let mut res = btreemap! {};

        for (path, content) in files {
            let path = MPath::new(path).unwrap();
            match content {
                Some((content, file_type)) => {
                    let size = content.len();
                    let content =
                        FileContents::Bytes(Bytes::copy_from_slice(content.as_bytes())).into_blob();
                    let content_id = content.store(&ctx, &repo.repo_blobstore).await?;
                    let file_change = FileChange::tracked(content_id, file_type, size as u64, None);
                    res.insert(path, file_change);
                }
                None => {
                    res.insert(path, FileChange::Deletion);
                }
            }
        }
        Ok(res)
    }

    #[async_trait]
    trait UnodeHistory {
        async fn get_parents<'a>(
            &'a self,
            ctx: &'a CoreContext,
            repo: &'a TestRepo,
        ) -> Result<Vec<UnodeEntry>, Error>;

        async fn get_linknode<'a>(
            &'a self,
            ctx: &'a CoreContext,
            repo: &'a TestRepo,
        ) -> Result<ChangesetId, Error>;
    }

    #[async_trait]
    impl UnodeHistory for UnodeEntry {
        async fn get_parents<'a>(
            &'a self,
            ctx: &'a CoreContext,
            repo: &'a TestRepo,
        ) -> Result<Vec<UnodeEntry>, Error> {
            match self {
                UnodeEntry::File(file_unode_id) => {
                    let unode_mf = file_unode_id.load(ctx, &repo.repo_blobstore).await?;
                    Ok(unode_mf
                        .parents()
                        .iter()
                        .cloned()
                        .map(UnodeEntry::File)
                        .collect())
                }
                UnodeEntry::Directory(mf_unode_id) => {
                    let unode_mf = mf_unode_id.load(ctx, &repo.repo_blobstore).await?;
                    Ok(unode_mf
                        .parents()
                        .iter()
                        .cloned()
                        .map(UnodeEntry::Directory)
                        .collect())
                }
            }
        }

        async fn get_linknode<'a>(
            &'a self,
            ctx: &'a CoreContext,
            repo: &'a TestRepo,
        ) -> Result<ChangesetId, Error> {
            match self {
                UnodeEntry::File(file_unode_id) => {
                    let unode_file = file_unode_id.load(ctx, &repo.repo_blobstore).await?;
                    Ok(unode_file.linknode().clone())
                }
                UnodeEntry::Directory(mf_unode_id) => {
                    let unode_mf = mf_unode_id.load(ctx, &repo.repo_blobstore).await?;
                    Ok(unode_mf.linknode().clone())
                }
            }
        }
    }

    async fn find_unode_history(
        fb: FacebookInit,
        repo: TestRepo,
        start: UnodeEntry,
    ) -> Result<Vec<ChangesetId>, Error> {
        let ctx = CoreContext::test_mock(fb);
        let mut q = VecDeque::new();
        q.push_back(start.clone());

        let mut visited = HashSet::new();
        visited.insert(start);
        let mut history = vec![];
        loop {
            let unode_entry = q.pop_front();
            let unode_entry = match unode_entry {
                Some(unode_entry) => unode_entry,
                None => {
                    break;
                }
            };
            let linknode = unode_entry.get_linknode(&ctx, &repo).await?;
            history.push(linknode);
            let parents = unode_entry.get_parents(&ctx, &repo).await?;
            q.extend(parents.into_iter().filter(|x| visited.insert(x.clone())));
        }

        Ok(history)
    }

    async fn find_filenode_history(
        fb: FacebookInit,
        repo: TestRepo,
        start: HgFileNodeId,
    ) -> Result<Vec<ChangesetId>, Error> {
        let ctx = CoreContext::test_mock(fb);

        let mut q = VecDeque::new();
        q.push_back(start);

        let mut visited = HashSet::new();
        visited.insert(start);
        let mut history = vec![];
        loop {
            let filenode_id = q.pop_front();
            let filenode_id = match filenode_id {
                Some(filenode_id) => filenode_id,
                None => {
                    break;
                }
            };

            let hg_linknode = repo
                .filenodes
                .get_filenode(&ctx.clone(), &RepoPath::RootPath, filenode_id)
                .await?
                .do_not_handle_disabled_filenodes()?
                .map(|filenode| filenode.linknode)
                .unwrap();

            let linknode = repo
                .bonsai_hg_mapping
                .get_bonsai_from_hg(&ctx, hg_linknode)
                .await?
                .unwrap();
            history.push(linknode);

            let mf = HgBlobManifest::load(
                &ctx,
                &repo.repo_blobstore,
                HgManifestId::new(filenode_id.into_nodehash()),
            )
            .await?
            .unwrap();

            q.extend(
                mf.p1()
                    .into_iter()
                    .map(HgFileNodeId::new)
                    .filter(|x| visited.insert(*x)),
            );
            q.extend(
                mf.p2()
                    .into_iter()
                    .map(HgFileNodeId::new)
                    .filter(|x| visited.insert(*x)),
            );
        }

        Ok(history)
    }

    async fn fetch_root_filenode_id(
        fb: FacebookInit,
        repo: TestRepo,
        bcs_id: ChangesetId,
    ) -> Result<HgFileNodeId, Error> {
        let ctx = CoreContext::test_mock(fb);
        let hg_cs_id = repo.derive_hg_changeset(&ctx, bcs_id).await?;
        let hg_cs = hg_cs_id.load(&ctx, &repo.repo_blobstore).await?;
        Ok(HgFileNodeId::new(hg_cs.manifestid().into_nodehash()))
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;

use anyhow::anyhow;
use anyhow::Error;
use anyhow::Result;
use cloned::cloned;
use context::CoreContext;
use derived_data_manager::DerivationContext;
use futures::future;
use futures::TryFutureExt;
use futures::TryStreamExt;
use manifest::ManifestOps;
use mononoke_types::BonsaiChangeset;
use mononoke_types::ChangesetId;
use mononoke_types::FileUnodeId;
use mononoke_types::MPath;
use thiserror::Error;

mod derive;
mod mapping;

pub use mapping::format_key;
pub use mapping::RootUnodeManifestId;

#[derive(Debug, Error)]
pub enum ErrorKind {
    #[error("Invalid bonsai changeset: {0}")]
    InvalidBonsai(String),
}

/// A rename source for a file that is renamed.
#[derive(Debug, Clone)]
pub struct UnodeRenameSource {
    /// Index of the parent changeset in the list of parents in the bonsai
    /// changeset.
    pub parent_index: usize,

    /// Path of the file in the parent changeset (i.e., the path it was
    /// renamed from).
    pub from_path: MPath,

    /// Unode ID of the file in the parent changeset.
    pub unode_id: FileUnodeId,
}

/// Given a bonsai changeset, find sources for all of the renames that
/// happened in this changeset.
///
/// Returns a mapping from paths in the current changeset to the source of the
/// rename in the parent changesets.
pub async fn find_unode_rename_sources(
    ctx: &CoreContext,
    derivation_ctx: &DerivationContext,
    bonsai: &BonsaiChangeset,
) -> Result<HashMap<MPath, UnodeRenameSource>, Error> {
    // Collect together a map of (source_path -> [dest_paths]) for each parent
    // changeset.
    let mut references: HashMap<ChangesetId, HashMap<&MPath, Vec<&MPath>>> = HashMap::new();
    for (to_path, file_change) in bonsai.file_changes() {
        if let Some((from_path, csid)) = file_change.copy_from() {
            references
                .entry(*csid)
                .or_insert_with(HashMap::new)
                .entry(from_path)
                .or_insert_with(Vec::new)
                .push(to_path);
        }
    }

    let blobstore = derivation_ctx.blobstore();
    let sources_futs = references.into_iter().map(|(csid, mut paths)| {
        cloned!(blobstore);
        async move {
            let parent_index = bonsai.parents().position(|p| p == csid).ok_or_else(|| {
                anyhow!(
                    "bonsai changeset {} contains invalid copy from parent: {}",
                    bonsai.get_changeset_id(),
                    csid
                )
            })?;
            let mf_root = derivation_ctx
                .derive_dependency::<RootUnodeManifestId>(ctx, csid)
                .await?;
            let from_paths: Vec<_> = paths.keys().cloned().cloned().collect();
            let unodes = mf_root
                .manifest_unode_id()
                .find_entries(ctx.clone(), blobstore, from_paths)
                .try_collect::<Vec<_>>()
                .await?;

            let mut sources = Vec::new();
            for (from_path, entry) in unodes {
                if let (Some(from_path), Some(unode_id)) = (from_path, entry.into_leaf()) {
                    if let Some(to_paths) = paths.remove(&from_path) {
                        for to_path in to_paths {
                            sources.push((
                                to_path.clone(),
                                UnodeRenameSource {
                                    parent_index,
                                    from_path: from_path.clone(),
                                    unode_id,
                                },
                            ));
                        }
                    }
                }
            }
            Ok(sources)
        }
    });

    future::try_join_all(sources_futs)
        .map_ok(|unodes| unodes.into_iter().flatten().collect())
        .await
}

/// Given bonsai changeset find unodes for all renames that happened in this changesest.
///
/// Returns mapping from paths in current changeset to file unodes in parents changesets
/// that were coppied to a given path.
///
/// This version of the function is incorrect: it fails to take into account
/// files that are copied multiple times.  The function is retained for
/// blame_v1 compatibility.
pub async fn find_unode_renames_incorrect_for_blame_v1(
    ctx: &CoreContext,
    derivation_ctx: &DerivationContext,
    bonsai: &BonsaiChangeset,
) -> Result<HashMap<MPath, FileUnodeId>, Error> {
    let mut references: HashMap<ChangesetId, HashMap<MPath, MPath>> = HashMap::new();
    for (to_path, file_change) in bonsai.file_changes() {
        if let Some((from_path, csid)) = file_change.copy_from() {
            references
                .entry(*csid)
                .or_default()
                .insert(from_path.clone(), to_path.clone());
        }
    }

    let unodes = references.into_iter().map(|(csid, mut paths)| async move {
        let mf_root = derivation_ctx
            .derive_dependency::<RootUnodeManifestId>(ctx, csid)
            .await?;
        let from_paths: Vec<_> = paths.keys().cloned().collect();
        let blobstore = derivation_ctx.blobstore();
        mf_root
            .manifest_unode_id()
            .clone()
            .find_entries(ctx.clone(), blobstore.clone(), from_paths)
            .map_ok(|(from_path, entry)| Some((from_path?, entry.into_leaf()?)))
            .try_filter_map(future::ok)
            .try_collect::<Vec<_>>()
            .map_ok(move |unodes| {
                unodes
                    .into_iter()
                    .filter_map(|(from_path, unode_id)| Some((paths.remove(&from_path)?, unode_id)))
                    .collect::<HashMap<_, _>>()
            })
            .await
    });

    future::try_join_all(unodes)
        .map_ok(|unodes| unodes.into_iter().flatten().collect())
        .await
}

#[cfg(test)]
mod tests {
    use anyhow::Result;
    use blobstore::Loadable;
    use bonsai_hg_mapping::BonsaiHgMapping;
    use bookmarks::Bookmarks;
    use borrowed::borrowed;
    use changeset_fetcher::ChangesetFetcher;
    use changesets::Changesets;
    use context::CoreContext;
    use fbinit::FacebookInit;
    use filenodes::Filenodes;
    use filestore::FilestoreConfig;
    use mononoke_types::MPath;
    use repo_blobstore::RepoBlobstore;
    use repo_derived_data::RepoDerivedData;
    use repo_identity::RepoIdentity;
    use tests_utils::CreateCommitContext;

    #[derive(Clone)]
    #[facet::container]
    pub(crate) struct TestRepo {
        #[facet]
        pub(crate) bonsai_hg_mapping: dyn BonsaiHgMapping,
        #[facet]
        pub(crate) bookmarks: dyn Bookmarks,
        #[facet]
        pub(crate) repo_blobstore: RepoBlobstore,
        #[facet]
        pub(crate) repo_derived_data: RepoDerivedData,
        #[facet]
        pub(crate) filestore_config: FilestoreConfig,
        #[facet]
        pub(crate) changeset_fetcher: dyn ChangesetFetcher,
        #[facet]
        pub(crate) changesets: dyn Changesets,
        #[facet]
        pub(crate) filenodes: dyn Filenodes,
        #[facet]
        pub(crate) repo_identity: RepoIdentity,
    }

    #[fbinit::test]
    async fn test_find_unode_rename_sources(fb: FacebookInit) -> Result<()> {
        let ctx = CoreContext::test_mock(fb);
        let repo: TestRepo = test_repo_factory::build_empty(ctx.fb).await?;
        borrowed!(ctx, repo);

        let c1 = CreateCommitContext::new_root(ctx, repo)
            .add_file("file1", "content")
            .commit()
            .await?;
        let c2 = CreateCommitContext::new(ctx, repo, vec![c1])
            .add_file("file2", "content")
            .commit()
            .await?;
        let c3 = CreateCommitContext::new(ctx, repo, vec![c1])
            .add_file("file3", "content")
            .commit()
            .await?;
        let c4 = CreateCommitContext::new(ctx, repo, vec![c2, c3])
            .add_file_with_copy_info("file1a", "content a", (c2, "file1"))
            .delete_file("file1")
            .add_file_with_copy_info("file2a", "content a", (c2, "file2"))
            .add_file_with_copy_info("file2b", "content b", (c2, "file2"))
            .add_file_with_copy_info("file3a", "content a", (c3, "file3"))
            .add_file_with_copy_info("file3b", "content b", (c3, "file3"))
            .commit()
            .await?;

        let bonsai = c4.load(ctx, &repo.repo_blobstore).await?;
        let derivation_ctx = repo.repo_derived_data.manager().derivation_context(None);
        let renames = crate::find_unode_rename_sources(ctx, &derivation_ctx, &bonsai).await?;

        let check = |path: &str, parent_index: usize, from_path: &str| {
            let source = renames
                .get(&MPath::new(path).unwrap())
                .expect("path should exist");
            assert_eq!(source.parent_index, parent_index);
            assert_eq!(source.from_path, MPath::new(from_path).unwrap());
        };

        check("file1a", 0, "file1");
        check("file2a", 0, "file2");
        check("file2b", 0, "file2");
        check("file3a", 1, "file3");
        check("file3b", 1, "file3");

        assert_eq!(renames.len(), 5);

        Ok(())
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;

use anyhow::anyhow;
use anyhow::Context;
use anyhow::Error;
use anyhow::Result;
use async_trait::async_trait;
use blobstore::Blobstore;
use blobstore::BlobstoreGetData;
use blobstore::Loadable;
use bytes::Bytes;
use context::CoreContext;
use derived_data::batch::split_bonsais_in_linear_stacks;
use derived_data::batch::FileConflicts;
use derived_data::impl_bonsai_derived_via_manager;
use derived_data_manager::dependencies;
use derived_data_manager::BonsaiDerivable;
use derived_data_manager::DerivableType;
use derived_data_manager::DerivationContext;
use derived_data_service_if::types as thrift;
use futures::future::try_join_all;
use futures::TryFutureExt;
use metaconfig_types::UnodeVersion;
use mononoke_types::BlobstoreBytes;
use mononoke_types::BonsaiChangeset;
use mononoke_types::ChangesetId;
use mononoke_types::ContentId;
use mononoke_types::FileType;
use mononoke_types::MPath;
use mononoke_types::ManifestUnodeId;
use slog::debug;
use stats::prelude::*;

use crate::derive::derive_unode_manifest;
use crate::derive::derive_unode_manifest_stack;

define_stats! {
    prefix = "mononoke.derived_data.unodes";
    new_parallel: timeseries(Rate, Sum),
}

#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
pub struct RootUnodeManifestId(pub ManifestUnodeId);

impl RootUnodeManifestId {
    pub fn manifest_unode_id(&self) -> &ManifestUnodeId {
        &self.0
    }
}

impl TryFrom<BlobstoreBytes> for RootUnodeManifestId {
    type Error = Error;

    fn try_from(blob_bytes: BlobstoreBytes) -> Result<Self> {
        ManifestUnodeId::from_bytes(&blob_bytes.into_bytes()).map(RootUnodeManifestId)
    }
}

impl TryFrom<BlobstoreGetData> for RootUnodeManifestId {
    type Error = Error;

    fn try_from(blob_val: BlobstoreGetData) -> Result<Self> {
        blob_val.into_bytes().try_into()
    }
}

impl From<RootUnodeManifestId> for BlobstoreBytes {
    fn from(root_mf_id: RootUnodeManifestId) -> Self {
        BlobstoreBytes::from_bytes(Bytes::copy_from_slice(root_mf_id.0.blake2().as_ref()))
    }
}

pub fn format_key(derivation_ctx: &DerivationContext, changeset_id: ChangesetId) -> String {
    let root_prefix = match derivation_ctx.config().unode_version {
        UnodeVersion::V2 => "derived_root_unode_v2.",
    };
    let key_prefix = derivation_ctx.mapping_key_prefix::<RootUnodeManifestId>();
    format!("{}{}{}", root_prefix, key_prefix, changeset_id)
}

#[async_trait]
impl BonsaiDerivable for RootUnodeManifestId {
    const VARIANT: DerivableType = DerivableType::Unodes;

    type Dependencies = dependencies![];

    async fn derive_single(
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        bonsai: BonsaiChangeset,
        parents: Vec<Self>,
    ) -> Result<Self> {
        let unode_version = derivation_ctx.config().unode_version;
        let csid = bonsai.get_changeset_id();
        derive_unode_manifest(
            ctx,
            derivation_ctx,
            csid,
            parents
                .into_iter()
                .map(|root_mf_id| root_mf_id.manifest_unode_id().clone())
                .collect(),
            get_file_changes(&bonsai),
            unode_version,
        )
        .map_ok(RootUnodeManifestId)
        .await
    }

    async fn derive_batch(
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        bonsais: Vec<BonsaiChangeset>,
        _gap_size: Option<usize>,
    ) -> Result<HashMap<ChangesetId, Self>> {
        if bonsais.is_empty() {
            return Ok(HashMap::new());
        }

        let mut res = HashMap::new();
        STATS::new_parallel.add_value(1);
        let batch_len = bonsais.len();
        let stacks = split_bonsais_in_linear_stacks(&bonsais, FileConflicts::ChangeDelete.into())?;

        let unode_version = derivation_ctx.config().unode_version;
        for stack in stacks {
            let derived_parents = try_join_all(
                stack
                    .parents
                    .into_iter()
                    .map(|p| derivation_ctx.fetch_unknown_dependency::<Self>(ctx, Some(&res), p)),
            )
            .await?;
            if let Some(item) = stack.stack_items.first() {
                debug!(
                    ctx.logger(),
                    "derive unode batch at {} (stack of {} from batch of {})",
                    item.cs_id.to_hex(),
                    stack.stack_items.len(),
                    batch_len,
                );
            }

            if derived_parents.len() > 1 {
                // we can't derive stack for a merge commit,
                // so let's derive it without batching
                for item in stack.stack_items {
                    let bonsai = item.cs_id.load(ctx, derivation_ctx.blobstore()).await?;
                    let parents = derivation_ctx
                        .fetch_unknown_parents(ctx, Some(&res), &bonsai)
                        .await?;
                    let derived = Self::derive_single(ctx, derivation_ctx, bonsai, parents).await?;
                    res.insert(item.cs_id, derived);
                }
            } else {
                let first = stack.stack_items.first().map(|item| item.cs_id);
                let last = stack.stack_items.last().map(|item| item.cs_id);
                let derived = derive_unode_manifest_stack(
                    ctx,
                    derivation_ctx,
                    stack
                        .stack_items
                        .into_iter()
                        .map(|item| (item.cs_id, item.per_commit_file_changes))
                        .collect(),
                    derived_parents
                        .get(0)
                        .map(|mf_id| *mf_id.manifest_unode_id()),
                    unode_version,
                )
                .await
                .with_context(|| format!("failed deriving stack of {:?} to {:?}", first, last,))?;

                res.extend(derived.into_iter().map(|(csid, mf_id)| (csid, Self(mf_id))));
            }
        }

        Ok(res)
    }

    async fn store_mapping(
        self,
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        changeset_id: ChangesetId,
    ) -> Result<()> {
        let key = format_key(derivation_ctx, changeset_id);
        derivation_ctx.blobstore().put(ctx, key, self.into()).await
    }

    async fn fetch(
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        changeset_id: ChangesetId,
    ) -> Result<Option<Self>> {
        let key = format_key(derivation_ctx, changeset_id);
        match derivation_ctx.blobstore().get(ctx, &key).await? {
            Some(blob) => Ok(Some(blob.try_into()?)),
            None => Ok(None),
        }
    }

    fn from_thrift(data: thrift::DerivedData) -> Result<Self> {
        if let thrift::DerivedData::unode(thrift::DerivedDataUnode::root_unode_manifest_id(id)) =
            data
        {
            ManifestUnodeId::from_thrift(id).map(Self)
        } else {
            Err(anyhow!(
                "Can't convert {} from provided thrift::DerivedData",
                Self::NAME.to_string(),
            ))
        }
    }

    fn into_thrift(data: Self) -> Result<thrift::DerivedData> {
        Ok(thrift::DerivedData::unode(
            thrift::DerivedDataUnode::root_unode_manifest_id(
                data.manifest_unode_id().into_thrift(),
            ),
        ))
    }
}

// For existing users of BonsaiDerived.
impl_bonsai_derived_via_manager!(RootUnodeManifestId);

pub(crate) fn get_file_changes(
    bcs: &BonsaiChangeset,
) -> Vec<(MPath, Option<(ContentId, FileType)>)> {
    bcs.file_changes()
        .map(|(mpath, file_change)| {
            let content_file_type = file_change
                .simplify()
                .map(|bc| (bc.content_id(), bc.file_type()));
            (mpath.clone(), content_file_type)
        })
        .collect()
}

#[cfg(test)]
mod test {
    use blobstore::Loadable;
    use bookmarks::BookmarkKey;
    use borrowed::borrowed;
    use cloned::cloned;
    use derived_data::BonsaiDerived;
    use derived_data_manager::BatchDeriveOptions;
    use derived_data_test_utils::iterate_all_manifest_entries;
    use fbinit::FacebookInit;
    use fixtures::BranchEven;
    use fixtures::BranchUneven;
    use fixtures::BranchWide;
    use fixtures::Linear;
    use fixtures::ManyDiamonds;
    use fixtures::ManyFilesDirs;
    use fixtures::MergeEven;
    use fixtures::MergeUneven;
    use fixtures::TestRepoFixture;
    use fixtures::UnsharedMergeEven;
    use fixtures::UnsharedMergeUneven;
    use futures::compat::Stream01CompatExt;
    use futures::Future;
    use futures::Stream;
    use futures::TryStreamExt;
    use manifest::Entry;
    use mercurial_derivation::DeriveHgChangeset;
    use mercurial_types::HgChangesetId;
    use mercurial_types::HgManifestId;
    use mononoke_types::ChangesetId;
    use repo_derived_data::RepoDerivedDataRef;
    use revset::AncestorsNodeStream;
    use tests_utils::CreateCommitContext;

    use super::*;
    use crate::tests::TestRepo;

    async fn fetch_manifest_by_cs_id(
        ctx: &CoreContext,
        repo: &TestRepo,
        hg_cs_id: HgChangesetId,
    ) -> Result<HgManifestId, Error> {
        Ok(hg_cs_id.load(ctx, &repo.repo_blobstore).await?.manifestid())
    }

    async fn verify_unode(
        ctx: &CoreContext,
        repo: &TestRepo,
        bcs_id: ChangesetId,
        hg_cs_id: HgChangesetId,
    ) -> Result<RootUnodeManifestId, Error> {
        let (unode_entries, mf_unode_id) = async move {
            let mf_unode_id = RootUnodeManifestId::derive(ctx, repo, bcs_id)
                .await?
                .manifest_unode_id()
                .clone();
            let mut paths = iterate_all_manifest_entries(ctx, repo, Entry::Tree(mf_unode_id))
                .map_ok(|(path, _)| path)
                .try_collect::<Vec<_>>()
                .await?;
            paths.sort();
            Result::<_, Error>::Ok((paths, RootUnodeManifestId(mf_unode_id)))
        }
        .await?;

        let filenode_entries = async move {
            let root_mf_id = fetch_manifest_by_cs_id(ctx, repo, hg_cs_id).await?;
            let mut paths = iterate_all_manifest_entries(ctx, repo, Entry::Tree(root_mf_id))
                .map_ok(|(path, _)| path)
                .try_collect::<Vec<_>>()
                .await?;
            paths.sort();
            Result::<_, Error>::Ok(paths)
        };

        let filenode_entries = filenode_entries.await?;
        assert_eq!(unode_entries, filenode_entries);

        Ok(mf_unode_id)
    }

    fn all_commits_descendants_to_ancestors(
        ctx: CoreContext,
        repo: TestRepo,
    ) -> impl Stream<Item = Result<(ChangesetId, HgChangesetId), Error>> {
        let master_book = BookmarkKey::new("master").unwrap();
        repo.bookmarks
            .get(ctx.clone(), &master_book)
            .map_ok(move |maybe_bcs_id| {
                let bcs_id = maybe_bcs_id.unwrap();
                AncestorsNodeStream::new(ctx.clone(), &repo.changeset_fetcher, bcs_id.clone())
                    .compat()
                    .and_then(move |new_bcs_id| {
                        cloned!(ctx, repo);
                        async move {
                            let hg_cs_id = repo.derive_hg_changeset(&ctx, new_bcs_id).await?;
                            Result::<_, Error>::Ok((new_bcs_id, hg_cs_id))
                        }
                    })
            })
            .try_flatten_stream()
    }

    async fn verify_repo<F, Fut>(fb: FacebookInit, repo_func: F)
    where
        F: Fn() -> Fut,
        Fut: Future<Output = TestRepo>,
    {
        let ctx = CoreContext::test_mock(fb);
        let repo = repo_func().await;
        println!("Processing {}", repo.repo_identity.name());
        borrowed!(ctx, repo);

        let commits_desc_to_anc = all_commits_descendants_to_ancestors(ctx.clone(), repo.clone())
            .and_then(move |(bcs_id, hg_cs_id)| async move {
                let unode_id = verify_unode(ctx, repo, bcs_id, hg_cs_id).await?;
                Ok((bcs_id, hg_cs_id, unode_id))
            })
            .try_collect::<Vec<_>>()
            .await
            .unwrap();

        // Recreate repo from scratch and derive everything again
        let repo = repo_func().await;
        let options = BatchDeriveOptions::Parallel { gap_size: None };
        let csids = commits_desc_to_anc
            .clone()
            .into_iter()
            .rev()
            .map(|(cs_id, _, _)| cs_id)
            .collect::<Vec<_>>();
        let manager = repo.repo_derived_data().manager();

        manager
            .derive_exactly_batch::<RootUnodeManifestId>(ctx, csids.clone(), options, None)
            .await
            .unwrap();
        let batch_derived = manager
            .fetch_derived_batch::<RootUnodeManifestId>(ctx, csids, None)
            .await
            .unwrap();

        for (cs_id, hg_cs_id, unode_id) in commits_desc_to_anc.into_iter().rev() {
            println!("{} {}", cs_id, hg_cs_id);
            println!("{:?} {:?}", batch_derived.get(&cs_id), Some(&unode_id));
            assert_eq!(batch_derived.get(&cs_id), Some(&unode_id));
        }
    }

    #[fbinit::test]
    async fn test_unode_derivation_on_multiple_repos(fb: FacebookInit) {
        verify_repo(fb, || Linear::get_custom_test_repo::<TestRepo>(fb)).await;
        verify_repo(fb, || BranchEven::get_custom_test_repo::<TestRepo>(fb)).await;
        verify_repo(fb, || BranchUneven::get_custom_test_repo::<TestRepo>(fb)).await;
        verify_repo(fb, || BranchWide::get_custom_test_repo::<TestRepo>(fb)).await;
        verify_repo(fb, || ManyDiamonds::get_custom_test_repo::<TestRepo>(fb)).await;
        verify_repo(fb, || ManyFilesDirs::get_custom_test_repo::<TestRepo>(fb)).await;
        verify_repo(fb, || MergeEven::get_custom_test_repo::<TestRepo>(fb)).await;
        verify_repo(fb, || MergeUneven::get_custom_test_repo::<TestRepo>(fb)).await;
        verify_repo(fb, || {
            UnsharedMergeEven::get_custom_test_repo::<TestRepo>(fb)
        })
        .await;
        verify_repo(fb, || {
            UnsharedMergeUneven::get_custom_test_repo::<TestRepo>(fb)
        })
        .await;
        // Create a repo with a few empty commits in a row
        verify_repo(fb, || async {
            let repo: TestRepo = test_repo_factory::build_empty(fb).await.unwrap();
            let ctx = CoreContext::test_mock(fb);
            let root_empty = CreateCommitContext::new_root(&ctx, &repo)
                .commit()
                .await
                .unwrap();
            let first_empty = CreateCommitContext::new(&ctx, &repo, vec![root_empty])
                .commit()
                .await
                .unwrap();
            let second_empty = CreateCommitContext::new(&ctx, &repo, vec![first_empty])
                .commit()
                .await
                .unwrap();
            let first_non_empty = CreateCommitContext::new(&ctx, &repo, vec![second_empty])
                .add_file("file", "a")
                .commit()
                .await
                .unwrap();
            let third_empty = CreateCommitContext::new(&ctx, &repo, vec![first_non_empty])
                .delete_file("file")
                .commit()
                .await
                .unwrap();
            let fourth_empty = CreateCommitContext::new(&ctx, &repo, vec![third_empty])
                .commit()
                .await
                .unwrap();
            let fifth_empty = CreateCommitContext::new(&ctx, &repo, vec![fourth_empty])
                .commit()
                .await
                .unwrap();

            tests_utils::bookmark(&ctx, &repo, "master")
                .set_to(fifth_empty)
                .await
                .unwrap();
            repo
        })
        .await;

        verify_repo(fb, || async {
            let repo: TestRepo = test_repo_factory::build_empty(fb).await.unwrap();
            let ctx = CoreContext::test_mock(fb);
            let root = CreateCommitContext::new_root(&ctx, &repo)
                .add_file("dir/subdir/to_replace", "one")
                .add_file("dir/subdir/file", "content")
                .add_file("somefile", "somecontent")
                .commit()
                .await
                .unwrap();
            let modify_unrelated = CreateCommitContext::new(&ctx, &repo, vec![root])
                .add_file("dir/subdir/file", "content2")
                .delete_file("somefile")
                .commit()
                .await
                .unwrap();
            let replace_file_with_dir =
                CreateCommitContext::new(&ctx, &repo, vec![modify_unrelated])
                    .delete_file("dir/subdir/to_replace")
                    .add_file("dir/subdir/to_replace/file", "newcontent")
                    .commit()
                    .await
                    .unwrap();

            tests_utils::bookmark(&ctx, &repo, "master")
                .set_to(replace_file_with_dir)
                .await
                .unwrap();
            repo
        })
        .await;

        // Weird case - let's delete a file that was already replaced with a directory
        verify_repo(fb, || async {
            let repo: TestRepo = test_repo_factory::build_empty(fb).await.unwrap();
            let ctx = CoreContext::test_mock(fb);
            let root = CreateCommitContext::new_root(&ctx, &repo)
                .add_file("dir/subdir/to_replace", "one")
                .commit()
                .await
                .unwrap();
            let replace_file_with_dir = CreateCommitContext::new(&ctx, &repo, vec![root])
                .delete_file("dir/subdir/to_replace")
                .add_file("dir/subdir/to_replace/file", "newcontent")
                .commit()
                .await
                .unwrap();
            let noop_delete = CreateCommitContext::new(&ctx, &repo, vec![replace_file_with_dir])
                .delete_file("dir/subdir/to_replace")
                .commit()
                .await
                .unwrap();
            let second_noop_delete = CreateCommitContext::new(&ctx, &repo, vec![noop_delete])
                .delete_file("dir/subdir/to_replace")
                .commit()
                .await
                .unwrap();

            tests_utils::bookmark(&ctx, &repo, "master")
                .set_to(second_noop_delete)
                .await
                .unwrap();
            repo
        })
        .await;
    }
}