/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use anyhow::format_err;
use anyhow::Context;
use anyhow::Error;
use anyhow::Result;
use blobstore::BlobstoreGetData;
use bytes::Bytes;
use derived_data_thrift as thrift;
use fbthrift::compact_protocol;
use mononoke_types::errors::ErrorKind;
use mononoke_types::BlobstoreBytes;
use mononoke_types::BonsaiChangeset;
use mononoke_types::BonsaiChangesetMut;
use mononoke_types::ChangesetId;
use mononoke_types::DateTime;
use smallvec::SmallVec;
use sorted_vector_map::SortedVectorMap;
use unicode_segmentation::UnicodeSegmentation;

/// Changeset Info is a derived data structure that represents a Bonsai changeset's
/// metadata.
///
/// Bonsai changeset consists of the commit metadata and a set of all the file
/// changes associated with the commit. Some of the changesets, usually for merge
/// commits, include thousands of file changes and touch history of even bigger
/// number of files. This makes fetching changesets an expensive operation due
/// to large commits taking many seconds to deserialize and affects performance
/// of the Source Control Service API methods.
///
/// Changeset Info data structure comes to resolve the necessity to waste time
/// deserializing lots of file changes, when commit's metadata is the main
/// reason the commit was fetched.
///
/// Q&A
///
/// Is the ChangesetInfo content any different from the Bonsai apart from the file changes?
///
/// Not really, ChangesetInfo includes all the metadata Bonsai has and its own "linknode" -
/// changeset id of the source Bonsai.
///
/// Why do we store commit message as union?
///
/// The commit message potentially can be quite large. So in the future we might want
/// to store the whole description separately from the changeset info. As an alternative
/// to the string containing message we could have a title and a message handler -
/// message id to fetch.
///
/// Why do we need title?
///
/// Some of the important SCS API methods only need title in most of the cases. Having
/// the whole heavy commit message in the store and a small title always available let's
/// us avoid fetching items from the blobstore twice: first commit info and then message.

#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub struct ChangesetInfo {
    /// changeset id of the source Bonsai changeset
    changeset_id: ChangesetId,
    parents: Vec<ChangesetId>,
    author: String,
    author_date: DateTime,
    committer: Option<String>,
    committer_date: Option<DateTime>,
    message: ChangesetMessage,
    hg_extra: SortedVectorMap<String, Vec<u8>>,
    git_extra_headers: Option<SortedVectorMap<SmallVec<[u8; 24]>, Bytes>>,
}

/// At some point we may like to store large commit messages as separate blobs
/// to make fetching changesets faster if there is no need in the whole description.
/// For example:
///     Handler((String /* title */, ChangesetMessageId /* message blob id */))
///
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub enum ChangesetMessage {
    Message(String),
}

impl ChangesetMessage {
    pub(crate) fn from_thrift(tc: thrift::ChangesetMessage) -> Result<Self> {
        match tc {
            thrift::ChangesetMessage::message(message) => Ok(ChangesetMessage::Message(message)),
            thrift::ChangesetMessage::UnknownField(other) => {
                Err(format_err!("Unknown ChangesetMessage field: {}", other))
            }
        }
    }

    pub(crate) fn into_thrift(self) -> thrift::ChangesetMessage {
        match self {
            ChangesetMessage::Message(msg) => thrift::ChangesetMessage::message(msg),
        }
    }
}

pub const DEFAULT_TITLE_LENGTH: usize = 1024;

impl ChangesetInfo {
    pub fn new(changeset_id: ChangesetId, changeset: BonsaiChangeset) -> Self {
        let BonsaiChangesetMut {
            parents,
            author,
            author_date,
            committer,
            committer_date,
            message,
            hg_extra,
            git_extra_headers,
            ..
        } = changeset.into_mut();

        Self {
            changeset_id,
            parents,
            author,
            author_date,
            committer,
            committer_date,
            message: ChangesetMessage::Message(message),
            hg_extra,
            git_extra_headers,
        }
    }

    /// Get id of the source Bonsai changeset.
    pub fn changeset_id(&self) -> &ChangesetId {
        &self.changeset_id
    }

    /// Get the changeset parents.
    pub fn parents<'a>(&'a self) -> impl Iterator<Item = ChangesetId> + 'a {
        self.parents.iter().cloned()
    }

    /// Get the author.
    pub fn author(&self) -> &str {
        &self.author
    }

    /// Get the author date (time and timezone).
    pub fn author_date(&self) -> &DateTime {
        &self.author_date
    }

    /// Get the committer.
    pub fn committer(&self) -> Option<&str> {
        self.committer.as_deref()
    }

    /// Get the committer date (time and timezone).
    pub fn committer_date(&self) -> Option<&DateTime> {
        self.committer_date.as_ref()
    }

    /// Get the commit title: the first line of the commit message or the first
    /// DEFAULT_TITLE_LENGTH characters.
    pub fn title(&self) -> &str {
        match &self.message {
            ChangesetMessage::Message(message) => get_title(message),
        }
    }

    /// Get the commit message.
    pub fn message(&self) -> &str {
        match &self.message {
            ChangesetMessage::Message(msg) => msg,
        }
    }

    /// Get the extra fields for this message.
    pub fn hg_extra(&self) -> impl Iterator<Item = (&str, &[u8])> {
        self.hg_extra
            .iter()
            .map(|(k, v)| (k.as_str(), v.as_slice()))
    }

    pub fn git_extra_headers(&self) -> Option<impl Iterator<Item = (&[u8], &[u8])>> {
        self.git_extra_headers
            .as_ref()
            .map(|extra| extra.iter().map(|(k, v)| (k.as_slice(), v.as_ref())))
    }

    pub(crate) fn from_thrift(tc: thrift::ChangesetInfo) -> Result<Self> {
        let catch_block = || -> Result<_> {
            Ok(ChangesetInfo {
                changeset_id: ChangesetId::from_thrift(tc.changeset_id)?,
                parents: tc
                    .parents
                    .into_iter()
                    .map(ChangesetId::from_thrift)
                    .collect::<Result<_>>()?,
                author: tc.author,
                author_date: DateTime::from_thrift(tc.author_date)?,
                committer: tc.committer,
                committer_date: match tc.committer_date {
                    Some(dt) => Some(DateTime::from_thrift(dt)?),
                    None => None,
                },
                message: ChangesetMessage::from_thrift(tc.message)?,
                hg_extra: tc.hg_extra,
                git_extra_headers: tc
                    .git_extra_headers
                    .map(|extra| extra.into_iter().map(|(k, v)| (k.0, v)).collect()),
            })
        };

        catch_block().with_context(|| {
            ErrorKind::InvalidThrift("ChangesetInfo".into(), "Invalid changeset info".into())
        })
    }

    pub fn into_thrift(self) -> thrift::ChangesetInfo {
        thrift::ChangesetInfo {
            changeset_id: self.changeset_id.into_thrift(),
            parents: self
                .parents
                .into_iter()
                .map(|parent| parent.into_thrift())
                .collect(),
            author: self.author,
            author_date: self.author_date.into_thrift(),
            committer: self.committer,
            committer_date: self.committer_date.map(|dt| dt.into_thrift()),
            message: self.message.into_thrift(),
            hg_extra: self.hg_extra,
            git_extra_headers: self.git_extra_headers.map(|extra| {
                extra
                    .into_iter()
                    .map(|(k, v)| (thrift::small_binary(k), v))
                    .collect()
            }),
        }
    }

    pub fn from_bytes(bytes: &[u8]) -> Result<Self> {
        let thrift_tc = compact_protocol::deserialize(bytes)
            .with_context(|| ErrorKind::BlobDeserializeError("ChangesetInfo".into()))?;
        Self::from_thrift(thrift_tc)
    }
}

impl TryFrom<BlobstoreBytes> for ChangesetInfo {
    type Error = Error;

    fn try_from(blob_bytes: BlobstoreBytes) -> Result<Self> {
        ChangesetInfo::from_bytes(&blob_bytes.into_bytes())
    }
}

impl TryFrom<BlobstoreGetData> for ChangesetInfo {
    type Error = Error;

    fn try_from(blob_get_data: BlobstoreGetData) -> Result<Self> {
        blob_get_data.into_bytes().try_into()
    }
}

impl From<ChangesetInfo> for BlobstoreBytes {
    fn from(info: ChangesetInfo) -> BlobstoreBytes {
        let data = compact_protocol::serialize(&info.into_thrift());
        BlobstoreBytes::from_bytes(data)
    }
}

/// Given a commit message returns the commit title: either the first line of the
/// message or the message itself, cropped by the DEFAULT_TITLE_LENGTH number of
/// characters.
fn get_title(message: &str) -> &str {
    // either first line or the whole message
    let title = message.trim_start().lines().next().unwrap_or("");
    match title.grapheme_indices(true).nth(DEFAULT_TITLE_LENGTH) {
        Some((i, _ch)) => &title[..i],
        None => title,
    }
}

#[cfg(test)]
mod test {
    use mononoke_types::BonsaiChangeset;
    use mononoke_types::BonsaiChangesetMut;
    use mononoke_types::DateTime;
    use mononoke_types::FileChange;
    use mononoke_types::MPath;
    use sorted_vector_map::sorted_vector_map;

    use super::*;

    #[fbinit::test]
    fn changeset_info_title_test() {
        {
            let bcs = create_bonsai_with_message("   \n\n  title \n\n summary\n");
            let info = ChangesetInfo::new(bcs.get_changeset_id(), bcs);

            check_info(&info, "title ");
        };

        {
            let bcs = create_bonsai_with_message("   \n  title - summary");
            let info = ChangesetInfo::new(bcs.get_changeset_id(), bcs);

            check_info(&info, "title - summary");
        };

        {
            let bcs = create_bonsai_with_message("  no title - no new lines ");
            let info = ChangesetInfo::new(bcs.get_changeset_id(), bcs);

            check_info(&info, "no title - no new lines ");
        };

        {
            let bcs = create_bonsai_with_message("  \n\n ");
            let info = ChangesetInfo::new(bcs.get_changeset_id(), bcs);

            check_info(&info, "");
        };
    }

    fn check_info(info: &ChangesetInfo, title: &str) {
        assert_eq!(info.title(), title);
    }

    fn create_bonsai_with_message(message: &str) -> BonsaiChangeset {
        BonsaiChangesetMut {
            parents: vec![],
            author: "author".to_string(),
            author_date: DateTime::now(),
            committer: None,
            committer_date: None,
            message: message.to_string(),
            hg_extra: Default::default(),
            git_extra_headers: Some(sorted_vector_map! { SmallVec::new() => Bytes::from_static(b"world")}),
            git_tree_hash: None,
            file_changes: sorted_vector_map! { MPath::new("file").unwrap() => FileChange::Deletion },
            is_snapshot: false,
            git_annotated_tag: None,
        }
        .freeze()
        .unwrap()
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;

use anyhow::anyhow;
use anyhow::Error;
use anyhow::Result;
use async_trait::async_trait;
use blobstore::Blobstore;
use context::CoreContext;
use derived_data::impl_bonsai_derived_via_manager;
use derived_data_manager::dependencies;
use derived_data_manager::BonsaiDerivable;
use derived_data_manager::DerivableType;
use derived_data_manager::DerivationContext;
use derived_data_service_if::types as thrift;
use mononoke_types::BonsaiChangeset;
use mononoke_types::ChangesetId;

use crate::ChangesetInfo;

pub fn format_key(derivation_ctx: &DerivationContext, changeset_id: ChangesetId) -> String {
    let root_prefix = "changeset_info.blake2.";
    let key_prefix = derivation_ctx.mapping_key_prefix::<ChangesetInfo>();
    format!("{}{}{}", root_prefix, key_prefix, changeset_id)
}

#[async_trait]
impl BonsaiDerivable for ChangesetInfo {
    const VARIANT: DerivableType = DerivableType::ChangesetInfo;

    type Dependencies = dependencies![];

    async fn derive_single(
        _ctx: &CoreContext,
        _derivation_ctx: &DerivationContext,
        bonsai: BonsaiChangeset,
        _parents: Vec<Self>,
    ) -> Result<Self, Error> {
        Ok(ChangesetInfo::new(bonsai.get_changeset_id(), bonsai))
    }

    async fn derive_batch(
        _ctx: &CoreContext,
        _derivation_ctx: &DerivationContext,
        bonsais: Vec<BonsaiChangeset>,
        _gap_size: Option<usize>,
    ) -> Result<HashMap<ChangesetId, Self>> {
        // Derivation with gaps doesn't make much sense for changeset info, so
        // ignore the gap size.
        Ok(bonsais
            .into_iter()
            .map(|bonsai| {
                let csid = bonsai.get_changeset_id();
                (csid, ChangesetInfo::new(csid, bonsai))
            })
            .collect())
    }

    async fn store_mapping(
        self,
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        changeset_id: ChangesetId,
    ) -> Result<()> {
        let key = format_key(derivation_ctx, changeset_id);
        derivation_ctx.blobstore().put(ctx, key, self.into()).await
    }

    async fn fetch(
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        changeset_id: ChangesetId,
    ) -> Result<Option<Self>> {
        let key = format_key(derivation_ctx, changeset_id);
        Ok(derivation_ctx
            .blobstore()
            .get(ctx, &key)
            .await?
            .map(TryInto::try_into)
            .transpose()?)
    }

    fn from_thrift(data: thrift::DerivedData) -> Result<Self> {
        if let thrift::DerivedData::changeset_info(
            thrift::DerivedDataChangesetInfo::changeset_info(data),
        ) = data
        {
            Self::from_thrift(data)
        } else {
            Err(anyhow!(
                "Can't convert {} from provided thrift::DerivedData",
                Self::NAME.to_string(),
            ))
        }
    }

    fn into_thrift(data: Self) -> Result<thrift::DerivedData> {
        Ok(thrift::DerivedData::changeset_info(
            thrift::DerivedDataChangesetInfo::changeset_info(data.into_thrift()),
        ))
    }
}

impl_bonsai_derived_via_manager!(ChangesetInfo);

#[cfg(test)]
mod test {
    use std::collections::BTreeMap;
    use std::str::FromStr;

    use blobstore::Loadable;
    use bonsai_hg_mapping::BonsaiHgMappingRef;
    use changeset_fetcher::ChangesetFetcherArc;
    use derived_data_manager::BatchDeriveOptions;
    use fbinit::FacebookInit;
    use fixtures::Linear;
    use fixtures::TestRepoFixture;
    use futures::compat::Stream01CompatExt;
    use futures::TryStreamExt;
    use mercurial_types::HgChangesetId;
    use mononoke_types::BonsaiChangeset;
    use repo_blobstore::RepoBlobstoreRef;
    use repo_derived_data::RepoDerivedDataRef;
    use revset::AncestorsNodeStream;
    use tests_utils::resolve_cs_id;

    use super::*;

    #[fbinit::test]
    async fn derive_info_test(fb: FacebookInit) -> Result<(), Error> {
        let repo = Linear::getrepo(fb).await;
        let ctx = CoreContext::test_mock(fb);
        let manager = repo.repo_derived_data().manager();

        let hg_cs_id = HgChangesetId::from_str("3c15267ebf11807f3d772eb891272b911ec68759").unwrap();
        let bcs_id = repo
            .bonsai_hg_mapping()
            .get_bonsai_from_hg(&ctx, hg_cs_id)
            .await?
            .unwrap();
        let bcs = bcs_id.load(&ctx, repo.repo_blobstore()).await?;
        // Make sure that the changeset info was saved in the blobstore
        let info = manager.derive(&ctx, bcs_id, None).await?;

        check_info(&info, &bcs);
        Ok(())
    }

    fn check_info(info: &ChangesetInfo, bcs: &BonsaiChangeset) {
        assert_eq!(*info.changeset_id(), bcs.get_changeset_id());
        assert_eq!(info.message(), bcs.message());
        assert_eq!(
            info.parents().collect::<Vec<_>>(),
            bcs.parents().collect::<Vec<_>>()
        );
        assert_eq!(info.author(), bcs.author());
        assert_eq!(info.author_date(), bcs.author_date());
        assert_eq!(info.committer(), bcs.committer());
        assert_eq!(info.committer_date(), bcs.committer_date());
        assert_eq!(
            info.hg_extra().collect::<BTreeMap<_, _>>(),
            bcs.hg_extra().collect::<BTreeMap<_, _>>()
        );
    }

    #[fbinit::test]
    async fn batch_derive(fb: FacebookInit) -> Result<(), Error> {
        let ctx = CoreContext::test_mock(fb);
        let repo = Linear::getrepo(fb).await;
        let master_cs_id = resolve_cs_id(&ctx, &repo, "master").await?;
        let manager = repo.repo_derived_data().manager();

        let mut cs_ids =
            AncestorsNodeStream::new(ctx.clone(), &repo.changeset_fetcher_arc(), master_cs_id)
                .compat()
                .try_collect::<Vec<_>>()
                .await?;
        cs_ids.reverse();
        manager
            .derive_exactly_batch::<ChangesetInfo>(
                &ctx,
                cs_ids.clone(),
                BatchDeriveOptions::Parallel { gap_size: None },
                None,
            )
            .await?;
        let cs_infos = manager
            .fetch_derived_batch(&ctx, cs_ids.clone(), None)
            .await?;

        for cs_id in cs_ids {
            let bonsai = cs_id.load(&ctx, repo.repo_blobstore()).await?;
            let cs_info = cs_infos
                .get(&cs_id)
                .expect("ChangesetInfo should have been derived");
            check_info(cs_info, &bonsai);
        }

        Ok(())
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

mod changeset_info;
mod derive;

pub use crate::changeset_info::ChangesetInfo;
pub use crate::changeset_info::ChangesetMessage;
pub use crate::derive::format_key;


// @generated by autocargo
use std::env;
use std::fs;
use std::path::Path;

use thrift_compiler::Config;
use thrift_compiler::GenContext;

#[rustfmt::skip]
fn main() {
    // Rerun if this gets rewritten.
    println!("cargo:rerun-if-changed=thrift_build.rs");

    let out_dir = env::var_os("OUT_DIR").expect("OUT_DIR env not provided");
    let out_dir: &Path = out_dir.as_ref();
    fs::write(
        out_dir.join("cratemap"),
        "changeset_info_thrift crate
mononoke_types_thrift mononoke_types_thrift",
    ).expect("Failed to write cratemap");

    let conf = {
        let mut conf = Config::from_env(GenContext::Lib).expect("Failed to instantiate thrift_compiler::Config");

        let path_from_manifest_to_base: &Path = "../../../../..".as_ref();
        let cargo_manifest_dir =
            env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not provided");
        let cargo_manifest_dir: &Path = cargo_manifest_dir.as_ref();
        let base_path = cargo_manifest_dir
            .join(path_from_manifest_to_base)
            .canonicalize()
            .expect("Failed to canonicalize base_path");
        // TODO: replace canonicalize() with std::path::absolute() when
        // https://github.com/rust-lang/rust/pull/91673 is available (~Rust 1.60)
        // and remove this block.
        #[cfg(windows)]
        let base_path = Path::new(
            base_path
                .as_path()
                .to_string_lossy()
                .trim_start_matches(r"\\?\"),
            )
            .to_path_buf();

        conf.base_path(base_path);

        let options = "types_crate=derived_data-thrift__types";
        if !options.is_empty() {
            conf.options(options);
        }

        let include_srcs = vec![

        ];
        conf.include_srcs(include_srcs);

        conf
    };

    conf
        .run(&[
            "changeset_info_thrift.thrift"
        ])
        .expect("Failed while running thrift compilation");
}


// @generated by autocargo
::codegen_includer_proc_macro::include!();


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

pub const DERIVATION_START: &str = "Generating derived data";
pub const DERIVATION_START_BATCH: &str = "Generating derived data batch";
pub const DERIVATION_END: &str = "Generated derived data";
pub const DERIVATION_END_BATCH: &str = "Generated derived data batch";
pub const FAILED_DERIVATION: &str = "Failed to generate derived data";
pub const FAILED_DERIVATION_BATCH: &str = "Failed to generate derived data batch";
pub const INSERTED_MAPPING: &str = "Inserted mapping";
pub const FAILED_INSERTING_MAPPING: &str = "Failed to insert mapping";

pub const SLOW_DERIVATION: &str = "Slow derivation";


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::BTreeMap;
use std::collections::HashMap;
use std::collections::HashSet;
use std::sync::Arc;

use anyhow::anyhow;
use anyhow::bail;
use anyhow::format_err;
use anyhow::Context;
use anyhow::Error;
use blobstore::Blobstore;
use blobstore::Loadable;
use borrowed::borrowed;
use bounded_traversal::bounded_traversal;
use cloned::cloned;
use context::CoreContext;
use derived_data::batch::split_bonsais_in_linear_stacks;
use derived_data::batch::FileConflicts;
use derived_data_manager::DerivationContext;
use futures::channel::mpsc;
use futures::future;
use futures::future::BoxFuture;
use futures::future::FutureExt;
use futures::stream;
use futures::stream::StreamExt;
use futures::stream::TryStreamExt;
use manifest::Diff;
use manifest::ManifestOps;
use manifest::PathTree;
use mononoke_types::deleted_manifest_common::DeletedManifestCommon;
use mononoke_types::BlobstoreKey;
use mononoke_types::BonsaiChangeset;
use mononoke_types::ChangesetId;
use mononoke_types::MPath;
use mononoke_types::MPathElement;
use mononoke_types::ManifestUnodeId;
use tokio::sync::Mutex;
use tunables::tunables;
use unodes::RootUnodeManifestId;

use crate::mapping::RootDeletedManifestIdCommon;

/// Derives deleted manifest for bonsai changeset `cs_id` given parent deleted files
/// manifests and the changes associated with the changeset. Parent deleted manifests should be
/// constructed for each parent of the given changeset.
///
/// Deleted manifest is a recursive data structure that starts with a root manifest and
/// points to the other manifests. Each node may represent either deleted directoty or deleted file.
/// Both directory's and file's manifest can have subentries, if a file has subentries it means
/// that this path was a directory earlier, then was deleted and reincarnated as a file.
///
/// Each manifest has an optional linknode. The initialized linknode points to the changeset where
/// the path was deleted. If linknode is not set, then manifest represents an existing
/// directory where some of the subentries (directories or files) have been deleted. There cannot
/// be a manifest without linknode and with no subentries.
///
/// Changes represent creations and deletions for both files and directories. They are applied
/// recursively starting from the root of parent manifest.
///
/// 1. If no files were deleted or created on the current path or any subpaths
///    - if there was corresponding deleted manifest, reuse it;
///    - otherwise, there is no need to create a new node.
/// 2. If no change ends on the current path BUT there are creations/deletions on the subpaths,
///    recurse to the parent subentries and the current subpaths' changes
///    - if there are deleted subpaths (subentries are not empty), create a live manifest (manifest
///      without an empty linknode);
///    - if subentries are empty (all subpaths were restored), delete the current node.
/// 3. If current path was deleted, recurse to the parent subentries and the current subpaths'
///    changes
///   - create a deleted manifest for the current path and set linknode to the current changeset id.
/// 4. If current path was created, recurse to the parent subentries and the current subpaths'
///    changes
///   - if there was a corresponding manifest and there are no subentries, delete the node;
///   - if there are subentries, create a live manifest or mark the existing node as live.
/// 5. If there was a file/dir conflict (file was replaced with directory or other way round),
///    recurse to the parent subentries and the current subpaths' changes
///   - if there are subentries, create a live manifest or mark the existing node as live.
///
pub(crate) struct DeletedManifestDeriver<Manifest: DeletedManifestCommon>(
    std::marker::PhantomData<Manifest>,
);

#[derive(Debug, Eq, PartialEq, Clone, Copy)]
pub(crate) enum PathChange {
    Add,
    Remove,
    FileDirConflict,
}

#[derive(Debug, Clone, Copy)]
pub(crate) enum DeletedManifestChangeType {
    /// Path was deleted, we create a node if not present.
    CreateDeleted,
    /// Path now exists, delete if it doesn't have any subentries that were
    /// previous deleted.
    RemoveIfNowEmpty,
    /// No changes to the path which has a single parent, reuse the parent.
    Reuse,
}

struct DeletedManifestChange<Manifest: DeletedManifestCommon> {
    /// Which change happened.
    change_type: DeletedManifestChangeType,
    /// Parent to base on. Result should be equivalent to copying the subentries
    /// of the parent and then applying the remanining modifications.
    copy_subentries_from: Option<Manifest>,
}

struct DeletedManifestUnfoldNode<Manifest: DeletedManifestCommon> {
    path_element: Option<MPathElement>,
    changes: PathTree<Option<PathChange>>,
    // set is used to automatically deduplicate parents that have equal ancestors
    parents: HashSet<Manifest::Id>,
}

impl<Manifest: DeletedManifestCommon> DeletedManifestDeriver<Manifest> {
    pub(crate) async fn derive(
        ctx: &CoreContext,
        blobstore: &Arc<dyn Blobstore>,
        cs_id: ChangesetId,
        parents: Vec<Manifest::Id>,
        changes: PathTree<Option<PathChange>>,
    ) -> Result<Manifest::Id, Error> {
        // Stream is used to batch writes to blobstore
        let (sender, receiver) = mpsc::unbounded();
        let created = Arc::new(Mutex::new(HashSet::new()));
        cloned!(blobstore, ctx);
        let f = async move {
            borrowed!(ctx, blobstore);
            let manifest_opt = bounded_traversal(
                256,
                DeletedManifestUnfoldNode {
                    path_element: None,
                    changes,
                    parents: parents.into_iter().collect(),
                },
                // unfold
                {
                    move |DeletedManifestUnfoldNode {
                              path_element,
                              changes,
                              parents,
                          }| {
                        // -> ((Option<MPathElement>, DeletedManifestChange), Vec<UnfoldNode>)
                        async move {
                            let (mf_change, next_states) =
                                Self::do_unfold(ctx, blobstore, changes, parents).await?;
                            Ok(((path_element, mf_change), next_states))
                        }
                        .boxed()
                    }
                },
                // fold
                {
                    cloned!(sender, created);
                    move |
                        (path, manifest_change): (
                            Option<MPathElement>,
                            DeletedManifestChange<Manifest>,
                        ),
                        // impl Iterator<Out>
                        subentries_iter,
                        // -> Out = (Option<MPathElement>, Option<Manifest::Id>)
                        // (_, None) means a leaf node was deleted because the file was recreated.
                        // (None, _) means the path is empty and should only happen on the root.
                    | {
                        cloned!(cs_id, sender, created);
                        async move {
                            let mut subentries_to_update = BTreeMap::new();
                            for entry in subentries_iter {
                                match entry {
                                    (None, _) => {
                                        return Err(anyhow!(concat!(
                                            "Failed to create deleted manifest: ",
                                            "subentry must have a path"
                                        )));
                                    }
                                    (Some(path), maybe_mf_id) => {
                                        subentries_to_update.insert(path, maybe_mf_id);
                                    }
                                }
                            }

                            let maybe_mf_id = Self::do_create(
                                ctx,
                                blobstore,
                                cs_id.clone(),
                                manifest_change,
                                subentries_to_update,
                                sender.clone(),
                                created.clone(),
                            )
                            .await?;

                            Ok((path, maybe_mf_id))
                        }
                        .boxed()
                    }
                },
            )
            .await?;

            debug_assert!(manifest_opt.0.is_none());
            match manifest_opt {
                (_, Some(mf_id)) => Ok(mf_id),
                (_, None) => {
                    // there are no deleted files, need to create an empty root manifest
                    match Manifest::copy_and_update_subentries(
                        ctx,
                        blobstore,
                        None,
                        None,
                        BTreeMap::new(),
                    )
                    .await
                    {
                        Ok(mf) => {
                            Self::save_manifest(mf, ctx, blobstore, sender.clone(), created.clone())
                                .await
                        }
                        Err(err) => Err(err),
                    }
                }
            }
        };

        let handle = tokio::spawn(f);

        receiver
            .buffered(1024)
            .try_for_each(|_| async { Ok(()) })
            .await?;
        handle.await?
    }

    async fn do_unfold(
        ctx: &CoreContext,
        blobstore: &Arc<dyn Blobstore>,
        changes: PathTree<Option<PathChange>>,
        parents: HashSet<Manifest::Id>,
    ) -> Result<
        (
            DeletedManifestChange<Manifest>,
            Vec<DeletedManifestUnfoldNode<Manifest>>,
        ),
        Error,
    > {
        let PathTree {
            value: change,
            subentries,
        } = changes;

        let parent_manifests =
            future::try_join_all(parents.iter().map(|mf_id| mf_id.load(ctx, blobstore))).await?;

        let check_consistency = |manifests: &[Manifest]| {
            let mut it = manifests.iter().map(|mf| mf.is_deleted());
            if let Some(status) = it.next() {
                if it.all(|st| st == status) {
                    return Ok(status);
                }
                return Err(format_err!(
                    "parent deleted manifests have different node status, but no changes were provided"
                ));
            }
            Ok(false)
        };

        let change_type = match change {
            None => {
                if subentries.is_empty() {
                    // nothing changed in the current node and in the subentries
                    // if parent manifests are equal, we can reuse them
                    match parent_manifests.as_slice() {
                        [] => {
                            return Ok((
                                DeletedManifestChange {
                                    change_type: DeletedManifestChangeType::Reuse,
                                    copy_subentries_from: None,
                                },
                                vec![],
                            ));
                        }
                        [parent] => {
                            return Ok((
                                DeletedManifestChange {
                                    change_type: DeletedManifestChangeType::Reuse,
                                    copy_subentries_from: Some(parent.clone()),
                                },
                                vec![],
                            ));
                        }
                        parents => {
                            // parent manifests are different, we need to merge them
                            // let's check that the node status is consistent across parents
                            let is_deleted = check_consistency(parents)?;
                            if is_deleted {
                                DeletedManifestChangeType::CreateDeleted
                            } else {
                                DeletedManifestChangeType::RemoveIfNowEmpty
                            }
                        }
                    }
                } else {
                    // some paths might be added/deleted
                    DeletedManifestChangeType::RemoveIfNowEmpty
                }
            }
            Some(PathChange::Add) => {
                // the path was added
                DeletedManifestChangeType::RemoveIfNowEmpty
            }
            Some(PathChange::Remove) => {
                // the path was removed
                DeletedManifestChangeType::CreateDeleted
            }
            Some(PathChange::FileDirConflict) => {
                // This is a file/dir conflict: either a file was replaced by directory or other way
                // round. In both cases one of the paths is being deleted and recreated as other
                // type. To keep this in history, we need to mark the path as deleted in the deleted
                // files manifest.
                DeletedManifestChangeType::RemoveIfNowEmpty
            }
        };

        // Base traversal for all modified subentries
        let mut recurse_entries = subentries
            .into_iter()
            .map(|(path, change_tree)| {
                (
                    path.clone(),
                    DeletedManifestUnfoldNode {
                        path_element: Some(path),
                        changes: change_tree,
                        parents: HashSet::new(),
                    },
                )
            })
            .collect::<BTreeMap<_, _>>();

        let fold_node = match parent_manifests.as_slice() {
            [] => DeletedManifestChange {
                change_type,
                copy_subentries_from: None,
            },
            [parent] => {
                // If there's one parent, we can "copy" its subentries
                // and modify only a few fields. Important if we're doing few
                // changes on a big node and need to optimise.
                stream::iter(recurse_entries.iter_mut().map(anyhow::Ok))
                    .try_for_each_concurrent(100, |(path, node)| async move {
                        if let Some(subentry_id) = parent.lookup(ctx, blobstore, path).await? {
                            node.parents.insert(subentry_id);
                        }
                        anyhow::Ok(())
                    })
                    .await?;

                DeletedManifestChange {
                    change_type,
                    copy_subentries_from: Some(parent.clone()),
                }
            }
            _ => {
                // If there are multiple parents and they're different, we need to
                // merge all different subentries. So let's just look at all of them.
                for parent in parent_manifests {
                    parent
                        .into_subentries(ctx, blobstore)
                        .try_for_each(|(path, mf_id)| {
                            let entry = recurse_entries.entry(path.clone()).or_insert_with(|| {
                                DeletedManifestUnfoldNode {
                                    path_element: Some(path),
                                    changes: Default::default(),
                                    parents: HashSet::new(),
                                }
                            });
                            entry.parents.insert(mf_id);
                            async { Ok(()) }
                        })
                        .await?;
                }
                DeletedManifestChange {
                    change_type,
                    copy_subentries_from: None,
                }
            }
        };

        Ok((fold_node, recurse_entries.into_values().collect::<Vec<_>>()))
    }

    async fn save_manifest(
        manifest: Manifest,
        ctx: &CoreContext,
        blobstore: &Arc<dyn Blobstore>,
        sender: mpsc::UnboundedSender<BoxFuture<'static, Result<(), Error>>>,
        created: Arc<Mutex<HashSet<String>>>,
    ) -> Result<Manifest::Id, Error> {
        let mf_id = manifest.id();

        let key = mf_id.blobstore_key();
        let mut created = created.lock().await;
        if created.insert(key.clone()) {
            let blob = manifest.into_blob();
            cloned!(ctx, blobstore);
            let f = async move { blobstore.put(&ctx, key, blob.into()).await }.boxed();

            sender
                .unbounded_send(f)
                .map_err(|err| anyhow!("failed to send manifest future {}", err))?;
        }
        Ok(mf_id)
    }

    async fn do_create(
        ctx: &CoreContext,
        blobstore: &Arc<dyn Blobstore>,
        cs_id: ChangesetId,
        change: DeletedManifestChange<Manifest>,
        subentries_to_update: BTreeMap<MPathElement, Option<Manifest::Id>>,
        sender: mpsc::UnboundedSender<BoxFuture<'static, Result<(), Error>>>,
        created: Arc<Mutex<HashSet<String>>>,
    ) -> Result<Option<Manifest::Id>, Error> {
        match change.change_type {
            DeletedManifestChangeType::Reuse => Ok(change.copy_subentries_from.map(|mf| mf.id())),
            DeletedManifestChangeType::CreateDeleted => Self::save_manifest(
                Manifest::copy_and_update_subentries(
                    ctx,
                    blobstore,
                    change.copy_subentries_from,
                    Some(cs_id),
                    subentries_to_update,
                )
                .await?,
                ctx,
                blobstore,
                sender,
                created,
            )
            .await
            .map(Some),
            DeletedManifestChangeType::RemoveIfNowEmpty => {
                let manifest = Manifest::copy_and_update_subentries(
                    ctx,
                    blobstore,
                    change.copy_subentries_from,
                    None,
                    subentries_to_update,
                )
                .await?;
                // some of the subentries were deleted, creating a new node but there is no need to
                // mark it as deleted
                if !manifest.is_empty() {
                    Self::save_manifest(manifest, ctx, blobstore, sender, created)
                        .await
                        .map(Some)
                } else {
                    Ok(None)
                }
            }
        }
    }
}

pub(crate) async fn get_changes(
    ctx: &CoreContext,
    derivation_ctx: &DerivationContext,
    bonsai: BonsaiChangeset,
) -> Result<PathTree<Option<PathChange>>, Error> {
    let changes = get_changes_list(ctx, derivation_ctx, bonsai).await?;
    Ok(PathTree::from_iter(
        changes
            .into_iter()
            .map(|(path, change)| (path, Some(change))),
    ))
}

async fn get_changes_list(
    ctx: &CoreContext,
    derivation_ctx: &DerivationContext,
    bonsai: BonsaiChangeset,
) -> Result<Vec<(MPath, PathChange)>, Error> {
    // Get file/directory changes between the current changeset and its parents
    //
    // get unode manifests first
    let bcs_id = bonsai.get_changeset_id();

    // get parent unodes
    let parent_cs_ids: Vec<_> = bonsai.parents().collect();
    let parent_unodes = parent_cs_ids.into_iter().map({
        move |cs_id| async move {
            let root_mf_id = derivation_ctx
                .derive_dependency::<RootUnodeManifestId>(ctx, cs_id)
                .await?;
            Ok(root_mf_id.manifest_unode_id().clone())
        }
    });

    let (root_unode_mf_id, parent_mf_ids) = future::try_join(
        derivation_ctx.derive_dependency::<RootUnodeManifestId>(ctx, bcs_id),
        future::try_join_all(parent_unodes),
    )
    .await?;

    // compute diff between changeset's and its parents' manifests
    let unode_mf_id = root_unode_mf_id.manifest_unode_id().clone();
    let changes = if parent_mf_ids.is_empty() {
        unode_mf_id
            .list_all_entries(ctx.clone(), derivation_ctx.blobstore().clone())
            .try_filter_map(move |(path, _)| async {
                match path {
                    Some(path) => Ok(Some((path, PathChange::Add))),
                    None => Ok(None),
                }
            })
            .try_collect::<Vec<_>>()
            .await
    } else {
        diff_against_parents(ctx, derivation_ctx, unode_mf_id, parent_mf_ids).await
    }?;

    Ok(changes)
}

async fn diff_against_parents(
    ctx: &CoreContext,
    derivation_ctx: &DerivationContext,
    unode: ManifestUnodeId,
    parents: Vec<ManifestUnodeId>,
) -> Result<Vec<(MPath, PathChange)>, Error> {
    let blobstore = derivation_ctx.blobstore();
    let parent_diffs_fut = parents.into_iter().map({
        cloned!(ctx, blobstore, unode);
        move |parent| {
            parent
                .diff(ctx.clone(), blobstore.clone(), unode.clone())
                .try_collect::<Vec<_>>()
        }
    });
    let parent_diffs = future::try_join_all(parent_diffs_fut).await?;
    let diffs = parent_diffs
        .into_iter()
        .flatten()
        .filter_map(|diff| match diff {
            Diff::Added(Some(path), _) => Some((path, PathChange::Add)),
            Diff::Removed(Some(path), _) => Some((path, PathChange::Remove)),
            _ => None,
        });

    let mut changes = BTreeMap::new();
    for (path, change) in diffs {
        // If the changeset has file/dir conflict the diff between
        // parent manifests and the current will have two entries
        // for the same path: one to remove the file/dir, another
        // to introduce new dir/file node.
        changes
            .entry(path)
            .and_modify(|e| {
                if *e != change {
                    *e = PathChange::FileDirConflict
                }
            })
            .or_insert(change);
    }
    let res: Vec<_> = changes.into_iter().collect();
    Ok(res)
}

pub(crate) struct RootDeletedManifestDeriver<Root: RootDeletedManifestIdCommon>(
    std::marker::PhantomData<Root>,
);

impl<Root: RootDeletedManifestIdCommon> RootDeletedManifestDeriver<Root> {
    pub(crate) async fn derive_single(
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        bonsai: BonsaiChangeset,
        parents: Vec<Root>,
    ) -> Result<Root, Error> {
        let bcs_id = bonsai.get_changeset_id();
        let changes = get_changes(ctx, derivation_ctx, bonsai).await?;
        let id = DeletedManifestDeriver::<Root::Manifest>::derive(
            ctx,
            derivation_ctx.blobstore(),
            bcs_id,
            parents
                .into_iter()
                .map(|root_mf_id| root_mf_id.id().clone())
                .collect(),
            changes,
        )
        .await
        .with_context(|| format!("Deriving {}", Root::NAME))?;
        Ok(Root::new(id))
    }

    pub(crate) async fn derive_batch(
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        bonsais: Vec<BonsaiChangeset>,
        gap_size: Option<usize>,
    ) -> Result<HashMap<ChangesetId, Root>, Error> {
        if gap_size.is_some() {
            bail!("Gap size not supported in deleted manifest")
        }
        let simple_stacks =
            split_bonsais_in_linear_stacks(&bonsais, FileConflicts::AnyChange.into())?;
        let id_to_bonsai: HashMap<ChangesetId, BonsaiChangeset> = bonsais
            .into_iter()
            .map(|bonsai| (bonsai.get_changeset_id(), bonsai))
            .collect();
        let use_new_parallel = !tunables()
            .deleted_manifest_disable_new_parallel_derivation()
            .unwrap_or_default();
        borrowed!(id_to_bonsai);
        // Map of ids to derived values.
        // We need to be careful to use this for self-references, since the intermediate derived
        // values don't get stored in blobstore until after this function returns.
        let mut derived: HashMap<ChangesetId, Root> = HashMap::with_capacity(id_to_bonsai.len());
        for stack in simple_stacks {
            let bonsais: Vec<BonsaiChangeset> = stack
                .stack_items
                .into_iter()
                // Panic safety: ids were created from the received bonsais
                .map(|item| id_to_bonsai.get(&item.cs_id).unwrap().clone())
                .collect();
            let parents: Vec<Root::Id> = stream::iter(stack.parents)
                .then(|p| match derived.get(&p) {
                    Some(root) => future::ok(root.clone()).left_future(),
                    None => derivation_ctx.fetch_dependency(ctx, p).right_future(),
                })
                .map_ok(|root: Root| root.id().clone())
                .try_collect()
                .await?;
            if use_new_parallel {
                Self::derive_single_stack(ctx, derivation_ctx, bonsais, parents, &mut derived)
                    .await?;
            } else {
                Self::derive_serially(ctx, derivation_ctx, bonsais, &mut derived).await?;
            }
        }
        Ok(derived)
    }

    async fn derive_serially(
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        stack: Vec<BonsaiChangeset>,
        derived: &mut HashMap<ChangesetId, Root>,
    ) -> Result<(), Error> {
        for bonsai in stack {
            let parents = derivation_ctx
                .fetch_unknown_parents(ctx, Some(derived), &bonsai)
                .await?;
            let id = bonsai.get_changeset_id();
            let root = Self::derive_single(ctx, derivation_ctx, bonsai, parents).await?;
            derived.insert(id, root);
        }
        Ok(())
    }

    async fn derive_single_stack(
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        stack: Vec<BonsaiChangeset>,
        parents: Vec<Root::Id>,
        derived: &mut HashMap<ChangesetId, Root>,
    ) -> Result<(), Error> {
        if parents.len() > 1 {
            // We can't derive stack for merge commits. Let's derive normally.
            // split_bonsais_in_linear_stacks promises us merges go in their own batch
            assert_eq!(stack.len(), 1);
            Self::derive_serially(ctx, derivation_ctx, stack, derived).await?;
        } else {
            let ids: Vec<_> = stack
                .iter()
                .map(|bonsai| bonsai.get_changeset_id())
                .collect();
            let all_changes = stream::iter(stack)
                .map(|bonsai| async move {
                    anyhow::Ok((
                        bonsai.get_changeset_id(),
                        get_changes_list(ctx, derivation_ctx, bonsai).await?,
                    ))
                })
                .buffered(100)
                .try_collect()
                .await?;
            let mf_ids = DeletedManifestDeriver::<Root::Manifest>::derive_simple_stack(
                ctx,
                derivation_ctx.blobstore(),
                parents.into_iter().next(),
                all_changes,
            )
            .await?;
            derived.extend(
                ids.into_iter()
                    .zip(mf_ids.into_iter().map(|id| Root::new(id))),
            );
        }
        Ok(())
    }

    pub(crate) async fn store_mapping(
        root: Root,
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        changeset_id: ChangesetId,
    ) -> Result<(), Error> {
        let key = Root::format_key(derivation_ctx, changeset_id);
        derivation_ctx.blobstore().put(ctx, key, root.into()).await
    }

    pub(crate) async fn fetch(
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        changeset_id: ChangesetId,
    ) -> Result<Option<Root>, Error> {
        let key = Root::format_key(derivation_ctx, changeset_id);
        derivation_ctx
            .blobstore()
            .get(ctx, &key)
            .await?
            .map(TryInto::try_into)
            .transpose()
    }
}

#[cfg(test)]
mod test {
    use anyhow::Result;
    use fbinit::FacebookInit;
    use maplit::btreemap;
    use memblob::Memblob;
    use mononoke_types::deleted_manifest_v2::DeletedManifestV2;
    use mononoke_types::hash::Blake2;
    use mononoke_types::DeletedManifestV2Id;
    use pretty_assertions::assert_eq;
    use PathChange::*;

    use super::*;
    type Id = DeletedManifestV2Id;

    fn csid(x: u8) -> ChangesetId {
        ChangesetId::new(Blake2::from_byte_array([x; 32]))
    }

    async fn assert_derive_stack(
        ctx: &CoreContext,
        blobstore: &Arc<dyn Blobstore>,
        parent: Option<Id>,
        changes_by_cs: BTreeMap<ChangesetId, BTreeMap<&str, PathChange>>,
    ) -> Result<Id> {
        let all_changes: Vec<(ChangesetId, Vec<(MPath, PathChange)>)> = changes_by_cs
            .iter()
            .map(|(k, changes)| {
                let changes = changes
                    .iter()
                    .map(|(name, change)| Ok((MPath::new(name)?, change.clone())))
                    .collect::<Result<Vec<_>>>()?;
                Ok((*k, changes))
            })
            .collect::<Result<_>>()?;
        let stack = DeletedManifestDeriver::<DeletedManifestV2>::derive_simple_stack(
            ctx,
            blobstore,
            parent,
            all_changes.clone(),
        )
        .await?;
        let mut parent = parent;
        let mut single = Vec::with_capacity(changes_by_cs.len());
        for (csid, changes) in all_changes {
            let node = DeletedManifestDeriver::<DeletedManifestV2>::derive(
                ctx,
                blobstore,
                csid,
                parent.iter().cloned().collect(),
                PathTree::from_iter(changes.into_iter().map(|(k, v)| (k, Some(v)))),
            )
            .await?;
            single.push(node);
            parent = Some(node);
        }
        assert_eq!(stack, single);
        let last = stack.last().unwrap().clone();
        Ok(last)
    }

    #[fbinit::test]
    async fn test_stack_derive(fb: FacebookInit) -> Result<()> {
        let blobstore: Arc<dyn Blobstore> = Arc::new(Memblob::default());
        let ctx = CoreContext::test_mock(fb);

        let derive =
            |parent: Option<Id>, changes| assert_derive_stack(&ctx, &blobstore, parent, changes);

        let id = derive(
            None,
            btreemap! {
                csid(1) => btreemap! {
                    "/dira/a" => Add,
                },
                csid(2) => btreemap!{
                    "/dirb/b" => Add,
                },
                csid(3) => btreemap!{
                    "/dira/a" => Remove,
                },
                csid(4) => btreemap!{
                    "/dira/a" => Add,
                    "/dirc/c" => Add,
                }
            },
        )
        .await?;

        derive(
            Some(id),
            btreemap! {
                csid(5) => btreemap! {
                    "/dirb/b" => Remove,
                    "/dirc/c" => Remove,
                },
                csid(6) => btreemap! {
                    "/dird/d" => Add,
                    "/dirc/c/inner_c" => Add,
                    "/new_file" => Add,
                },
                csid(7) => btreemap! {
                    "/dird/d" => FileDirConflict,
                    "/dird/d/inner_d" => Add,
                    "/dir/c/inner_c" => Remove,
                },
                csid(8) => btreemap! {
                    "/dird/d/inner_d" => Remove,
                    "/new_file" => Remove,
                    "/newer_file" => Add,
                }
            },
        )
        .await?;

        // Let's try to get many operations on a single node, basically
        // by deleting and re-adding the same files over and over again
        let files = ["/dir/a", "/dir/b", "/dir/c", "/dir/d", "/other_dir/e"];
        let mut has = vec![true; files.len()];
        let mut changes: BTreeMap<ChangesetId, BTreeMap<&str, PathChange>> = BTreeMap::new();
        // Initially add all files
        changes.insert(csid(0), files.iter().map(|name| (*name, Add)).collect());
        for idx in 1..100usize {
            let idx1 = idx % files.len();
            let idx2 = (idx * 173) % files.len();
            let mut change = BTreeMap::new();
            let has1 = has[idx1];
            change.insert(files[idx1], if has1 { Remove } else { Add });
            has[idx1] = !has1;
            if idx2 != idx1 {
                let has2 = has[idx2];
                change.insert(files[idx2], if has2 { Remove } else { Add });
                has[idx2] = !has2;
            }
            changes.insert(csid(idx as u8), change);
        }
        let id = derive(None, changes.clone()).await?;
        // DM format shouldn't change easily, let's store it in a test.
        assert_eq!(
            id.blobstore_key(),
            "deletedmanifest2.blake2.e95435b8be02a31dcc28465f7e9ba5d9eddd67be782f0c900b00b214d39f0395"
        );
        // Let's also try it in two batches and see if it works the same.
        let mut i = 0usize;
        let (batch1, batch2) = changes.into_iter().partition::<BTreeMap<_, _>, _>(|_| {
            i += 1;
            i <= 50
        });
        let id1 = derive(None, batch1).await?;
        let id2 = derive(Some(id1), batch2).await?;
        assert_eq!(id, id2);

        Ok(())
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::BTreeMap;
use std::collections::HashMap;
use std::sync::Arc;

use anyhow::bail;
use anyhow::Result;
use blobstore::Blobstore;
use blobstore::Loadable;
use borrowed::borrowed;
use bounded_traversal::bounded_traversal;
use context::CoreContext;
use futures::future::FutureExt;
use futures::stream;
use futures::stream::StreamExt;
use futures::stream::TryStreamExt;
use itertools::Itertools;
use manifest::PathTree;
use mononoke_types::deleted_manifest_common::DeletedManifestCommon;
use mononoke_types::BlobstoreKey;
use mononoke_types::ChangesetId;
use mononoke_types::MPath;
use mononoke_types::MPathElement;

use crate::derive::DeletedManifestChangeType;
use crate::derive::DeletedManifestDeriver;
use crate::derive::PathChange;

struct UnfoldNode<M: DeletedManifestCommon> {
    path_element: Option<MPathElement>,
    all_changes: PathTree<Vec<(ChangesetId, PathChange)>>,
    parent: Option<M::Id>,
}

struct FoldNode<M> {
    change_by_cs: ByChangeset<DeletedManifestChangeType>,
    parent: Option<M>,
    base_name: Option<MPathElement>,
}

struct FoldOutput<M: DeletedManifestCommon>(Option<MPathElement>, ByChangeset<Option<M::Id>>);

struct ByChangeset<V>(HashMap<ChangesetId, V>);

impl<Manifest: DeletedManifestCommon> DeletedManifestDeriver<Manifest> {
    /// Derives DM for a simple stack of commits
    /// The idea is similar to manifest::derive_manifests_for_simple_stack_of_commits (https://fburl.com/code/5fy838h2)
    /// A simple stack of commits is a stack of consecutive commits that does not include a merge commit.
    // Explanation: We can't really parallelise DM derivation over commits, as each change creates a new
    // node, and it depends on the previous node. However, we can parallelise the derivation of the nodes
    // themselves. That is, if a node (for a particular file) was only edited on commit A on the stack,
    // and another node was modified only on commit B, we can derive those two new nodes in parallel.
    // The root will of course be edited always, and the parallelisation won't help there, and in general
    // this optimisation does not improve the worst case (all commits modify the same files), but it
    // optimises the usual case (commits touch mostly different files) quite well.
    pub(crate) async fn derive_simple_stack(
        ctx: &CoreContext,
        blobstore: &Arc<dyn Blobstore>,
        parent: Option<Manifest::Id>,
        all_changes: Vec<(ChangesetId, Vec<(MPath, PathChange)>)>,
    ) -> Result<Vec<Manifest::Id>> {
        let (path_tree, commit_stack) = {
            let mut path_tree: PathTree<Vec<(ChangesetId, PathChange)>> = PathTree::default();
            let mut commit_stack: Vec<ChangesetId> = vec![];
            for (csid, cs_changes) in all_changes {
                for (path, change) in cs_changes {
                    path_tree.insert_and_merge(Some(path), (csid, change));
                }
                commit_stack.push(csid);
            }
            (path_tree, commit_stack)
        };

        let FoldOutput(name, mfid_by_cs) = bounded_traversal(
            256,
            UnfoldNode {
                path_element: None,
                all_changes: path_tree,
                parent,
            },
            |unfold_node| Self::unfold_batch(ctx, blobstore, unfold_node).boxed(),
            |fold_node, subentries| {
                Self::fold_batch(ctx, blobstore, &commit_stack, fold_node, subentries).boxed()
            },
        )
        .await?;
        assert!(name.is_none());
        Self::finalize_stack(ctx, blobstore, commit_stack, parent, mfid_by_cs).await
    }

    /// Given a stack of commits and a manifest ids for some of those, create
    /// manifests ids for ALL of them, by reusing from previous nodes in the stack.
    /// If a commit doesn't have an id, the manifest is empty and an empty root
    /// should be created for it.
    async fn finalize_stack(
        ctx: &CoreContext,
        blobstore: &Arc<dyn Blobstore>,
        commit_stack: Vec<ChangesetId>,
        parent_id: Option<Manifest::Id>,
        mfid_by_cs: ByChangeset<Option<Manifest::Id>>,
    ) -> Result<Vec<Manifest::Id>> {
        let mut mfids: Vec<Manifest::Id> = Vec::with_capacity(commit_stack.len());
        let mut cur_mfid = parent_id;
        // This is a "cache", so we only create the empty manifest once, if necessary
        let mut cached_empty_mf_id = None;
        for csid in commit_stack {
            if let Some(maybe_mfid) = mfid_by_cs.0.get(&csid) {
                cur_mfid = *maybe_mfid;
            }
            let mfid = if let Some(mfid) = cur_mfid.clone() {
                mfid
            } else {
                // Empty dm, we need to create an empty root
                // if it's not cached already
                if let Some(mfid) = cached_empty_mf_id {
                    mfid
                } else {
                    let mf = Manifest::copy_and_update_subentries(
                        ctx,
                        blobstore,
                        None,
                        None,
                        BTreeMap::new(),
                    )
                    .await?;
                    let mfid = Self::save_mf(mf, ctx, blobstore).await?;
                    cached_empty_mf_id = Some(mfid);
                    mfid
                }
            };
            mfids.push(mfid);
        }
        Ok(mfids)
    }

    async fn unfold_batch(
        ctx: &CoreContext,
        blobstore: &Arc<dyn Blobstore>,
        unfold_node: UnfoldNode<Manifest>,
    ) -> Result<(
        FoldNode<Manifest>,
        impl IntoIterator<Item = UnfoldNode<Manifest>>,
    )> {
        let UnfoldNode {
            path_element,
            all_changes,
            parent,
        } = unfold_node;
        let PathTree {
            value: change_by_cs,
            subentries,
        } = all_changes;

        let parent = match parent {
            None => None,
            Some(p) => Some(p.load(ctx, blobstore).await?),
        };

        let change_by_cs = ByChangeset(
            change_by_cs
                .into_iter()
                .map(|(cs_id, change)| {
                    (
                        cs_id,
                        match change {
                            PathChange::Add | PathChange::FileDirConflict => {
                                DeletedManifestChangeType::RemoveIfNowEmpty
                            }
                            PathChange::Remove => DeletedManifestChangeType::CreateDeleted,
                        },
                    )
                })
                .collect(),
        );

        let children = {
            borrowed!(parent);
            stream::iter(
                subentries
                    .into_iter()
                    .map({
                        |(path, path_tree)| async move {
                            let parent = match parent {
                                None => None,
                                Some(p) => p.lookup(ctx, blobstore, &path).await?,
                            };
                            anyhow::Ok(UnfoldNode {
                                path_element: Some(path),
                                all_changes: path_tree,
                                parent,
                            })
                        }
                    })
                    .collect::<Vec<_>>(),
            )
            .buffer_unordered(100)
            .try_collect::<Vec<_>>()
            .await?
        };

        let fold_node = FoldNode {
            change_by_cs,
            parent,
            base_name: path_element,
        };

        Ok((fold_node, children))
    }

    /// Given a stack of commits and changes applied to some of those commits, create
    /// the manifests for the commits with modifications.
    async fn apply_changes_to_stack_and_create_mfs(
        ctx: &CoreContext,
        blobstore: &Arc<dyn Blobstore>,
        commit_stack: &[ChangesetId],
        mut parent: Option<Manifest>,
        mut modified_subentries_by_cs: ByChangeset<BTreeMap<MPathElement, Option<Manifest::Id>>>,
        mut change_by_cs: ByChangeset<DeletedManifestChangeType>,
    ) -> Result<ByChangeset<Option<Manifest::Id>>> {
        let mut mfid_by_cs: ByChangeset<Option<Manifest::Id>> = ByChangeset(HashMap::new());
        // TODO: We can optimise here even further by only iterating over the commits that are
        // keys in either modified_subentries_by_cs or change_by_cs, but we need to do so in
        // the correct order. This optimisation likely doesn't help much as iterating over the
        // whole array likely doesn't add much overhead.
        for csid in commit_stack {
            let modified_subentries = modified_subentries_by_cs.0.remove(csid);
            let change = change_by_cs.0.remove(csid);
            if change.is_none() && modified_subentries.is_none() {
                // No changes since last commit, continuing here just reuses the
                // same node from the previous commit
                continue;
            }
            let cur_subentries_to_update = modified_subentries.unwrap_or_default();
            let mf = match change.unwrap_or(DeletedManifestChangeType::RemoveIfNowEmpty) {
                DeletedManifestChangeType::Reuse => {
                    bail!("Reuse is implicit on batch derivation")
                }
                DeletedManifestChangeType::CreateDeleted => {
                    let mf = Manifest::copy_and_update_subentries(
                        ctx,
                        blobstore,
                        parent,
                        Some(*csid),
                        cur_subentries_to_update.clone(),
                    )
                    .await?;
                    Self::save_mf(mf.clone(), ctx, blobstore).await?;
                    Some(mf)
                }
                DeletedManifestChangeType::RemoveIfNowEmpty => {
                    let mf = Manifest::copy_and_update_subentries(
                        ctx,
                        blobstore,
                        parent,
                        None,
                        cur_subentries_to_update.clone(),
                    )
                    .await?;
                    if mf.is_empty() {
                        None
                    } else {
                        Self::save_mf(mf.clone(), ctx, blobstore).await?;
                        Some(mf)
                    }
                }
            };
            let previous = mfid_by_cs.0.insert(*csid, mf.as_ref().map(|m| m.id()));
            assert!(previous.is_none());
            parent = mf;
        }
        assert!(change_by_cs.0.is_empty());
        assert!(modified_subentries_by_cs.0.is_empty());
        Ok(mfid_by_cs)
    }

    async fn fold_batch(
        ctx: &CoreContext,
        blobstore: &Arc<dyn Blobstore>,
        commit_stack: &[ChangesetId],
        fold_node: FoldNode<Manifest>,
        subentries: impl Iterator<Item = FoldOutput<Manifest>>,
    ) -> Result<FoldOutput<Manifest>> {
        let FoldNode {
            change_by_cs,
            parent,
            base_name,
        } = fold_node;
        let modified_subentries_by_cs = ByChangeset(
            subentries
                .filter_map(|FoldOutput(subname, mfid_by_cs)| {
                    Some(std::iter::repeat(subname?).zip(mfid_by_cs.0))
                })
                .flatten()
                .map(|(sub_name, (csid, maybe_mfid))| (csid, (sub_name, maybe_mfid)))
                .into_grouping_map()
                .collect(),
        );

        let mfid_by_cs = Self::apply_changes_to_stack_and_create_mfs(
            ctx,
            blobstore,
            commit_stack,
            parent,
            modified_subentries_by_cs,
            change_by_cs,
        )
        .await?;
        Ok(FoldOutput(base_name, mfid_by_cs))
    }

    async fn save_mf(
        mf: Manifest,
        ctx: &CoreContext,
        blobstore: &Arc<dyn Blobstore>,
    ) -> Result<Manifest::Id> {
        let mf_id = mf.id();
        let key = mf_id.blobstore_key();
        blobstore.put(ctx, key, mf.into_blob().into()).await?;
        Ok(mf_id)
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

#![feature(trait_alias)]

mod derive;
mod derive_batch;
mod mapping;
mod mapping_v2;
mod ops;
#[cfg(test)]
pub mod test_utils;

pub use mapping::RootDeletedManifestIdCommon;
pub use mapping_v2::format_key;
pub use mapping_v2::RootDeletedManifestV2Id;
pub use ops::DeletedManifestOps;
pub use ops::PathState;


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use blobstore::BlobstoreGetData;
use blobstore::Loadable;
use derived_data_manager::BonsaiDerivable;
use derived_data_manager::DerivationContext;
use mononoke_types::deleted_manifest_common::DeletedManifestCommon;
use mononoke_types::BlobstoreBytes;
use mononoke_types::ChangesetId;
use mononoke_types::MononokeId;

/// Common trait for the root id of all deleted manifest types
pub trait RootDeletedManifestIdCommon:
    BonsaiDerivable
    + std::fmt::Debug
    + Clone
    + Into<BlobstoreBytes>
    + TryFrom<BlobstoreGetData, Error = anyhow::Error>
{
    /// The manifest type
    type Manifest: DeletedManifestCommon<Id = Self::Id>;
    /// The id type (Manifest::Id)
    // Basically just a type alias to Manifest::Id, but Rust makes us be a bit more verbose
    type Id: MononokeId<Value = Self::Manifest> + Loadable<Value = Self::Manifest>;

    /// Create a root DM id
    fn new(id: Self::Id) -> Self;

    /// Get the id of the root deleted manifest node
    fn id(&self) -> &Self::Id;

    /// Create a key for this DM in the given Changeset
    fn format_key(derivation_ctx: &DerivationContext, changeset_id: ChangesetId) -> String;
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;

use anyhow::anyhow;
use anyhow::Error;
use anyhow::Result;
use async_trait::async_trait;
use blobstore::BlobstoreGetData;
use bytes::Bytes;
use context::CoreContext;
use derived_data::impl_bonsai_derived_via_manager;
use derived_data_manager::dependencies;
use derived_data_manager::BonsaiDerivable;
use derived_data_manager::DerivableType;
use derived_data_manager::DerivationContext;
use derived_data_service_if::types as thrift;
use mononoke_types::deleted_manifest_v2::DeletedManifestV2;
use mononoke_types::BlobstoreBytes;
use mononoke_types::BonsaiChangeset;
use mononoke_types::ChangesetId;
use mononoke_types::DeletedManifestV2Id;
use unodes::RootUnodeManifestId;

use crate::derive::RootDeletedManifestDeriver;
use crate::mapping::RootDeletedManifestIdCommon;

#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
pub struct RootDeletedManifestV2Id(DeletedManifestV2Id);

pub fn format_key(derivation_ctx: &DerivationContext, changeset_id: ChangesetId) -> String {
    RootDeletedManifestV2Id::format_key(derivation_ctx, changeset_id)
}

impl RootDeletedManifestIdCommon for RootDeletedManifestV2Id {
    type Manifest = DeletedManifestV2;
    type Id = DeletedManifestV2Id;

    fn id(&self) -> &Self::Id {
        &self.0
    }

    fn new(id: Self::Id) -> Self {
        Self(id)
    }

    fn format_key(derivation_ctx: &DerivationContext, changeset_id: ChangesetId) -> String {
        let root_prefix = "derived_root_deleted_manifest2.";
        let key_prefix = derivation_ctx.mapping_key_prefix::<RootDeletedManifestV2Id>();
        format!("{}{}{}", root_prefix, key_prefix, changeset_id)
    }
}

impl TryFrom<BlobstoreBytes> for RootDeletedManifestV2Id {
    type Error = Error;
    fn try_from(blob_bytes: BlobstoreBytes) -> Result<Self> {
        DeletedManifestV2Id::from_bytes(&blob_bytes.into_bytes()).map(RootDeletedManifestV2Id)
    }
}

impl TryFrom<BlobstoreGetData> for RootDeletedManifestV2Id {
    type Error = Error;
    fn try_from(blob_val: BlobstoreGetData) -> Result<Self> {
        blob_val.into_bytes().try_into()
    }
}

impl From<RootDeletedManifestV2Id> for BlobstoreBytes {
    fn from(root_mf_id: RootDeletedManifestV2Id) -> Self {
        BlobstoreBytes::from_bytes(Bytes::copy_from_slice(root_mf_id.0.blake2().as_ref()))
    }
}

#[async_trait]
impl BonsaiDerivable for RootDeletedManifestV2Id {
    const VARIANT: DerivableType = DerivableType::DeletedManifests;

    type Dependencies = dependencies![RootUnodeManifestId];

    async fn derive_single(
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        bonsai: BonsaiChangeset,
        parents: Vec<Self>,
    ) -> Result<Self, Error> {
        RootDeletedManifestDeriver::derive_single(ctx, derivation_ctx, bonsai, parents).await
    }

    async fn store_mapping(
        self,
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        changeset_id: ChangesetId,
    ) -> Result<()> {
        RootDeletedManifestDeriver::store_mapping(self, ctx, derivation_ctx, changeset_id).await
    }

    async fn fetch(
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        changeset_id: ChangesetId,
    ) -> Result<Option<Self>> {
        RootDeletedManifestDeriver::fetch(ctx, derivation_ctx, changeset_id).await
    }

    async fn derive_batch(
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        bonsais: Vec<BonsaiChangeset>,
        gap_size: Option<usize>,
    ) -> Result<HashMap<ChangesetId, Self>> {
        RootDeletedManifestDeriver::derive_batch(ctx, derivation_ctx, bonsais, gap_size).await
    }

    fn from_thrift(data: thrift::DerivedData) -> Result<Self> {
        if let thrift::DerivedData::deleted_manifest_v2(
            thrift::DerivedDataDeletedManifestV2::root_deleted_manifest_v2_id(id),
        ) = data
        {
            DeletedManifestV2Id::from_thrift(id).map(Self)
        } else {
            Err(anyhow!(
                "Can't convert {} from provided thrift::DerivedData",
                Self::NAME.to_string(),
            ))
        }
    }

    fn into_thrift(data: Self) -> Result<thrift::DerivedData> {
        Ok(thrift::DerivedData::deleted_manifest_v2(
            thrift::DerivedDataDeletedManifestV2::root_deleted_manifest_v2_id(
                data.id().into_thrift(),
            ),
        ))
    }
}

impl_bonsai_derived_via_manager!(RootDeletedManifestV2Id);

#[cfg(test)]
crate::test_utils::impl_deleted_manifest_tests!(RootDeletedManifestV2Id);


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::borrow::Borrow;
use std::collections::HashSet;
use std::collections::VecDeque;

use anyhow::Error;
use blobstore::Blobstore;
use blobstore::Loadable;
use bounded_traversal::bounded_traversal_stream;
use changeset_fetcher::ChangesetFetcherRef;
use cloned::cloned;
use context::CoreContext;
use futures::future;
use futures::pin_mut;
use futures::stream;
use futures::stream::BoxStream;
use futures::FutureExt;
use futures::StreamExt;
use futures::TryStreamExt;
use manifest::Entry;
use manifest::ManifestOps;
use manifest::PathOrPrefix;
use manifest::PathTree;
use maplit::hashset;
use mononoke_types::deleted_manifest_common::DeletedManifestCommon;
use mononoke_types::ChangesetId;
use mononoke_types::FileUnodeId;
use mononoke_types::MPath;
use mononoke_types::ManifestUnodeId;
use repo_blobstore::RepoBlobstoreArc;
use repo_derived_data::RepoDerivedDataRef;
use unodes::RootUnodeManifestId;

//use time_ext::DurationExt;
use crate::mapping::RootDeletedManifestIdCommon;

type UnodeEntry = Entry<ManifestUnodeId, FileUnodeId>;

pub enum PathState {
    // Changeset where the path was deleted and unode where the path was last changed.
    Deleted(Vec<(ChangesetId, UnodeEntry)>),
    // Unode if the path exists.
    Exists(UnodeEntry),
}

pub trait Repo =
    RepoDerivedDataRef + RepoBlobstoreArc + ChangesetFetcherRef + Clone + Copy + Send + Sync;

#[async_trait::async_trait]
pub trait DeletedManifestOps: RootDeletedManifestIdCommon {
    /// Find if and when the path deleted.
    ///
    /// Given a changeset and a path returns:
    ///  * if the paths exists in the commit: the unode corresponding to the path
    ///  * if it doesn't the unodes and changeset where the path last existed (there might be more than
    ///    one if deletion happened in separe merge branches)
    ///  * if the path never existed returns None
    ///
    /// Returns None for deleted files if deleted file manifests are not enabled in a given repo.
    ///
    /// This is the high-level public API of this crate i.e. what clients should use if they want to
    /// fetch find where the path was deleted.
    async fn resolve_path_state<'a>(
        ctx: &'a CoreContext,
        repo: impl Repo + 'a,
        cs_id: ChangesetId,
        path: &'a Option<MPath>,
    ) -> Result<Option<PathState>, Error> {
        // if unode exists return entry
        let unode_entry = derive_unode_entry(ctx, repo, cs_id.clone(), path).await?;
        if let Some(unode_entry) = unode_entry {
            return Ok(Some(PathState::Exists(unode_entry)));
        }

        let use_deleted_manifest = repo.repo_derived_data().config().is_enabled(Self::NAME);
        if !use_deleted_manifest {
            return Ok(None);
        }

        // if there is no unode for the commit:path, check deleted manifest
        // the path might be deleted
        let deleted = stream::try_unfold(
            // starting state
            (VecDeque::from(vec![cs_id.clone()]), hashset! { cs_id }),
            // unfold
            {
                cloned!(ctx, repo, path);
                move |(queue, visited)| {
                    Self::resolve_path_state_unfold(
                        ctx.clone(),
                        repo.clone(),
                        path.clone(),
                        queue,
                        visited,
                    )
                }
            },
        )
        .try_collect::<Vec<_>>()
        .await?
        .into_iter()
        .flatten()
        .collect::<Vec<_>>();
        if deleted.is_empty() {
            Ok(None)
        } else {
            Ok(Some(PathState::Deleted(deleted)))
        }
    }

    async fn resolve_path_state_unfold(
        ctx: CoreContext,
        repo: impl Repo,
        path: Option<MPath>,
        mut queue: VecDeque<ChangesetId>,
        mut visited: HashSet<ChangesetId>,
    ) -> Result<
        Option<(
            Vec<(ChangesetId, UnodeEntry)>,
            (VecDeque<ChangesetId>, HashSet<ChangesetId>),
        )>,
        Error,
    > {
        let manager = repo.repo_derived_data().manager();
        // let's get deleted manifests for each changeset id
        // and try to find the given path
        if let Some(cs_id) = queue.pop_front() {
            let root_dfm_id = manager.derive::<Self>(&ctx, cs_id, None).await?;
            let entry = root_dfm_id
                .find_entry(&ctx, repo.repo_blobstore(), path.clone())
                .await?;

            if let Some(mf_id) = entry {
                // we need to get the linknode, so let's load the deleted manifest
                // if the linknodes is None it means that file should exist
                // but it doesn't, let's throw an error
                let mf = mf_id.load(&ctx, repo.repo_blobstore()).await?;
                let linknode = mf.linknode().ok_or_else(|| {
                let message = format!(
                    "there is no unode for the path '{}' and changeset {:?}, but it exists as a live entry in deleted manifest",
                    MPath::display_opt(path.as_ref()),
                    cs_id,
                );
                Error::msg(message)
            })?;

                // to get last change before deletion we have to look at the liknode
                // parents for the deleted path
                let parents = &repo
                    .changeset_fetcher()
                    .get_parents(&ctx, linknode.clone())
                    .await?;

                // checking parent unodes
                let parent_unodes = parents.iter().map({
                    cloned!(ctx, repo, path);
                    move |parent| {
                        cloned!(ctx, repo, path);
                        async move {
                            let unode_entry =
                                derive_unode_entry(&ctx, repo, parent.clone(), &path).await?;
                            Ok::<_, Error>((parent, unode_entry))
                        }
                    }
                });
                let parent_unodes = future::try_join_all(parent_unodes).await?;
                return match *parent_unodes {
                    [] => {
                        // the linknode must have a parent, otherwise the path couldn't be deleted
                        let message = format!(
                            "the path '{}' was deleted in {:?}, but the changeset doesn't have parents",
                            MPath::display_opt(path.as_ref()),
                            linknode,
                        );
                        Err(Error::msg(message))
                    }
                    [(_parent, unode_entry)] => {
                        if let Some(unode_entry) = unode_entry {
                            // we've found the last path change before deletion
                            Ok(Some((vec![(*linknode, unode_entry)], (queue, visited))))
                        } else {
                            // the unode entry must exist
                            let message = format!(
                                "the path '{}' was deleted in {:?}, but the parent changeset doesn't have a unode",
                                MPath::display_opt(path.as_ref()),
                                linknode,
                            );
                            Err(Error::msg(message))
                        }
                    }
                    _ => {
                        let mut last_changes = vec![];
                        for (parent, unode_entry) in parent_unodes.into_iter() {
                            if let Some(unode_entry) = unode_entry {
                                // this is one of the last changes
                                last_changes.push((*linknode, unode_entry));
                            } else {
                                // the path could have been already deleted here
                                // need to add this node into the queue
                                if visited.insert(parent.clone()) {
                                    queue.push_back(*parent);
                                }
                            }
                        }
                        Ok(Some((last_changes, (queue, visited))))
                    }
                };
            }

            // the path was not deleted here, but could be deleted in other branches
            return Ok(Some((vec![], (queue, visited))));
        }

        Ok(None)
    }

    /// List all Deleted Manifest paths recursively that were deleted and match specified paths
    /// and/or prefixes.
    ///
    fn find_entries<'a>(
        &self,
        ctx: &'a CoreContext,
        blobstore: &'a impl Blobstore,
        paths_or_prefixes: impl IntoIterator<Item = impl Into<PathOrPrefix>>,
    ) -> BoxStream<'a, Result<(Option<MPath>, Self::Id), Error>> {
        let root_id = self.id().clone();
        enum Pattern {
            Path,
            Prefix,
        }

        enum Selector {
            Selector(PathTree<Option<Pattern>>),
            Recursive,
        }

        let path_tree = PathTree::from_iter(paths_or_prefixes.into_iter().map(|path_or_prefix| {
            match path_or_prefix.into() {
                PathOrPrefix::Path(path) => (path, Some(Pattern::Path)),
                PathOrPrefix::Prefix(prefix) => (prefix, Some(Pattern::Prefix)),
            }
        }));

        (async_stream::stream! {
            let ctx = ctx.borrow();
            let blobstore = &blobstore;
            let s: BoxStream<'_, Result<(Option<MPath>, Self::Id), Error>> = bounded_traversal_stream(
                256,
                // starting point
                Some((None, Selector::Selector(path_tree), root_id)),
                move |(path, selector, manifest_id)| {
                    async move {
                        let mf = manifest_id.load(ctx, blobstore).await?;
                        let return_entry = if mf.is_deleted() {
                            vec![(path.clone(), manifest_id)]
                        } else {
                            vec![]
                        };

                        match selector {
                            Selector::Recursive => {
                                // collect subentries to recurse into
                                let recurse = mf.into_subentries(ctx, blobstore).map_ok(|(name, mf_id)| {
                                    let next_path = MPath::join_opt_element(path.as_ref(), &name);
                                    (Some(next_path), Selector::Recursive, mf_id)
                                }).try_collect::<Vec<_>>().await?;

                                Ok((return_entry, recurse))
                            }
                            Selector::Selector(path_tree) => {
                                let PathTree { value, subentries } = path_tree;

                                match value {
                                    Some(Pattern::Prefix) => {
                                        // collect subentries to recurse into
                                        let recurse = mf.into_subentries(ctx, blobstore).map_ok(|(name, mf_id)| {
                                            let next_path = MPath::join_opt_element(path.as_ref(), &name);
                                            (Some(next_path), Selector::Recursive, mf_id)
                                        }).try_collect::<Vec<_>>().await?;

                                        Ok((return_entry, recurse))
                                    }
                                    // Rustc bug: 1.50.0 considers the None pattern wrongly unreachable.
                                    // https://github.com/rust-lang/rust/issues/82012
                                    #[allow(unreachable_patterns)]
                                    None | Some(Pattern::Path) => {
                                        // need to recurse
                                        let mut recurse = vec![];
                                        // add path tree selectors
                                        for (name, tree) in subentries {
                                            if let Some(mf_id) = mf.lookup(ctx, blobstore, &name).await? {
                                                let next_path =
                                                    MPath::join_opt_element(path.as_ref(), &name);
                                                recurse.push((
                                                    Some(next_path),
                                                    Selector::Selector(tree),
                                                    mf_id,
                                                ));
                                            }
                                        }

                                        let return_path = if let Some(Pattern::Path) = value {
                                            return_entry
                                        } else {
                                            vec![]
                                        };
                                        Result::<_, Error>::Ok((return_path, recurse))
                                    }
                                }
                            }
                        }
                    }.boxed()
                },
            )
            .map_ok(|entries| stream::iter(entries.into_iter().map(Ok)))
            .try_flatten()
            .boxed();

            pin_mut!(s);
            while let Some(value) = s.next().await {
                yield value;
            }
        }).boxed()
    }

    /// Return Deleted Manifest entry for the given path
    ///
    async fn find_entry(
        &self,
        ctx: &CoreContext,
        blobstore: &impl Blobstore,
        path: Option<MPath>,
    ) -> Result<Option<Self::Id>, Error> {
        let s = self.find_entries(ctx, blobstore, vec![PathOrPrefix::Path(path)]);
        pin_mut!(s);
        match s.next().await.transpose()? {
            Some((_path, mf_id)) => Ok(Some(mf_id)),
            None => Ok(None),
        }
    }

    /// List all Deleted manifest entries recursively, that represent deleted paths.
    ///
    fn list_all_entries<'a>(
        &self,
        ctx: &'a CoreContext,
        blobstore: &'a impl Blobstore,
    ) -> BoxStream<'a, Result<(Option<MPath>, Self::Id), Error>> {
        let root_id = self.id().clone();
        (async_stream::stream! {
            let ctx = ctx.borrow();
            let blobstore = &blobstore;
            let s = bounded_traversal_stream(256, Some((None, root_id)), move |(path, manifest_id)| {
                async move {
                    let manifest = manifest_id.load(ctx, blobstore).await?;
                    let entry = if manifest.is_deleted() {
                        vec![(path.clone(), manifest_id)]
                    } else {
                        vec![]
                    };
                    let recurse_subentries = manifest
                        .into_subentries(ctx, blobstore)
                        .map_ok(|(name, mf_id)| {
                            let full_path = MPath::join_opt_element(path.as_ref(), &name);
                            (Some(full_path), mf_id)
                        })
                        .try_collect::<Vec<_>>().await?;

                    Result::<_, Error>::Ok((entry, recurse_subentries))
                }.boxed()
            })
            .map_ok(|entries| stream::iter(entries.into_iter().map(Ok)))
            .try_flatten();

            pin_mut!(s);
            while let Some(value) = s.next().await {
                yield value;
            }
        }).boxed()
    }
}

impl<Root: RootDeletedManifestIdCommon> DeletedManifestOps for Root {}

async fn derive_unode_entry(
    ctx: &CoreContext,
    repo: impl RepoDerivedDataRef + RepoBlobstoreArc,
    cs_id: ChangesetId,
    path: &Option<MPath>,
) -> Result<Option<UnodeEntry>, Error> {
    let root_unode_mf_id = repo
        .repo_derived_data()
        .manager()
        .derive::<RootUnodeManifestId>(ctx, cs_id, None)
        .await?;
    root_unode_mf_id
        .manifest_unode_id()
        .find_entry(ctx.clone(), repo.repo_blobstore_arc(), path.clone())
        .await
}