/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::BTreeMap;
use std::sync::Arc;

use anyhow::Error;
use blobstore::Blobstore;
use blobstore::Loadable;
use bonsai_hg_mapping::BonsaiHgMapping;
use bookmarks::Bookmarks;
use bounded_traversal::bounded_traversal_stream;
use changeset_fetcher::ChangesetFetcher;
use changesets::Changesets;
use changesets_creation::save_changesets;
use cloned::cloned;
use context::CoreContext;
use derived_data_test_utils::bonsai_changeset_from_hg;
use fbinit::FacebookInit;
use filestore::FilestoreConfig;
use fixtures::store_files;
use fixtures::ManyFilesDirs;
use fixtures::TestRepoFixture;
use futures::pin_mut;
use futures::stream::iter;
use futures::FutureExt;
use futures::Stream;
use futures::StreamExt;
use futures::TryStreamExt;
use manifest::PathOrPrefix;
use maplit::btreemap;
use mononoke_types::deleted_manifest_common::DeletedManifestCommon;
use mononoke_types::BonsaiChangeset;
use mononoke_types::BonsaiChangesetMut;
use mononoke_types::ChangesetId;
use mononoke_types::DateTime;
use mononoke_types::FileChange;
use mononoke_types::MPath;
use pretty_assertions::assert_eq;
use repo_blobstore::RepoBlobstore;
use repo_blobstore::RepoBlobstoreArc;
use repo_blobstore::RepoBlobstoreRef;
use repo_derived_data::RepoDerivedData;
use repo_derived_data::RepoDerivedDataRef;
use sorted_vector_map::SortedVectorMap;
use tests_utils::CreateCommitContext;

use crate::derive::get_changes;
use crate::derive::DeletedManifestDeriver;
use crate::mapping::RootDeletedManifestIdCommon;
use crate::ops::DeletedManifestOps;

#[derive(Clone)]
#[facet::container]
struct TestRepo {
    #[facet]
    bonsai_hg_mapping: dyn BonsaiHgMapping,
    #[facet]
    bookmarks: dyn Bookmarks,
    #[facet]
    repo_blobstore: RepoBlobstore,
    #[facet]
    repo_derived_data: RepoDerivedData,
    #[facet]
    filestore_config: FilestoreConfig,
    #[facet]
    changeset_fetcher: dyn ChangesetFetcher,
    #[facet]
    changesets: dyn Changesets,
}

/// Defines all common DM tests.
// Why a macro and not a function? So we get different tests that are paralellised
// and have separate signal, instead of a single catch-all test.
macro_rules! impl_deleted_manifest_tests {
    ($manifest:ty) => {
        mod tests {
            use ::anyhow::Result;
            use ::fbinit::FacebookInit;

            use super::*;

            #[fbinit::test]
            async fn linear_test(fb: FacebookInit) {
                $crate::test_utils::linear_test::<$manifest>(fb).await
            }
            #[fbinit::test]
            async fn many_file_dirs_test(fb: FacebookInit) {
                $crate::test_utils::many_file_dirs_test::<$manifest>(fb).await
            }
            #[fbinit::test]
            async fn merged_history_test(fb: FacebookInit) -> Result<()> {
                $crate::test_utils::merged_history_test::<$manifest>(fb).await
            }
            #[fbinit::test]
            async fn test_find_entries(fb: FacebookInit) {
                $crate::test_utils::test_find_entries::<$manifest>(fb).await
            }
            #[fbinit::test]
            async fn test_list_all_entries(fb: FacebookInit) {
                $crate::test_utils::test_list_all_entries::<$manifest>(fb).await
            }
        }
    };
}
pub(crate) use impl_deleted_manifest_tests;

async fn build_repo(fb: FacebookInit) -> Result<TestRepo, Error> {
    Ok(test_repo_factory::TestRepoFactory::new(fb)?.build().await?)
}

pub(crate) async fn linear_test<Root: RootDeletedManifestIdCommon>(fb: FacebookInit) {
    // Test simple separate files and whole dir deletions
    let repo: TestRepo = build_repo(fb).await.unwrap();
    let ctx = CoreContext::test_mock(fb);

    // create parent deleted manifest
    let (bcs_id_1, mf_id_1) = {
        let file_changes = btreemap! {
            "file.txt" => Some("1\n"),
            "file-2.txt" => Some("2\n"),
            "dir/sub/f-1" => Some("3\n"),
            "dir/f-2" => Some("4\n"),
            "dir-2/sub/f-3" => Some("5\n"),
            "dir-2/f-4" => Some("6\n"),
        };
        let (bcs_id, mf_id, deleted_nodes) =
            create_cs_and_derive_manifest::<Root>(ctx.clone(), repo.clone(), file_changes, vec![])
                .await;

        // nothing was deleted yet
        let expected_nodes = vec![(None, Status::Live)];
        assert_eq!(deleted_nodes, expected_nodes);

        (bcs_id, mf_id)
    };

    // delete some files and dirs
    let (bcs_id_2, mf_id_2) = {
        let file_changes = btreemap! {
            "file.txt" => None,
            "file-2.txt" => Some("2\n2\n"),
            "file-3.txt" => Some("3\n3\n"),
            "dir/sub/f-1" => None,
            "dir/f-2" => None,
            "dir-2/sub/f-3" => None,
        };
        let (bcs_id, mf_id, deleted_nodes) = create_cs_and_derive_manifest::<Root>(
            ctx.clone(),
            repo.clone(),
            file_changes,
            vec![(bcs_id_1, mf_id_1)],
        )
        .await;

        let expected_nodes = vec![
            (None, Status::Live),
            (Some(path("dir")), Status::Deleted(bcs_id)),
            (Some(path("dir/f-2")), Status::Deleted(bcs_id)),
            (Some(path("dir/sub")), Status::Deleted(bcs_id)),
            (Some(path("dir/sub/f-1")), Status::Deleted(bcs_id)),
            (Some(path("dir-2")), Status::Live),
            (Some(path("dir-2/sub")), Status::Deleted(bcs_id)),
            (Some(path("dir-2/sub/f-3")), Status::Deleted(bcs_id)),
            (Some(path("file.txt")), Status::Deleted(bcs_id)),
        ];
        assert_eq!(deleted_nodes, expected_nodes);

        (bcs_id, mf_id)
    };

    // reincarnate file and directory
    let (bcs_id_3, mf_id_3) = {
        let file_changes = btreemap! {
            "file.txt" => Some("1\n1\n1\n"),
            "file-2.txt" => None,
            "dir/sub/f-4" => Some("4\n4\n4\n"),
        };
        let (bcs_id, mf_id, deleted_nodes) = create_cs_and_derive_manifest::<Root>(
            ctx.clone(),
            repo.clone(),
            file_changes,
            vec![(bcs_id_2, mf_id_2)],
        )
        .await;

        let expected_nodes = vec![
            (None, Status::Live),
            (Some(path("dir")), Status::Live),
            (Some(path("dir/f-2")), Status::Deleted(bcs_id_2)),
            (Some(path("dir/sub")), Status::Live),
            (Some(path("dir/sub/f-1")), Status::Deleted(bcs_id_2)),
            (Some(path("dir-2")), Status::Live),
            (Some(path("dir-2/sub")), Status::Deleted(bcs_id_2)),
            (Some(path("dir-2/sub/f-3")), Status::Deleted(bcs_id_2)),
            (Some(path("file-2.txt")), Status::Deleted(bcs_id)),
        ];
        assert_eq!(deleted_nodes, expected_nodes);

        (bcs_id, mf_id)
    };

    // reincarnate file as dir and dir as file
    let (bcs_id_4, mf_id_4) = {
        let file_changes = btreemap! {
            // file as dir
            "file-2.txt/subfile.txt" => Some("2\n2\n1\n"),
            // dir as file
            "dir-2/sub" => Some("file now!\n"),
        };
        let (bcs_id, mf_id, deleted_nodes) = create_cs_and_derive_manifest::<Root>(
            ctx.clone(),
            repo.clone(),
            file_changes,
            vec![(bcs_id_3, mf_id_3)],
        )
        .await;

        let expected_nodes = vec![
            (None, Status::Live),
            (Some(path("dir")), Status::Live),
            (Some(path("dir/f-2")), Status::Deleted(bcs_id_2)),
            (Some(path("dir/sub")), Status::Live),
            (Some(path("dir/sub/f-1")), Status::Deleted(bcs_id_2)),
            (Some(path("dir-2")), Status::Live),
            (Some(path("dir-2/sub")), Status::Live),
            (Some(path("dir-2/sub/f-3")), Status::Deleted(bcs_id_2)),
        ];
        assert_eq!(deleted_nodes, expected_nodes);

        (bcs_id, mf_id)
    };

    // delete everything
    {
        let file_changes = btreemap! {
            "file.txt" => None,
            "file-2.txt/subfile.txt" => None,
            "file-3.txt" => None,
            "dir-2/f-4" => None,
            "dir-2/sub" => None,
            "dir/sub/f-4" => None,
        };
        let (bcs_id, mf_id, deleted_nodes) = create_cs_and_derive_manifest::<Root>(
            ctx.clone(),
            repo.clone(),
            file_changes,
            vec![(bcs_id_4, mf_id_4)],
        )
        .await;

        let expected_nodes = vec![
            (None, Status::Live),
            (Some(path("dir")), Status::Deleted(bcs_id)),
            (Some(path("dir/f-2")), Status::Deleted(bcs_id_2)),
            (Some(path("dir/sub")), Status::Deleted(bcs_id)),
            (Some(path("dir/sub/f-1")), Status::Deleted(bcs_id_2)),
            (Some(path("dir/sub/f-4")), Status::Deleted(bcs_id)),
            (Some(path("dir-2")), Status::Deleted(bcs_id)),
            (Some(path("dir-2/f-4")), Status::Deleted(bcs_id)),
            (Some(path("dir-2/sub")), Status::Deleted(bcs_id)),
            (Some(path("dir-2/sub/f-3")), Status::Deleted(bcs_id_2)),
            (Some(path("file-2.txt")), Status::Deleted(bcs_id)),
            (
                Some(path("file-2.txt/subfile.txt")),
                Status::Deleted(bcs_id),
            ),
            (Some(path("file-3.txt")), Status::Deleted(bcs_id)),
            (Some(path("file.txt")), Status::Deleted(bcs_id)),
        ];
        assert_eq!(deleted_nodes, expected_nodes);

        (bcs_id, mf_id)
    };
}

pub(crate) async fn many_file_dirs_test<Root: RootDeletedManifestIdCommon>(fb: FacebookInit) {
    let repo: TestRepo = build_repo(fb).await.unwrap();
    ManyFilesDirs::initrepo(fb, &repo).await;
    let ctx = CoreContext::test_mock(fb);

    let mf_id_1 = {
        let hg_cs = "5a28e25f924a5d209b82ce0713d8d83e68982bc8";
        let (_, bcs) = bonsai_changeset_from_hg(&ctx, &repo, hg_cs).await.unwrap();

        let (_, mf_id, deleted_nodes) = derive_manifest::<Root>(&ctx, &repo, bcs, vec![]).await;

        // nothing was deleted yet
        let expected_nodes = vec![(None, Status::Live)];
        assert_eq!(deleted_nodes, expected_nodes);
        mf_id
    };

    let mf_id_2 = {
        let hg_cs = "2f866e7e549760934e31bf0420a873f65100ad63";
        let (_, bcs) = bonsai_changeset_from_hg(&ctx, &repo, hg_cs).await.unwrap();

        let (_, mf_id, deleted_nodes) =
            derive_manifest::<Root>(&ctx, &repo, bcs, vec![mf_id_1]).await;

        // nothing was deleted yet
        let expected_nodes = vec![(None, Status::Live)];
        assert_eq!(deleted_nodes, expected_nodes);
        mf_id
    };

    let mf_id_3 = {
        let hg_cs = "d261bc7900818dea7c86935b3fb17a33b2e3a6b4";
        let (_, bcs) = bonsai_changeset_from_hg(&ctx, &repo, hg_cs).await.unwrap();

        let (_, mf_id, deleted_nodes) =
            derive_manifest::<Root>(&ctx, &repo, bcs, vec![mf_id_2]).await;

        // nothing was deleted yet
        let expected_nodes = vec![(None, Status::Live)];
        assert_eq!(deleted_nodes, expected_nodes);
        mf_id
    };

    {
        let hg_cs = "051946ed218061e925fb120dac02634f9ad40ae2";
        let (bcs_id, bcs) = bonsai_changeset_from_hg(&ctx, &repo, hg_cs).await.unwrap();

        let (_, mf_id, deleted_nodes) =
            derive_manifest::<Root>(&ctx, &repo, bcs, vec![mf_id_3]).await;

        let expected_nodes = vec![
            (None, Status::Live),
            (Some(path("dir1")), Status::Live),
            (Some(path("dir1/file_1_in_dir1")), Status::Deleted(bcs_id)),
            (Some(path("dir1/file_2_in_dir1")), Status::Deleted(bcs_id)),
            (Some(path("dir1/subdir1")), Status::Deleted(bcs_id)),
            (Some(path("dir1/subdir1/file_1")), Status::Deleted(bcs_id)),
            (
                Some(path("dir1/subdir1/subsubdir1")),
                Status::Deleted(bcs_id),
            ),
            (
                Some(path("dir1/subdir1/subsubdir1/file_1")),
                Status::Deleted(bcs_id),
            ),
            (
                Some(path("dir1/subdir1/subsubdir2")),
                Status::Deleted(bcs_id),
            ),
            (
                Some(path("dir1/subdir1/subsubdir2/file_1")),
                Status::Deleted(bcs_id),
            ),
            (
                Some(path("dir1/subdir1/subsubdir2/file_2")),
                Status::Deleted(bcs_id),
            ),
        ];
        assert_eq!(deleted_nodes, expected_nodes);
        mf_id
    };
}

pub(crate) async fn merged_history_test<Root: RootDeletedManifestIdCommon>(
    fb: FacebookInit,
) -> Result<(), Error> {
    //
    //  N
    //  | \
    //  K  M
    //  |  |
    //  J  L
    //  | /
    //  I
    //  | \
    //  |  H
    //  |  |
    //  |  G
    //  |  | \
    //  |  D  F
    //  |  |  |
    //  B  C  E
    //  | /
    //  A
    //
    let repo: TestRepo = build_repo(fb).await.unwrap();
    let ctx = CoreContext::test_mock(fb);

    let a = CreateCommitContext::new_root(&ctx, &repo)
        .add_file("file", "1")
        .add_file("dir/file", "2")
        .add_file("dir_2/file", "3")
        .add_file("dir_3/file_1", "1")
        .add_file("dir_3/file_2", "2")
        .commit()
        .await?;

    let b = CreateCommitContext::new(&ctx, &repo, vec![a.clone()])
        .delete_file("file")
        .delete_file("dir/file")
        .delete_file("dir_3/file_1")
        .add_file("dir/file_2", "file->file_2")
        .commit()
        .await?;
    let deleted_nodes = gen_deleted_manifest_nodes::<Root>(&ctx, &repo, b.clone()).await?;
    let expected_nodes = vec![
        (None, Status::Live),
        (Some(path("dir")), Status::Live),
        (Some(path("dir/file")), Status::Deleted(b)),
        (Some(path("dir_3")), Status::Live),
        (Some(path("dir_3/file_1")), Status::Deleted(b)),
        (Some(path("file")), Status::Deleted(b)),
    ];
    assert_eq!(deleted_nodes, expected_nodes);

    let c = CreateCommitContext::new(&ctx, &repo, vec![a.clone()])
        .add_file("file", "1->2")
        .commit()
        .await?;

    let d = CreateCommitContext::new(&ctx, &repo, vec![c.clone()])
        .delete_file("dir/file")
        .delete_file("dir_2/file")
        .commit()
        .await?;

    let deleted_nodes = gen_deleted_manifest_nodes::<Root>(&ctx, &repo, d.clone()).await?;
    let expected_nodes = vec![
        (None, Status::Live),
        (Some(path("dir")), Status::Deleted(d)),
        (Some(path("dir/file")), Status::Deleted(d)),
        (Some(path("dir_2")), Status::Deleted(d)),
        (Some(path("dir_2/file")), Status::Deleted(d)),
    ];
    assert_eq!(deleted_nodes, expected_nodes);

    let e = CreateCommitContext::new_root(&ctx, &repo)
        .add_file("file", "3")
        .add_file("dir_2/file", "4")
        .commit()
        .await?;

    let f = CreateCommitContext::new(&ctx, &repo, vec![e.clone()])
        .delete_file("file")
        .add_file("dir_2/file", "4->5")
        .commit()
        .await?;

    // first merge commit:
    // * dir_2/file - was deleted in branch D and modified in F, merge commit
    //   accepts modification. It means the file must be restored.
    // * file - was changed in branch D and deleted in F, merge commit accepts
    //   deletion. It means new deleted manifet node must be created and must
    //   point to the merge commit.
    // * dir/file - existed and was deleted in the one branch and never
    //   existed in the other, but still must be discoverable.
    let g = CreateCommitContext::new(&ctx, &repo, vec![d.clone(), f.clone()])
        .delete_file("file")
        .add_file("dir_2/file", "4->5")
        .add_file("dir_2/file_2", "5")
        .commit()
        .await?;

    let deleted_nodes = gen_deleted_manifest_nodes::<Root>(&ctx, &repo, g.clone()).await?;
    let expected_nodes = vec![
        (None, Status::Live),
        (Some(path("dir")), Status::Deleted(d)),
        (Some(path("dir/file")), Status::Deleted(d)),
        (Some(path("file")), Status::Deleted(g)),
    ];
    assert_eq!(deleted_nodes, expected_nodes);

    let h = CreateCommitContext::new(&ctx, &repo, vec![g.clone()])
        .delete_file("dir_3/file_2")
        .add_file("dir_2/file", "4->5")
        .add_file("dir_2/file_2", "5")
        .commit()
        .await?;

    let deleted_nodes = gen_deleted_manifest_nodes::<Root>(&ctx, &repo, h.clone()).await?;
    let expected_nodes = vec![
        (None, Status::Live),
        (Some(path("dir")), Status::Deleted(d)),
        (Some(path("dir/file")), Status::Deleted(d)),
        (Some(path("dir_3")), Status::Live),
        (Some(path("dir_3/file_2")), Status::Deleted(h)),
        (Some(path("file")), Status::Deleted(g)),
    ];
    assert_eq!(deleted_nodes, expected_nodes);

    // second merge commit
    // * dir/file - is deleted in both branches, new manifest node must
    //   have linknode pointed to the merge commit
    // * file - same as for dir/file
    // * dir - still exists because of dir/file_2
    let i = CreateCommitContext::new(&ctx, &repo, vec![b.clone(), h.clone()])
        .delete_file("dir_3/file_1")
        .delete_file("dir_3/file_2")
        .add_file("dir_2/file", "4->5")
        .add_file("dir_5/file_1", "5.1")
        .add_file("dir_5/file_2", "5.2")
        .commit()
        .await?;
    let deleted_nodes = gen_deleted_manifest_nodes::<Root>(&ctx, &repo, i.clone()).await?;
    let expected_nodes = vec![
        (None, Status::Live),
        (Some(path("dir")), Status::Live),
        (Some(path("dir/file")), Status::Deleted(i)),
        (Some(path("dir_3")), Status::Deleted(i)),
        (Some(path("dir_3/file_1")), Status::Deleted(i)),
        (Some(path("dir_3/file_2")), Status::Deleted(i)),
        (Some(path("file")), Status::Deleted(i)),
    ];
    assert_eq!(deleted_nodes, expected_nodes);

    // this commit creates a file in a new dir
    // and deletes one of the dir_5 files
    let j = CreateCommitContext::new(&ctx, &repo, vec![i.clone()])
        .delete_file("dir_5/file_1")
        .add_file("dir_4/file_1", "new")
        .commit()
        .await?;

    // this commit deletes the file created in its parent j
    // and adds a new file and dir
    let k = CreateCommitContext::new(&ctx, &repo, vec![j.clone()])
        .delete_file("dir_4/file_1")
        .add_file("dir_to_file/file", "will be replaced")
        .commit()
        .await?;

    // this commit creates a file in the same dir as the other branch
    // and deletes one of the dir_5 files
    let l = CreateCommitContext::new(&ctx, &repo, vec![i.clone()])
        .delete_file("dir_5/file_2")
        .add_file("dir_4/file_2", "new")
        .commit()
        .await?;

    // this commit deletes the file created in its parent l
    let m = CreateCommitContext::new(&ctx, &repo, vec![l.clone()])
        .delete_file("dir_4/file_2")
        .commit()
        .await?;

    // third merge commit
    // * dir_4/file_1 - is created and then deleted in the branch K,
    //   linknode for the merge commit N must point to the commit K
    // * dir_4/file_2 - is created and then deleted in the branch M,
    //   linknode for the merge commit N must point to the commit M
    // * dir_4 - existed in both branches, linknode should point to
    //   the merge commit itself
    // * dir_5/file_1 - existed in both branches, but deleted in J,
    //   linknode for the merge commit N must point to the N itself
    // * dir_5/file_2 - existed in both branches, but deleted in L,
    //   linknode for the merge commit N must point to the N itself
    // * dir_5 - existed in both branches, but as a result of merge
    //   must be deleted, linknode should point to N
    // * dir_to_file/file is replaced here with dir_to_file, this
    //   should result in dir_to_file node live and dir_to_file/file
    //   deleted
    let n = CreateCommitContext::new(&ctx, &repo, vec![k.clone(), m.clone()])
        .delete_file("dir_5/file_1")
        .delete_file("dir_5/file_2")
        .add_file("dir_to_file", "replaced!")
        .commit()
        .await?;

    let deleted_nodes = gen_deleted_manifest_nodes::<Root>(&ctx, &repo, n.clone()).await?;
    let expected_nodes = vec![
        (None, Status::Live),
        (Some(path("dir")), Status::Live),
        (Some(path("dir/file")), Status::Deleted(i)),
        (Some(path("dir_3")), Status::Deleted(i)),
        (Some(path("dir_3/file_1")), Status::Deleted(i)),
        (Some(path("dir_3/file_2")), Status::Deleted(i)),
        (Some(path("dir_4")), Status::Deleted(n)),
        (Some(path("dir_4/file_1")), Status::Deleted(k)),
        (Some(path("dir_4/file_2")), Status::Deleted(m)),
        (Some(path("dir_5")), Status::Deleted(n)),
        (Some(path("dir_5/file_1")), Status::Deleted(n)),
        (Some(path("dir_5/file_2")), Status::Deleted(n)),
        (Some(path("dir_to_file")), Status::Live),
        (Some(path("dir_to_file/file")), Status::Deleted(n)),
        (Some(path("file")), Status::Deleted(i)),
    ];
    assert_eq!(deleted_nodes, expected_nodes);

    Ok(())
}

pub(crate) async fn test_find_entries<Root: RootDeletedManifestIdCommon>(fb: FacebookInit) {
    // Test simple separate files and whole dir deletions
    let repo: TestRepo = build_repo(fb).await.unwrap();
    let ctx = CoreContext::test_mock(fb);

    // create parent deleted manifest
    let (bcs_id_1, mf_id_1) = {
        let file_changes = btreemap! {
            "file.txt" => Some("1\n"),
            "file-2.txt" => Some("2\n"),
            "dir/sub/f-1" => Some("3\n"),
            "dir/sub/f-6" => Some("3\n"),
            "dir/f-2" => Some("4\n"),
            "dir-2/sub/f-3" => Some("5\n"),
            "dir-2/f-4" => Some("6\n"),
            "dir-2/f-5" => Some("7\n"),
        };
        let files = store_files(&ctx, file_changes, &repo).await;
        let bcs = create_bonsai_changeset(ctx.fb, repo.clone(), files, vec![]).await;

        let bcs_id = bcs.get_changeset_id();
        let mf_id = derive_manifest::<Root>(&ctx, &repo, bcs, vec![]).await.1;

        (bcs_id, mf_id)
    };

    // delete some files and dirs
    {
        let file_changes = btreemap! {
            "dir/sub/f-1" => None,
            "dir/sub/f-6" => None,
            "dir/f-2" => None,
            "dir-2/sub/f-3" => None,
            "dir-2/f-4" => None,
        };
        let files = store_files(&ctx, file_changes, &repo).await;
        let bcs = create_bonsai_changeset(ctx.fb, repo.clone(), files, vec![bcs_id_1]).await;

        let _bcs_id = bcs.get_changeset_id();
        let mf_id = derive_manifest::<Root>(&ctx, &repo, bcs, vec![mf_id_1])
            .await
            .1;

        {
            // check that it will yield only two deleted paths
            let mut entries = Root::new(mf_id)
                .find_entries(
                    &ctx,
                    repo.repo_blobstore(),
                    vec![
                        PathOrPrefix::Path(Some(path("file.txt"))),
                        PathOrPrefix::Path(Some(path("dir/f-2"))),
                        PathOrPrefix::Path(Some(path("dir/sub/f-1"))),
                    ],
                )
                .map_ok(|(path, _)| path)
                .try_collect::<Vec<_>>()
                .await
                .unwrap();

            entries.sort();
            let expected_entries = vec![Some(path("dir/f-2")), Some(path("dir/sub/f-1"))];
            assert_eq!(entries, expected_entries);
        }

        {
            // check that it will yield recursively all deleted paths including dirs
            let mut entries = Root::new(mf_id)
                .find_entries(
                    &ctx,
                    repo.repo_blobstore(),
                    vec![PathOrPrefix::Prefix(Some(path("dir-2")))],
                )
                .map_ok(|(path, _)| path)
                .try_collect::<Vec<_>>()
                .await
                .unwrap();

            entries.sort();
            let expected_entries = vec![
                Some(path("dir-2/f-4")),
                Some(path("dir-2/sub")),
                Some(path("dir-2/sub/f-3")),
            ];
            assert_eq!(entries, expected_entries);
        }

        {
            // check that it will yield recursively even having a path patterns
            let mut entries = Root::new(mf_id)
                .find_entries(
                    &ctx,
                    repo.repo_blobstore(),
                    vec![
                        PathOrPrefix::Prefix(Some(path("dir/sub"))),
                        PathOrPrefix::Path(Some(path("dir/sub/f-1"))),
                    ],
                )
                .map_ok(|(path, _)| path)
                .try_collect::<Vec<_>>()
                .await
                .unwrap();

            entries.sort();
            let expected_entries = vec![
                Some(path("dir/sub")),
                Some(path("dir/sub/f-1")),
                Some(path("dir/sub/f-6")),
            ];
            assert_eq!(entries, expected_entries);
        }
    }
}

pub(crate) async fn test_list_all_entries<Root: RootDeletedManifestIdCommon>(fb: FacebookInit) {
    // Test simple separate files and whole dir deletions
    let repo: TestRepo = build_repo(fb).await.unwrap();
    let ctx = CoreContext::test_mock(fb);

    // create parent deleted manifest
    let (bcs_id_1, mf_id_1) = {
        let file_changes = btreemap! {
            "file.txt" => Some("1\n"),
            "dir/sub/f-1" => Some("3\n"),
            "dir/sub/f-3" => Some("3\n"),
            "dir/f-2" => Some("4\n"),
        };
        let files = store_files(&ctx, file_changes, &repo).await;
        let bcs = create_bonsai_changeset(ctx.fb, repo.clone(), files, vec![]).await;

        let bcs_id = bcs.get_changeset_id();
        let mf_id = derive_manifest::<Root>(&ctx, &repo, bcs, vec![]).await.1;

        (bcs_id, mf_id)
    };

    {
        let file_changes = btreemap! {
            "dir/sub/f-1" => None,
            "dir/sub/f-3" => None,
        };
        let files = store_files(&ctx, file_changes, &repo).await;
        let bcs = create_bonsai_changeset(ctx.fb, repo.clone(), files, vec![bcs_id_1]).await;

        let _bcs_id = bcs.get_changeset_id();
        let mf_id = derive_manifest::<Root>(&ctx, &repo, bcs, vec![mf_id_1])
            .await
            .1;

        {
            // check that it will yield only two deleted paths
            let entries = Root::new(mf_id)
                .list_all_entries(&ctx, repo.repo_blobstore())
                .try_collect::<Vec<_>>()
                .await
                .unwrap();

            let mut entries = entries
                .into_iter()
                .map(|(path, _)| path)
                .collect::<Vec<_>>();
            entries.sort();
            let expected_entries = vec![
                Some(path("dir/sub")),
                Some(path("dir/sub/f-1")),
                Some(path("dir/sub/f-3")),
            ];
            assert_eq!(entries, expected_entries);
        }
    }
}

async fn gen_deleted_manifest_nodes<Root: RootDeletedManifestIdCommon>(
    ctx: &CoreContext,
    repo: &TestRepo,
    bonsai: ChangesetId,
) -> Result<Vec<(Option<MPath>, Status)>, Error> {
    let manifest = repo
        .repo_derived_data()
        .manager()
        .derive::<Root>(ctx, bonsai, None)
        .await?;
    let mut deleted_nodes = iterate_all_entries::<Root>(ctx.clone(), repo.clone(), *manifest.id())
        .map_ok(|(path, st, ..)| (path, st))
        .try_collect::<Vec<_>>()
        .await?;
    deleted_nodes.sort_by_key(|(path, ..)| path.clone());
    Ok(deleted_nodes)
}

async fn create_cs_and_derive_manifest<Root: RootDeletedManifestIdCommon>(
    ctx: CoreContext,
    repo: TestRepo,
    file_changes: BTreeMap<&str, Option<&str>>,
    parent_ids: Vec<(ChangesetId, Root::Id)>,
) -> (ChangesetId, Root::Id, Vec<(Option<MPath>, Status)>) {
    let parent_bcs_ids = parent_ids
        .iter()
        .map(|(bs, _)| bs.clone())
        .collect::<Vec<_>>();
    let parent_mf_ids = parent_ids.into_iter().map(|(_, mf)| mf).collect::<Vec<_>>();

    let files = store_files(&ctx, file_changes, &repo).await;

    let bcs = create_bonsai_changeset(ctx.fb, repo.clone(), files, parent_bcs_ids).await;

    derive_manifest::<Root>(&ctx, &repo, bcs, parent_mf_ids).await
}

async fn derive_manifest<Root: RootDeletedManifestIdCommon>(
    ctx: &CoreContext,
    repo: &TestRepo,
    bcs: BonsaiChangeset,
    parent_mf_ids: Vec<Root::Id>,
) -> (ChangesetId, Root::Id, Vec<(Option<MPath>, Status)>) {
    let blobstore = repo.repo_blobstore_arc() as Arc<dyn Blobstore>;
    let bcs_id = bcs.get_changeset_id();

    let changes = get_changes(
        ctx,
        &repo.repo_derived_data().manager().derivation_context(None),
        bcs,
    )
    .await
    .unwrap();
    let f = DeletedManifestDeriver::<Root::Manifest>::derive(
        ctx,
        &blobstore,
        bcs_id,
        parent_mf_ids,
        changes,
    );

    let dfm_id = f.await.unwrap();
    // Make sure it's saved in the blobstore
    dfm_id.load(ctx, &blobstore).await.unwrap();

    let mut deleted_nodes = iterate_all_entries::<Root>(ctx.clone(), repo.clone(), dfm_id.clone())
        .map_ok(|(path, st, ..)| (path, st))
        .try_collect::<Vec<_>>()
        .await
        .unwrap();
    deleted_nodes.sort_by_key(|(path, ..)| path.clone());

    (bcs_id, dfm_id, deleted_nodes)
}

async fn create_bonsai_changeset(
    fb: FacebookInit,
    repo: TestRepo,
    file_changes: SortedVectorMap<MPath, FileChange>,
    parents: Vec<ChangesetId>,
) -> BonsaiChangeset {
    let bcs = BonsaiChangesetMut {
        parents,
        author: "author".to_string(),
        author_date: DateTime::now(),
        committer: None,
        committer_date: None,
        message: "message".to_string(),
        hg_extra: Default::default(),
        git_extra_headers: None,
        git_tree_hash: None,
        file_changes,
        is_snapshot: false,
        git_annotated_tag: None,
    }
    .freeze()
    .unwrap();

    save_changesets(&CoreContext::test_mock(fb), &repo, vec![bcs.clone()])
        .await
        .unwrap();
    bcs
}

#[derive(Debug, Clone, Eq, PartialEq)]
enum Status {
    Deleted(ChangesetId),
    Live,
}

impl From<Option<ChangesetId>> for Status {
    fn from(linknode: Option<ChangesetId>) -> Self {
        linknode.map_or(Status::Live, Status::Deleted)
    }
}

fn iterate_all_entries<Root: RootDeletedManifestIdCommon>(
    ctx: CoreContext,
    repo: TestRepo,
    manifest_id: Root::Id,
) -> impl Stream<Item = Result<(Option<MPath>, Status, Root::Id), Error>> {
    async_stream::stream! {
        let blobstore = repo.repo_blobstore();
        let s = bounded_traversal_stream(256, Some((None, manifest_id)), move |(path, manifest_id)| {
            cloned!(ctx, blobstore);
            async move {
                let manifest = manifest_id.load(&ctx, &blobstore).await?;
                let entry = (
                    path.clone(),
                    Status::from(manifest.linknode().cloned()),
                    manifest_id,
                );
                let recurse_subentries = manifest
                    .into_subentries(&ctx, &blobstore)
                    .map_ok(|(name, mf_id)| {
                        let full_path = MPath::join_opt_element(path.as_ref(), &name);
                        (Some(full_path), mf_id)
                    })
                    .try_collect::<Vec<_>>().await?;

                Result::<_, Error>::Ok((vec![entry], recurse_subentries))
            }.boxed()
        })
        .map_ok(|entries| iter(entries.into_iter().map(Ok)))
        .try_flatten();

        pin_mut!(s);
        while let Some(value) = s.next().await {
            yield value;
        }
    }
}

fn path(path_str: &str) -> MPath {
    MPath::new(path_str).unwrap()
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;
use std::collections::HashSet;
use std::collections::VecDeque;
use std::sync::Arc;

use anyhow::Error;
use blobstore::Blobstore;
use blobstore::BlobstoreBytes;
use context::CoreContext;
use futures::future::try_join_all;
use manifest::Entry;
use maplit::hashset;
use mononoke_types::fastlog_batch::FastlogBatch;
use mononoke_types::fastlog_batch::ParentOffset;
use mononoke_types::ChangesetId;
use mononoke_types::FileUnodeId;
use mononoke_types::ManifestUnodeId;

use crate::ErrorKind;
use crate::FastlogParent;

pub(crate) async fn create_new_batch(
    ctx: &CoreContext,
    blobstore: &Arc<dyn Blobstore>,
    unode_parents: Vec<Entry<ManifestUnodeId, FileUnodeId>>,
    linknode: ChangesetId,
) -> Result<FastlogBatch, Error> {
    let parent_batches = try_join_all(unode_parents.clone().into_iter().map({
        move |entry| async move {
            let maybe_batch = fetch_fastlog_batch_by_unode_id(ctx, blobstore, &entry).await?;
            maybe_batch.ok_or_else(|| Error::from(ErrorKind::NotFound(entry)))
        }
    }))
    .await?;

    if parent_batches.len() < 2 {
        match parent_batches.get(0) {
            Some(parent_batch) => {
                parent_batch
                    .prepend_child_with_single_parent(ctx, blobstore, linknode)
                    .await
            }
            None => {
                let mut d = VecDeque::new();
                d.push_back((linknode, vec![]));
                FastlogBatch::new_from_raw_list(ctx, blobstore, d).await
            }
        }
    } else {
        let parents_flattened =
            try_join_all(parent_batches.into_iter().map({
                move |batch| async move { fetch_flattened(&batch, ctx, blobstore).await }
            }))
            .await?;
        let raw_list = convert_to_raw_list(create_merged_list(linknode, parents_flattened));
        FastlogBatch::new_from_raw_list(ctx, blobstore, raw_list).await
    }
}

// This function creates a FastlogBatch list for a merge unode.
// It does so by taking a merge_cs_id (i.e. a linknode of this merge unode) and
// FastlogBatches for it's parents and merges them together in BFS order
//
// For example, let's say we have a unode whose history graph is the following:
//
//             o <- commit A
//            / \
// commit B  o   \
//           \   o <- commit C
//            \ /
//             o <- commit D
//
// create_merged_list() accepts commit A as merge_cs_id, [B, D] as a first parent's list
// and [C, D] as the second parent's list. The expected output is [A, B, C, D].
fn create_merged_list(
    merge_cs_id: ChangesetId,
    parents_lists: Vec<Vec<(ChangesetId, Vec<FastlogParent>)>>,
) -> Vec<(ChangesetId, Vec<FastlogParent>)> {
    // parents_of_merge_commits preserve the order of `parents_lists`
    let mut parents_of_merge_commit = vec![];
    for list in parents_lists.iter() {
        if let Some((p, _)) = list.get(0) {
            parents_of_merge_commit.push(FastlogParent::Known(*p));
        }
    }
    {
        // Make sure we have unique parents
        let mut used = HashSet::new();
        parents_of_merge_commit.retain(move |p| used.insert(p.clone()));
    }

    let mut cs_id_to_parents: HashMap<_, _> = parents_lists
        .into_iter()
        .flat_map(|list| list.into_iter())
        .collect();
    cs_id_to_parents.insert(merge_cs_id, parents_of_merge_commit.clone());

    let mut q = VecDeque::new();
    q.push_back((merge_cs_id, parents_of_merge_commit));

    let mut res = vec![];
    let mut used = hashset! {merge_cs_id};
    while let Some((cs_id, parents)) = q.pop_front() {
        res.push((cs_id, parents.clone()));

        for p in parents {
            if let FastlogParent::Known(p) = p {
                if let Some(parents) = cs_id_to_parents.get(&p) {
                    if used.insert(p) {
                        q.push_back((p, parents.clone()));
                    }
                }
            }
        }
    }

    res
}

// Converts from an "external" representation (i.e. the one used by users of this library)
// to an "internal" representation (i.e. the one that we store in the blobstore).
fn convert_to_raw_list(
    list: Vec<(ChangesetId, Vec<FastlogParent>)>,
) -> Vec<(ChangesetId, Vec<ParentOffset>)> {
    let cs_to_idx: HashMap<_, _> = list
        .iter()
        .enumerate()
        .map(|(idx, (cs_id, _))| (*cs_id, idx as i32))
        .collect();

    // Special offset that points outside of the list.
    // It's used for unknown parents
    let max_idx = (list.len() + 1) as i32;
    let mut res = vec![];
    for (current_idx, (cs_id, fastlog_parents)) in list.into_iter().enumerate() {
        let current_idx = current_idx as i32;
        let mut parent_offsets = vec![];
        for p in fastlog_parents {
            let maybe_idx = match p {
                FastlogParent::Known(cs_id) => {
                    cs_to_idx.get(&cs_id).cloned().map(|idx| idx - current_idx)
                }
                FastlogParent::Unknown => None,
            };

            parent_offsets.push(ParentOffset::new(maybe_idx.unwrap_or(max_idx)))
        }
        res.push((cs_id, parent_offsets));
    }

    res
}

pub async fn fetch_fastlog_batch_by_unode_id<B: Blobstore>(
    ctx: &CoreContext,
    blobstore: &B,
    unode_entry: &Entry<ManifestUnodeId, FileUnodeId>,
) -> Result<Option<FastlogBatch>, Error> {
    let fastlog_batch_key = unode_entry_to_fastlog_batch_key(unode_entry);

    let maybe_bytes = blobstore.get(ctx, &fastlog_batch_key).await?;

    match maybe_bytes {
        Some(serialized) => FastlogBatch::from_bytes(serialized.as_raw_bytes()).map(Some),
        None => Ok(None),
    }
}

pub(crate) async fn save_fastlog_batch_by_unode_id<B: Blobstore>(
    ctx: &CoreContext,
    blobstore: &B,
    unode_entry: Entry<ManifestUnodeId, FileUnodeId>,
    batch: FastlogBatch,
) -> Result<(), Error> {
    let fastlog_batch_key = unode_entry_to_fastlog_batch_key(&unode_entry);
    let serialized = batch.into_bytes();

    blobstore
        .put(
            ctx,
            fastlog_batch_key,
            BlobstoreBytes::from_bytes(serialized),
        )
        .await
}

pub fn unode_entry_to_fastlog_batch_key(
    unode_entry: &Entry<ManifestUnodeId, FileUnodeId>,
) -> String {
    let key_part = match unode_entry {
        Entry::Leaf(file_unode_id) => format!("fileunode.{}", file_unode_id),
        Entry::Tree(mf_unode_id) => format!("manifestunode.{}", mf_unode_id),
    };
    format!("fastlogbatch.{}", key_part)
}

pub async fn fetch_flattened<B: Blobstore>(
    batch: &FastlogBatch,
    ctx: &CoreContext,
    blobstore: &B,
) -> Result<Vec<(ChangesetId, Vec<FastlogParent>)>, Error> {
    let raw_list = batch.fetch_raw_list(ctx, blobstore).await?;
    Ok(flatten_raw_list(raw_list))
}

fn flatten_raw_list(
    raw_list: Vec<(ChangesetId, Vec<ParentOffset>)>,
) -> Vec<(ChangesetId, Vec<FastlogParent>)> {
    let mut res = vec![];
    for (index, (cs_id, parent_offsets)) in raw_list.iter().enumerate() {
        let mut batch_parents = vec![];
        for offset in parent_offsets {
            // NOTE: Offset can be negative!
            let parent_index = index as i32 + offset.num();
            let batch_parent = if parent_index >= 0 {
                match raw_list.get(parent_index as usize) {
                    Some((p_cs_id, _)) => FastlogParent::Known(*p_cs_id),
                    None => FastlogParent::Unknown,
                }
            } else {
                FastlogParent::Unknown
            };
            batch_parents.push(batch_parent);
        }

        res.push((*cs_id, batch_parents));
    }

    res
}

#[cfg(test)]
mod test {
    use borrowed::borrowed;
    use fbinit::FacebookInit;
    use fixtures::Linear;
    use fixtures::TestRepoFixture;
    use mononoke_types_mocks::changesetid::ONES_CSID;
    use mononoke_types_mocks::changesetid::THREES_CSID;
    use mononoke_types_mocks::changesetid::TWOS_CSID;
    use repo_blobstore::RepoBlobstoreRef;

    use super::*;

    #[fbinit::test]
    async fn fetch_flattened_simple(fb: FacebookInit) -> Result<(), Error> {
        let ctx = CoreContext::test_mock(fb);
        let repo = Linear::getrepo(fb).await;
        let blobstore = repo.repo_blobstore();
        borrowed!(ctx);
        let mut d = VecDeque::new();
        d.push_back((ONES_CSID, vec![]));
        let batch = FastlogBatch::new_from_raw_list(ctx, blobstore, d).await?;

        assert_eq!(
            vec![(ONES_CSID, vec![])],
            fetch_flattened(&batch, ctx, blobstore).await.unwrap()
        );
        Ok(())
    }

    #[fbinit::test]
    async fn fetch_flattened_prepend(fb: FacebookInit) -> Result<(), Error> {
        let ctx = CoreContext::test_mock(fb);
        let repo = Linear::getrepo(fb).await;
        let blobstore = repo.repo_blobstore();
        borrowed!(ctx);
        let mut d = VecDeque::new();
        d.push_back((ONES_CSID, vec![]));
        let batch = FastlogBatch::new_from_raw_list(ctx, blobstore, d).await?;

        assert_eq!(
            vec![(ONES_CSID, vec![])],
            fetch_flattened(&batch, ctx, blobstore).await.unwrap()
        );

        let prepended = batch
            .prepend_child_with_single_parent(ctx, blobstore, TWOS_CSID)
            .await
            .unwrap();
        assert_eq!(
            vec![
                (TWOS_CSID, vec![FastlogParent::Known(ONES_CSID)]),
                (ONES_CSID, vec![])
            ],
            fetch_flattened(&prepended, ctx, blobstore).await.unwrap()
        );

        let prepended = prepended
            .prepend_child_with_single_parent(ctx, &blobstore, THREES_CSID)
            .await
            .unwrap();
        assert_eq!(
            vec![
                (THREES_CSID, vec![FastlogParent::Known(TWOS_CSID)]),
                (TWOS_CSID, vec![FastlogParent::Known(ONES_CSID)]),
                (ONES_CSID, vec![])
            ],
            fetch_flattened(&prepended, ctx, blobstore).await.unwrap()
        );

        Ok(())
    }

    #[test]
    fn test_create_merged_list() -> Result<(), Error> {
        assert_eq!(
            create_merged_list(ONES_CSID, vec![]),
            vec![(ONES_CSID, vec![])]
        );

        let first_parent = vec![(TWOS_CSID, vec![])];
        let second_parent = vec![(THREES_CSID, vec![])];
        assert_eq!(
            create_merged_list(ONES_CSID, vec![first_parent, second_parent]),
            vec![
                (
                    ONES_CSID,
                    vec![
                        FastlogParent::Known(TWOS_CSID),
                        FastlogParent::Known(THREES_CSID)
                    ]
                ),
                (TWOS_CSID, vec![]),
                (THREES_CSID, vec![]),
            ]
        );
        Ok(())
    }

    #[test]
    fn test_create_merged_list_same_commit() -> Result<(), Error> {
        assert_eq!(
            create_merged_list(ONES_CSID, vec![]),
            vec![(ONES_CSID, vec![])]
        );

        let first_parent = vec![(TWOS_CSID, vec![])];
        let second_parent = vec![(TWOS_CSID, vec![])];
        assert_eq!(
            create_merged_list(ONES_CSID, vec![first_parent, second_parent]),
            vec![
                (ONES_CSID, vec![FastlogParent::Known(TWOS_CSID),]),
                (TWOS_CSID, vec![]),
            ]
        );
        Ok(())
    }

    #[test]
    fn test_convert_to_raw_list_simple() -> Result<(), Error> {
        let list = vec![
            (
                ONES_CSID,
                vec![
                    FastlogParent::Known(TWOS_CSID),
                    FastlogParent::Known(THREES_CSID),
                ],
            ),
            (TWOS_CSID, vec![]),
            (THREES_CSID, vec![]),
        ];

        let raw_list = convert_to_raw_list(list.clone());
        let expected = vec![
            (ONES_CSID, vec![ParentOffset::new(1), ParentOffset::new(2)]),
            (TWOS_CSID, vec![]),
            (THREES_CSID, vec![]),
        ];
        assert_eq!(raw_list, expected);
        assert_eq!(flatten_raw_list(raw_list), list);

        let list = vec![
            (ONES_CSID, vec![FastlogParent::Known(TWOS_CSID)]),
            (TWOS_CSID, vec![FastlogParent::Known(THREES_CSID)]),
            (THREES_CSID, vec![]),
        ];

        let raw_list = convert_to_raw_list(list.clone());
        let expected = vec![
            (ONES_CSID, vec![ParentOffset::new(1)]),
            (TWOS_CSID, vec![ParentOffset::new(1)]),
            (THREES_CSID, vec![]),
        ];
        assert_eq!(raw_list, expected);
        assert_eq!(flatten_raw_list(raw_list), list);

        Ok(())
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

/// This library is used to efficiently store file and directory history.
/// For each unode we store a FastlogBatch - thrift structure that stores latest commits and their
/// parents that modified this file or directory. Commits are stored in BFS order.
/// All FastlogBatches are stored in blobstore.
///
/// Commits also store pointers to their parents, however they are stored as an offset to the
/// commit hash in batch. I.e. if we have two commits A and B and A is an ancestor of B, then
/// batch will look like:
/// B, vec![ParentOffset(1)]
/// A, vec![]
///
/// Note that commits where a file was deleted are not stored in FastlogBatch. It also doesn't
/// store a history across deletions i.e. if a file was added, then deleted then added again in
/// commit A, FastlogBatch in commit A will contain only one entry.
///
/// RootFastlog is a derived data which derives FastlogBatch for each unode
/// that was created or modified in this commit.
mod fastlog_impl;
mod mapping;
mod thrift {
    pub use mononoke_types_thrift::*;
}

pub use fastlog_impl::fetch_fastlog_batch_by_unode_id;
pub use fastlog_impl::fetch_flattened;
pub use fastlog_impl::unode_entry_to_fastlog_batch_key;
pub use mapping::format_key;
pub use mapping::ErrorKind;
pub use mapping::FastlogParent;
pub use mapping::RootFastlog;


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use anyhow::anyhow;
use anyhow::Error;
use anyhow::Result;
use async_trait::async_trait;
use blobstore::Blobstore;
use blobstore::BlobstoreBytes;
use blobstore::Loadable;
use cloned::cloned;
use context::CoreContext;
use derived_data::impl_bonsai_derived_via_manager;
use derived_data_manager::dependencies;
use derived_data_manager::BonsaiDerivable;
use derived_data_manager::DerivableType;
use derived_data_manager::DerivationContext;
use derived_data_service_if::types as thrift;
use futures::stream::TryStreamExt;
use manifest::find_intersection_of_diffs;
use manifest::Entry;
use mononoke_types::BonsaiChangeset;
use mononoke_types::ChangesetId;
use mononoke_types::FileUnodeId;
use mononoke_types::ManifestUnodeId;
use thiserror::Error;
use unodes::RootUnodeManifestId;

use crate::fastlog_impl::create_new_batch;
use crate::fastlog_impl::save_fastlog_batch_by_unode_id;

#[derive(Clone, Debug, Hash, PartialEq, Eq)]
pub enum FastlogParent {
    /// Parent exists and it's stored in the batch
    Known(ChangesetId),
    /// Parent exists, but it's not stored in the batch (including previous_batches).
    /// It needs to be fetched separately
    Unknown,
}

#[derive(Debug, Error)]
pub enum ErrorKind {
    #[error("invalid Thrift structure '{0}': {1}")]
    InvalidThrift(String, String),
    #[error("Fastlog batch for {0:?} unode not found")]
    NotFound(Entry<ManifestUnodeId, FileUnodeId>),
    #[error("Failed to deserialize FastlogBatch for {0}: {1}")]
    DeserializationError(String, String),
}

#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RootFastlog(ChangesetId);

impl RootFastlog {
    pub fn changeset_id(&self) -> &ChangesetId {
        &self.0
    }
}

impl From<ChangesetId> for RootFastlog {
    fn from(csid: ChangesetId) -> RootFastlog {
        RootFastlog(csid)
    }
}

pub fn format_key(derivation_ctx: &DerivationContext, changeset_id: ChangesetId) -> String {
    let root_prefix = "derived_rootfastlog.";
    let key_prefix = derivation_ctx.mapping_key_prefix::<RootFastlog>();
    format!("{}{}{}", root_prefix, key_prefix, changeset_id)
}

#[async_trait]
impl BonsaiDerivable for RootFastlog {
    const VARIANT: DerivableType = DerivableType::Fastlog;

    type Dependencies = dependencies![RootUnodeManifestId];

    async fn derive_single(
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        bonsai: BonsaiChangeset,
        _parents: Vec<Self>,
    ) -> Result<Self, Error> {
        let bcs_id = bonsai.get_changeset_id();
        let unode_mf_id = derivation_ctx
            .derive_dependency::<RootUnodeManifestId>(ctx, bonsai.get_changeset_id())
            .await?
            .manifest_unode_id()
            .clone();
        let parents = derivation_ctx
            .fetch_parents::<RootUnodeManifestId>(ctx, &bonsai)
            .await?
            .into_iter()
            .map(|id| id.manifest_unode_id().clone())
            .collect::<Vec<_>>();

        let blobstore = derivation_ctx.blobstore();

        find_intersection_of_diffs(ctx.clone(), blobstore.clone(), unode_mf_id, parents)
            .map_ok(move |(_, entry)| {
                cloned!(blobstore, ctx);
                async move {
                    tokio::spawn(async move {
                        let parents = fetch_unode_parents(&ctx, &blobstore, entry).await?;

                        let fastlog_batch =
                            create_new_batch(&ctx, &blobstore, parents, bcs_id).await?;

                        save_fastlog_batch_by_unode_id(&ctx, &blobstore, entry, fastlog_batch).await
                    })
                    .await?
                }
            })
            .try_buffer_unordered(100)
            .try_for_each(|_| async { Ok(()) })
            .await?;

        Ok(RootFastlog(bcs_id))
    }

    async fn store_mapping(
        self,
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        changeset_id: ChangesetId,
    ) -> Result<()> {
        let key = format_key(derivation_ctx, changeset_id);
        derivation_ctx
            .blobstore()
            .put(ctx, key, BlobstoreBytes::empty())
            .await
    }

    async fn fetch(
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        changeset_id: ChangesetId,
    ) -> Result<Option<Self>> {
        let key = format_key(derivation_ctx, changeset_id);
        match derivation_ctx.blobstore().get(ctx, &key).await? {
            Some(_) => Ok(Some(RootFastlog(changeset_id))),
            None => Ok(None),
        }
    }

    fn from_thrift(data: thrift::DerivedData) -> Result<Self> {
        if let thrift::DerivedData::fastlog(thrift::DerivedDataFastlog::root_fastlog_id(id)) = data
        {
            ChangesetId::from_thrift(id).map(Self)
        } else {
            Err(anyhow!(
                "Can't convert {} from provided thrift::DerivedData",
                Self::NAME.to_string(),
            ))
        }
    }

    fn into_thrift(data: Self) -> Result<thrift::DerivedData> {
        Ok(thrift::DerivedData::fastlog(
            thrift::DerivedDataFastlog::root_fastlog_id(data.changeset_id().into_thrift()),
        ))
    }
}

async fn fetch_unode_parents<B: Blobstore>(
    ctx: &CoreContext,
    blobstore: &B,
    unode_entry_id: Entry<ManifestUnodeId, FileUnodeId>,
) -> Result<Vec<Entry<ManifestUnodeId, FileUnodeId>>, Error> {
    let unode_entry = unode_entry_id.load(ctx, blobstore).await?;

    let res = match unode_entry {
        Entry::Tree(tree) => tree
            .parents()
            .clone()
            .into_iter()
            .map(Entry::Tree)
            .collect(),
        Entry::Leaf(leaf) => leaf
            .parents()
            .clone()
            .into_iter()
            .map(Entry::Leaf)
            .collect(),
    };
    Ok(res)
}

impl_bonsai_derived_via_manager!(RootFastlog);

#[cfg(test)]
mod tests {
    use std::collections::BTreeMap;
    use std::collections::HashSet;
    use std::collections::VecDeque;
    use std::str::FromStr;
    use std::sync::Arc;

    use blobrepo::save_bonsai_changesets;
    use bonsai_hg_mapping::BonsaiHgMapping;
    use bookmarks::BookmarkKey;
    use bookmarks::Bookmarks;
    use changeset_fetcher::ChangesetFetcher;
    use changesets::Changesets;
    use context::CoreContext;
    use fbinit::FacebookInit;
    use filestore::FilestoreConfig;
    use fixtures::create_bonsai_changeset;
    use fixtures::create_bonsai_changeset_with_author;
    use fixtures::create_bonsai_changeset_with_files;
    use fixtures::store_files;
    use fixtures::Linear;
    use fixtures::MergeEven;
    use fixtures::MergeUneven;
    use fixtures::TestRepoFixture;
    use fixtures::UnsharedMergeEven;
    use fixtures::UnsharedMergeUneven;
    use futures::compat::Stream01CompatExt;
    use futures::Stream;
    use futures::TryFutureExt;
    use manifest::ManifestOps;
    use maplit::btreemap;
    use mercurial_derivation::DeriveHgChangeset;
    use mercurial_types::HgChangesetId;
    use mononoke_types::fastlog_batch::max_entries_in_fastlog_batch;
    use mononoke_types::fastlog_batch::MAX_BATCHES;
    use mononoke_types::fastlog_batch::MAX_LATEST_LEN;
    use mononoke_types::MPath;
    use mononoke_types::ManifestUnodeId;
    use pretty_assertions::assert_eq;
    use rand::SeedableRng;
    use rand_xorshift::XorShiftRng;
    use repo_blobstore::RepoBlobstore;
    use repo_derived_data::RepoDerivedData;
    use repo_derived_data::RepoDerivedDataRef;
    use revset::AncestorsNodeStream;

    use super::*;
    use crate::fastlog_impl::fetch_fastlog_batch_by_unode_id;
    use crate::fastlog_impl::fetch_flattened;

    #[derive(Clone)]
    #[facet::container]
    struct TestRepo {
        #[facet]
        bonsai_hg_mapping: dyn BonsaiHgMapping,
        #[facet]
        bookmarks: dyn Bookmarks,
        #[facet]
        repo_blobstore: RepoBlobstore,
        #[facet]
        repo_derived_data: RepoDerivedData,
        #[facet]
        filestore_config: FilestoreConfig,
        #[facet]
        changeset_fetcher: dyn ChangesetFetcher,
        #[facet]
        changesets: dyn Changesets,
    }

    #[fbinit::test]
    async fn test_derive_single_empty_commit_no_parents(fb: FacebookInit) {
        let repo: TestRepo = Linear::get_custom_test_repo(fb).await;
        let ctx = CoreContext::test_mock(fb);
        let bcs = create_bonsai_changeset(vec![]);
        let bcs_id = bcs.get_changeset_id();
        save_bonsai_changesets(vec![bcs], ctx.clone(), &repo)
            .await
            .unwrap();

        let root_unode_mf_id = derive_fastlog_batch_and_unode(&ctx, bcs_id.clone(), &repo).await;

        let list = fetch_list(&ctx, &repo, Entry::Tree(root_unode_mf_id)).await;
        assert_eq!(list, vec![(bcs_id, vec![])]);
    }

    #[fbinit::test]
    async fn test_derive_single_commit_no_parents(fb: FacebookInit) {
        let repo: TestRepo = Linear::get_custom_test_repo(fb).await;
        let ctx = CoreContext::test_mock(fb);

        // This is the initial diff with no parents
        // See tests/fixtures/src/lib.rs
        let hg_cs_id = HgChangesetId::from_str("2d7d4ba9ce0a6ffd222de7785b249ead9c51c536").unwrap();
        let bcs_id = repo
            .bonsai_hg_mapping
            .get_bonsai_from_hg(&ctx, hg_cs_id)
            .await
            .unwrap()
            .unwrap();

        let root_unode_mf_id = derive_fastlog_batch_and_unode(&ctx, bcs_id.clone(), &repo).await;
        let list = fetch_list(&ctx, &repo, Entry::Tree(root_unode_mf_id.clone())).await;
        assert_eq!(list, vec![(bcs_id, vec![])]);

        let blobstore = Arc::new(repo.repo_blobstore.clone());
        let path_1 = MPath::new("1").unwrap();
        let path_files = MPath::new("files").unwrap();
        let entries: Vec<_> = root_unode_mf_id
            .find_entries(ctx.clone(), blobstore.clone(), vec![path_1, path_files])
            .try_collect()
            .await
            .unwrap();

        let list = fetch_list(&ctx, &repo, entries.get(0).unwrap().1.clone()).await;
        assert_eq!(list, vec![(bcs_id, vec![])]);

        let list = fetch_list(&ctx, &repo, entries.get(1).unwrap().1.clone()).await;
        assert_eq!(list, vec![(bcs_id, vec![])]);
    }

    #[fbinit::test]
    async fn test_derive_linear(fb: FacebookInit) {
        let repo: TestRepo = Linear::get_custom_test_repo(fb).await;
        let ctx = CoreContext::test_mock(fb);

        let hg_cs_id = HgChangesetId::from_str("79a13814c5ce7330173ec04d279bf95ab3f652fb").unwrap();
        let bcs_id = repo
            .bonsai_hg_mapping
            .get_bonsai_from_hg(&ctx, hg_cs_id)
            .await
            .unwrap()
            .unwrap();

        let root_unode_mf_id = derive_fastlog_batch_and_unode(&ctx, bcs_id.clone(), &repo).await;

        let blobstore = Arc::new(repo.repo_blobstore.clone());
        let entries: Vec<_> = root_unode_mf_id
            .list_all_entries(ctx.clone(), blobstore)
            .map_ok(|(_, entry)| entry)
            .try_collect()
            .await
            .unwrap();

        for entry in entries {
            verify_list(&ctx, &repo, entry).await;
        }
    }

    #[fbinit::test]
    async fn test_derive_overflow(fb: FacebookInit) {
        let repo: TestRepo = Linear::get_custom_test_repo(fb).await;
        let ctx = CoreContext::test_mock(fb);

        let mut bonsais = vec![];
        let mut parents = vec![];
        for i in 1..max_entries_in_fastlog_batch() {
            let filename = String::from("1");
            let content = format!("{}", i);
            let stored_files = store_files(
                &ctx,
                btreemap! { filename.as_str() => Some(content.as_str()) },
                &repo,
            )
            .await;

            let bcs = create_bonsai_changeset_with_files(parents, stored_files);
            let bcs_id = bcs.get_changeset_id();
            bonsais.push(bcs);
            parents = vec![bcs_id];
        }

        let latest = parents.get(0).unwrap();
        save_bonsai_changesets(bonsais, ctx.clone(), &repo)
            .await
            .unwrap();

        verify_all_entries_for_commit(&ctx, &repo, *latest).await;
    }

    #[fbinit::test]
    async fn test_random_repo(fb: FacebookInit) {
        let repo: TestRepo = Linear::get_custom_test_repo(fb).await;
        let ctx = CoreContext::test_mock(fb);

        let mut rng = XorShiftRng::seed_from_u64(0); // reproducable Rng
        let (latest, _) =
            tests_utils::random::create_random_stack(&ctx, &repo, &mut rng, None, [10; 200])
                .await
                .unwrap();

        verify_all_entries_for_commit(&ctx, &repo, latest).await;
    }

    #[fbinit::test]
    async fn test_derive_empty_commits(fb: FacebookInit) {
        let repo: TestRepo = Linear::get_custom_test_repo(fb).await;
        let ctx = CoreContext::test_mock(fb);

        let mut bonsais = vec![];
        let mut parents = vec![];
        for _ in 1..max_entries_in_fastlog_batch() {
            let bcs = create_bonsai_changeset(parents);
            let bcs_id = bcs.get_changeset_id();
            bonsais.push(bcs);
            parents = vec![bcs_id];
        }

        let latest = parents.get(0).unwrap();
        save_bonsai_changesets(bonsais, ctx.clone(), &repo)
            .await
            .unwrap();

        verify_all_entries_for_commit(&ctx, &repo, *latest).await;
    }

    #[fbinit::test]
    async fn test_find_intersection_of_diffs_unodes_linear(fb: FacebookInit) -> Result<(), Error> {
        let repo: TestRepo = Linear::get_custom_test_repo(fb).await;
        let ctx = CoreContext::test_mock(fb);

        // This commit creates file "1" and "files"
        // See eden/mononoke/tests/fixtures
        let parent_root_unode =
            derive_unode(&ctx, &repo, "2d7d4ba9ce0a6ffd222de7785b249ead9c51c536").await?;

        // This commit creates file "2" and modifies "files"
        // See eden/mononoke/tests/fixtures
        let child_root_unode =
            derive_unode(&ctx, &repo, "3e0e761030db6e479a7fb58b12881883f9f8c63f").await?;

        let mut entries: Vec<_> = find_intersection_of_diffs(
            ctx,
            Arc::new(repo.repo_blobstore),
            child_root_unode,
            vec![parent_root_unode],
        )
        .map_ok(|(path, _)| match path {
            Some(path) => String::from_utf8(path.to_vec()).unwrap(),
            None => String::new(),
        })
        .try_collect()
        .await?;
        entries.sort();

        assert_eq!(
            entries,
            vec![String::new(), String::from("2"), String::from("files")]
        );
        Ok(())
    }

    #[fbinit::test]
    async fn test_find_intersection_of_diffs_merge(fb: FacebookInit) -> Result<(), Error> {
        async fn test_single_find_unodes_merge(
            fb: FacebookInit,
            parent_files: Vec<BTreeMap<&str, Option<&str>>>,
            merge_files: BTreeMap<&str, Option<&str>>,
            expected: Vec<String>,
        ) -> Result<(), Error> {
            let repo: TestRepo = Linear::get_custom_test_repo(fb).await;
            let ctx = CoreContext::test_mock(fb);
            let manager = repo.repo_derived_data().manager();

            let mut bonsais = vec![];
            let mut parents = vec![];

            for (i, p) in parent_files.into_iter().enumerate() {
                println!("parent {}, {:?} ", i, p);
                let stored_files = store_files(&ctx, p, &repo).await;
                let bcs = create_bonsai_changeset_with_files(vec![], stored_files);
                parents.push(bcs.get_changeset_id());
                bonsais.push(bcs);
            }

            println!("merge {:?} ", merge_files);
            let merge_stored_files = store_files(&ctx, merge_files, &repo).await;
            let bcs = create_bonsai_changeset_with_files(parents.clone(), merge_stored_files);
            let merge_bcs_id = bcs.get_changeset_id();

            bonsais.push(bcs);
            save_bonsai_changesets(bonsais, ctx.clone(), &repo)
                .await
                .unwrap();

            let mut parent_unodes = vec![];

            for p in parents {
                let parent_unode = manager.derive::<RootUnodeManifestId>(&ctx, p, None).await?;
                let parent_unode = parent_unode.manifest_unode_id().clone();
                parent_unodes.push(parent_unode);
            }

            let merge_unode = manager
                .derive::<RootUnodeManifestId>(&ctx, merge_bcs_id, None)
                .await?;
            let merge_unode = merge_unode.manifest_unode_id().clone();

            let mut entries: Vec<_> = find_intersection_of_diffs(
                ctx,
                Arc::new(repo.repo_blobstore),
                merge_unode,
                parent_unodes,
            )
            .map_ok(|(path, _)| match path {
                Some(path) => String::from_utf8(path.to_vec()).unwrap(),
                None => String::new(),
            })
            .try_collect()
            .await?;
            entries.sort();

            assert_eq!(entries, expected);
            Ok(())
        }

        test_single_find_unodes_merge(
            fb,
            vec![
                btreemap! {
                    "1" => Some("1"),
                },
                btreemap! {
                    "2" => Some("2"),
                },
            ],
            btreemap! {},
            vec![String::new()],
        )
        .await?;

        test_single_find_unodes_merge(
            fb,
            vec![
                btreemap! {
                    "1" => Some("1"),
                },
                btreemap! {
                    "2" => Some("2"),
                },
                btreemap! {
                    "3" => Some("3"),
                },
            ],
            btreemap! {},
            vec![String::new()],
        )
        .await?;

        test_single_find_unodes_merge(
            fb,
            vec![
                btreemap! {
                    "1" => Some("1"),
                },
                btreemap! {
                    "2" => Some("2"),
                },
            ],
            btreemap! {
                "inmerge" => Some("1"),
            },
            vec![String::new(), String::from("inmerge")],
        )
        .await?;

        test_single_find_unodes_merge(
            fb,
            vec![
                btreemap! {
                    "file" => Some("contenta"),
                },
                btreemap! {
                    "file" => Some("contentb"),
                },
            ],
            btreemap! {
                "file" => Some("mergecontent"),
            },
            vec![String::new(), String::from("file")],
        )
        .await?;
        Ok(())
    }

    #[fbinit::test]
    async fn test_derive_merges(fb: FacebookInit) -> Result<(), Error> {
        let ctx = CoreContext::test_mock(fb);

        {
            let repo: TestRepo = MergeUneven::get_custom_test_repo(fb).await;
            let all_commits: Vec<_> = all_commits(ctx.clone(), repo.clone()).try_collect().await?;

            for (bcs_id, _hg_cs_id) in all_commits {
                verify_all_entries_for_commit(&ctx, &repo, bcs_id).await;
            }
        }

        {
            let repo: TestRepo = MergeEven::get_custom_test_repo(fb).await;
            let all_commits: Vec<_> = all_commits(ctx.clone(), repo.clone()).try_collect().await?;

            for (bcs_id, _hg_cs_id) in all_commits {
                verify_all_entries_for_commit(&ctx, &repo, bcs_id).await;
            }
        }

        {
            let repo: TestRepo = UnsharedMergeEven::get_custom_test_repo(fb).await;
            let all_commits: Vec<_> = all_commits(ctx.clone(), repo.clone()).try_collect().await?;

            for (bcs_id, _hg_cs_id) in all_commits {
                verify_all_entries_for_commit(&ctx, &repo, bcs_id).await;
            }
        }

        {
            let repo: TestRepo = UnsharedMergeUneven::get_custom_test_repo(fb).await;
            let all_commits: Vec<_> = all_commits(ctx.clone(), repo.clone()).try_collect().await?;

            for (bcs_id, _hg_cs_id) in all_commits {
                verify_all_entries_for_commit(&ctx, &repo, bcs_id).await;
            }
        }

        Ok(())
    }

    #[fbinit::test]
    async fn test_bfs_order(fb: FacebookInit) -> Result<(), Error> {
        let repo: TestRepo = Linear::get_custom_test_repo(fb).await;
        let ctx = CoreContext::test_mock(fb);

        //            E
        //           / \
        //          D   C
        //         /   / \
        //        F   A   B
        //       /
        //      G
        //
        //   Expected order [E, D, C, F, A, B, G]

        let mut bonsais = vec![];

        let a = create_bonsai_changeset_with_author(vec![], "author1".to_string());
        println!("a = {}", a.get_changeset_id());
        bonsais.push(a.clone());
        let b = create_bonsai_changeset_with_author(vec![], "author2".to_string());
        println!("b = {}", b.get_changeset_id());
        bonsais.push(b.clone());

        let c = create_bonsai_changeset(vec![a.get_changeset_id(), b.get_changeset_id()]);
        println!("c = {}", c.get_changeset_id());
        bonsais.push(c.clone());

        let g = create_bonsai_changeset_with_author(vec![], "author3".to_string());
        println!("g = {}", g.get_changeset_id());
        bonsais.push(g.clone());

        let stored_files = store_files(&ctx, btreemap! { "file" => Some("f") }, &repo).await;
        let f = create_bonsai_changeset_with_files(vec![g.get_changeset_id()], stored_files);
        println!("f = {}", f.get_changeset_id());
        bonsais.push(f.clone());

        let stored_files = store_files(&ctx, btreemap! { "file" => Some("d") }, &repo).await;
        let d = create_bonsai_changeset_with_files(vec![f.get_changeset_id()], stored_files);
        println!("d = {}", d.get_changeset_id());
        bonsais.push(d.clone());

        let stored_files = store_files(&ctx, btreemap! { "file" => Some("e") }, &repo).await;
        let e = create_bonsai_changeset_with_files(
            vec![d.get_changeset_id(), c.get_changeset_id()],
            stored_files,
        );
        println!("e = {}", e.get_changeset_id());
        bonsais.push(e.clone());

        save_bonsai_changesets(bonsais, ctx.clone(), &repo).await?;

        verify_all_entries_for_commit(&ctx, &repo, e.get_changeset_id()).await;
        Ok(())
    }

    fn all_commits(
        ctx: CoreContext,
        repo: TestRepo,
    ) -> impl Stream<Item = Result<(ChangesetId, HgChangesetId), Error>> {
        let master_book = BookmarkKey::new("master").unwrap();
        repo.bookmarks
            .get(ctx.clone(), &master_book)
            .map_ok(move |maybe_bcs_id| {
                let bcs_id = maybe_bcs_id.unwrap();
                AncestorsNodeStream::new(ctx.clone(), &repo.changeset_fetcher, bcs_id.clone())
                    .compat()
                    .and_then(move |new_bcs_id| {
                        cloned!(ctx, repo);
                        async move {
                            let hg_cs_id = repo.derive_hg_changeset(&ctx, new_bcs_id).await?;
                            Ok((new_bcs_id, hg_cs_id))
                        }
                    })
            })
            .try_flatten_stream()
    }

    async fn verify_all_entries_for_commit(
        ctx: &CoreContext,
        repo: &TestRepo,
        bcs_id: ChangesetId,
    ) {
        let root_unode_mf_id = derive_fastlog_batch_and_unode(ctx, bcs_id.clone(), repo).await;

        let blobstore = Arc::new(repo.repo_blobstore.clone());
        let entries: Vec<_> = root_unode_mf_id
            .list_all_entries(ctx.clone(), blobstore.clone())
            .try_collect()
            .await
            .unwrap();

        for (path, entry) in entries {
            println!("verifying: path: {:?} unode: {:?}", path, entry);
            verify_list(ctx, repo, entry).await;
        }
    }

    async fn derive_unode(
        ctx: &CoreContext,
        repo: &TestRepo,
        hg_cs: &str,
    ) -> Result<ManifestUnodeId, Error> {
        let manager = repo.repo_derived_data().manager();
        let hg_cs_id = HgChangesetId::from_str(hg_cs)?;
        let bcs_id = repo
            .bonsai_hg_mapping
            .get_bonsai_from_hg(ctx, hg_cs_id)
            .await?
            .unwrap();
        let root_unode = manager
            .derive::<RootUnodeManifestId>(ctx, bcs_id, None)
            .await?;
        Ok(root_unode.manifest_unode_id().clone())
    }

    async fn derive_fastlog_batch_and_unode(
        ctx: &CoreContext,
        bcs_id: ChangesetId,
        repo: &TestRepo,
    ) -> ManifestUnodeId {
        let manager = repo.repo_derived_data().manager();
        manager
            .derive::<RootFastlog>(ctx, bcs_id, None)
            .await
            .unwrap();

        let root_unode = manager
            .derive::<RootUnodeManifestId>(ctx, bcs_id, None)
            .await
            .unwrap();
        root_unode.manifest_unode_id().clone()
    }

    async fn verify_list(
        ctx: &CoreContext,
        repo: &TestRepo,
        entry: Entry<ManifestUnodeId, FileUnodeId>,
    ) {
        let list = fetch_list(ctx, repo, entry).await;
        let actual_bonsais: Vec<_> = list.into_iter().map(|(bcs_id, _)| bcs_id).collect();

        let expected_bonsais = find_unode_history(ctx.fb, repo, entry).await;
        assert_eq!(actual_bonsais, expected_bonsais);
    }

    async fn fetch_list(
        ctx: &CoreContext,
        repo: &TestRepo,
        entry: Entry<ManifestUnodeId, FileUnodeId>,
    ) -> Vec<(ChangesetId, Vec<FastlogParent>)> {
        let blobstore = repo.repo_blobstore.clone();
        let batch = fetch_fastlog_batch_by_unode_id(ctx, &blobstore, &entry)
            .await
            .unwrap()
            .expect("batch hasn't been generated yet");

        println!(
            "batch for {:?}: latest size: {}, previous batches size: {}",
            entry,
            batch.latest().len(),
            batch.previous_batches().len(),
        );
        assert!(batch.latest().len() <= MAX_LATEST_LEN);
        assert!(batch.previous_batches().len() <= MAX_BATCHES);
        fetch_flattened(&batch, ctx, &blobstore).await.unwrap()
    }

    async fn find_unode_history(
        fb: FacebookInit,
        repo: &TestRepo,
        start: Entry<ManifestUnodeId, FileUnodeId>,
    ) -> Vec<ChangesetId> {
        let ctx = CoreContext::test_mock(fb);
        let mut q = VecDeque::new();
        q.push_back(start.clone());

        let mut visited = HashSet::new();
        visited.insert(start);
        let mut history = vec![];
        loop {
            let unode_entry = q.pop_front();
            let unode_entry = match unode_entry {
                Some(unode_entry) => unode_entry,
                None => {
                    break;
                }
            };
            let linknode = unode_entry.get_linknode(&ctx, repo).await.unwrap();
            history.push(linknode);
            if history.len() >= max_entries_in_fastlog_batch() {
                break;
            }
            let parents = unode_entry.get_parents(&ctx, repo).await.unwrap();
            q.extend(parents.into_iter().filter(|x| visited.insert(x.clone())));
        }

        history
    }

    #[async_trait]
    trait UnodeHistory {
        async fn get_parents<'a>(
            &'a self,
            ctx: &'a CoreContext,
            repo: &'a TestRepo,
        ) -> Result<Vec<Entry<ManifestUnodeId, FileUnodeId>>, Error>;

        async fn get_linknode<'a>(
            &'a self,
            ctx: &'a CoreContext,
            repo: &'a TestRepo,
        ) -> Result<ChangesetId, Error>;
    }

    #[async_trait]
    impl UnodeHistory for Entry<ManifestUnodeId, FileUnodeId> {
        async fn get_parents<'a>(
            &'a self,
            ctx: &'a CoreContext,
            repo: &'a TestRepo,
        ) -> Result<Vec<Entry<ManifestUnodeId, FileUnodeId>>, Error> {
            match self {
                Entry::Leaf(file_unode_id) => {
                    let unode_mf = file_unode_id.load(ctx, &repo.repo_blobstore).await?;
                    Ok(unode_mf
                        .parents()
                        .iter()
                        .cloned()
                        .map(Entry::Leaf)
                        .collect())
                }
                Entry::Tree(mf_unode_id) => {
                    let unode_mf = mf_unode_id.load(ctx, &repo.repo_blobstore).await?;
                    Ok(unode_mf
                        .parents()
                        .iter()
                        .cloned()
                        .map(Entry::Tree)
                        .collect())
                }
            }
        }

        async fn get_linknode<'a>(
            &'a self,
            ctx: &'a CoreContext,
            repo: &'a TestRepo,
        ) -> Result<ChangesetId, Error> {
            match self {
                Entry::Leaf(file_unode_id) => {
                    let unode_file = file_unode_id.load(ctx, &repo.repo_blobstore).await?;
                    Ok(unode_file.linknode().clone())
                }
                Entry::Tree(mf_unode_id) => {
                    let unode_mf = mf_unode_id.load(ctx, &repo.repo_blobstore).await?;
                    Ok(unode_mf.linknode().clone())
                }
            }
        }
    }
}