/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;
use std::collections::HashSet;
use std::sync::Arc;

use ::manifest::Entry;
use anyhow::bail;
use anyhow::Context;
use anyhow::Error;
use blobrepo::BlobRepo;
use blobrepo_hg::create_bonsai_changeset_hook;
use blobrepo_hg::ChangesetHandle;
use blobrepo_hg::CreateChangeset;
use bytes::Bytes;
use cloned::cloned;
use context::CoreContext;
use failure_ext::Compat;
use failure_ext::FutureErrorContext;
use failure_ext::StreamErrorContext;
use filestore::FilestoreConfigRef;
use futures::compat::Future01CompatExt;
use futures::compat::Stream01CompatExt;
use futures::FutureExt;
use futures::StreamExt;
use futures::TryFutureExt;
use futures_01_ext::try_boxfuture;
use futures_01_ext::try_boxstream;
use futures_01_ext::BoxFuture;
use futures_01_ext::BoxStream;
use futures_01_ext::FutureExt as _;
use futures_01_ext::StreamExt as _;
use futures_ext::FbTryFutureExt;
use futures_old::future;
use futures_old::future::SharedItem;
use futures_old::stream;
use futures_old::stream::Stream;
use futures_old::Future;
use futures_old::IntoFuture;
use lfs_import_lib::lfs_upload;
use mercurial_revlog::manifest;
use mercurial_revlog::revlog::RevIdx;
use mercurial_revlog::RevlogChangeset;
use mercurial_revlog::RevlogEntry;
use mercurial_revlog::RevlogRepo;
use mercurial_types::blobs::ChangesetMetadata;
use mercurial_types::blobs::ContentBlobMeta;
use mercurial_types::blobs::File;
use mercurial_types::blobs::HgBlobChangeset;
use mercurial_types::blobs::LFSContent;
use mercurial_types::blobs::UploadHgFileContents;
use mercurial_types::blobs::UploadHgFileEntry;
use mercurial_types::blobs::UploadHgNodeHash;
use mercurial_types::blobs::UploadHgTreeEntry;
use mercurial_types::HgBlob;
use mercurial_types::HgChangesetId;
use mercurial_types::HgFileNodeId;
use mercurial_types::HgManifestId;
use mercurial_types::HgNodeHash;
use mercurial_types::MPath;
use mercurial_types::RepoPath;
use mercurial_types::Type;
use mercurial_types::NULL_HASH;
use mononoke_types::BonsaiChangeset;
use mononoke_types::ContentMetadataV2;
use phases::PhasesArc;
use repo_blobstore::RepoBlobstoreArc;
use repo_identity::RepoIdentityRef;
use slog::info;
use tokio::runtime::Handle;
use wireproto_handler::BackupSourceRepo;

use crate::concurrency::JobProcessor;

struct ParseChangeset {
    revlogcs: BoxFuture<SharedItem<RevlogChangeset>, Error>,
    rootmf:
        BoxFuture<Option<(HgManifestId, HgBlob, Option<HgNodeHash>, Option<HgNodeHash>)>, Error>,
    entries: BoxStream<(Option<MPath>, RevlogEntry), Error>,
}

// Extracts all the data from revlog repo that commit API may need.
fn parse_changeset(revlog_repo: RevlogRepo, csid: HgChangesetId) -> ParseChangeset {
    let revlogcs = revlog_repo
        .get_changeset(csid)
        .with_context(move || format!("While reading changeset {:?}", csid))
        .map_err(Compat)
        .boxify()
        .shared();

    let rootmf = revlogcs
        .clone()
        .map_err(Error::from)
        .and_then({
            let revlog_repo = revlog_repo.clone();
            move |cs| {
                if cs.manifestid().into_nodehash() == NULL_HASH {
                    future::ok(None).boxify()
                } else {
                    revlog_repo
                        .get_root_manifest(cs.manifestid())
                        .map({
                            let manifest_id = cs.manifestid();
                            move |rootmf| Some((manifest_id, rootmf))
                        })
                        .boxify()
                }
            }
        })
        .with_context(move || format!("While reading root manifest for {:?}", csid))
        .map_err(Compat)
        .boxify()
        .shared();

    let entries = revlogcs
        .clone()
        .map_err(Error::from)
        .and_then({
            let revlog_repo = revlog_repo.clone();
            move |cs| {
                let mut parents = cs
                    .parents()
                    .into_iter()
                    .map(HgChangesetId::new)
                    .map(|csid| {
                        let revlog_repo = revlog_repo.clone();
                        revlog_repo
                            .get_changeset(csid)
                            .and_then(move |cs| {
                                if cs.manifestid().into_nodehash() == NULL_HASH {
                                    future::ok(None).boxify()
                                } else {
                                    revlog_repo
                                        .get_root_manifest(cs.manifestid())
                                        .map(Some)
                                        .boxify()
                                }
                            })
                            .boxify()
                    });

                let p1 = parents
                    .next()
                    .unwrap_or_else(|| Ok(None).into_future().boxify());
                let p2 = parents
                    .next()
                    .unwrap_or_else(|| Ok(None).into_future().boxify());

                p1.join(p2)
                    .with_context(move || format!("While reading parents of {:?}", csid))
                    .from_err()
            }
        })
        .join(rootmf.clone().from_err())
        .map(|((p1, p2), rootmf_shared)| match *rootmf_shared {
            None => stream::empty().boxify(),
            Some((_, ref rootmf)) => {
                manifest::new_entry_intersection_stream(rootmf, p1.as_ref(), p2.as_ref())
            }
        })
        .flatten_stream()
        .with_context(move || format!("While reading entries for {:?}", csid))
        .from_err()
        .boxify();

    let revlogcs = revlogcs.map_err(Error::from).boxify();

    let rootmf = rootmf
        .map_err(Error::from)
        .and_then(move |rootmf_shared| match *rootmf_shared {
            None => Ok(None),
            Some((manifest_id, ref mf)) => {
                let mut bytes = Vec::new();
                mf.generate(&mut bytes).with_context(|| {
                    format!("While generating root manifest blob for {:?}", csid)
                })?;

                let (p1, p2) = mf.parents().get_nodes();
                Ok(Some((
                    manifest_id,
                    HgBlob::from(Bytes::from(bytes)),
                    p1,
                    p2,
                )))
            }
        })
        .boxify();

    ParseChangeset {
        revlogcs,
        rootmf,
        entries,
    }
}

fn upload_entry(
    ctx: CoreContext,
    blobrepo: &BlobRepo,
    lfs_uploader: Arc<JobProcessor<LFSContent, ContentMetadataV2>>,
    entry: RevlogEntry,
    path: Option<MPath>,
) -> BoxFuture<(Entry<HgManifestId, HgFileNodeId>, RepoPath), Error> {
    let blobrepo = blobrepo.clone();

    let ty = entry.get_type();

    let path = MPath::join_element_opt(path.as_ref(), entry.get_name());
    let path = match path {
        // XXX this shouldn't be possible -- encode this in the type system
        None => {
            return future::err(Error::msg(
                "internal error: joined root path with root manifest",
            ))
            .boxify();
        }
        Some(path) => path,
    };

    let content = entry.get_raw_content();
    let is_ext = entry.is_ext();
    let parents = entry.get_parents();

    (content, is_ext, parents)
        .into_future()
        .and_then(move |(content, is_ext, parents)| {
            let (p1, p2) = parents.get_nodes();
            let upload_node_id = UploadHgNodeHash::Checked(entry.get_hash().into_nodehash());
            let blobstore = blobrepo.repo_blobstore_arc();
            let filestore_config = *blobrepo.filestore_config();
            match (ty, is_ext) {
                (Type::Tree, false) => {
                    let upload = UploadHgTreeEntry {
                        upload_node_id,
                        contents: content.into_inner(),
                        p1,
                        p2,
                        path: RepoPath::DirectoryPath(path),
                    };
                    let (_, upload_fut) = try_boxfuture!(upload.upload_as_entry(ctx, blobstore));
                    upload_fut
                }
                (Type::Tree, true) => Err(Error::msg("Inconsistent data: externally stored Tree"))
                    .into_future()
                    .boxify(),
                (Type::File(..), false) => {
                    let upload = UploadHgFileEntry {
                        upload_node_id,
                        contents: UploadHgFileContents::RawBytes(
                            content.into_inner(),
                            filestore_config,
                        ),
                        p1: p1.map(HgFileNodeId::new),
                        p2: p2.map(HgFileNodeId::new),
                    };
                    let upload_fut = upload.upload_as_entry(ctx, blobstore, path);
                    tokio::task::spawn(upload_fut)
                        .flatten_err()
                        .boxed()
                        .compat()
                        .boxify()
                }
                (Type::File(..), true) => {
                    let p1 = p1.map(HgFileNodeId::new);
                    let p2 = p2.map(HgFileNodeId::new);

                    let file = File::new(content, p1.clone(), p2.clone());
                    let lfs_content = try_boxfuture!(file.get_lfs_content());

                    lfs_uploader
                        .process(lfs_content.clone())
                        .and_then(move |meta| {
                            let cbmeta = ContentBlobMeta {
                                id: meta.content_id,
                                size: meta.total_size,
                                copy_from: lfs_content.copy_from(),
                            };

                            let upload = UploadHgFileEntry {
                                upload_node_id,
                                contents: UploadHgFileContents::ContentUploaded(cbmeta),
                                p1,
                                p2,
                            };
                            let upload_fut = upload.upload_as_entry(ctx, blobstore, path);
                            tokio::task::spawn(upload_fut)
                                .flatten_err()
                                .boxed()
                                .compat()
                                .boxify()
                        })
                        .boxify()
                }
            }
        })
        .boxify()
}

pub struct UploadChangesets {
    pub ctx: CoreContext,
    pub blobrepo: BlobRepo,
    pub revlogrepo: RevlogRepo,
    pub lfs_helper: Option<String>,
    pub concurrent_changesets: usize,
    pub concurrent_blobs: usize,
    pub concurrent_lfs_imports: usize,
    pub fixed_parent_order: HashMap<HgChangesetId, Vec<HgChangesetId>>,
}

impl UploadChangesets {
    pub fn upload(
        self,
        changesets: impl Stream<Item = (RevIdx, HgNodeHash), Error = Error> + Send + 'static,
        is_import_from_beggining: bool,
        origin_repo: Option<BackupSourceRepo>,
    ) -> BoxStream<(RevIdx, (BonsaiChangeset, HgBlobChangeset)), Error> {
        let Self {
            ctx,
            blobrepo,
            revlogrepo,
            lfs_helper,
            concurrent_changesets,
            concurrent_blobs,
            concurrent_lfs_imports,
            fixed_parent_order,
        } = self;

        let mut parent_changeset_handles: HashMap<HgNodeHash, ChangesetHandle> = HashMap::new();

        let handle = try_boxstream!(Handle::try_current().context("No tokio runtime available"));

        let mut scuba_logger = ctx.scuba().clone();
        scuba_logger
            .add("Repo Id", blobrepo.repo_identity().id().to_string())
            .add("Repo name", blobrepo.repo_identity().name().to_string());

        let lfs_uploader = Arc::new(try_boxstream!(JobProcessor::new(
            {
                cloned!(ctx, blobrepo);
                move |lfs_content| match &lfs_helper {
                    Some(lfs_helper) => {
                        cloned!(ctx, blobrepo, lfs_helper);
                        async move { lfs_upload(&ctx, &blobrepo, &lfs_helper, &lfs_content).await }
                            .boxed()
                            .compat()
                    }
                    .boxify(),
                    None => Err(Error::msg("Cannot blobimport LFS without LFS helper"))
                        .into_future()
                        .boxify(),
                }
            },
            &handle,
            concurrent_lfs_imports,
        )));

        let blob_uploader = Arc::new(try_boxstream!(JobProcessor::new(
            {
                cloned!(ctx, blobrepo, lfs_uploader);
                move |(entry, path)| {
                    upload_entry(ctx.clone(), &blobrepo, lfs_uploader.clone(), entry, path).boxify()
                }
            },
            &handle,
            concurrent_blobs,
        )));

        let create_and_verify_bonsai = create_bonsai_changeset_hook(origin_repo);

        changesets
            .and_then({
                cloned!(ctx, revlogrepo, blobrepo);
                move |(revidx, csid)| {
                    let ParseChangeset {
                        revlogcs,
                        rootmf,
                        entries,
                    } = parse_changeset(revlogrepo.clone(), HgChangesetId::new(csid));

                    let rootmf = rootmf.map({
                        cloned!(ctx, blobrepo);
                        move |rootmf| {
                            match rootmf {
                                None => future::ok(None).boxify(),
                                Some((manifest_id, blob, p1, p2)) => {
                                    let upload = UploadHgTreeEntry {
                                        // The root tree manifest is expected to have the wrong hash in
                                        // hybrid mode. This will probably never go away for
                                        // compatibility with old repositories.
                                        upload_node_id: UploadHgNodeHash::Supplied(
                                            manifest_id.into_nodehash(),
                                        ),
                                        contents: blob.into_inner(),
                                        p1,
                                        p2,
                                        path: RepoPath::root(),
                                    };
                                    upload
                                        .upload(ctx, blobrepo.repo_blobstore_arc())
                                        .into_future()
                                        .and_then(|(_, entry)| entry)
                                        .map(Some)
                                        .boxify()
                                }
                            }
                        }
                    });

                    let entries = entries.map({
                        cloned!(blob_uploader);
                        move |(path, entry)| blob_uploader.process((entry, path))
                    });

                    revlogcs
                        .join3(rootmf, entries.collect())
                        .map(move |(cs, rootmf, entries)| (revidx, csid, cs, rootmf, entries))
                }
            })
            .and_then({
                cloned!(ctx);
                move |(revidx, csid, cs, rootmf, entries)| {
                    let parents_from_revlog: Vec<_> =
                        cs.parents().into_iter().map(HgChangesetId::new).collect();

                    if let Some(parent_order) =
                        fixed_parent_order.get(&HgChangesetId::new(csid.clone()))
                    {
                        let actual: HashSet<_> = parents_from_revlog.into_iter().collect();
                        let expected: HashSet<_> = parent_order.iter().copied().collect();
                        if actual != expected {
                            bail!(
                                "Changeset {} has unexpected parents: actual {:?}\nexpected {:?}",
                                csid,
                                actual,
                                expected
                            );
                        }

                        info!(
                            ctx.logger(),
                            "fixing parent order for {}: {:?}", csid, parent_order
                        );
                        Ok((revidx, csid, cs, rootmf, entries, parent_order.clone()))
                    } else {
                        Ok((revidx, csid, cs, rootmf, entries, parents_from_revlog))
                    }
                }
            })
            .map(move |(revidx, csid, cs, rootmf, entries, parents)| {
                let entries = stream::futures_unordered(entries).boxify();

                let (p1handle, p2handle) = {
                    let mut parents = parents.into_iter().map(|p| {
                        let p = p.into_nodehash();
                        let maybe_handle = parent_changeset_handles.get(&p).cloned();

                        if is_import_from_beggining {
                            maybe_handle
                                .unwrap_or_else(|| panic!("parent {} not found for {}", p, csid))
                        } else {
                            let hg_cs_id = HgChangesetId::new(p);

                            maybe_handle.unwrap_or_else({
                                cloned!(ctx, blobrepo);
                                move || ChangesetHandle::ready_cs_handle(ctx, blobrepo, hg_cs_id)
                            })
                        }
                    });

                    (parents.next(), parents.next())
                };

                let cs_metadata = ChangesetMetadata {
                    user: String::from_utf8(Vec::from(cs.user()))
                        .unwrap_or_else(|_| panic!("non-utf8 username for {}", csid)),
                    time: cs.time().clone(),
                    extra: cs.extra().clone(),
                    message: String::from_utf8(Vec::from(cs.message()))
                        .unwrap_or_else(|_| panic!("non-utf8 message for {}", csid)),
                };
                let create_changeset = CreateChangeset {
                    expected_nodeid: Some(csid),
                    expected_files: Some(Vec::from(cs.files())),
                    p1: p1handle,
                    p2: p2handle,
                    root_manifest: rootmf.compat().boxed(),
                    sub_entries: entries.compat().boxed(),
                    cs_metadata,
                    create_bonsai_changeset_hook: Some(create_and_verify_bonsai.clone()),
                    upload_to_blobstore_only: false,
                };
                let cshandle =
                    create_changeset.create(ctx.clone(), &blobrepo, scuba_logger.clone());
                parent_changeset_handles.insert(csid, cshandle.clone());

                cloned!(ctx);
                let phases = blobrepo.phases_arc();

                // Uploading changeset and populate phases
                // We know they are public.
                tokio::task::spawn(async move {
                    cshandle
                        .get_completed_changeset()
                        .await
                        .with_context(move || format!("While uploading changeset: {}", csid))
                })
                .flatten_err()
                .boxed()
                .compat()
                .and_then(move |shared| {
                    cloned!(ctx, phases);
                    async move {
                        phases
                            .add_reachable_as_public(&ctx, vec![shared.0.get_changeset_id()])
                            .await?;
                        Ok((revidx, shared))
                    }
                    .boxed()
                    .compat()
                })
                .boxify()
            })
            // This is the number of changesets to upload in parallel. Keep it small to keep the database
            // load under control
            .buffer_unordered(concurrent_changesets)
            .boxify()
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use anyhow::format_err;
use anyhow::Error;
use anyhow::Result;
use futures::compat::Future01CompatExt;
use futures_01_ext::BoxFuture;
use futures_01_ext::FutureExt;
use futures_old::sync::mpsc;
use futures_old::sync::oneshot;
use futures_old::Future;
use futures_old::IntoFuture;
use futures_old::Stream;
use tokio::runtime::Handle;

type Job<In, Out> = (In, oneshot::Sender<Result<Out>>);

/// JobProcessor allows for limiting concurrency for a particular kind of action, implemented in
/// the handler passed when instantiating a JobProcessor. The JobProcessor does not enforce any
/// limits on buffering of jobs: only on their execution! This is useful when different pieces of
/// code need to share an underlying resource but aren't modelled as an individual stream that can
/// be buffered.
pub struct JobProcessor<In, Out> {
    sender: mpsc::UnboundedSender<Job<In, Out>>,
}

impl<In, Out> JobProcessor<In, Out>
where
    In: Send + 'static,
    Out: Send + 'static,
{
    pub fn new<H>(handler: H, handle: &Handle, concurrency: usize) -> Result<Self>
    where
        H: Fn(In) -> BoxFuture<Out, Error> + Send + 'static,
    {
        // NOTE: This buffer is unbounded, because we allow buffering as many entries as possible
        // on this stream, we just don't process all of them at once. We do implicitly have some
        // limits on the size of this buffer set e.g. the number of concurrent changesets we
        // process.
        let (sender, receiver) = mpsc::unbounded::<Job<In, Out>>();

        let processor = receiver
            .map(move |(input, sender)| {
                handler(input).then(move |res| {
                    let _ = sender.send(res); // Don't kill the stream if one receiver is gone.
                    Ok(())
                })
            })
            .buffer_unordered(concurrency)
            .for_each(|()| Ok(()))
            .discard()
            .compat();

        handle.spawn(processor);

        Ok(Self { sender })
    }

    pub fn process(&self, input: In) -> impl Future<Item = Out, Error = Error> {
        let (sender, receiver) = oneshot::channel::<Result<Out>>();

        match self.sender.unbounded_send((input, sender)) {
            Ok(()) => receiver
                .into_future()
                .map_err(|e| format_err!("JobProcessor: Receiver failed: {:?}", e))
                .and_then(|r| r)
                .left_future(),
            Err(e) => Err(format_err!("JobProcessor Sender failed: {:?}", e))
                .into_future()
                .right_future(),
        }
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

mod bookmark;
mod changeset;
mod concurrency;

use std::cmp;
use std::collections::HashMap;
use std::error::Error as StdError;
use std::ops::Deref;
use std::path::PathBuf;
use std::sync::Arc;

use anyhow::format_err;
use anyhow::Context;
use anyhow::Error;
use ascii::AsciiString;
use blobrepo::BlobRepo;
use blobrepo_hg::BlobRepoHg;
use bonsai_git_mapping::BonsaiGitMappingRef;
use bonsai_globalrev_mapping::bulk_import_globalrevs;
use bonsai_globalrev_mapping::BonsaiGlobalrevMapping;
use bookmarks::BookmarksMaybeStaleExt;
use bookmarks::BookmarksRef;
pub use consts::HIGHEST_IMPORTED_GEN_NUM;
use context::CoreContext;
use derived_data_utils::derive_data_for_csids;
use futures::compat::Future01CompatExt;
use futures::compat::Stream01CompatExt;
use futures::future;
use futures::stream;
use futures::stream::StreamExt;
use futures::stream::TryStreamExt;
use futures::Stream;
use futures_01_ext::StreamExt as OldStreamExt;
use mercurial_revlog::revlog::RevIdx;
use mercurial_revlog::RevlogRepo;
use mercurial_types::HgChangesetId;
use mercurial_types::HgNodeHash;
use mononoke_types::ChangesetId;
use mononoke_types::RepositoryId;
use phases::PhasesRef;
use repo_identity::RepoIdentityRef;
use slog::debug;
use slog::error;
use slog::info;
use synced_commit_mapping::SyncedCommitMapping;
use synced_commit_mapping::SyncedCommitMappingEntry;
use synced_commit_mapping::SyncedCommitSourceRepo;
use wireproto_handler::BackupSourceRepo;

use crate::changeset::UploadChangesets;

// What to do with bookmarks when blobimporting a repo
pub enum BookmarkImportPolicy {
    // Do not import bookmarks
    Ignore,
    // Prefix bookmark names when importing
    Prefix(AsciiString),
}

pub struct Blobimport<'a> {
    pub ctx: &'a CoreContext,
    pub blobrepo: BlobRepo,
    pub revlogrepo_path: PathBuf,
    pub changeset: Option<HgNodeHash>,
    pub skip: Option<usize>,
    pub commits_limit: Option<usize>,
    pub bookmark_import_policy: BookmarkImportPolicy,
    pub globalrevs_store: Arc<dyn BonsaiGlobalrevMapping>,
    pub synced_commit_mapping: Arc<dyn SyncedCommitMapping>,
    pub lfs_helper: Option<String>,
    pub concurrent_changesets: usize,
    pub concurrent_blobs: usize,
    pub concurrent_lfs_imports: usize,
    pub fixed_parent_order: HashMap<HgChangesetId, Vec<HgChangesetId>>,
    pub has_globalrev: bool,
    pub populate_git_mapping: bool,
    pub small_repo_id: Option<RepositoryId>,
    pub derived_data_types: Vec<String>,
    pub origin_repo: Option<BackupSourceRepo>,
}

impl<'a> Blobimport<'a> {
    pub async fn import(self) -> Result<Option<(RevIdx, ChangesetId)>, Error> {
        let Self {
            ctx,
            blobrepo,
            revlogrepo_path,
            changeset,
            skip,
            commits_limit,
            bookmark_import_policy,
            globalrevs_store,
            synced_commit_mapping,
            lfs_helper,
            concurrent_changesets,
            concurrent_blobs,
            concurrent_lfs_imports,
            fixed_parent_order,
            has_globalrev,
            populate_git_mapping,
            small_repo_id,
            derived_data_types,
            origin_repo,
        } = self;

        // Take refs to avoid `async move` blocks capturing data data
        // in async move blocks
        let blobrepo = &blobrepo;
        let globalrevs_store = &globalrevs_store;
        let synced_commit_mapping = &synced_commit_mapping;
        let derived_data_types = &derived_data_types;

        let repo_id = blobrepo.repo_identity().id();

        let revlogrepo = RevlogRepo::open(&revlogrepo_path)
            .with_context(|| format!("While opening revlog repo at {:?}", revlogrepo_path))?;
        let stale_bookmarks_fut = bookmark::read_bookmarks(&revlogrepo).compat();

        let log_step = match commits_limit {
            Some(commits_limit) => cmp::max(1, commits_limit / 10),
            None => 5000,
        };

        let chunk_size = 100;

        let is_import_from_beggining = changeset.is_none() && skip.is_none();
        let changesets = get_changeset_stream(&revlogrepo, changeset, skip, commits_limit)
            .boxed()
            .compat();

        let mut upload_changesets = UploadChangesets {
            ctx: ctx.clone(),
            blobrepo: blobrepo.clone(),
            revlogrepo: revlogrepo.clone(),
            lfs_helper,
            concurrent_changesets,
            concurrent_blobs,
            concurrent_lfs_imports,
            fixed_parent_order,
        }
        .upload(changesets, is_import_from_beggining, origin_repo)
        .enumerate()
        .compat()
        .map_ok({
            move |(cs_count, (revidx, cs))| {
                debug!(
                    ctx.logger(),
                    "{} inserted: {}",
                    cs_count,
                    cs.1.get_changeset_id()
                );
                if cs_count % log_step == 0 {
                    info!(ctx.logger(), "inserted commits # {}", cs_count);
                }
                (revidx, cs.0)
            }
        })
        .chunks(chunk_size)
        .map(|chunk: Vec<Result<_, _>>| chunk.into_iter().collect::<Result<Vec<_>, _>>())
        .map_err({
            move |err| {
                let msg = format!("failed to blobimport: {}", err);
                error!(ctx.logger(), "{}", msg);

                let mut err = err.deref() as &dyn StdError;
                while let Some(cause) = failure_ext::cause(err) {
                    info!(ctx.logger(), "cause: {}", cause);
                    err = cause;
                }
                info!(ctx.logger(), "root cause: {:?}", err);

                Error::msg(msg)
            }
        });

        // Blobimport does not see scratch bookmarks in Mercurial, so we use
        // PublishingOrPullDefaultPublishing here, which is the non-scratch set in Mononoke.
        let mononoke_bookmarks_fut = blobrepo
            .bookmarks()
            .get_publishing_bookmarks_maybe_stale(ctx.clone())
            .map_ok(|(bookmark, changeset_id)| (bookmark.into_key(), changeset_id))
            .try_collect::<Vec<_>>();

        let (stale_bookmarks, mononoke_bookmarks) =
            future::try_join(stale_bookmarks_fut, mononoke_bookmarks_fut).await?;

        let mut max_rev_and_bcs_id = None;
        while let Some(chunk_result) = upload_changesets.next().await {
            let chunk = chunk_result?;
            for (rev, cs) in chunk.iter() {
                let max_rev = max_rev_and_bcs_id.map_or_else(RevIdx::zero, |(revidx, _)| revidx);
                if rev >= &max_rev {
                    max_rev_and_bcs_id = Some((*rev, cs.get_changeset_id()))
                }
            }

            let changesets: &Vec<_> = &chunk.into_iter().map(|(_, cs)| cs).collect();

            let synced_commit_mapping_work = async {
                if let Some(small_repo_id) = small_repo_id {
                    let entries = changesets
                        .iter()
                        .map(|cs| SyncedCommitMappingEntry {
                            large_repo_id: repo_id,
                            large_bcs_id: cs.get_changeset_id(),
                            small_repo_id,
                            small_bcs_id: cs.get_changeset_id(),
                            version_name: None,
                            source_repo: Some(SyncedCommitSourceRepo::Small),
                        })
                        .collect();
                    synced_commit_mapping
                        .add_bulk(ctx, entries)
                        .await
                        .map(|_| ())
                } else {
                    Ok(())
                }
            };

            let globalrevs_work = async {
                if has_globalrev {
                    bulk_import_globalrevs(ctx, globalrevs_store.as_ref(), changesets.iter()).await
                } else {
                    Ok(())
                }
            };

            let git_mapping_work = async move {
                if populate_git_mapping {
                    let git_mapping_store = blobrepo.bonsai_git_mapping();
                    git_mapping_store
                        .bulk_import_from_bonsai(ctx, changesets)
                        .await
                } else {
                    Ok(())
                }
            };

            if !derived_data_types.is_empty() {
                info!(ctx.logger(), "Deriving data for: {:?}", derived_data_types);
            }

            let derivation_work = derive_data_for_csids(
                ctx,
                blobrepo,
                changesets.iter().map(|cs| cs.get_changeset_id()).collect(),
                &derived_data_types[..],
            )?;

            future::try_join4(
                synced_commit_mapping_work,
                globalrevs_work,
                git_mapping_work,
                derivation_work,
            )
            .await?;
        }

        info!(
            ctx.logger(),
            "finished uploading changesets, globalrevs and deriving data"
        );

        match bookmark_import_policy {
            BookmarkImportPolicy::Ignore => {
                info!(
                    ctx.logger(),
                    "since --no-bookmark was provided, bookmarks won't be imported"
                );
            }
            BookmarkImportPolicy::Prefix(prefix) => {
                bookmark::upload_bookmarks(
                    ctx.clone(),
                    ctx.logger(),
                    revlogrepo,
                    blobrepo.clone(),
                    stale_bookmarks,
                    mononoke_bookmarks,
                    bookmark::get_bookmark_prefixer(prefix),
                )
                .compat()
                .await?
            }
        };

        Ok(max_rev_and_bcs_id)
    }

    /// From a stream of revisions returns the latest revision that's already imported and public
    /// From this revision it's safe to restart blobimport next time.
    /// Note that we might have a situation where revision i is imported, i+1 is not and i+2 is imported.
    /// In that case this function would return i.
    pub async fn find_already_imported_revision(
        self,
    ) -> Result<Option<(RevIdx, ChangesetId)>, Error> {
        let Self {
            ctx,
            blobrepo,
            revlogrepo_path,
            changeset,
            skip,
            commits_limit,
            ..
        } = self;

        let blobrepo = &blobrepo;
        let revlogrepo = RevlogRepo::open(revlogrepo_path)?;
        let imported: Vec<_> = get_changeset_stream(&revlogrepo, changeset, skip, commits_limit)
            .chunks(100)
            .then(|chunk| async move {
                let chunk: Result<Vec<_>, Error> = chunk.into_iter().collect();
                let chunk = chunk?;
                let hg_to_bcs_ids = blobrepo
                    .get_hg_bonsai_mapping(
                        ctx.clone(),
                        chunk
                            .clone()
                            .into_iter()
                            .map(|(_, hg_cs_id)| HgChangesetId::new(hg_cs_id))
                            .collect::<Vec<_>>(),
                    )
                    .await?;

                let public = blobrepo
                    .phases()
                    .get_public(
                        ctx,
                        hg_to_bcs_ids
                            .clone()
                            .into_iter()
                            .map(|(_, bcs_id)| bcs_id)
                            .collect(),
                        false, /* ephemeral_derive */
                    )
                    .await?;

                let hg_cs_ids: HashMap<_, _> = hg_to_bcs_ids.into_iter().collect();
                let s = chunk.into_iter().map(move |(revidx, hg_cs_id)| {
                    match hg_cs_ids.get(&HgChangesetId::new(hg_cs_id)) {
                        Some(bcs_id) => {
                            if public.contains(bcs_id) {
                                Some((revidx, *bcs_id))
                            } else {
                                None
                            }
                        }
                        None => None,
                    }
                });

                Result::<_, Error>::Ok(stream::iter(s).map(Result::<_, Error>::Ok))
            })
            .try_flatten()
            .take_while(|res| match res {
                Ok(maybe_public_bcs_id) => future::ready(maybe_public_bcs_id.is_some()),
                // Take errors since they will just be propagated to the caller,
                // and if we don't take them then the caller won't know
                // about the error.
                Err(_) => future::ready(true),
            })
            .try_collect()
            .await?;

        let mut max_rev_and_bcs_id = None;
        for maybe_rev_cs in imported {
            if let Some((rev, bcs_id)) = maybe_rev_cs {
                let max_rev = max_rev_and_bcs_id.map_or_else(RevIdx::zero, |(revidx, _)| revidx);
                if rev >= max_rev {
                    max_rev_and_bcs_id = Some((rev, bcs_id))
                }
            }
        }

        Ok(max_rev_and_bcs_id)
    }
}

fn get_changeset_stream(
    revlogrepo: &RevlogRepo,
    changeset: Option<HgNodeHash>,
    skip: Option<usize>,
    commits_limit: Option<usize>,
) -> impl Stream<Item = Result<(RevIdx, HgNodeHash), Error>> + 'static {
    let changesets = match changeset {
        Some(hash) => {
            let maybe_idx = revlogrepo.get_rev_idx_for_changeset(HgChangesetId::new(hash));
            stream::once(async move {
                match maybe_idx {
                    Ok(idx) => Ok((idx, hash)),
                    Err(err) => Err(format_err!("{} not found in revlog repo: {}", hash, err)),
                }
            })
            .left_stream()
        }
        None => revlogrepo.changesets().compat().right_stream(),
    };

    let changesets = match skip {
        None => changesets.left_stream(),
        Some(skip) => changesets.skip(skip).right_stream(),
    };

    match commits_limit {
        None => changesets.left_stream(),
        Some(limit) => changesets.take(limit).right_stream(),
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashSet;

use anyhow::Context;
use anyhow::Error;
use anyhow::Result;
use blobstore::Blobstore;
use blobstore::Loadable;
use cloned::cloned;
use context::CoreContext;
use futures::stream;
use futures::stream::StreamExt;
use futures::stream::TryStream;
use futures::stream::TryStreamExt;
use manifest::bonsai_diff;
use manifest::BonsaiDiffFileChange;
use manifest::ManifestOps;
use mercurial_types::blobs::HgBlobChangeset;
use mercurial_types::blobs::HgBlobEnvelope;
use mercurial_types::HgFileEnvelope;
use mercurial_types::HgFileNodeId;
use mercurial_types::HgManifestId;
use mercurial_types::MPath;
use mercurial_types::RepoPath;
use mononoke_types::BlobstoreKey;
use mononoke_types::BlobstoreValue;
use mononoke_types::BonsaiChangeset;
use mononoke_types::BonsaiChangesetMut;
use mononoke_types::ChangesetId;
use mononoke_types::FileChange;
use repo_blobstore::RepoBlobstore;
use sorted_vector_map::SortedVectorMap;

use crate::errors::*;

/// Creates bonsai changeset from already created HgBlobChangeset.
pub async fn create_bonsai_changeset_object(
    ctx: &CoreContext,
    cs: HgBlobChangeset,
    parent_manifests: Vec<HgManifestId>,
    bonsai_parents: Vec<ChangesetId>,
    blobstore: &RepoBlobstore,
) -> Result<BonsaiChangeset, Error> {
    let file_changes = find_file_changes(
        ctx,
        cs.clone(),
        parent_manifests,
        blobstore,
        bonsai_parents.clone(),
    )
    .await?;

    let extra = cs
        .extra()
        .iter()
        .map(|(key, value)| {
            // Extra keys must be valid UTF-8.   Mercurial supports arbitrary
            // bytes, but that is not supported in Mononoke.  Extra values can
            // be arbitrary bytes.
            let key = String::from_utf8(key.clone())?;
            Ok((key, value.clone()))
        })
        .collect::<Result<SortedVectorMap<_, _>, Error>>()?;

    let author = String::from_utf8(cs.user().to_vec())
        .with_context(|| format!("While converting author name {:?}", cs.user()))?;
    let message = String::from_utf8(cs.message().to_vec())
        .with_context(|| format!("While converting commit message {:?}", cs.message()))?;
    BonsaiChangesetMut {
        parents: bonsai_parents,
        author,
        author_date: *cs.time(),
        committer: None,
        committer_date: None,
        message,
        hg_extra: extra,
        git_extra_headers: None,
        git_tree_hash: None,
        file_changes,
        is_snapshot: false,
        git_annotated_tag: None,
    }
    .freeze()
}

pub async fn save_bonsai_changeset_object(
    ctx: &CoreContext,
    blobstore: &RepoBlobstore,
    bonsai_cs: BonsaiChangeset,
) -> Result<(), Error> {
    let bonsai_blob = bonsai_cs.into_blob();
    let bcs_id = bonsai_blob.id().clone();
    let blobstore_key = bcs_id.blobstore_key();

    blobstore.put(ctx, blobstore_key, bonsai_blob.into()).await
}

fn find_bonsai_diff(
    ctx: &CoreContext,
    blobstore: RepoBlobstore,
    cs: HgBlobChangeset,
    parent_manifests: HashSet<HgManifestId>,
) -> Result<impl TryStream<Ok = BonsaiDiffFileChange<HgFileNodeId>, Error = Error>> {
    Ok(bonsai_diff(
        ctx.clone(),
        blobstore,
        cs.manifestid(),
        parent_manifests,
    ))
}

// Finds files that were changed in the commit and returns it in the format suitable for BonsaiChangeset
async fn find_file_changes(
    ctx: &CoreContext,
    cs: HgBlobChangeset,
    parent_manifests: Vec<HgManifestId>,
    blobstore: &RepoBlobstore,
    bonsai_parents: Vec<ChangesetId>,
) -> Result<SortedVectorMap<MPath, FileChange>, Error> {
    let diff: Result<_, Error> = find_bonsai_diff(
        ctx,
        blobstore.clone(),
        cs,
        parent_manifests.iter().cloned().collect(),
    )
    .context("While finding bonsai diff")?
    .map_ok(|diff| {
        cloned!(parent_manifests, bonsai_parents);
        async move {
            match diff {
                BonsaiDiffFileChange::Changed(path, ty, entry_id) => {
                    let file_node_id = HgFileNodeId::new(entry_id.into_nodehash());
                    let envelope = file_node_id
                        .load(ctx, blobstore)
                        .await
                        .context("While fetching bonsai file changes")?;
                    let size = envelope.content_size();
                    let content_id = envelope.content_id();

                    let copyinfo = get_copy_info(
                        ctx.clone(),
                        blobstore.clone(),
                        bonsai_parents,
                        path.clone(),
                        envelope,
                        parent_manifests,
                    )
                    .await
                    .context("While fetching copy information")?;
                    Ok((path, FileChange::tracked(content_id, ty, size, copyinfo)))
                }
                BonsaiDiffFileChange::ChangedReusedId(path, ty, entry_id) => {
                    let file_node_id = HgFileNodeId::new(entry_id.into_nodehash());
                    let envelope = file_node_id
                        .load(ctx, blobstore)
                        .await
                        .context("While fetching bonsai file changes")?;
                    let size = envelope.content_size();
                    let content_id = envelope.content_id();

                    // Reused ID means copy info is *not* stored.
                    Ok((path, FileChange::tracked(content_id, ty, size, None)))
                }
                BonsaiDiffFileChange::Deleted(path) => Ok((path, FileChange::Deletion)),
            }
        }
    })
    .try_buffer_unordered(100) // TODO(stash): magic number?
    .try_collect::<std::collections::BTreeMap<_, _>>()
    .await;

    Ok(SortedVectorMap::from_iter(diff?))
}

// Returns copy information for a given path and node if this file was copied.
// This function is quite complicated because hg and bonsai store copy information differently.
// In hg copy information is (path, filenode), in bonsai it's (path, parent cs id). That means that
// we need to find a parent from which this filenode was copied.
async fn get_copy_info(
    ctx: CoreContext,
    blobstore: RepoBlobstore,
    bonsai_parents: Vec<ChangesetId>,
    copy_from_path: MPath,
    envelope: HgFileEnvelope,
    parent_manifests: Vec<HgManifestId>,
) -> Result<Option<(MPath, ChangesetId)>, Error> {
    let node_id = envelope.node_id();

    let maybecopy = envelope
        .get_copy_info()?
        .map(|(path, hash)| (RepoPath::FilePath(path), hash));

    match maybecopy {
        Some((repopath, copyfromnode)) => {
            let repopath = repopath.mpath().ok_or(ErrorKind::UnexpectedRootPath)?;

            let parents_bonsai_and_mfs =
                stream::iter(bonsai_parents.into_iter().zip(parent_manifests.into_iter()));

            let get_bonsai_cs_copied_from =
                |(bonsai_parent, parent_mf): (ChangesetId, HgManifestId)| {
                    cloned!(ctx, blobstore);
                    async move {
                        let entry = parent_mf
                            .find_entry(ctx, blobstore, Some(repopath.clone()))
                            .await
                            .ok()?;
                        if entry?.into_leaf()?.1 == copyfromnode {
                            Some(bonsai_parent)
                        } else {
                            None
                        }
                    }
                };

            let copied_from = parents_bonsai_and_mfs
                .then(get_bonsai_cs_copied_from)
                .filter_map(|x| async move { x })
                .collect::<Vec<ChangesetId>>()
                .await;

            match copied_from.get(0) {
                Some(bonsai_cs_copied_from) => {
                    Ok(Some((repopath.clone(), bonsai_cs_copied_from.clone())))
                }
                None => Err(ErrorKind::IncorrectCopyInfo {
                    from_path: copy_from_path,
                    from_node: node_id,
                    to_path: repopath.clone(),
                    to_node: copyfromnode,
                }
                .into()),
            }
        }
        None => Ok(None),
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::sync::Arc;
use std::sync::Mutex;

use ::manifest::Entry;
use anyhow::anyhow;
use anyhow::format_err;
use anyhow::Context;
use anyhow::Error;
use anyhow::Result;
use blobrepo::BlobRepo;
use blobstore::Loadable;
use bonsai_hg_mapping::BonsaiHgMappingArc;
use bonsai_hg_mapping::BonsaiHgMappingEntry;
use bonsai_hg_mapping::BonsaiHgMappingRef;
use changesets::ChangesetInsert;
use changesets::Changesets;
use changesets::ChangesetsArc;
use cloned::cloned;
use context::CoreContext;
use futures::channel::oneshot;
use futures::future;
use futures::future::BoxFuture;
use futures::future::FutureExt;
use futures::future::TryFutureExt;
use futures::stream::BoxStream;
use futures_ext::FbTryFutureExt;
use futures_stats::TimedTryFutureExt;
use mercurial_types::blobs::ChangesetMetadata;
use mercurial_types::blobs::HgBlobChangeset;
use mercurial_types::HgFileNodeId;
use mercurial_types::HgManifestId;
use mercurial_types::HgNodeHash;
use mercurial_types::RepoPath;
use mononoke_types::BlobstoreValue;
use mononoke_types::BonsaiChangeset;
use mononoke_types::ChangesetId;
use mononoke_types::MPath;
use repo_blobstore::RepoBlobstoreArc;
use repo_blobstore::RepoBlobstoreRef;
use scuba_ext::MononokeScubaSampleBuilder;
use stats::prelude::*;
use uuid::Uuid;
use wireproto_handler::BackupSourceRepo;

use crate::bonsai_generation::create_bonsai_changeset_object;
use crate::bonsai_generation::save_bonsai_changeset_object;
use crate::repo_commit::*;
use crate::ErrorKind;

type BonsaiChangesetHook = dyn Fn(
        CoreContext,
        HgBlobChangeset,
        Vec<HgManifestId>,
        Vec<ChangesetId>,
        BlobRepo,
    ) -> BoxFuture<'static, Result<BonsaiChangeset>>
    + Send
    + Sync;

define_stats! {
    prefix = "mononoke.blobrepo";
    create_changeset: timeseries(Rate, Sum),
    create_changeset_compute_cf: timeseries("create_changeset.compute_changed_files"; Rate, Sum),
    create_changeset_expected_cf: timeseries("create_changeset.expected_changed_files"; Rate, Sum),
    create_changeset_cf_count: timeseries("create_changeset.changed_files_count"; Average, Sum),
}

async fn verify_bonsai_changeset_with_origin(
    ctx: CoreContext,
    bcs: BonsaiChangeset,
    cs: HgBlobChangeset,
    origin_repo: Option<BackupSourceRepo>,
) -> Result<BonsaiChangeset, Error> {
    match origin_repo {
        Some(origin_repo) => {
            // There are some non-canonical bonsai changesets in the prod repos.
            // To make the blobimported backup repos exactly the same, we will
            // fetch bonsai from the prod in case of mismatch
            let origin_bonsai_id = origin_repo
                .bonsai_hg_mapping()
                .get_bonsai_from_hg(&ctx, cs.get_changeset_id())
                .await?;
            match origin_bonsai_id {
                Some(id) if id != bcs.get_changeset_id() => {
                    id.load(&ctx, origin_repo.repo_blobstore())
                        .map_err(|e| anyhow!(e))
                        .await
                }
                _ => Ok(bcs),
            }
        }
        None => Ok(bcs),
    }
}

pub fn create_bonsai_changeset_hook(
    origin_repo: Option<BackupSourceRepo>,
) -> Arc<BonsaiChangesetHook> {
    Arc::new(
        move |ctx: CoreContext,
              hg_cs: HgBlobChangeset,
              parent_manifest_hashes: Vec<HgManifestId>,
              bonsai_parents: Vec<ChangesetId>,
              repo: BlobRepo| {
            cloned!(origin_repo);
            async move {
                let bonsai_cs = create_bonsai_changeset_object(
                    &ctx,
                    hg_cs.clone(),
                    parent_manifest_hashes,
                    bonsai_parents,
                    repo.repo_blobstore(),
                )
                .await?;
                verify_bonsai_changeset_with_origin(ctx, bonsai_cs, hg_cs, origin_repo).await
            }
            .boxed()
        },
    )
}

pub struct CreateChangeset {
    /// This should always be provided, keeping it an Option for tests
    pub expected_nodeid: Option<HgNodeHash>,
    pub expected_files: Option<Vec<MPath>>,
    pub p1: Option<ChangesetHandle>,
    pub p2: Option<ChangesetHandle>,
    // root_manifest can be None f.e. when commit removes all the content of the repo
    pub root_manifest: BoxFuture<'static, Result<Option<(HgManifestId, RepoPath)>>>,
    pub sub_entries: BoxStream<'static, Result<(Entry<HgManifestId, HgFileNodeId>, RepoPath)>>,
    pub cs_metadata: ChangesetMetadata,
    pub create_bonsai_changeset_hook: Option<Arc<BonsaiChangesetHook>>,
    /// If set to true, don't update Changesets or BonsaiHgMapping, which should be done
    /// manually after this call. Effectively, the commit will be in the blobstore, but
    /// unreachable.
    pub upload_to_blobstore_only: bool,
}

impl CreateChangeset {
    pub fn create(
        self,
        ctx: CoreContext,
        repo: &BlobRepo,
        mut scuba_logger: MononokeScubaSampleBuilder,
    ) -> ChangesetHandle {
        STATS::create_changeset.add_value(1);
        // This is used for logging, so that we can tie up all our pieces without knowing about
        // the final commit hash
        let uuid = Uuid::new_v4();
        scuba_logger.add("changeset_uuid", format!("{}", uuid));

        let entry_processor =
            UploadEntries::new(repo.repo_blobstore().clone(), scuba_logger.clone());
        let (signal_parent_ready, can_be_parent) = oneshot::channel();
        let signal_parent_ready = Arc::new(Mutex::new(Some(signal_parent_ready)));
        let expected_nodeid = self.expected_nodeid;

        let upload_entries = {
            cloned!(ctx, entry_processor);
            let root_manifest = self.root_manifest;
            let sub_entries = self.sub_entries;
            async move {
                process_entries(&ctx, &entry_processor, root_manifest, sub_entries)
                    .await
                    .context("While processing entries")
            }
        };

        let parents_complete = extract_parents_complete(&self.p1, &self.p2)
            .try_timed()
            .map({
                let mut scuba_logger = scuba_logger.clone();
                move |result| match result {
                    Err(err) => Err(err.context("While waiting for parents to complete")),
                    Ok((stats, result)) => {
                        scuba_logger
                            .add_future_stats(&stats)
                            .log_with_msg("Parents completed", None);
                        Ok(result)
                    }
                }
            });
        let parents_data = handle_parents(scuba_logger.clone(), self.p1, self.p2)
            .map_err(|err| err.context("While waiting for parents to upload data"));

        let create_bonsai_changeset_object = match self.create_bonsai_changeset_hook {
            Some(hook) => Arc::clone(&hook),
            None => Arc::new(
                |ctx: CoreContext,
                 hg_cs: HgBlobChangeset,
                 parent_manifest_hashes: Vec<HgManifestId>,
                 bonsai_parents: Vec<ChangesetId>,
                 repo: BlobRepo| {
                    async move {
                        create_bonsai_changeset_object(
                            &ctx,
                            hg_cs,
                            parent_manifest_hashes,
                            bonsai_parents,
                            repo.repo_blobstore(),
                        )
                        .await
                    }
                    .boxed()
                },
            ),
        };

        let changeset = {
            cloned!(ctx, repo, signal_parent_ready, mut scuba_logger);
            let expected_files = self.expected_files;
            let cs_metadata = self.cs_metadata;
            let blobstore = repo.repo_blobstore().clone();

            async move {
                let (root_mf_id, (parents, parent_manifest_hashes, bonsai_parents)) =
                    future::try_join(upload_entries, parents_data).await?;
                let files = if let Some(expected_files) = expected_files {
                    STATS::create_changeset_expected_cf.add_value(1);
                    // We are trusting the callee to provide a list of changed files, used
                    // by the import job
                    expected_files
                } else {
                    STATS::create_changeset_compute_cf.add_value(1);
                    compute_changed_files(
                        ctx.clone(),
                        repo.repo_blobstore_arc(),
                        root_mf_id,
                        parent_manifest_hashes.get(0).cloned(),
                        parent_manifest_hashes.get(1).cloned(),
                    )
                    .await?
                };

                STATS::create_changeset_cf_count.add_value(files.len() as i64);
                let hg_cs = make_new_changeset(parents, root_mf_id, cs_metadata, files)?;
                let bonsai_cs = create_bonsai_changeset_object(
                    ctx.clone(),
                    hg_cs.clone(),
                    parent_manifest_hashes.clone(),
                    bonsai_parents,
                    repo.clone(),
                )
                .await?;

                let bonsai_blob = bonsai_cs.clone().into_blob();
                let bcs_id = bonsai_blob.id().clone();

                let cs_id = hg_cs.get_changeset_id().into_nodehash();
                let manifest_id = hg_cs.manifestid();

                if let Some(expected_nodeid) = expected_nodeid {
                    if cs_id != expected_nodeid {
                        return Err(ErrorKind::InconsistentChangesetHash(
                            expected_nodeid,
                            cs_id,
                            hg_cs,
                        )
                        .into());
                    }
                }

                scuba_logger
                    .add("changeset_id", format!("{}", cs_id))
                    .log_with_msg("Changeset uuid to hash mapping", None);
                // NOTE(luk): an attempt was made in D8187210 to split the
                // upload_entries signal into upload_entries and
                // processed_entries and to signal_parent_ready after
                // upload_entries, so that one doesn't need to wait for the
                // entries to be processed. There were no performance gains
                // from that experiment
                //
                // We deliberately eat this error - this is only so that
                // another changeset can start verifying data in the blob
                // store while we verify this one
                let _ = signal_parent_ready
                    .lock()
                    .expect("poisoned lock")
                    .take()
                    .expect("signal_parent_ready cannot be taken yet")
                    .send(Ok((bcs_id, cs_id, manifest_id)));

                futures::try_join!(
                    save_bonsai_changeset_object(&ctx, &blobstore, bonsai_cs.clone()),
                    hg_cs.save(&ctx, &blobstore),
                    entry_processor
                        .finalize(&ctx, root_mf_id, parent_manifest_hashes)
                        .map_err(|err| err.context("While finalizing processing")),
                )?;

                Ok::<_, Error>((hg_cs, bonsai_cs))
            }
        }
        .try_timed()
        .map({
            cloned!(mut scuba_logger);
            move |result| {
                match result {
                    Ok((stats, result)) => {
                        scuba_logger
                            .add_future_stats(&stats)
                            .log_with_msg("Changeset created", None);
                        Ok(result)
                    }
                    Err(err) => {
                        let err =
                            err.context("While creating and verifying Changeset for blobstore");
                        let trigger = signal_parent_ready.lock().expect("poisoned lock").take();
                        if let Some(trigger) = trigger {
                            // Ignore errors if the receiving end has gone away.
                            let e = format_err!("signal_parent_ready failed: {:?}", err);
                            let _ = trigger.send(Err(e));
                        }
                        Err(err)
                    }
                }
            }
        });

        let complete_changesets = repo.changesets_arc();
        let bonsai_hg_mapping = repo.bonsai_hg_mapping_arc();
        let changeset_complete_fut = async move {
            let ((hg_cs, bonsai_cs), _) = future::try_join(changeset, parents_complete).await?;

            if !self.upload_to_blobstore_only {
                // update changeset mapping
                let completion_record = ChangesetInsert {
                    cs_id: bonsai_cs.get_changeset_id(),
                    parents: bonsai_cs.parents().collect(),
                };
                complete_changesets
                    .add(&ctx, completion_record)
                    .await
                    .context("While inserting into changeset table")?;

                // update bonsai mapping
                let bcs_id = bonsai_cs.get_changeset_id();
                let bonsai_hg_entry = BonsaiHgMappingEntry {
                    hg_cs_id: hg_cs.get_changeset_id(),
                    bcs_id,
                };
                bonsai_hg_mapping
                    .add(&ctx, bonsai_hg_entry)
                    .await
                    .context("While inserting mapping")?;
            }

            Ok::<_, Error>((bonsai_cs, hg_cs))
        }
        .try_timed()
        .map({
            cloned!(mut scuba_logger);
            move |result| match result {
                Ok((stats, result)) => {
                    scuba_logger
                        .add_future_stats(&stats)
                        .log_with_msg("CreateChangeset Finished", None);
                    Ok(result)
                }
                Err(err) => Err(err.context(format!(
                    "While creating Changeset {:?}, uuid: {}",
                    expected_nodeid, uuid
                ))),
            }
        });

        let can_be_parent = can_be_parent
            .map(|r| match r {
                Ok(res) => res,
                Err(e) => Err(format_err!("can_be_parent: {:?}", e)),
            })
            .boxed()
            .try_shared();

        let completion_future = tokio::spawn(changeset_complete_fut)
            .map(|result| result?)
            .boxed()
            .try_shared();

        ChangesetHandle::new_pending(can_be_parent, completion_future)
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

mod bonsai_generation;
mod create_changeset;
pub mod repo_commit;
pub use changeset_fetcher::ChangesetFetcher;

pub use crate::bonsai_generation::create_bonsai_changeset_object;
pub use crate::bonsai_generation::save_bonsai_changeset_object;
// TODO: This is exported for testing - is this the right place for it?
pub use crate::repo_commit::compute_changed_files;
pub use crate::repo_commit::ChangesetHandle;
pub use crate::repo_commit::UploadEntries;
pub mod errors {
    pub use blobrepo_errors::*;
}
pub use create_changeset::create_bonsai_changeset_hook;
pub use create_changeset::CreateChangeset;
pub mod file_history {
    pub use blobrepo_common::file_history::*;
}

use std::collections::HashMap;
use std::collections::HashSet;

use anyhow::Error;
use async_trait::async_trait;
use blobrepo_errors::ErrorKind;
use bonsai_hg_mapping::BonsaiHgMappingRef;
use bonsai_hg_mapping::BonsaiOrHgChangesetIds;
use bookmarks::Bookmark;
use bookmarks::BookmarkCategory;
use bookmarks::BookmarkKey;
use bookmarks::BookmarkKind;
use bookmarks::BookmarkPagination;
use bookmarks::BookmarkPrefix;
use bookmarks::BookmarksRef;
use bookmarks::Freshness;
use changesets::ChangesetsRef;
use cloned::cloned;
use context::CoreContext;
use filenodes::FilenodeInfo;
use filenodes::FilenodeRange;
use filenodes::FilenodeResult;
use filenodes::FilenodesRef;
use futures::future;
use futures::stream;
use futures::stream::BoxStream;
use futures::Stream;
use futures::StreamExt;
use futures::TryFutureExt;
use futures::TryStreamExt;
use mercurial_derivation::DeriveHgChangeset;
use mercurial_types::HgChangesetId;
use mercurial_types::HgFileNodeId;
use mononoke_types::ChangesetId;
use mononoke_types::RepoPath;
use repo_derived_data::RepoDerivedDataRef;
use stats::prelude::*;

/// `BlobRepoHg` is an extension trait for repo facet containers which contains
/// mercurial specific methods.
#[async_trait]
pub trait BlobRepoHg: Send + Sync {
    async fn get_hg_bonsai_mapping<'a>(
        &'a self,
        ctx: CoreContext,
        bonsai_or_hg_cs_ids: impl Into<BonsaiOrHgChangesetIds> + 'a + Send,
    ) -> Result<Vec<(HgChangesetId, ChangesetId)>, Error>
    where
        Self: ChangesetsRef + RepoDerivedDataRef + BonsaiHgMappingRef;

    fn get_hg_heads_maybe_stale(
        &self,
        ctx: CoreContext,
    ) -> BoxStream<'_, Result<HgChangesetId, Error>>
    where
        Self: BookmarksRef + RepoDerivedDataRef + Send + Sync;

    async fn hg_changeset_exists(
        &self,
        ctx: CoreContext,
        changesetid: HgChangesetId,
    ) -> Result<bool, Error>
    where
        Self: BonsaiHgMappingRef + ChangesetsRef;

    async fn get_hg_changeset_and_parents_from_bonsai(
        &self,
        ctx: CoreContext,
        csid: ChangesetId,
    ) -> Result<(HgChangesetId, Vec<HgChangesetId>), Error>
    where
        Self: BonsaiHgMappingRef + ChangesetsRef + RepoDerivedDataRef;

    async fn get_hg_changeset_parents(
        &self,
        ctx: CoreContext,
        changesetid: HgChangesetId,
    ) -> Result<Vec<HgChangesetId>, Error>
    where
        Self: BonsaiHgMappingRef + ChangesetsRef + RepoDerivedDataRef;

    async fn get_bookmark_hg(
        &self,
        ctx: CoreContext,
        name: &BookmarkKey,
    ) -> Result<Option<HgChangesetId>, Error>
    where
        Self: BookmarksRef + RepoDerivedDataRef;

    fn get_publishing_bookmarks_maybe_stale_hg(
        &self,
        ctx: CoreContext,
    ) -> BoxStream<'_, Result<(Bookmark, HgChangesetId), Error>>
    where
        Self: ChangesetsRef
            + BonsaiHgMappingRef
            + RepoDerivedDataRef
            + BookmarksRef
            + Clone
            + Send
            + Sync;

    async fn get_filenode_opt(
        &self,
        ctx: CoreContext,
        path: &RepoPath,
        node: HgFileNodeId,
    ) -> Result<FilenodeResult<Option<FilenodeInfo>>, Error>
    where
        Self: FilenodesRef;

    async fn get_filenode(
        &self,
        ctx: CoreContext,
        path: &RepoPath,
        node: HgFileNodeId,
    ) -> Result<FilenodeResult<FilenodeInfo>, Error>
    where
        Self: FilenodesRef;

    async fn get_all_filenodes_maybe_stale(
        &self,
        ctx: CoreContext,
        path: RepoPath,
        limit: Option<u64>,
    ) -> Result<FilenodeResult<FilenodeRange>, Error>
    where
        Self: FilenodesRef;
}

define_stats! {
    prefix = "mononoke.blobrepo";
    hg_changeset_exists: timeseries(Rate, Sum),
    get_all_filenodes: timeseries(Rate, Sum),
    get_bookmark_hg: timeseries(Rate, Sum),
    get_hg_changeset_and_parents_from_bonsai: timeseries(Rate, Sum),
    get_hg_changeset_parents: timeseries(Rate, Sum),
    get_hg_heads_maybe_stale: timeseries(Rate, Sum),
    get_hg_bonsai_mapping: timeseries(Rate, Sum),
    get_publishing_bookmarks_maybe_stale_hg: timeseries(Rate, Sum),
}

#[async_trait]
impl<T: ChangesetsRef + BonsaiHgMappingRef + Send + Sync> BlobRepoHg for T {
    // Returns only the mapping for valid changests that are known to the server.
    // For Bonsai -> Hg conversion, missing Hg changesets will be derived (so all Bonsais will be
    // in the output).
    // For Hg -> Bonsai conversion, missing Bonsais will not be returned, since they cannot be
    // derived from Hg Changesets.
    async fn get_hg_bonsai_mapping<'a>(
        &'a self,
        ctx: CoreContext,
        bonsai_or_hg_cs_ids: impl Into<BonsaiOrHgChangesetIds> + 'a + Send,
    ) -> Result<Vec<(HgChangesetId, ChangesetId)>, Error>
    where
        Self: ChangesetsRef + RepoDerivedDataRef + BonsaiHgMappingRef,
    {
        STATS::get_hg_bonsai_mapping.add_value(1);

        let bonsai_or_hg_cs_ids = bonsai_or_hg_cs_ids.into();
        let hg_bonsai_list = self
            .bonsai_hg_mapping()
            .get(&ctx, bonsai_or_hg_cs_ids.clone())
            .await?
            .into_iter()
            .map(|entry| (entry.hg_cs_id, entry.bcs_id))
            .collect::<Vec<_>>();

        use BonsaiOrHgChangesetIds::*;
        match bonsai_or_hg_cs_ids {
            Bonsai(bonsais) => {
                // If a bonsai commit doesn't exist in the bonsai_hg_mapping,
                // that might mean two things: 1) Bonsai commit just doesn't exist
                // 2) Bonsai commit exists but hg changesets weren't generated for it
                // Normally the callers of get_hg_bonsai_mapping would expect that hg
                // changesets will be lazily generated, so the
                // code below explicitly checks if a commit exists and if yes then
                // generates hg changeset for it.
                let mapping: HashMap<_, _> = hg_bonsai_list
                    .iter()
                    .map(|(hg_id, bcs_id)| (bcs_id, hg_id))
                    .collect();

                let mut notfound = vec![];
                for b in bonsais {
                    if !mapping.contains_key(&b) {
                        notfound.push(b);
                    }
                }

                let existing: HashSet<_> = self
                    .changesets()
                    .get_many(&ctx, notfound.clone())
                    .await?
                    .into_iter()
                    .map(|entry| entry.cs_id)
                    .collect();

                let mut newmapping: Vec<_> = stream::iter(
                    notfound
                        .into_iter()
                        .filter(|csid| existing.contains(csid))
                        .map(Ok),
                )
                .map_ok(|csid| {
                    self.derive_hg_changeset(&ctx, csid)
                        .map_ok(move |hgcsid| (hgcsid, csid))
                })
                .try_buffer_unordered(100)
                .try_collect()
                .await?;

                newmapping.extend(hg_bonsai_list);
                Ok(newmapping)
            }
            Hg(_) => Ok(hg_bonsai_list),
        }
        // TODO(stash, luk): T37303879 also need to check that entries exist in changeset table
    }

    /// Get Mercurial heads, which we approximate as publishing Bonsai Bookmarks.
    fn get_hg_heads_maybe_stale(
        &self,
        ctx: CoreContext,
    ) -> BoxStream<'_, Result<HgChangesetId, Error>>
    where
        Self: BookmarksRef + RepoDerivedDataRef + Send + Sync,
    {
        STATS::get_hg_heads_maybe_stale.add_value(1);

        self.bookmarks()
            .list(
                ctx.clone(),
                Freshness::MaybeStale,
                &BookmarkPrefix::empty(),
                BookmarkCategory::ALL,
                BookmarkKind::ALL_PUBLISHING,
                &BookmarkPagination::FromStart,
                std::u64::MAX,
            )
            .map_ok(move |(_, cs)| {
                cloned!(ctx);
                async move { self.derive_hg_changeset(&ctx, cs).await }
            })
            .try_buffer_unordered(100)
            .boxed()
    }

    async fn hg_changeset_exists(
        &self,
        ctx: CoreContext,
        changesetid: HgChangesetId,
    ) -> Result<bool, Error>
    where
        Self: BonsaiHgMappingRef + ChangesetsRef,
    {
        STATS::hg_changeset_exists.add_value(1);
        let csid = self
            .bonsai_hg_mapping()
            .get_bonsai_from_hg(&ctx, changesetid)
            .await?;
        match csid {
            Some(bonsai) => {
                let res = self.changesets().get(&ctx, bonsai).await?;
                Ok(res.is_some())
            }
            None => Ok(false),
        }
    }

    async fn get_hg_changeset_and_parents_from_bonsai(
        &self,
        ctx: CoreContext,
        csid: ChangesetId,
    ) -> Result<(HgChangesetId, Vec<HgChangesetId>), Error>
    where
        Self: BonsaiHgMappingRef + ChangesetsRef + RepoDerivedDataRef,
    {
        STATS::get_hg_changeset_and_parents_from_bonsai.add_value(1);

        let parents = self
            .changesets()
            .get(&ctx, csid)
            .await?
            .ok_or(ErrorKind::BonsaiNotFound(csid))?
            .parents
            .into_iter()
            .map(|parent| self.derive_hg_changeset(&ctx, parent));

        let changesetid = self
            .bonsai_hg_mapping()
            .get_hg_from_bonsai(&ctx, csid)
            .await?
            .ok_or(ErrorKind::BonsaiNotFound(csid))?;

        Ok((changesetid, future::try_join_all(parents).await?))
    }

    async fn get_hg_changeset_parents(
        &self,
        ctx: CoreContext,
        changesetid: HgChangesetId,
    ) -> Result<Vec<HgChangesetId>, Error>
    where
        Self: BonsaiHgMappingRef + ChangesetsRef + RepoDerivedDataRef,
    {
        STATS::get_hg_changeset_parents.add_value(1);

        let csid = self
            .bonsai_hg_mapping()
            .get_bonsai_from_hg(&ctx, changesetid)
            .await?
            .ok_or(ErrorKind::BonsaiMappingNotFound(changesetid))?;

        let parents = self
            .changesets()
            .get(&ctx, csid)
            .await?
            .ok_or(ErrorKind::BonsaiNotFound(csid))?
            .parents
            .into_iter()
            .map(|parent| self.derive_hg_changeset(&ctx, parent));

        Ok(future::try_join_all(parents).await?)
    }

    async fn get_bookmark_hg(
        &self,
        ctx: CoreContext,
        name: &BookmarkKey,
    ) -> Result<Option<HgChangesetId>, Error>
    where
        Self: BookmarksRef + RepoDerivedDataRef,
    {
        STATS::get_bookmark_hg.add_value(1);
        let cs_opt = self.bookmarks().get(ctx.clone(), name).await?;
        match cs_opt {
            None => Ok(None),
            Some(cs) => {
                let hg_csid = self.derive_hg_changeset(&ctx, cs).await?;
                Ok(Some(hg_csid))
            }
        }
    }

    /// Get Publishing (Publishing is a Mercurial concept) bookmarks by prefix, they will be read
    /// from cache or a replica, so they might be stale.
    fn get_publishing_bookmarks_maybe_stale_hg(
        &self,
        ctx: CoreContext,
    ) -> BoxStream<'_, Result<(Bookmark, HgChangesetId), Error>>
    where
        Self: ChangesetsRef
            + BonsaiHgMappingRef
            + RepoDerivedDataRef
            + BlobRepoHg
            + BookmarksRef
            + Clone
            + Send
            + Sync,
    {
        STATS::get_publishing_bookmarks_maybe_stale_hg.add_value(1);
        let stream = self.bookmarks().list(
            ctx.clone(),
            Freshness::MaybeStale,
            &BookmarkPrefix::empty(),
            BookmarkCategory::ALL,
            BookmarkKind::ALL_PUBLISHING,
            &BookmarkPagination::FromStart,
            std::u64::MAX,
        );
        to_hg_bookmark_stream(self, &ctx, stream)
    }

    async fn get_filenode_opt(
        &self,
        ctx: CoreContext,
        path: &RepoPath,
        node: HgFileNodeId,
    ) -> Result<FilenodeResult<Option<FilenodeInfo>>, Error>
    where
        Self: FilenodesRef,
    {
        self.filenodes().get_filenode(&ctx, path, node).await
    }

    async fn get_filenode(
        &self,
        ctx: CoreContext,
        path: &RepoPath,
        node: HgFileNodeId,
    ) -> Result<FilenodeResult<FilenodeInfo>, Error>
    where
        Self: FilenodesRef,
    {
        match self.get_filenode_opt(ctx, path, node).await? {
            FilenodeResult::Present(maybe_filenode) => {
                let filenode = maybe_filenode
                    .ok_or_else(|| Error::from(ErrorKind::MissingFilenode(path.clone(), node)))?;
                Ok(FilenodeResult::Present(filenode))
            }
            FilenodeResult::Disabled => Ok(FilenodeResult::Disabled),
        }
    }

    async fn get_all_filenodes_maybe_stale(
        &self,
        ctx: CoreContext,
        path: RepoPath,
        limit: Option<u64>,
    ) -> Result<FilenodeResult<FilenodeRange>, Error>
    where
        Self: FilenodesRef,
    {
        STATS::get_all_filenodes.add_value(1);
        self.filenodes()
            .get_all_filenodes_maybe_stale(&ctx, &path, limit)
            .await
    }
}

pub fn to_hg_bookmark_stream<'repo, BookmarkType>(
    repo: &(
         impl ChangesetsRef
         + BonsaiHgMappingRef
         + RepoDerivedDataRef
         + BlobRepoHg
         + Clone
         + Send
         + Sync
         + 'repo
     ),
    ctx: &CoreContext,
    stream: impl Stream<Item = Result<(BookmarkType, ChangesetId), Error>> + Send + 'repo,
) -> BoxStream<'repo, Result<(BookmarkType, HgChangesetId), Error>>
where
    BookmarkType: Send,
{
    stream
        .chunks(100)
        .map({
            cloned!(repo, ctx);
            move |chunk| {
                cloned!(repo, ctx);
                async move {
                    let chunk: Vec<_> = chunk.into_iter().collect::<Result<_, _>>()?;
                    let cs_ids = chunk.iter().map(|(_, cs_id)| *cs_id).collect::<Vec<_>>();
                    let mapping: HashMap<_, _> = repo
                        .get_hg_bonsai_mapping(ctx.clone(), cs_ids)
                        .await?
                        .into_iter()
                        .map(|(hg_cs_id, cs_id)| (cs_id, hg_cs_id))
                        .collect();
                    let res = chunk.into_iter().map(move |(bookmark, cs_id)| {
                        let hg_cs_id = mapping.get(&cs_id).ok_or_else(|| {
                            anyhow::format_err!("cs_id was missing from mapping: {:?}", cs_id)
                        })?;
                        Ok((bookmark, *hg_cs_id))
                    });
                    Ok(stream::iter(res))
                }
                .try_flatten_stream()
            }
        })
        .flatten()
        .boxed()
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;
use std::collections::HashSet;
use std::sync::Arc;
use std::sync::Mutex;

use ::manifest::find_intersection_of_diffs;
use ::manifest::Entry;
use anyhow::format_err;
use anyhow::Context;
use anyhow::Error;
use anyhow::Result;
pub use blobrepo_common::changed_files::compute_changed_files;
use blobstore::Blobstore;
use blobstore::ErrorKind as BlobstoreError;
use blobstore::Loadable;
use bonsai_hg_mapping::BonsaiHgMappingRef;
use cloned::cloned;
use context::CoreContext;
use futures::channel::oneshot;
use futures::future;
use futures::future::BoxFuture;
use futures::future::FutureExt;
use futures::future::TryFutureExt;
use futures::stream;
use futures::stream::BoxStream;
use futures::stream::TryStreamExt;
use futures::StreamExt;
use futures_ext::future::TryShared;
use futures_ext::FbTryFutureExt;
use futures_stats::TimedTryFutureExt;
use mercurial_types::blobs::fetch_manifest_envelope;
use mercurial_types::blobs::ChangesetMetadata;
use mercurial_types::blobs::HgBlobChangeset;
use mercurial_types::blobs::HgBlobEnvelope;
use mercurial_types::blobs::HgChangesetContent;
use mercurial_types::nodehash::HgFileNodeId;
use mercurial_types::nodehash::HgManifestId;
use mercurial_types::HgChangesetId;
use mercurial_types::HgNodeHash;
use mercurial_types::HgParents;
use mercurial_types::MPath;
use mercurial_types::RepoPath;
use mercurial_types::NULL_HASH;
use mononoke_types::BlobstoreKey;
use mononoke_types::BonsaiChangeset;
use mononoke_types::ChangesetId;
use repo_blobstore::RepoBlobstore;
use repo_blobstore::RepoBlobstoreRef;
use scuba_ext::MononokeScubaSampleBuilder;
use stats::prelude::*;

use crate::errors::*;

define_stats! {
    prefix = "mononoke.blobrepo_commit";
    process_file_entry: timeseries(Rate, Sum),
    process_tree_entry: timeseries(Rate, Sum),
    parents_checked: timeseries(Rate, Average, Sum),
    finalize_parent: timeseries(Rate, Average, Sum),
    finalize_uploaded: timeseries(Rate, Average, Sum),
    finalize_uploaded_filenodes: timeseries(Rate, Average, Sum),
    finalize_uploaded_manifests: timeseries(Rate, Average, Sum),
    finalize_compute_copy_from_info: timeseries(Rate, Sum),
}

/// A handle to a possibly incomplete HgBlobChangeset. This is used instead of
/// Future<Item = HgBlobChangeset> where we don't want to fully serialize waiting for completion.
/// For example, `create_changeset` takes these as p1/p2 so that it can handle the blobstore side
/// of creating a new changeset before its parent changesets are complete.
/// See `get_completed_changeset()` for the public API you can use to extract the final changeset
#[derive(Clone)]
pub struct ChangesetHandle {
    can_be_parent:
        TryShared<BoxFuture<'static, Result<(ChangesetId, HgNodeHash, HgManifestId), Error>>>,
    // * Shared is required here because a single changeset can have more than one child, and
    //   all of those children will want to refer to the corresponding future for their parents.
    // * The Compat<Error> here is because the error type for Shared (a cloneable wrapper called
    //   SharedError) doesn't implement Fail, and only implements Error if the wrapped type
    //   implements Error.
    completion_future:
        TryShared<BoxFuture<'static, Result<(BonsaiChangeset, HgBlobChangeset), Error>>>,
}

impl ChangesetHandle {
    pub fn new_pending(
        can_be_parent: TryShared<
            BoxFuture<'static, Result<(ChangesetId, HgNodeHash, HgManifestId), Error>>,
        >,
        completion_future: TryShared<
            BoxFuture<'static, Result<(BonsaiChangeset, HgBlobChangeset), Error>>,
        >,
    ) -> Self {
        Self {
            can_be_parent,
            completion_future,
        }
    }

    pub fn ready_cs_handle(
        ctx: CoreContext,
        repo: impl RepoBlobstoreRef + BonsaiHgMappingRef + Clone + Send + Sync + 'static,
        hg_cs: HgChangesetId,
    ) -> Self {
        let (trigger, can_be_parent) = oneshot::channel();
        let can_be_parent = can_be_parent
            .map_err(|e| format_err!("can_be_parent: {:?}", e))
            .boxed()
            .try_shared();

        let bonsai_cs = {
            cloned!(ctx, repo);
            async move {
                let csid = repo
                    .bonsai_hg_mapping()
                    .get_bonsai_from_hg(&ctx, hg_cs)
                    .await?
                    .ok_or(ErrorKind::BonsaiMappingNotFound(hg_cs))?;
                let bonsai_cs = csid.load(&ctx, repo.repo_blobstore()).await?;
                Ok::<_, Error>(bonsai_cs)
            }
        };

        let completion_future = async move {
            let (bonsai_cs, hg_cs) = future::try_join(
                bonsai_cs,
                hg_cs.load(&ctx, repo.repo_blobstore()).map_err(Error::from),
            )
            .await?;
            let _ = trigger.send((
                bonsai_cs.get_changeset_id(),
                hg_cs.get_changeset_id().into_nodehash(),
                hg_cs.manifestid(),
            ));
            Ok((bonsai_cs, hg_cs))
        }
        .boxed()
        .try_shared();

        Self {
            can_be_parent,
            completion_future,
        }
    }

    pub fn get_completed_changeset(
        self,
    ) -> TryShared<BoxFuture<'static, Result<(BonsaiChangeset, HgBlobChangeset), Error>>> {
        self.completion_future
    }
}

/// State used while tracking uploaded entries, to ensure that a changeset ends up with the right
/// set of blobs uploaded, and all filenodes present.
struct UploadEntriesState {
    /// All the blobs that have been uploaded in this changeset
    uploaded_entries: HashMap<RepoPath, Entry<HgManifestId, HgFileNodeId>>,
    /// Parent hashes (if any) of the blobs that have been uploaded in this changeset. Used for
    /// validation of this upload - all parents must either have been uploaded in this changeset,
    /// or be present in the blobstore before the changeset can complete.
    parents: HashSet<Entry<HgManifestId, HgFileNodeId>>,
}

#[derive(Clone)]
pub struct UploadEntries {
    scuba_logger: MononokeScubaSampleBuilder,
    inner: Arc<Mutex<UploadEntriesState>>,
    blobstore: RepoBlobstore,
}

impl UploadEntries {
    pub fn new(blobstore: RepoBlobstore, scuba_logger: MononokeScubaSampleBuilder) -> Self {
        Self {
            scuba_logger,
            inner: Arc::new(Mutex::new(UploadEntriesState {
                uploaded_entries: HashMap::new(),
                parents: HashSet::new(),
            })),
            blobstore,
        }
    }

    fn scuba_logger(&self) -> MononokeScubaSampleBuilder {
        self.scuba_logger.clone()
    }

    /// The root manifest needs special processing - unlike all other entries, it is required even
    /// if no other manifest references it. Otherwise, this function is the same as
    /// `process_one_entry` and can be called after it.
    /// It is safe to call this multiple times, but not recommended - every manifest passed to
    /// this function is assumed required for this commit, even if it is not the root.
    pub async fn process_root_manifest<'a>(
        &'a self,
        ctx: &'a CoreContext,
        entry: HgManifestId,
    ) -> Result<()> {
        if entry.clone().into_nodehash() == NULL_HASH {
            Ok(())
        } else {
            self.process_one_entry(ctx, Entry::Tree(entry), RepoPath::root())
                .await
        }
    }

    pub async fn process_one_entry<'a>(
        &'a self,
        ctx: &'a CoreContext,
        entry: Entry<HgManifestId, HgFileNodeId>,
        path: RepoPath,
    ) -> Result<()> {
        {
            let mut inner = self.inner.lock().expect("Lock poisoned");
            inner.uploaded_entries.insert(path.clone(), entry.clone());
        }

        let parents = match entry {
            Entry::Tree(manifest_id) => {
                STATS::process_tree_entry.add_value(1);

                // NOTE: Just fetch the envelope here, because we don't actually need the
                // deserialized manifest: just the parents will do.
                let envelope = fetch_manifest_envelope(ctx, &self.blobstore, manifest_id)
                    .await
                    .with_context(|| {
                        format!(
                            "Error processing manifest with id {} and path {}",
                            manifest_id, path
                        )
                    })?;

                envelope
                    .get_parents()
                    .into_iter()
                    .map(|p| Entry::Tree(HgManifestId::new(p)))
                    .collect::<Vec<_>>()
            }
            Entry::Leaf(filenode_id) => {
                STATS::process_file_entry.add_value(1);

                let envelope = filenode_id
                    .load(ctx, &self.blobstore)
                    .await
                    .with_context(|| {
                        format!(
                            "Error processing file with id {} and path {}",
                            filenode_id, path
                        )
                    })?;

                envelope
                    .get_parents()
                    .into_iter()
                    .map(|p| Entry::Leaf(HgFileNodeId::new(p)))
                    .collect::<Vec<_>>()
            }
        };

        {
            let mut inner = self.inner.lock().expect("Lock poisoned");
            inner.parents.extend(parents.into_iter());
        }

        Ok(())
    }

    // Check the blobstore to see whether a particular node is present.
    async fn assert_in_blobstore(
        ctx: &CoreContext,
        blobstore: &RepoBlobstore,
        entry: Entry<HgManifestId, HgFileNodeId>,
    ) -> Result<(), Error> {
        match entry {
            Entry::Tree(mfid) => {
                if mfid.into_nodehash() == NULL_HASH {
                    return Ok(());
                }

                let key = mfid.blobstore_key();
                if !blobstore
                    .is_present(ctx, &key)
                    .await?
                    .assume_not_found_if_unsure()
                {
                    return Err(BlobstoreError::NotFound(key).into());
                }
            }
            Entry::Leaf(fnid) => {
                if fnid.into_nodehash() == NULL_HASH {
                    return Ok(());
                }

                let envelope = fnid.load(ctx, &blobstore).await?;

                let key = envelope.content_id().blobstore_key();
                if !blobstore.is_present(ctx, &key).await?.fail_if_unsure()? {
                    return Err(BlobstoreError::NotFound(key).into());
                }
            }
        }

        Ok(())
    }

    pub async fn finalize(
        self,
        ctx: &CoreContext,
        mf_id: HgManifestId,
        parent_manifest_ids: Vec<HgManifestId>,
    ) -> Result<(), Error> {
        // NOTE: we consume self.entries hence the signature, even if we don't actually need
        // mutable access
        let this = &self;

        let required_checks = {
            async move {
                let (stats, ()) = find_intersection_of_diffs(
                    ctx.clone(),
                    this.blobstore.clone().boxed(),
                    mf_id,
                    parent_manifest_ids,
                )
                .try_for_each_concurrent(100, {
                    move |(path, entry)| {
                        async move {
                            let entry = entry.map_leaf(|(_, fnid)| fnid);
                            Self::assert_in_blobstore(ctx, &this.blobstore, entry)
                                .await
                                .with_context(|| format!("Error checking for path: {:?}", path))?;
                            Ok(())
                        }
                        .boxed()
                    }
                })
                .try_timed()
                .await?;

                this.scuba_logger()
                    .add_future_stats(&stats)
                    .log_with_msg("Required checks", None);

                Ok::<_, Error>(())
            }
        };

        let parent_checks = async move {
            let checks: Vec<_> = {
                let inner = this.inner.lock().expect("Lock poisoned");

                inner
                    .parents
                    .iter()
                    .copied()
                    .map(|entry| async move {
                        Self::assert_in_blobstore(ctx, &this.blobstore, entry)
                            .await
                            .with_context(|| {
                                format!("Error checking for a parent node: {:?}", entry)
                            })?;
                        STATS::parents_checked.add_value(1);
                        Result::<_, Error>::Ok(())
                    })
                    .collect()
            };

            STATS::finalize_parent.add_value(checks.len() as i64);

            let (stats, ()) = stream::iter(checks)
                .map(Ok)
                .try_for_each_concurrent(100, |f| f)
                .try_timed()
                .await?;
            this.scuba_logger()
                .add_future_stats(&stats)
                .log_with_msg("Parent checks", None);
            Ok(())
        };

        {
            let mut inner = this.inner.lock().expect("Lock poisoned");
            let uploaded_entries = std::mem::take(&mut inner.uploaded_entries);

            let uploaded_filenodes_cnt = uploaded_entries
                .iter()
                .filter(|&(path, _)| path.is_file())
                .count();
            let uploaded_manifests_cnt = uploaded_entries
                .iter()
                .filter(|&(path, _)| !path.is_file())
                .count();

            STATS::finalize_uploaded.add_value(uploaded_entries.len() as i64);
            STATS::finalize_uploaded_filenodes.add_value(uploaded_filenodes_cnt as i64);
            STATS::finalize_uploaded_manifests.add_value(uploaded_manifests_cnt as i64);

            this.scuba_logger()
                .add("manifests_count", uploaded_manifests_cnt)
                .add("filelogs_count", uploaded_filenodes_cnt)
                .log_with_msg("Size of changeset", None);
        }

        future::try_join(parent_checks, required_checks).await?;
        Ok(())
    }
}

pub async fn process_entries<'a>(
    ctx: &'a CoreContext,
    entry_processor: &'a UploadEntries,
    root_manifest: BoxFuture<'a, Result<Option<(HgManifestId, RepoPath)>>>,
    new_child_entries: BoxStream<'a, Result<(Entry<HgManifestId, HgFileNodeId>, RepoPath)>>,
) -> Result<HgManifestId> {
    let root_manifest_fut = async move {
        let root_manifest = root_manifest
            .await
            .context("While uploading root manifest")?;
        match root_manifest {
            None => Ok(None),
            Some((mfid, path)) => {
                if path == RepoPath::RootPath {
                    entry_processor.process_root_manifest(ctx, mfid).await?;
                    Ok(Some(mfid))
                } else {
                    Err(Error::from(ErrorKind::BadRootManifest(mfid)))
                }
            }
        }
    };

    // Not wrapping this future in "async move" causes mismatched opaque types
    // error Â¯\_(ãƒ„)_/Â¯
    let child_entries_fut = async move {
        new_child_entries
            .map_err(|err| err.context("While uploading child entries"))
            .try_for_each_concurrent(100, move |(entry, path)| {
                entry_processor.process_one_entry(ctx, entry, path)
            })
            .await
    };

    let (stats, (root_hash, ())) = future::try_join(root_manifest_fut, child_entries_fut)
        .try_timed()
        .await?;

    entry_processor
        .scuba_logger
        .clone()
        .add_future_stats(&stats)
        .log_with_msg("Upload entries", None);

    match root_hash {
        None => Ok(HgManifestId::new(NULL_HASH)),
        Some(root_hash) => Ok(root_hash),
    }
}

pub fn extract_parents_complete(
    p1: &Option<ChangesetHandle>,
    p2: &Option<ChangesetHandle>,
) -> BoxFuture<'static, Result<(), Error>> {
    // DO NOT replace and_then() with join() or futures_ordered()!
    // It may result in a combinatoral explosion in mergy repos, like the following:
    //  o
    //  |\
    //  | o
    //  |/|
    //  o |
    //  |\|
    //  | o
    //  |/|
    //  o |
    //  |\|
    //  ...
    //  |/|
    //  | ~
    //  o
    //  |\
    //  ~ ~
    //
    let p1 = p1.as_ref().map(|p1| p1.completion_future.clone());
    let p2 = p2.as_ref().map(|p2| p2.completion_future.clone());
    async move {
        if let Some(p1) = p1 {
            p1.await?;
        }
        if let Some(p2) = p2 {
            p2.await?;
        }
        Ok::<(), Error>(())
    }
    .boxed()
}

pub async fn handle_parents(
    mut scuba_logger: MononokeScubaSampleBuilder,
    p1: Option<ChangesetHandle>,
    p2: Option<ChangesetHandle>,
) -> Result<(HgParents, Vec<HgManifestId>, Vec<ChangesetId>), Error> {
    // DO NOT replace and_then() with join() or futures_ordered()!
    // It may result in a combinatoral explosion in mergy repos, like the following:
    //  o
    //  |\
    //  | o
    //  |/|
    //  o |
    //  |\|
    //  | o
    //  |/|
    //  o |
    //  |\|
    //  ...
    //  |/|
    //  | ~
    //  o
    //  |\
    //  ~ ~
    //
    let (stats, result) = async move {
        let mut bonsai_parents = Vec::new();
        let mut parent_manifest_hashes = Vec::new();
        let p1_hash = match p1 {
            Some(p1) => {
                let (bonsai_cs_id, hash, manifest) = p1.can_be_parent.await?;
                bonsai_parents.push(bonsai_cs_id);
                parent_manifest_hashes.push(manifest);
                Some(hash)
            }
            None => None,
        };
        let p2_hash = match p2 {
            Some(p2) => {
                let (bonsai_cs_id, hash, manifest) = p2.can_be_parent.await?;
                bonsai_parents.push(bonsai_cs_id);
                parent_manifest_hashes.push(manifest);
                Some(hash)
            }
            None => None,
        };
        let parents = HgParents::new(p1_hash, p2_hash);
        Ok::<_, Error>((parents, parent_manifest_hashes, bonsai_parents))
    }
    .try_timed()
    .await?;
    scuba_logger
        .add_future_stats(&stats)
        .log_with_msg("Wait for parents ready", None);
    Ok(result)
}

pub fn make_new_changeset(
    parents: HgParents,
    root_hash: HgManifestId,
    cs_metadata: ChangesetMetadata,
    files: Vec<MPath>,
) -> Result<HgBlobChangeset> {
    let changeset = HgChangesetContent::new_from_parts(parents, root_hash, cs_metadata, files);
    HgBlobChangeset::new(changeset)
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

#![type_length_limit = "1817182"]

use anyhow::anyhow;
use anyhow::Result;
use context::CoreContext;
use filestore::FetchKey;
use filestore::FilestoreConfig;
use mononoke_types::ContentId;
use repo_blobstore::RepoBlobstore;

pub async fn copy_content(
    ctx: &CoreContext,
    src_blobstore: &RepoBlobstore,
    dst_blobstore: &RepoBlobstore,
    dst_filestore_config: FilestoreConfig,
    key: ContentId,
) -> Result<()> {
    let fetch_key = FetchKey::Canonical(key.clone());
    if filestore::exists(dst_blobstore, ctx, &fetch_key).await? {
        return Ok(());
    }

    let content_metadata = filestore::get_metadata(src_blobstore, ctx, &fetch_key)
        .await?
        .ok_or_else(|| anyhow!("File not found for fetch key: {:?}", fetch_key))?;

    filestore::copy(
        src_blobstore,
        &src_blobstore.copier_to(dst_blobstore),
        dst_filestore_config,
        ctx,
        &content_metadata,
    )
    .await
}

#[cfg(test)]
mod test {
    #![allow(warnings)]

    use std::collections::HashMap;
    use std::sync::Arc;

    use borrowed::borrowed;
    use bytes::Bytes;
    use context::CoreContext;
    use fbinit::FacebookInit;
    use filestore::StoreRequest;
    use futures::stream;
    use memblob::Memblob;
    use mononoke_types::typed_hash;
    use mononoke_types::BlobstoreBytes;
    use mononoke_types::ContentMetadataV2;
    use mononoke_types::RepositoryId;
    use redactedblobstore::RedactedBlobstore;
    use repo_blobstore::RepoBlobstore;
    use scuba_ext::MononokeScubaSampleBuilder;

    use super::*;

    fn request(data: impl AsRef<[u8]>) -> StoreRequest {
        StoreRequest::new(data.as_ref().len() as u64)
    }

    fn canonical(data: impl AsRef<[u8]>) -> ContentId {
        let mut ctx = typed_hash::ContentIdContext::new();
        ctx.update(data.as_ref());
        ctx.finish()
    }

    #[fbinit::test]
    async fn test_copy_content(fb: FacebookInit) -> Result<()> {
        let ctx = CoreContext::test_mock(fb);
        let inner1 = Arc::new(Memblob::default());
        let inner2 = Arc::new(Memblob::default());

        let bs1 = RepoBlobstore::new(
            inner1,
            None,
            RepositoryId::new(1),
            MononokeScubaSampleBuilder::with_discard(),
        );

        let bs2 = RepoBlobstore::new(
            inner2,
            None,
            RepositoryId::new(2),
            MononokeScubaSampleBuilder::with_discard(),
        );

        borrowed!(ctx, bs1, bs2);

        let default_filestore_config = FilestoreConfig::no_chunking_filestore();

        let bytes = b"hello world";
        let req = request(bytes);
        let cid = canonical(bytes);

        filestore::store(
            bs1,
            default_filestore_config,
            ctx,
            &req,
            stream::once(async { Ok(Bytes::from(&bytes[..])) }),
        )
        .await?;
        copy_content(ctx, bs1, bs2, default_filestore_config.clone(), cid).await?;
        let maybe_copy_meta = filestore::get_metadata(bs2, ctx, &FetchKey::Canonical(cid)).await?;

        let copy_meta =
            maybe_copy_meta.expect("Copied file not found in the destination filestore");
        assert_eq!(copy_meta.total_size, bytes.len() as u64);
        assert_eq!(copy_meta.content_id, cid);
        Ok(())
    }
}