/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

mod mem_writes_changesets;
use std::collections::HashMap;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;

use anyhow::Context;
use anyhow::Error;
use blobrepo::BlobRepo;
use blobrepo_override::DangerousOverride;
use blobstore::Blobstore;
use blobstore::Loadable;
use bonsai_hg_mapping::ArcBonsaiHgMapping;
use bonsai_hg_mapping::MemWritesBonsaiHgMapping;
use cacheblob::dummy::DummyLease;
use cacheblob::LeaseOps;
use cacheblob::MemWritesBlobstore;
use changesets::ArcChangesets;
use clap::Parser;
use clap::Subcommand;
use context::CoreContext;
use fbinit::FacebookInit;
use futures::future;
use import_tools::create_changeset_for_annotated_tag;
use import_tools::import_tree_as_single_bonsai_changeset;
use import_tools::upload_git_tag;
use import_tools::GitimportPreferences;
use import_tools::GitimportTarget;
use linked_hash_map::LinkedHashMap;
use mercurial_derivation::get_manifest_from_bonsai;
use mercurial_derivation::DeriveHgChangeset;
use mononoke_api::BookmarkCategory;
use mononoke_api::BookmarkFreshness;
use mononoke_api::BookmarkKey;
use mononoke_app::args::RepoArgs;
use mononoke_app::fb303::AliveService;
use mononoke_app::fb303::Fb303AppExtension;
use mononoke_app::MononokeApp;
use mononoke_app::MononokeAppBuilder;
use mononoke_types::ChangesetId;
use repo_authorization::AuthorizationContext;
use repo_blobstore::RepoBlobstoreArc;
use repo_blobstore::RepoBlobstoreRef;
use repo_identity::RepoIdentityRef;
use slog::info;

use crate::mem_writes_changesets::MemWritesChangesets;

// Refactor this a bit. Use a thread pool for git operations. Pass that wherever we use store repo.
// Transform the walk into a stream of commit + file changes.

async fn derive_hg(
    ctx: &CoreContext,
    repo: &BlobRepo,
    import_map: impl Iterator<Item = (&git_hash::ObjectId, &ChangesetId)>,
) -> Result<(), Error> {
    let mut hg_manifests = HashMap::new();

    for (id, bcs_id) in import_map {
        let bcs = bcs_id.load(ctx, repo.repo_blobstore()).await?;
        let parent_manifests = future::try_join_all(bcs.parents().map({
            let hg_manifests = &hg_manifests;
            move |p| async move {
                let manifest = if let Some(manifest) = hg_manifests.get(&p) {
                    *manifest
                } else {
                    repo.derive_hg_changeset(ctx, p)
                        .await?
                        .load(ctx, repo.repo_blobstore())
                        .await?
                        .manifestid()
                };
                Result::<_, Error>::Ok(manifest)
            }
        }))
        .await?;

        let manifest = get_manifest_from_bonsai(
            ctx.clone(),
            repo.repo_blobstore_arc(),
            bcs.clone(),
            parent_manifests,
        )
        .await?;

        hg_manifests.insert(*bcs_id, manifest);

        info!(ctx.logger(), "Hg: {:?}: {:?}", id, manifest);
    }

    Ok(())
}

/// Mononoke Git Importer
#[derive(Parser)]
struct GitimportArgs {
    #[clap(long)]
    derive_hg: bool,
    /// This is used to suppress the printing of the potentially really long git Reference -> BonzaiID mapping.
    #[clap(long)]
    suppress_ref_mapping: bool,
    /// **Dangerous** Generate bookmarks for all git refs (tags and branches)
    /// Make sure not to use this on a mononoke repo in production or you will overwhelm any
    /// service doing backfilling on public changesets!
    /// Use at your own risk!
    #[clap(long)]
    generate_bookmarks: bool,
    /// When set, the gitimport tool would bypass the read-only check while creating and moving bookmarks.
    #[clap(long)]
    bypass_readonly: bool,
    /// Set the path to the git binary - preset to git.real
    #[clap(long)]
    git_command_path: Option<String>,
    /// Path to a git repository to import
    git_repository_path: String,
    /// Reupload git commits, even if they already exist in Mononoke
    #[clap(long)]
    reupload_commits: bool,
    #[clap(subcommand)]
    subcommand: GitimportSubcommand,
    #[clap(flatten)]
    repo_args: RepoArgs,
}

#[derive(Subcommand)]
enum GitimportSubcommand {
    /// Import all of the commits in this repo
    FullRepo,
    /// Import all commits between <GIT_FROM> and <GIT_TO>
    GitRange {
        git_from: String,
        git_to: String,
    },
    /// Import <GIT_COMMIT> and all its history that's not yet been imported.
    /// Makes a pass over the repo on construction to find missing history
    MissingForCommit {
        git_commit: String,
    },
    ImportTreeAsSingleBonsaiChangeset {
        git_commit: String,
    },
}

#[fbinit::main]
fn main(fb: FacebookInit) -> Result<(), Error> {
    let app = MononokeAppBuilder::new(fb)
        .with_app_extension(Fb303AppExtension {})
        .build::<GitimportArgs>()?;

    app.run_with_monitoring_and_logging(async_main, "gitimport", AliveService)
}

async fn async_main(app: MononokeApp) -> Result<(), Error> {
    let logger = app.logger();
    let ctx = CoreContext::new_with_logger(app.fb, logger.clone());
    let args: GitimportArgs = app.args()?;
    let mut prefs = GitimportPreferences::default();

    // if we are readonly, then we'll set up some overrides to still be able to do meaningful
    // things below.
    let dry_run = app.readonly_storage().0;
    prefs.dry_run = dry_run;

    if let Some(path) = args.git_command_path {
        prefs.git_command_path = PathBuf::from(path);
    }

    let path = Path::new(&args.git_repository_path);

    let reupload = if args.reupload_commits {
        import_direct::ReuploadCommits::Always
    } else {
        import_direct::ReuploadCommits::Never
    };

    let repo: BlobRepo = app.open_repo(&args.repo_args).await?;
    info!(
        logger,
        "using repo \"{}\" repoid {:?}",
        repo.repo_identity().name(),
        repo.repo_identity().id(),
    );

    let repo = if dry_run {
        repo.dangerous_override(|blobstore| -> Arc<dyn Blobstore> {
            Arc::new(MemWritesBlobstore::new(blobstore))
        })
        .dangerous_override(|changesets| -> ArcChangesets {
            Arc::new(MemWritesChangesets::new(changesets))
        })
        .dangerous_override(|bonsai_hg_mapping| -> ArcBonsaiHgMapping {
            Arc::new(MemWritesBonsaiHgMapping::new(bonsai_hg_mapping))
        })
        .dangerous_override(|_| Arc::new(DummyLease {}) as Arc<dyn LeaseOps>)
    } else {
        repo
    };

    let uploader = import_direct::DirectUploader::new(repo.clone(), reupload);

    let target = match args.subcommand {
        GitimportSubcommand::FullRepo {} => GitimportTarget::full(),
        GitimportSubcommand::GitRange { git_from, git_to } => {
            let from = git_from.parse()?;
            let to = git_to.parse()?;
            import_direct::range(from, to, &ctx, &repo).await?
        }
        GitimportSubcommand::MissingForCommit { git_commit } => {
            let commit = git_commit.parse()?;
            import_direct::missing_for_commit(commit, &ctx, &repo, &prefs.git_command_path, path)
                .await?
        }
        GitimportSubcommand::ImportTreeAsSingleBonsaiChangeset { git_commit } => {
            let commit = git_commit.parse()?;
            let bcs_id =
                import_tree_as_single_bonsai_changeset(&ctx, path, uploader, commit, &prefs)
                    .await?;
            info!(ctx.logger(), "imported as {}", bcs_id);
            if args.derive_hg {
                derive_hg(&ctx, &repo, [(&commit, &bcs_id)].into_iter()).await?;
            }
            return Ok(());
        }
    };

    let gitimport_result: LinkedHashMap<_, _> =
        import_tools::gitimport(&ctx, path, &uploader, &target, &prefs)
            .await
            .context("gitimport failed")?;
    if args.derive_hg {
        derive_hg(&ctx, &repo, gitimport_result.iter())
            .await
            .context("derive_hg failed")?;
    }

    if !args.suppress_ref_mapping || args.generate_bookmarks {
        let refs = import_tools::read_git_refs(path, &prefs)
            .await
            .context("read_git_refs failed")?;
        let mapping = refs
            .iter()
            .map(|(git_ref, commit)| {
                (
                    git_ref.maybe_tag_id,
                    String::from_utf8_lossy(&git_ref.name),
                    gitimport_result.get(commit),
                )
            })
            .collect::<Vec<_>>();
        if !args.suppress_ref_mapping {
            for (_, name, changeset) in &mapping {
                info!(ctx.logger(), "Ref: {:?}: {:?}", name, changeset);
            }
        }
        if args.generate_bookmarks {
            let authz = AuthorizationContext::new_bypass_access_control();
            let repo_context = app
                .open_managed_repo_arg(&args.repo_args)
                .await
                .context("failed to create mononoke app")?
                .make_mononoke_api()?
                .repo_by_id(ctx.clone(), repo.repo_identity().id())
                .await
                .with_context(|| format!("failed to access repo: {}", repo.repo_identity().id()))?
                .expect("repo exists")
                .with_authorization_context(authz)
                .build()
                .await
                .context("failed to build RepoContext")?;
            for (maybe_tag_id, name, changeset) in
                mapping
                    .iter()
                    .filter_map(|(maybe_tag_id, name, changeset)| {
                        changeset.map(|cs| (maybe_tag_id, name, cs))
                    })
            {
                let final_changeset = changeset.clone();
                let mut name = name
                    .strip_prefix("refs/")
                    .context("Ref does not start with refs/")?
                    .to_string();
                if name.starts_with("remotes/origin/") {
                    name = name.replacen("remotes/origin/", "heads/", 1);
                };
                if name.as_str() == "heads/HEAD" {
                    // Skip the HEAD revision: it shouldn't be imported as a bookmark in mononoke
                    continue;
                }
                if let Some(tag_id) = maybe_tag_id {
                    // The ref getting imported is a tag, so store the raw git Tag object.
                    upload_git_tag(&ctx, &uploader, path, &prefs, tag_id).await?;
                    // Create the changeset corresponding to the commit pointed to by the tag.
                    create_changeset_for_annotated_tag(
                        &ctx, &uploader, path, &prefs, tag_id, changeset,
                    )
                    .await?;
                }
                // Set the appropriate category for branch and tag bookmarks
                let bookmark_key = if maybe_tag_id.is_some() {
                    BookmarkKey::with_name_and_category(name.parse()?, BookmarkCategory::Tag)
                } else {
                    BookmarkKey::new(&name)?
                };

                let pushvars = if args.bypass_readonly {
                    Some(HashMap::from_iter([(
                        "BYPASS_READONLY".to_string(),
                        bytes::Bytes::from("true"),
                    )]))
                } else {
                    None
                };
                let old_changeset = repo_context
                    .resolve_bookmark(&bookmark_key, BookmarkFreshness::MostRecent)
                    .await
                    .with_context(|| format!("failed to resolve bookmark {name}"))?
                    .map(|context| context.id());
                match old_changeset {
                    // The bookmark already exists. Instead of creating it, we need to move it.
                    Some(old_changeset) => {
                        if old_changeset != final_changeset {
                            let allow_non_fast_forward = true;
                            repo_context
                                .move_bookmark(
                                    &bookmark_key,
                                    final_changeset,
                                    Some(old_changeset),
                                    allow_non_fast_forward,
                                    pushvars.as_ref(),
                                )
                                .await
                                .with_context(|| format!("failed to move bookmark {name} from {old_changeset:?} to {final_changeset:?}"))?;
                            info!(
                                ctx.logger(),
                                "Bookmark: \"{name}\": {final_changeset:?} (moved from {old_changeset:?})"
                            );
                        } else {
                            info!(
                                ctx.logger(),
                                "Bookmark: \"{name}\": {final_changeset:?} (already up-to-date)"
                            );
                        }
                    }
                    // The bookmark doesn't yet exist. Create it.
                    None => {
                        repo_context
                            .create_bookmark(&bookmark_key, final_changeset, pushvars.as_ref())
                            .await
                            .with_context(|| {
                                format!("failed to create bookmark {name} during gitimport")
                            })?;
                        info!(
                            ctx.logger(),
                            "Bookmark: \"{name}\": {final_changeset:?} (created)"
                        )
                    }
                }
            }
        };
    }
    Ok(())
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;
use std::sync::Arc;
use std::sync::Mutex;

use anyhow::Error;
use async_trait::async_trait;
use changesets::ChangesetEntry;
use changesets::ChangesetInsert;
use changesets::Changesets;
use changesets::SortOrder;
use context::CoreContext;
use futures::future;
use futures::stream::BoxStream;
use lock_ext::LockExt;
use mononoke_types::ChangesetId;
use mononoke_types::ChangesetIdPrefix;
use mononoke_types::ChangesetIdsResolvedFromPrefix;
use mononoke_types::Generation;
use mononoke_types::RepositoryId;
use vec1::Vec1;

#[derive(Clone)]
pub struct MemWritesChangesets<T: Changesets + Clone + 'static> {
    repo_id: RepositoryId,
    inner: T,
    cache: Arc<Mutex<HashMap<ChangesetId, ChangesetEntry>>>,
}

impl<T: Changesets + Clone + 'static> MemWritesChangesets<T> {
    pub fn new(inner: T) -> Self {
        Self {
            repo_id: inner.repo_id(),
            inner,
            cache: Default::default(),
        }
    }
}

#[async_trait]
impl<T: Changesets + Clone + 'static> Changesets for MemWritesChangesets<T> {
    fn repo_id(&self) -> RepositoryId {
        self.repo_id
    }

    async fn add(&self, ctx: &CoreContext, ci: ChangesetInsert) -> Result<bool, Error> {
        let ChangesetInsert { cs_id, parents } = ci;

        let cs = self.get(ctx, cs_id);
        let parent_css = self.get_many(ctx, parents.clone());
        let (cs, parent_css) = future::try_join(cs, parent_css).await?;

        if cs.is_some() {
            Ok(false)
        } else {
            let gen = parent_css.into_iter().map(|p| p.gen).max().unwrap_or(0);

            let entry = ChangesetEntry {
                repo_id: self.repo_id,
                cs_id,
                parents,
                gen,
            };

            self.cache.with(|cache| cache.insert(cs_id, entry));

            Ok(true)
        }
    }

    async fn add_many(
        &self,
        ctx: &CoreContext,
        css: Vec1<(ChangesetInsert, Generation)>,
    ) -> Result<(), Error> {
        for (cs, _) in css {
            self.add(ctx, cs).await?;
        }
        Ok(())
    }

    async fn get(
        &self,
        ctx: &CoreContext,
        cs_id: ChangesetId,
    ) -> Result<Option<ChangesetEntry>, Error> {
        match self.cache.with(|cache| cache.get(&cs_id).cloned()) {
            Some(entry) => Ok(Some(entry)),
            None => self.inner.get(ctx, cs_id).await,
        }
    }

    async fn get_many(
        &self,
        ctx: &CoreContext,
        cs_ids: Vec<ChangesetId>,
    ) -> Result<Vec<ChangesetEntry>, Error> {
        let mut from_cache = vec![];
        let mut from_inner = vec![];

        for cs_id in cs_ids {
            match self.cache.with(|cache| cache.get(&cs_id).cloned()) {
                Some(entry) => from_cache.push(entry),
                None => from_inner.push(cs_id),
            };
        }

        let from_inner = self.inner.get_many(ctx, from_inner).await?;
        from_cache.extend(from_inner);
        Ok(from_cache)
    }

    async fn get_many_by_prefix(
        &self,
        _ctx: &CoreContext,
        _cs_prefix: ChangesetIdPrefix,
        _limit: usize,
    ) -> Result<ChangesetIdsResolvedFromPrefix, Error> {
        unimplemented!("This is not currently implemented in Gitimport")
    }

    fn prime_cache(&self, ctx: &CoreContext, changesets: &[ChangesetEntry]) {
        self.inner.prime_cache(ctx, changesets)
    }

    async fn enumeration_bounds(
        &self,
        ctx: &CoreContext,
        read_from_master: bool,
        known_heads: Vec<ChangesetId>,
    ) -> Result<Option<(u64, u64)>, Error> {
        self.inner
            .enumeration_bounds(ctx, read_from_master, known_heads)
            .await
    }

    fn list_enumeration_range(
        &self,
        ctx: &CoreContext,
        min_id: u64,
        max_id: u64,
        sort_and_limit: Option<(SortOrder, u64)>,
        read_from_master: bool,
    ) -> BoxStream<'_, Result<(ChangesetId, u64), Error>> {
        self.inner
            .list_enumeration_range(ctx, min_id, max_id, sort_and_limit, read_from_master)
    }
}


// @generated by autocargo
use std::env;
use std::fs;
use std::path::Path;

use thrift_compiler::Config;
use thrift_compiler::GenContext;

#[rustfmt::skip]
fn main() {
    // Rerun if this gets rewritten.
    println!("cargo:rerun-if-changed=thrift_build.rs");

    let out_dir = env::var_os("OUT_DIR").expect("OUT_DIR env not provided");
    let out_dir: &Path = out_dir.as_ref();
    fs::write(
        out_dir.join("cratemap"),
        "git_types_thrift crate
mononoke_types_thrift mononoke_types_thrift",
    ).expect("Failed to write cratemap");

    let conf = {
        let mut conf = Config::from_env(GenContext::Lib).expect("Failed to instantiate thrift_compiler::Config");

        let path_from_manifest_to_base: &Path = "../../../../..".as_ref();
        let cargo_manifest_dir =
            env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not provided");
        let cargo_manifest_dir: &Path = cargo_manifest_dir.as_ref();
        let base_path = cargo_manifest_dir
            .join(path_from_manifest_to_base)
            .canonicalize()
            .expect("Failed to canonicalize base_path");
        // TODO: replace canonicalize() with std::path::absolute() when
        // https://github.com/rust-lang/rust/pull/91673 is available (~Rust 1.60)
        // and remove this block.
        #[cfg(windows)]
        let base_path = Path::new(
            base_path
                .as_path()
                .to_string_lossy()
                .trim_start_matches(r"\\?\"),
            )
            .to_path_buf();

        conf.base_path(base_path);

        let options = "types_crate=git-types-thrift__types";
        if !options.is_empty() {
            conf.options(options);
        }

        let include_srcs = vec![

        ];
        conf.include_srcs(include_srcs);

        conf
    };

    conf
        .run(&[
            "git_types_thrift.thrift"
        ])
        .expect("Failed while running thrift compilation");
}


// @generated by autocargo
::codegen_includer_proc_macro::include!();


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use anyhow::Error;
use mononoke_types::hash::RichGitSha1;
use mononoke_types::ContentMetadataV2;
use mononoke_types::FileType;

use crate::mode;
use crate::thrift;
use crate::ObjectKind;

#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq)]
pub struct BlobHandle {
    oid: RichGitSha1,
    file_type: FileType,
}

impl BlobHandle {
    pub fn new(metadata: ContentMetadataV2, file_type: FileType) -> Self {
        Self {
            oid: metadata.git_sha1,
            file_type,
        }
    }

    pub fn filemode(&self) -> i32 {
        match self.file_type {
            FileType::Regular => mode::GIT_FILEMODE_BLOB,
            FileType::Executable => mode::GIT_FILEMODE_BLOB_EXECUTABLE,
            FileType::Symlink => mode::GIT_FILEMODE_LINK,
        }
    }

    pub fn oid(&self) -> &RichGitSha1 {
        &self.oid
    }
}

impl TryFrom<thrift::BlobHandle> for BlobHandle {
    type Error = Error;

    fn try_from(t: thrift::BlobHandle) -> Result<Self, Error> {
        let size = t.size.try_into()?;
        let oid = RichGitSha1::from_bytes(&t.oid.0, ObjectKind::Blob.as_str(), size)?;

        Ok(Self {
            oid,
            file_type: FileType::from_thrift(t.file_type)?,
        })
    }
}

impl From<BlobHandle> for thrift::BlobHandle {
    fn from(bh: BlobHandle) -> thrift::BlobHandle {
        let size = bh.oid.size();

        thrift::BlobHandle {
            oid: bh.oid.into_thrift(),
            size: size
                .try_into()
                .expect("Blob size must fit in a i64 for Thrift serialization"),
            file_type: bh.file_type.into_thrift(),
        }
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use anyhow::anyhow;
use anyhow::bail;
use anyhow::Error;
use anyhow::Result;
use async_trait::async_trait;
use blobstore::Blobstore;
use blobstore::Storable;
use cloned::cloned;
use context::CoreContext;
use derived_data::impl_bonsai_derived_via_manager;
use derived_data_manager::dependencies;
use derived_data_manager::BonsaiDerivable;
use derived_data_manager::DerivableType;
use derived_data_manager::DerivationContext;
use derived_data_service_if::types as thrift;
use filestore::FetchKey;
use futures::future::ready;
use futures::stream::FuturesUnordered;
use futures::stream::TryStreamExt;
use manifest::derive_manifest;
use mononoke_types::BonsaiChangeset;
use mononoke_types::ChangesetId;
use mononoke_types::MPath;

use crate::errors::ErrorKind;
use crate::BlobHandle;
use crate::Tree;
use crate::TreeBuilder;
use crate::TreeHandle;

fn format_key(derivation_ctx: &DerivationContext, changeset_id: ChangesetId) -> String {
    let root_prefix = "git.derived_root.";
    let key_prefix = derivation_ctx.mapping_key_prefix::<TreeHandle>();
    format!("{}{}{}", root_prefix, key_prefix, changeset_id)
}

#[async_trait]
impl BonsaiDerivable for TreeHandle {
    const VARIANT: DerivableType = DerivableType::GitTree;

    type Dependencies = dependencies![];

    async fn derive_single(
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        bonsai: BonsaiChangeset,
        parents: Vec<Self>,
    ) -> Result<Self> {
        if bonsai.is_snapshot() {
            bail!("Can't derive TreeHandle for snapshot")
        }
        let blobstore = derivation_ctx.blobstore().clone();
        let changes = get_file_changes(&blobstore, ctx, bonsai).await?;
        derive_git_manifest(ctx, blobstore, parents, changes).await
    }

    async fn store_mapping(
        self,
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        changeset_id: ChangesetId,
    ) -> Result<()> {
        let key = format_key(derivation_ctx, changeset_id);
        derivation_ctx.blobstore().put(ctx, key, self.into()).await
    }

    async fn fetch(
        ctx: &CoreContext,
        derivation_ctx: &DerivationContext,
        changeset_id: ChangesetId,
    ) -> Result<Option<Self>> {
        let key = format_key(derivation_ctx, changeset_id);
        Ok(derivation_ctx
            .blobstore()
            .get(ctx, &key)
            .await?
            .map(TryInto::try_into)
            .transpose()?)
    }

    fn from_thrift(data: thrift::DerivedData) -> Result<Self> {
        if let thrift::DerivedData::tree_handle(thrift::DerivedDataTreeHandle::tree_handle(id)) =
            data
        {
            Self::try_from(id)
        } else {
            Err(anyhow!(
                "Can't convert {} from provided thrift::DerivedData",
                Self::NAME.to_string(),
            ))
        }
    }

    fn into_thrift(data: Self) -> Result<thrift::DerivedData> {
        Ok(thrift::DerivedData::tree_handle(
            thrift::DerivedDataTreeHandle::tree_handle(data.into()),
        ))
    }
}

impl_bonsai_derived_via_manager!(TreeHandle);

async fn derive_git_manifest<B: Blobstore + Clone + 'static>(
    ctx: &CoreContext,
    blobstore: B,
    parents: Vec<TreeHandle>,
    changes: Vec<(MPath, Option<BlobHandle>)>,
) -> Result<TreeHandle, Error> {
    let handle = derive_manifest(
        ctx.clone(),
        blobstore.clone(),
        parents,
        changes,
        {
            cloned!(ctx, blobstore);
            move |tree_info| {
                let members = tree_info
                    .subentries
                    .into_iter()
                    .map(|(p, (_, entry))| (p, entry.into()))
                    .collect();

                let tree: Tree = TreeBuilder::new(members).into();

                cloned!(ctx, blobstore);
                async move {
                    let handle = tree.store(&ctx, &blobstore).await?;
                    Ok(((), handle))
                }
            }
        },
        {
            // NOTE: A None leaf will happen in derive_manifest if the parents have conflicting
            // leaves. However, since we're deriving from a Bonsai changeset and using our Git Tree
            // manifest which has leaves that are equivalent derived to their Bonsai
            // representation, that won't happen.
            |leaf_info| {
                let leaf = leaf_info
                    .leaf
                    .ok_or_else(|| ErrorKind::TreeDerivationFailed.into())
                    .map(|l| ((), l));
                ready(leaf)
            }
        },
    )
    .await?;

    match handle {
        Some(handle) => Ok(handle),
        None => {
            let tree: Tree = TreeBuilder::default().into();
            tree.store(ctx, &blobstore).await
        }
    }
}

pub async fn get_file_changes<B: Blobstore + Clone>(
    blobstore: &B,
    ctx: &CoreContext,
    bcs: BonsaiChangeset,
) -> Result<Vec<(MPath, Option<BlobHandle>)>, Error> {
    bcs.into_mut()
        .file_changes
        .into_iter()
        .map(|(mpath, file_change)| {
            cloned!(ctx, blobstore);
            async move {
                match file_change.simplify() {
                    Some(fc) => {
                        let t = fc.file_type();
                        let k = FetchKey::Canonical(fc.content_id());

                        let r = filestore::get_metadata(&blobstore, &ctx, &k).await?;
                        let m = r.ok_or(ErrorKind::ContentMissing(k))?;
                        Ok((mpath, Some(BlobHandle::new(m, t))))
                    }
                    None => Ok((mpath, None)),
                }
            }
        })
        .collect::<FuturesUnordered<_>>()
        .try_collect()
        .await
}

#[cfg(test)]
mod test {
    use std::fs::File;
    use std::io::Write;
    use std::path::Path;
    use std::str::FromStr;

    use anyhow::format_err;
    use bookmarks::BookmarkKey;
    use bookmarks::BookmarksRef;
    use derived_data::BonsaiDerived;
    use fbinit::FacebookInit;
    use filestore::Alias;
    use fixtures::TestRepoFixture;
    use futures_util::stream::TryStreamExt;
    use git2::Oid;
    use git2::Repository;
    use manifest::ManifestOps;
    use repo_blobstore::RepoBlobstoreArc;
    use repo_derived_data::RepoDerivedDataRef;
    use tempdir::TempDir;

    use super::*;

    /// This function creates a new Git tree from the fixture's master Bonsai bookmark,
    /// materializes it to disk, then verifies that libgit produces the same Git tree for it.
    async fn run_tree_derivation_for_fixture(
        fb: FacebookInit,
        repo: impl BookmarksRef + RepoBlobstoreArc + RepoDerivedDataRef + Send + Sync,
    ) -> Result<(), Error> {
        let ctx = CoreContext::test_mock(fb);

        let bcs_id = repo
            .bookmarks()
            .get(ctx.clone(), &BookmarkKey::from_str("master")?)
            .await?
            .ok_or_else(|| format_err!("no master"))?;

        let tree = TreeHandle::derive(&ctx, &repo, bcs_id).await?;

        let leaves = tree
            .list_leaf_entries(ctx.clone(), repo.repo_blobstore_arc())
            .try_collect::<Vec<_>>()
            .await?;

        let tmp_dir = TempDir::new("git_types_test")?;
        let root_path = tmp_dir.path();
        let git = Repository::init(root_path)?;
        let mut index = git.index()?;

        for (mpath, blob_handle) in leaves.into_iter() {
            let blob = filestore::fetch_concat(
                repo.repo_blobstore(),
                &ctx,
                FetchKey::Aliased(Alias::GitSha1(blob_handle.oid().sha1())),
            )
            .await?;

            let path = &mpath.to_string();
            let path = Path::new(&path);
            File::create(&root_path.join(path))?.write_all(&blob)?;

            index.add_path(path)?;
        }

        let git_oid = index.write_tree()?;
        let derived_tree_oid = Oid::from_bytes(tree.oid().as_ref())?;
        assert_eq!(git_oid, derived_tree_oid);

        tmp_dir.close()?;

        Ok(())
    }

    macro_rules! impl_test {
        ($test_name:ident, $fixture:ident) => {
            #[fbinit::test]
            fn $test_name(fb: FacebookInit) -> Result<(), Error> {
                let runtime = tokio::runtime::Runtime::new()?;
                runtime.block_on(async move {
                    let repo = fixtures::$fixture::getrepo(fb).await;
                    run_tree_derivation_for_fixture(fb, repo).await
                })
            }
        };
    }

    impl_test!(linear, Linear);
    impl_test!(branch_even, BranchEven);
    impl_test!(branch_uneven, BranchUneven);
    impl_test!(branch_wide, BranchWide);
    impl_test!(merge_even, MergeEven);
    impl_test!(many_files_dirs, ManyFilesDirs);
    impl_test!(merge_uneven, MergeUneven);
    impl_test!(unshared_merge_even, UnsharedMergeEven);
    impl_test!(unshared_merge_uneven, UnsharedMergeUneven);
    impl_test!(many_diamonds, ManyDiamonds);
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use filestore::FetchKey;
use thiserror::Error;

#[derive(Debug, Error)]
pub enum ErrorKind {
    #[error("Could not locate content: {0:?}")]
    ContentMissing(FetchKey),
    #[error("Tree Derivation Failed")]
    TreeDerivationFailed,
    #[error("Invalid Thrift")]
    InvalidThrift,
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

pub mod mode;

mod thrift {
    pub use git_types_thrift::*;
}

mod blob;
mod derive_tree;
mod errors;
mod manifest;
mod nodehash;
mod object;
mod store;
mod tree;

pub use object::ObjectKind;

pub use crate::blob::BlobHandle;
pub use crate::nodehash::GitSha1Prefix;
pub use crate::nodehash::GitSha1sResolvedFromPrefix;
pub use crate::tree::Tree;
pub use crate::tree::TreeBuilder;
pub use crate::tree::TreeHandle;
pub use crate::tree::TreeMember;
pub use crate::tree::Treeish;


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use ::manifest::Entry;
use ::manifest::Manifest;
use mononoke_types::MPathElement;

use crate::BlobHandle;
use crate::Tree;
use crate::TreeHandle;
use crate::Treeish;

impl Manifest for Tree {
    type TreeId = TreeHandle;
    type LeafId = BlobHandle;

    fn list(&self) -> Box<dyn Iterator<Item = (MPathElement, Entry<Self::TreeId, Self::LeafId>)>> {
        let members: Vec<_> = self
            .members()
            .iter()
            .map(|(k, v)| (k.clone(), v.clone().into()))
            .collect();

        Box::new(members.into_iter())
    }

    fn lookup(&self, name: &MPathElement) -> Option<Entry<Self::TreeId, Self::LeafId>> {
        self.members().get(name).map(|e| e.clone().into())
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

// Filemodes as per:
// https://github.com/libgit2/libgit2/blob/68cfb580e19c419992ba0b0a299e5fd6dc60ed99/include/git2/types.h#L210-L217
pub const GIT_FILEMODE_UNREADABLE: i32 = 0o000000;
pub const GIT_FILEMODE_TREE: i32 = 0o040000;
pub const GIT_FILEMODE_BLOB: i32 = 0o100644;
pub const GIT_FILEMODE_BLOB_EXECUTABLE: i32 = 0o100755;
pub const GIT_FILEMODE_LINK: i32 = 0o120000;
pub const GIT_FILEMODE_COMMIT: i32 = 0o160000;


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

//! A hash of a node (changeset, manifest or file).

use std::fmt;
use std::fmt::Display;
use std::result;
use std::str::FromStr;

use abomonation_derive::Abomonation;
use anyhow::Result;
use mononoke_types::hash::GitSha1;
use mononoke_types::sha1_hash::Sha1;
use mononoke_types::sha1_hash::Sha1Prefix;

/// An identifier for a changeset hash prefix in Nercurial.
#[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Debug, Hash)]
#[derive(Abomonation)]
pub struct GitSha1Prefix(Sha1Prefix);

impl GitSha1Prefix {
    pub const fn new(sha1prefix: Sha1Prefix) -> Self {
        GitSha1Prefix(sha1prefix)
    }

    pub fn from_bytes<B: AsRef<[u8]> + ?Sized>(bytes: &B) -> Result<Self> {
        Sha1Prefix::from_bytes(bytes).map(Self::new)
    }

    #[inline]
    pub fn min_cs(&self) -> GitSha1 {
        GitSha1::from_bytes(self.min_as_ref()).expect("Min sha1 is a valid sha1")
    }

    #[inline]
    pub fn max_cs(&self) -> GitSha1 {
        GitSha1::from_bytes(self.max_as_ref()).expect("Max sha1 is a valid sha1")
    }

    #[inline]
    pub fn min_as_ref(&self) -> &[u8] {
        self.0.min_as_ref()
    }

    #[inline]
    pub fn max_as_ref(&self) -> &[u8] {
        self.0.max_as_ref()
    }

    #[inline]
    pub fn into_git_sha1(self) -> Option<GitSha1> {
        self.0
            .into_sha1()
            .map(Sha1::into_byte_array)
            .map(GitSha1::from_byte_array)
    }
}

impl FromStr for GitSha1Prefix {
    type Err = <Sha1Prefix as FromStr>::Err;
    fn from_str(s: &str) -> result::Result<GitSha1Prefix, Self::Err> {
        Sha1Prefix::from_str(s).map(GitSha1Prefix)
    }
}

impl Display for GitSha1Prefix {
    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
        self.0.fmt(fmt)
    }
}

#[derive(Debug, Clone, Eq, PartialEq, Hash)]
/// The type for resolving changesets by prefix of the hash
pub enum GitSha1sResolvedFromPrefix {
    /// Found single changeset
    Single(GitSha1),
    /// Found several changesets within the limit provided
    Multiple(Vec<GitSha1>),
    /// Found too many changesets exceeding the limit provided
    TooMany(Vec<GitSha1>),
    /// Changeset was not found
    NoMatch,
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use digest::Digest;
use mononoke_types::hash::RichGitSha1;
use sha1::Sha1;

#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq)]
pub enum ObjectKind {
    Blob,
    Tree,
    Commit,
}

impl ObjectKind {
    pub fn as_str(&self) -> &'static str {
        match self {
            Self::Blob => "blob",
            Self::Tree => "tree",
            Self::Commit => "commit",
        }
    }

    pub fn is_tree(&self) -> bool {
        match self {
            Self::Blob => false,
            Self::Tree => true,
            Self::Commit => false,
        }
    }

    pub fn create_oid(&self, object_buff: impl AsRef<[u8]>) -> RichGitSha1 {
        let object_buff = object_buff.as_ref();
        let size = object_buff
            .len()
            .try_into()
            .expect("Object size must fit in a u64");

        let mut sha1 = Sha1::new();
        sha1.update(&format!("{} {}", self.as_str(), size));
        sha1.update([0]);
        sha1.update(<[u8] as AsRef<[u8]>>::as_ref(object_buff));

        let hash: [u8; 20] = sha1.finalize().into();

        RichGitSha1::from_byte_array(hash, self.as_str(), size)
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use blobstore::impl_loadable_storable;

use crate::thrift::Tree as ThriftTree;
use crate::thrift::TreeHandle as ThriftTreeHandle;
use crate::Tree;
use crate::TreeHandle;

impl_loadable_storable! {
    handle_type => TreeHandle,
    handle_thrift_type => ThriftTreeHandle,
    value_type => Tree,
    value_thrift_type => ThriftTree,
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::cmp;
use std::cmp::Ordering;
use std::collections::HashMap;
use std::fmt;
use std::fmt::Display;
use std::io;
use std::io::Write;

use ::manifest::Entry;
use anyhow::Error;
use mononoke_types::hash::RichGitSha1;
use mononoke_types::MPathElement;

use crate::errors::ErrorKind;
use crate::mode;
use crate::thrift;
use crate::BlobHandle;
use crate::ObjectKind;

#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq)]
pub struct TreeHandle {
    oid: RichGitSha1,
}

impl TreeHandle {
    pub fn filemode(&self) -> i32 {
        mode::GIT_FILEMODE_TREE
    }

    pub fn oid(&self) -> &RichGitSha1 {
        &self.oid
    }

    pub fn blobstore_key(&self) -> String {
        format!("git.tree.{}", self.oid)
    }
}

impl TryFrom<thrift::TreeHandle> for TreeHandle {
    type Error = Error;

    fn try_from(t: thrift::TreeHandle) -> Result<Self, Error> {
        let size = t.size.try_into()?;
        let oid = RichGitSha1::from_bytes(&t.oid.0, ObjectKind::Tree.as_str(), size)?;
        Ok(Self { oid })
    }
}

impl From<TreeHandle> for thrift::TreeHandle {
    fn from(th: TreeHandle) -> thrift::TreeHandle {
        let size = th.oid.size();

        thrift::TreeHandle {
            oid: th.oid.into_thrift(),
            size: size.try_into().expect("Tree size must fit in a i64"),
        }
    }
}

#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq)]
pub enum TreeMember {
    Blob(BlobHandle),
    Tree(TreeHandle),
}

impl From<TreeMember> for Entry<TreeHandle, BlobHandle> {
    fn from(tm: TreeMember) -> Entry<TreeHandle, BlobHandle> {
        match tm {
            TreeMember::Blob(handle) => Entry::Leaf(handle),
            TreeMember::Tree(handle) => Entry::Tree(handle),
        }
    }
}

impl From<Entry<TreeHandle, BlobHandle>> for TreeMember {
    fn from(entry: Entry<TreeHandle, BlobHandle>) -> Self {
        match entry {
            Entry::Leaf(handle) => Self::Blob(handle),
            Entry::Tree(handle) => Self::Tree(handle),
        }
    }
}

impl TreeMember {
    pub fn filemode(&self) -> i32 {
        match self {
            Self::Blob(ref blob) => blob.filemode(),
            Self::Tree(ref tree) => tree.filemode(),
        }
    }

    pub fn oid(&self) -> &RichGitSha1 {
        match self {
            Self::Blob(ref blob) => blob.oid(),
            Self::Tree(ref tree) => tree.oid(),
        }
    }

    pub fn kind(&self) -> ObjectKind {
        match self {
            Self::Blob(..) => ObjectKind::Blob,
            Self::Tree(..) => ObjectKind::Tree,
        }
    }
}

impl TryFrom<thrift::TreeMember> for TreeMember {
    type Error = Error;

    fn try_from(t: thrift::TreeMember) -> Result<Self, Error> {
        match t {
            thrift::TreeMember::Blob(blob) => Ok(Self::Blob(blob.try_into()?)),
            thrift::TreeMember::Tree(tree) => Ok(Self::Tree(tree.try_into()?)),
            thrift::TreeMember::UnknownField(..) => Err(ErrorKind::InvalidThrift.into()),
        }
    }
}

impl From<TreeMember> for thrift::TreeMember {
    fn from(tm: TreeMember) -> thrift::TreeMember {
        match tm {
            TreeMember::Blob(blob) => thrift::TreeMember::Blob(blob.into()),
            TreeMember::Tree(tree) => thrift::TreeMember::Tree(tree.into()),
        }
    }
}

#[derive(Debug, Clone)]
pub struct Tree {
    handle: TreeHandle,
    members: HashMap<MPathElement, TreeMember>,
}

impl Tree {
    pub fn handle(&self) -> &TreeHandle {
        &self.handle
    }
}

impl TryFrom<thrift::Tree> for Tree {
    type Error = Error;

    fn try_from(t: thrift::Tree) -> Result<Self, Error> {
        let handle = t.handle.try_into()?;

        let members = t
            .members
            .into_iter()
            .map(|(path, member)| {
                let path = MPathElement::from_thrift(path)?;
                let member = member.try_into()?;
                Ok((path, member))
            })
            .collect::<Result<HashMap<_, _>, Error>>()?;

        Ok(Self { handle, members })
    }
}

impl From<Tree> for thrift::Tree {
    fn from(t: Tree) -> thrift::Tree {
        let Tree { handle, members } = t;

        let members = members
            .into_iter()
            .map(|(path, member)| (path.into_thrift(), member.into()))
            .collect();

        thrift::Tree {
            handle: handle.into(),
            members,
        }
    }
}

#[derive(Debug, Clone, Default)]
pub struct TreeBuilder {
    members: HashMap<MPathElement, TreeMember>,
}

impl TreeBuilder {
    // TODO: Can we verify members here (git_path_isvalid)
    pub fn new(members: HashMap<MPathElement, TreeMember>) -> Self {
        Self { members }
    }
}

impl From<TreeBuilder> for Tree {
    fn from(tb: TreeBuilder) -> Tree {
        let mut object_buff = Vec::new();
        tb.write_serialized_object(&mut object_buff)
            .expect("Writes to Vec cannot fail");

        let oid = ObjectKind::Tree.create_oid(&object_buff);

        Tree {
            handle: TreeHandle { oid },
            members: tb.members,
        }
    }
}

pub trait Treeish {
    fn members(&self) -> &HashMap<MPathElement, TreeMember>;

    fn write_serialized_object(&self, writer: &mut impl Write) -> Result<(), io::Error> {
        for (path, member) in iter_members_git_path_order(self.members()) {
            write!(writer, "{:o} ", member.filemode())?;
            writer.write_all(path.as_ref())?;
            writer.write_all(&[0])?;
            writer.write_all(member.oid().as_ref())?;
        }

        Ok(())
    }

    fn write_humanized_representation(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        for (path, member) in iter_members_git_path_order(self.members()) {
            write!(
                f,
                "{:06o} {} {}\t{}\n",
                member.filemode(),
                member.kind().as_str(),
                member.oid(),
                path
            )?;
        }

        Ok(())
    }
}

impl Treeish for Tree {
    fn members(&self) -> &HashMap<MPathElement, TreeMember> {
        &self.members
    }
}

impl Treeish for TreeBuilder {
    fn members(&self) -> &HashMap<MPathElement, TreeMember> {
        &self.members
    }
}

impl Display for Tree {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.write_humanized_representation(f)
    }
}

impl Display for TreeBuilder {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        self.write_humanized_representation(f)
    }
}

fn iter_members_git_path_order(
    members: &HashMap<MPathElement, TreeMember>,
) -> impl Iterator<Item = (&MPathElement, &TreeMember)> {
    let mut members: Vec<_> = members.iter().collect();
    members.sort_by(|(p1, e1), (p2, e2)| git_path_cmp(p1, e1, p2, e2));
    members.into_iter()
}

// TODO: Expose git_path_cmp from libgit2 and use it here
// https://github.com/libgit2/libgit2/blob/fb439c975a2de33f5b0c317f3fdea49dc94b27dc/src/path.c#L850
fn git_path_cmp(
    p1: &MPathElement,
    e1: &TreeMember,
    p2: &MPathElement,
    e2: &TreeMember,
) -> Ordering {
    const NULL: u8 = 0;
    const SLASH: u8 = b'/';

    let p1 = p1.as_ref();
    let p2 = p2.as_ref();
    let len = cmp::min(p1.len(), p2.len());

    let ordering = p1[..len].cmp(&p2[..len]);
    if ordering != Ordering::Equal {
        return ordering;
    }

    let c1 = p1
        .get(len)
        .unwrap_or(if e1.kind().is_tree() { &SLASH } else { &NULL });

    let c2 = p2
        .get(len)
        .unwrap_or(if e2.kind().is_tree() { &SLASH } else { &NULL });

    c1.cmp(c2)
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;
use std::collections::HashSet;
use std::path::Path;
use std::time::Instant;

use anyhow::format_err;
use anyhow::Error;
use bonsai_git_mapping::BonsaiGitMappingRef;
use context::CoreContext;
use git_hash::ObjectId;
use import_tools::oid_to_sha1;
use import_tools::GitRepoReader;
use import_tools::GitimportTarget;
use mononoke_types::typed_hash::ChangesetId;
use slog::debug;

mod uploader;

pub use uploader::DirectUploader;
pub use uploader::ReuploadCommits;

/// Import starting at from (known to be in Mononoke) and ending with to
pub async fn range(
    from: ObjectId,
    to: ObjectId,
    ctx: &CoreContext,
    repo: &impl BonsaiGitMappingRef,
) -> Result<GitimportTarget, Error> {
    let from_csid = repo
        .bonsai_git_mapping()
        .get_bonsai_from_git_sha1(ctx, oid_to_sha1(&from)?)
        .await?
        .ok_or_else(|| {
            format_err!(
                "Cannot start import from root {}: commit does not exist in repo",
                from
            )
        })?;
    let known = [(from, from_csid)].into();
    GitimportTarget::new(to, known)
}

/// Import commit and all its history that's not yet been imported
/// Makes a pass over the repo on construction to find missing history
pub async fn missing_for_commit(
    commit: ObjectId,
    ctx: &CoreContext,
    hg_repo: &impl BonsaiGitMappingRef,
    git_command_path: &Path,
    repo_path: &Path,
) -> Result<GitimportTarget, Error> {
    let reader = GitRepoReader::new(git_command_path, repo_path).await?;
    let ta = Instant::now();

    // Starting from the specified commit. We need to get the boundaries of what already is imported into Mononoke.
    // We do this by doing a bfs search from the specified commit.
    let mut known = HashMap::<ObjectId, ChangesetId>::new();
    let mut visited = HashSet::new();
    let mut q = vec![commit];
    while let Some(id) = q.pop() {
        if visited.insert(id) {
            if let Some(changeset) = commit_in_mononoke(ctx, hg_repo, &id).await? {
                known.insert(id, changeset);
            } else {
                let object = reader.get_object(&id).await?;
                let commit = object
                    .parsed
                    .try_into_commit()
                    .map_err(|_| format_err!("oid {} is not a commit", id))?;
                q.extend(commit.parents);
            }
        }
    }

    let tb = Instant::now();
    debug!(
        ctx.logger(),
        "Time to find missing commits {:?}",
        tb.duration_since(ta)
    );

    GitimportTarget::new(commit, known)
}

async fn commit_in_mononoke(
    ctx: &CoreContext,
    hg_repo: &impl BonsaiGitMappingRef,
    commit_id: &git_hash::oid,
) -> Result<Option<ChangesetId>, Error> {
    let changeset = hg_repo
        .bonsai_git_mapping()
        .get_bonsai_from_git_sha1(ctx, oid_to_sha1(commit_id)?)
        .await?;
    if let Some(existing_changeset) = changeset {
        debug!(
            ctx.logger(),
            "Commit found in Mononoke Oid:{} -> ChangesetId:{}",
            oid_to_sha1(commit_id)?.to_brief(),
            existing_changeset.to_brief()
        );
    }
    Ok(changeset)
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::sync::Arc;

use anyhow::Context;
use anyhow::Error;
use async_trait::async_trait;
use blobrepo::save_bonsai_changesets;
use bonsai_git_mapping::BonsaiGitMappingEntry;
use bonsai_git_mapping::BonsaiGitMappingRef;
use bonsai_tag_mapping::BonsaiTagMappingRef;
use bytes::Bytes;
use changesets::ChangesetsRef;
use cloned::cloned;
use context::CoreContext;
use filestore::FilestoreConfigRef;
use filestore::StoreRequest;
use futures::stream;
use futures::stream::Stream;
use futures_stats::TimedTryFutureExt;
use git_hash::ObjectId;
use import_tools::CommitMetadata;
use import_tools::GitImportLfs;
use import_tools::GitUploader;
use import_tools::TagMetadata;
use import_tools::HGGIT_COMMIT_ID_EXTRA;
use import_tools::HGGIT_MARKER_EXTRA;
use import_tools::HGGIT_MARKER_VALUE;
use mononoke_api::repo::git::create_annotated_tag;
use mononoke_api::repo::upload_git_object;
use mononoke_types::bonsai_changeset::BonsaiAnnotatedTag;
use mononoke_types::bonsai_changeset::BonsaiAnnotatedTagTarget;
use mononoke_types::hash;
use mononoke_types::BonsaiChangeset;
use mononoke_types::BonsaiChangesetMut;
use mononoke_types::ChangesetId;
use mononoke_types::FileChange;
use mononoke_types::FileType;
use mononoke_types::MPath;
use repo_blobstore::RepoBlobstoreRef;
use slog::debug;
use slog::info;
use sorted_vector_map::SortedVectorMap;

#[derive(Clone, Copy, Debug)]
pub enum ReuploadCommits {
    Never,
    Always,
}

impl ReuploadCommits {
    fn reupload_commit(&self) -> bool {
        match self {
            ReuploadCommits::Never => false,
            ReuploadCommits::Always => true,
        }
    }
}

#[derive(Clone)]
pub struct DirectUploader<R> {
    inner: Arc<R>,
    reupload_commits: ReuploadCommits,
}

impl<R> DirectUploader<R> {
    pub fn new(repo: R, reupload_commits: ReuploadCommits) -> Self {
        Self {
            inner: Arc::new(repo),
            reupload_commits,
        }
    }
}

#[async_trait]
impl<R> GitUploader for DirectUploader<R>
where
    R: ChangesetsRef
        + RepoBlobstoreRef
        + BonsaiGitMappingRef
        + BonsaiTagMappingRef
        + FilestoreConfigRef
        + Clone
        + Send
        + Sync
        + 'static,
{
    type Change = FileChange;
    type IntermediateChangeset = BonsaiChangeset;

    fn deleted() -> Self::Change {
        FileChange::Deletion
    }

    async fn check_commit_uploaded(
        &self,
        ctx: &CoreContext,
        oid: &git_hash::oid,
    ) -> Result<Option<ChangesetId>, Error> {
        if self.reupload_commits.reupload_commit() {
            return Ok(None);
        }

        self.inner
            .bonsai_git_mapping()
            .get_bonsai_from_git_sha1(ctx, hash::GitSha1::from_bytes(oid.as_bytes())?)
            .await
    }

    async fn upload_file(
        &self,
        ctx: &CoreContext,
        lfs: &GitImportLfs,
        path: &MPath,
        ty: FileType,
        oid: ObjectId,
        git_bytes: Bytes,
    ) -> Result<Self::Change, Error> {
        let meta_ret = if let Some(lfs_meta) = lfs.is_lfs_file(&git_bytes, oid) {
            let blobstore = self.inner.repo_blobstore();
            let filestore_config = *self.inner.filestore_config();
            cloned!(ctx, lfs, blobstore, path);
            Ok(lfs
                .with(
                    ctx,
                    lfs_meta,
                    move |ctx, lfs_meta, req, bstream| async move {
                        info!(
                            ctx.logger(),
                            "Uploading LFS {} sha256:{} size:{}",
                            path,
                            lfs_meta.sha256.to_brief(),
                            lfs_meta.size,
                        );
                        filestore::store(&blobstore, filestore_config, &ctx, &req, bstream).await
                    },
                )
                .await?)
        } else {
            let (req, bstream) = git_store_request(ctx, oid, git_bytes)?;
            Ok(filestore::store(
                self.inner.repo_blobstore(),
                *self.inner.filestore_config(),
                ctx,
                &req,
                bstream,
            )
            .await?)
        };
        debug!(
            ctx.logger(),
            "Uploaded {} blob {}",
            path,
            oid.to_hex_with_len(8),
        );
        meta_ret.map(|meta| FileChange::tracked(meta.content_id, ty, meta.total_size, None))
    }

    async fn generate_changeset_for_commit(
        &self,
        _ctx: &CoreContext,
        bonsai_parents: Vec<ChangesetId>,
        metadata: CommitMetadata,
        changes: SortedVectorMap<MPath, Self::Change>,
        _dry_run: bool,
    ) -> Result<(Self::IntermediateChangeset, ChangesetId), Error> {
        let bcs = generate_bonsai_changeset(metadata, bonsai_parents, changes)?;
        let bcs_id = bcs.get_changeset_id();
        Ok((bcs, bcs_id))
    }

    async fn finalize_batch(
        &self,
        ctx: &CoreContext,
        dry_run: bool,
        changesets: Vec<(Self::IntermediateChangeset, hash::GitSha1)>,
    ) -> Result<(), Error> {
        let oid_to_bcsid = changesets
            .iter()
            .map(|(bcs, git_sha1)| BonsaiGitMappingEntry::new(*git_sha1, bcs.get_changeset_id()))
            .collect::<Vec<BonsaiGitMappingEntry>>();
        let vbcs = changesets.into_iter().map(|x| x.0).collect();

        // We know that the commits are in order (this is guaranteed by the Walk), so we
        // can insert them as-is, one by one, without extra dependency / ordering checks.
        let (stats, ()) = save_bonsai_changesets(vbcs, ctx.clone(), &*self.inner)
            .try_timed()
            .await?;
        debug!(
            ctx.logger(),
            "save_bonsai_changesets for {} commits in {:?}",
            oid_to_bcsid.len(),
            stats.completion_time
        );

        if !dry_run {
            self.inner
                .bonsai_git_mapping()
                .bulk_add(ctx, &oid_to_bcsid)
                .await?;
        }

        Ok(())
    }

    async fn upload_object(
        &self,
        ctx: &CoreContext,
        oid: ObjectId,
        git_bytes: Bytes,
    ) -> Result<(), Error> {
        upload_git_object(ctx, &*self.inner, &oid, git_bytes.to_vec())
            .await
            .map_err(|e| {
                anyhow::anyhow!(
                    "Failure in uploading raw git object. Cause: {}",
                    e.to_string()
                )
            })
    }

    async fn generate_changeset_for_annotated_tag(
        &self,
        ctx: &CoreContext,
        target_changeset_id: ChangesetId,
        mut tag: TagMetadata,
    ) -> Result<ChangesetId, Error> {
        let annotated_tag = BonsaiAnnotatedTag {
            target: BonsaiAnnotatedTagTarget::Changeset(target_changeset_id),
            pgp_signature: tag.pgp_signature.take(),
        };
        let tag_name = format!("tags/{}", tag.name);
        create_annotated_tag(
            ctx,
            &*self.inner,
            tag_name,
            tag.author.take(),
            tag.author_date.take().map(|date| date.into()),
            tag.message,
            annotated_tag,
        )
        .await
        .context("Failure in creating changeset for tag")
    }
}

fn git_store_request(
    ctx: &CoreContext,
    git_id: ObjectId,
    git_bytes: Bytes,
) -> Result<(StoreRequest, impl Stream<Item = Result<Bytes, Error>>), Error> {
    let size = git_bytes.len().try_into()?;
    let git_sha1 =
        hash::RichGitSha1::from_bytes(Bytes::copy_from_slice(git_id.as_bytes()), "blob", size)?;
    let req = StoreRequest::with_git_sha1(size, git_sha1);
    debug!(
        ctx.logger(),
        "Uploading git-blob:{} size:{}",
        git_sha1.sha1().to_brief(),
        size
    );
    Ok((req, stream::once(async move { Ok(git_bytes) })))
}

fn generate_bonsai_changeset(
    metadata: CommitMetadata,
    parents: Vec<ChangesetId>,
    file_changes: SortedVectorMap<MPath, FileChange>,
) -> Result<BonsaiChangeset, Error> {
    let CommitMetadata {
        oid,
        message,
        author,
        author_date,
        committer,
        committer_date,
        git_extra_headers,
        ..
    } = metadata;
    let git_extra_headers = if git_extra_headers.is_empty() {
        None
    } else {
        Some(git_extra_headers)
    };
    let mut extra = SortedVectorMap::new();
    extra.insert(
        HGGIT_COMMIT_ID_EXTRA.to_string(),
        oid.to_string().into_bytes(),
    );
    extra.insert(HGGIT_MARKER_EXTRA.to_string(), HGGIT_MARKER_VALUE.to_vec());

    BonsaiChangesetMut {
        parents,
        author,
        author_date,
        committer: Some(committer),
        committer_date: Some(committer_date),
        message,
        hg_extra: extra,
        git_extra_headers,
        // TODO(rajshar): Populate these fields correctly instead of using empty values
        git_tree_hash: None,
        file_changes,
        is_snapshot: false,
        git_annotated_tag: None,
    }
    .freeze()
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;
use std::collections::HashSet;
use std::path::Path;
use std::path::PathBuf;
use std::process::Stdio;

use anyhow::anyhow;
use anyhow::bail;
use anyhow::format_err;
use anyhow::Context;
use anyhow::Error;
use async_trait::async_trait;
use blobstore::LoadableError;
use bytes::Bytes;
use context::CoreContext;
use encoding_rs::Encoding;
use futures::stream::Stream;
use futures::stream::TryStreamExt;
use git_hash::ObjectId;
use git_object::bstr::BString;
use git_object::tree;
use git_object::Commit;
use git_object::Tag;
use git_object::Tree;
use manifest::Entry;
use manifest::Manifest;
use manifest::StoreLoadable;
use mononoke_types::hash;
use mononoke_types::ChangesetId;
use mononoke_types::DateTime;
use mononoke_types::FileType;
use mononoke_types::MPath;
use mononoke_types::MPathElement;
use slog::debug;
use slog::Logger;
use smallvec::SmallVec;
use sorted_vector_map::SortedVectorMap;
use tokio::io::AsyncBufReadExt;
use tokio::io::AsyncWriteExt;
use tokio::io::BufReader;
use tokio::process::Child;
use tokio::process::Command;
use tokio_stream::wrappers::LinesStream;

use crate::git_reader::GitRepoReader;
use crate::gitlfs::GitImportLfs;

#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
pub struct GitTree(pub ObjectId);

#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
pub struct GitLeaf(pub ObjectId);

pub struct GitManifest(HashMap<MPathElement, Entry<GitTree, (FileType, GitLeaf)>>);

impl Manifest for GitManifest {
    type TreeId = GitTree;
    type LeafId = (FileType, GitLeaf);

    fn lookup(&self, name: &MPathElement) -> Option<Entry<Self::TreeId, Self::LeafId>> {
        self.0.get(name).cloned()
    }

    fn list(&self) -> Box<dyn Iterator<Item = (MPathElement, Entry<Self::TreeId, Self::LeafId>)>> {
        Box::new(self.0.clone().into_iter())
    }
}

async fn read_tree(reader: &GitRepoReader, oid: &git_hash::oid) -> Result<Tree, Error> {
    let object = reader.get_object(oid).await?;
    object
        .parsed
        .try_into_tree()
        .map_err(|_| format_err!("{} is not a tree", oid))
}

async fn load_git_tree(oid: &git_hash::oid, reader: &GitRepoReader) -> Result<GitManifest, Error> {
    let tree = read_tree(reader, oid).await?;

    let elements = tree
        .entries
        .into_iter()
        .filter_map(
            |tree::Entry {
                 mode,
                 filename,
                 oid,
             }| {
                let name = match MPathElement::new(filename.into()) {
                    Ok(name) => name,
                    Err(e) => return Some(Err(e)),
                };

                let r = match mode {
                    tree::EntryMode::Blob => {
                        Some((name, Entry::Leaf((FileType::Regular, GitLeaf(oid)))))
                    }
                    tree::EntryMode::BlobExecutable => {
                        Some((name, Entry::Leaf((FileType::Executable, GitLeaf(oid)))))
                    }
                    tree::EntryMode::Link => {
                        Some((name, Entry::Leaf((FileType::Symlink, GitLeaf(oid)))))
                    }
                    tree::EntryMode::Tree => Some((name, Entry::Tree(GitTree(oid)))),

                    // git-sub-modules are represented as ObjectType::Commit inside the tree.
                    // For now we do not support git-sub-modules but we still need to import
                    // repositories that has sub-modules in them (just not synchronized), so
                    // ignoring any sub-module for now.
                    tree::EntryMode::Commit => None,
                };
                anyhow::Ok(r).transpose()
            },
        )
        .collect::<Result<_, Error>>()?;

    anyhow::Ok(GitManifest(elements))
}

#[async_trait]
impl StoreLoadable<GitRepoReader> for GitTree {
    type Value = GitManifest;

    async fn load<'a>(
        &'a self,
        _ctx: &'a CoreContext,
        reader: &'a GitRepoReader,
    ) -> Result<Self::Value, LoadableError> {
        load_git_tree(&self.0, reader)
            .await
            .map_err(LoadableError::from)
    }
}

#[derive(Clone, Debug)]
pub struct GitimportPreferences {
    pub dry_run: bool,
    /// Only for logging purpuses,
    /// useful when several repos are imported simultainously.
    pub gitrepo_name: Option<String>,
    pub concurrency: usize,
    pub lfs: GitImportLfs,
    pub git_command_path: PathBuf,
}

impl Default for GitimportPreferences {
    fn default() -> Self {
        GitimportPreferences {
            dry_run: false,
            gitrepo_name: None,
            concurrency: 20,
            lfs: GitImportLfs::default(),
            git_command_path: PathBuf::from("/usr/bin/git.real"),
        }
    }
}

pub fn oid_to_sha1(oid: &git_hash::oid) -> Result<hash::GitSha1, Error> {
    hash::GitSha1::from_bytes(oid.as_bytes())
}

/// Determines which commits to import
pub struct GitimportTarget {
    // If both are empty, we'll grab all commits
    // TODO: The None case is only used by Mononoke - see if we can get the Mononoke team
    // to let us remove this and just store the ObjectId directly
    wanted: Option<ObjectId>,
    known: HashMap<ObjectId, ChangesetId>,
}

impl GitimportTarget {
    /// Import the full repo
    pub fn full() -> Self {
        Self {
            wanted: None,
            known: HashMap::new(),
        }
    }

    pub fn new(wanted: ObjectId, known: HashMap<ObjectId, ChangesetId>) -> Result<Self, Error> {
        Ok(Self {
            wanted: Some(wanted),
            known,
        })
    }

    /// Roots are the Oid -> ChangesetId mappings that already are
    /// imported into Mononoke.
    pub fn get_roots(&self) -> &HashMap<ObjectId, ChangesetId> {
        &self.known
    }

    /// Returns true if wanted commit is already imported
    pub fn is_already_imported(&self) -> bool {
        if let Some(wanted) = self.wanted.as_ref() {
            self.known.contains_key(wanted)
        } else {
            false
        }
    }

    /// Returns the number of commits to import
    pub async fn get_nb_commits(
        &self,
        git_command_path: &Path,
        repo_path: &Path,
    ) -> Result<usize, Error> {
        let mut rev_list = self
            .build_rev_list(git_command_path, repo_path)
            .arg("--count")
            .spawn()?;

        self.write_filter_list(&mut rev_list).await?;

        // stdout is a single line that parses as number of commits
        let stdout = BufReader::new(rev_list.stdout.take().context("stdout not set up")?);
        let mut lines = stdout.lines();
        if let Some(line) = lines.next_line().await? {
            Ok(line.parse()?)
        } else {
            bail!("No lines returned by git rev-list");
        }
    }

    /// Returns a stream of commit hashes to import, ordered so that all
    /// of a commit's parents are listed first
    pub(crate) async fn list_commits(
        &self,
        git_command_path: &Path,
        repo_path: &Path,
    ) -> Result<impl Stream<Item = Result<ObjectId, Error>>, Error> {
        let mut rev_list = self
            .build_rev_list(git_command_path, repo_path)
            .arg("--topo-order")
            .arg("--reverse")
            .spawn()?;

        self.write_filter_list(&mut rev_list).await?;

        let stdout = BufReader::new(rev_list.stdout.take().context("stdout not set up")?);
        let lines_stream = LinesStream::new(stdout.lines());

        Ok(lines_stream.err_into().and_then(|line| async move {
            // rev-list with --boundary option returns boundary commits prefixed with `-`
            // here we remove that prefix to get uniformed list of commits
            line.replace('-', "")
                .parse()
                .context("Reading from git rev-list")
        }))
    }

    async fn write_filter_list(&self, rev_list: &mut Child) -> Result<(), Error> {
        if let Some(wanted) = self.wanted.as_ref() {
            let mut stdin = rev_list.stdin.take().context("stdin not set up properly")?;
            stdin.write_all(format!("{}\n", wanted).as_bytes()).await?;
            for commit in self.known.keys() {
                stdin.write_all(format!("^{}\n", commit).as_bytes()).await?;
            }
        }

        Ok(())
    }

    fn build_rev_list(&self, git_command_path: &Path, repo_path: &Path) -> Command {
        let mut command = Command::new(git_command_path);
        command
            .current_dir(repo_path)
            .env_clear()
            .kill_on_drop(false)
            .stdout(Stdio::piped())
            .arg("rev-list")
            .arg("--boundary");

        if self.wanted.is_none() {
            command.arg("--all").stdin(Stdio::null());
        } else {
            command.arg("--stdin").stdin(Stdio::piped());
        }
        command
    }
}

#[derive(Debug)]
pub struct TagMetadata {
    pub message: String,
    pub author: Option<String>,
    pub author_date: Option<DateTime>,
    pub name: String,
    pub pgp_signature: Option<Bytes>,
}

impl TagMetadata {
    pub async fn new(
        ctx: &CoreContext,
        oid: ObjectId,
        reader: &GitRepoReader,
    ) -> Result<Self, Error> {
        let Tag {
            name,
            mut tagger,
            message,
            mut pgp_signature,
            ..
        } = read_tag(reader, &oid).await?;

        let author_date = tagger
            .take()
            .map(|tagger| convert_time_to_datetime(&tagger.time))
            .transpose()?;
        let author = tagger
            .take()
            .map(|tagger| format_signature(tagger.to_ref()));
        let message = decode_message(&message, &None, ctx.logger())?;
        let name = decode_message(&name, &None, ctx.logger())?;
        let pgp_signature = pgp_signature
            .take()
            .map(|signature| Bytes::from(signature.to_vec()));
        Result::<_, Error>::Ok(TagMetadata {
            author,
            author_date,
            name,
            message,
            pgp_signature,
        })
    }
}

pub struct CommitMetadata {
    pub oid: ObjectId,
    pub parents: Vec<ObjectId>,
    pub message: String,
    pub author: String,
    pub author_date: DateTime,
    pub committer: String,
    pub committer_date: DateTime,
    pub git_extra_headers: SortedVectorMap<SmallVec<[u8; 24]>, Bytes>,
}

pub struct ExtractedCommit {
    pub metadata: CommitMetadata,
    pub tree: GitTree,
    pub parent_trees: HashSet<GitTree>,
    pub original_commit: Bytes,
}

pub(crate) async fn read_tag(reader: &GitRepoReader, oid: &git_hash::oid) -> Result<Tag, Error> {
    let object = reader.get_object(oid).await?;
    object
        .parsed
        .try_into_tag()
        .map_err(|_| format_err!("{} is not a tag", oid))
}

pub(crate) async fn read_commit(
    reader: &GitRepoReader,
    oid: &git_hash::oid,
) -> Result<Commit, Error> {
    let object = reader.get_object(oid).await?;
    object
        .parsed
        .try_into_commit()
        .map_err(|_| format_err!("{} is not a commit", oid))
}

pub(crate) async fn read_raw_object(
    reader: &GitRepoReader,
    oid: &git_hash::oid,
) -> Result<Bytes, Error> {
    reader
        .get_object(oid)
        .await
        .map(|obj| obj.raw)
        .with_context(|| format!("Error while fetching Git object for ID {}", oid))
}

fn format_signature(sig: git_actor::SignatureRef) -> String {
    format!("{} <{}>", sig.name, sig.email)
}

/// Decode a git commit message
///
/// Git choses to keep the raw user-provided bytes for the commit message.
/// That is to avoid a possibly lossy conversion to UTF-8.
/// Git provides an option to set the encoding by setting i18n.commitEncoding in .git/config.
/// See [the git documentation](https://git-scm.com/docs/git-commit#_discussion) for a discussion
/// of that design choice.
///
/// In contrast, mononoke stores commit messages in UTF-8.
///
/// This means that importing a git commit message can be lossy. For instance, if a git user used a
/// non-UTF-8 compatible encoding such as latin1, but didn't set the `commitEncoding` setting
/// accordingly, the conversion will be lossy.
/// These latin1-encoded bytes: `b"Hello, R\xe9mi-\xc9tienne!"` will convert to `"Hello, Rï¿½mi-ï¿½tienne!"`
/// if the encoding is not specified (so it will default to UTF-8).
fn decode_message(
    message: &[u8],
    encoding: &Option<BString>,
    logger: &Logger,
) -> Result<String, Error> {
    let encoding = Encoding::for_label(&encoding.clone().unwrap_or_else(|| BString::from("utf-8")))
        .ok_or_else(|| anyhow!("Failed to parse git commit encoding: {encoding:?}"))?;
    let (decoded, actual_encoding, replacement) = encoding.decode(message);
    let message = decoded.to_string();
    if actual_encoding != encoding {
        // Decode performs BOM sniffing to detect the actual encoding for this byte string.
        // We expect it to match the encoding declared in the commit metadata.
        bail!("Unexpected encoding: expected {encoding:?}, got {actual_encoding:?}");
    } else if replacement {
        // If the input string contains malformed sequences, they get replaced with the
        // REPLACEMENT CHARACTER.
        // In this situation, don't fail but log the occurrence.
        debug!(
            logger,
            "Failed to decode git message:\n{message:?}\nwith encoding: {encoding:?}.\nThe offending characters were replaced"
        );
    }
    Ok(message)
}

impl ExtractedCommit {
    pub async fn new(
        ctx: &CoreContext,
        oid: ObjectId,
        reader: &GitRepoReader,
    ) -> Result<Self, Error> {
        let Commit {
            tree,
            parents,
            author,
            committer,
            encoding,
            message,
            extra_headers,
            ..
        } = read_commit(reader, &oid).await?;

        let tree = GitTree(tree);

        let parent_trees = {
            let mut trees = HashSet::new();
            for parent in &parents {
                let commit = read_commit(reader, parent).await?;
                trees.insert(GitTree(commit.tree));
            }
            trees
        };

        let author_date = convert_time_to_datetime(&author.time)?;
        let committer_date = convert_time_to_datetime(&committer.time)?;
        let author = format_signature(author.to_ref());
        let committer = format_signature(committer.to_ref());
        let message = decode_message(&message, &encoding, ctx.logger())?;
        let parents = parents.into_vec();
        let git_extra_headers = extra_headers
            .into_iter()
            .map(|(k, v)| {
                (
                    SmallVec::from(k.as_slice()),
                    Bytes::copy_from_slice(v.as_slice()),
                )
            })
            .collect();
        let original_commit = read_raw_object(reader, &oid).await?;
        Result::<_, Error>::Ok(ExtractedCommit {
            original_commit,
            metadata: CommitMetadata {
                oid,
                parents,
                message,
                author,
                author_date,
                committer,
                committer_date,
                git_extra_headers,
            },
            tree,
            parent_trees,
        })
    }
}

pub fn convert_time_to_datetime(time: &git_actor::Time) -> Result<DateTime, Error> {
    DateTime::from_timestamp(
        time.seconds_since_unix_epoch.into(),
        -time.offset_in_seconds,
    )
}

#[async_trait]
pub trait GitUploader: Clone + Send + Sync + 'static {
    /// The type of a file change to be uploaded
    type Change: Clone + Send + Sync + 'static;

    /// The type of a changeset returned by generate_changeset
    type IntermediateChangeset: Send + Sync;

    /// Returns a change representing a deletion
    fn deleted() -> Self::Change;

    /// Looks to see if we can elide importing a commit
    /// If you can give us the ChangesetId for a given git object,
    /// then we assume that it's already imported and skip it
    async fn check_commit_uploaded(
        &self,
        ctx: &CoreContext,
        oid: &git_hash::oid,
    ) -> Result<Option<ChangesetId>, Error>;

    /// Upload a single file to the repo
    async fn upload_file(
        &self,
        ctx: &CoreContext,
        lfs: &GitImportLfs,
        path: &MPath,
        ty: FileType,
        oid: ObjectId,
        git_bytes: Bytes,
    ) -> Result<Self::Change, Error>;

    /// Upload a single git object to the repo blobstore of the mercurial mirror.
    /// Use this method for uploading non-blob git objects (e.g. tree, commit, etc)
    async fn upload_object(
        &self,
        ctx: &CoreContext,
        oid: ObjectId,
        git_bytes: Bytes,
    ) -> Result<(), Error>;

    /// Generate a single Bonsai changeset ID for corresponding Git commit
    /// This should delay saving the changeset if possible
    /// but may save it if required.
    ///
    /// You are guaranteed that all parents of the given changeset
    /// have been generated by this point.
    async fn generate_changeset_for_commit(
        &self,
        ctx: &CoreContext,
        bonsai_parents: Vec<ChangesetId>,
        metadata: CommitMetadata,
        changes: SortedVectorMap<MPath, Self::Change>,
        dry_run: bool,
    ) -> Result<(Self::IntermediateChangeset, ChangesetId), Error>;

    /// Generate a single Bonsai changeset ID for corresponding Git
    /// annotated tag.
    async fn generate_changeset_for_annotated_tag(
        &self,
        ctx: &CoreContext,
        target_changeset_id: ChangesetId,
        tag: TagMetadata,
    ) -> Result<ChangesetId, Error>;

    /// Finalize a batch of generated changesets. The supplied batch is
    /// topologically sorted so that parents are all present before children
    /// If you did not finalize the changeset in generate_changeset,
    /// you must do so here.
    async fn finalize_batch(
        &self,
        ctx: &CoreContext,
        dry_run: bool,
        changesets: Vec<(Self::IntermediateChangeset, hash::GitSha1)>,
    ) -> Result<(), Error>;
}

#[cfg(test)]
mod tests {
    use slog::o;

    use super::decode_message;
    use super::BString;
    use super::Logger;

    const ASCII_BSTR: &[u8] = b"Hello, World!".as_slice();
    const ASCII_STR: &str = "Hello, World!";
    const UTF8_UNICODE_BSTR: &[u8] =
        b"Hello, \xce\xba\xe1\xbd\xb9\xcf\x83\xce\xbc\xce\xB5!".as_slice();
    const UTF8_UNICODE_STR: &str = "Hello, Îºá½¹ÏƒÎ¼Îµ!";
    const LATIN1_ACCENTED_BSTR: &[u8] = b"Hello, R\xe9mi-\xc9tienne!".as_slice();
    const UTF8_ACCENTED_BSTR: &[u8] = b"Hello, R\xc3\xa9mi-\xc3\x89tienne!".as_slice();
    const BROKEN_LATIN1_FROM_UTF8_ACCENTED_STR: &str = "Hello, RÃƒÂ©mi-Ãƒâ€°tienne!";
    const UTF8_ACCENTED_STR: &str = "Hello, RÃ©mi-Ã‰tienne!";
    const UTF8_ACCENTED_STR_WITH_REPLACEMENT_CHARACTER: &str = "Hello, Rï¿½mi-ï¿½tienne!";

    fn should_decode_into(message: &[u8], encoding: &Option<BString>, expected: &str) {
        let logger = Logger::root(slog::Discard, o!());
        let m = decode_message(message, encoding, &logger);
        assert!(m.is_ok());
        assert_eq!(expected, &m.unwrap())
    }
    fn should_fail_to_decode(message: &[u8], encoding: &Option<BString>) {
        let logger = Logger::root(slog::Discard, o!());
        let m = decode_message(message, encoding, &logger);
        assert!(m.is_err());
    }

    #[test]
    fn test_decode_commit_message_given_invalid_encoding_should_fail() {
        should_fail_to_decode(
            ASCII_BSTR,
            &Some(BString::from("not a valid encoding label")),
        );
    }
    #[test]
    fn test_decode_commit_message_given_ascii_as_utf8() {
        for encoding in [None, Some(BString::from("utf-8"))] {
            should_decode_into(ASCII_BSTR, &encoding, ASCII_STR);
        }
    }
    #[test]
    fn test_decode_commit_message_given_valid_utf8() {
        for encoding in [None, Some(BString::from("utf-8"))] {
            should_decode_into(UTF8_UNICODE_BSTR, &encoding, UTF8_UNICODE_STR);
            should_decode_into(UTF8_ACCENTED_BSTR, &encoding, UTF8_ACCENTED_STR);
        }
    }
    #[test]
    fn test_decode_commit_message_given_malformed_utf8() {
        for encoding in [None, Some(BString::from("utf-8"))] {
            should_decode_into(
                LATIN1_ACCENTED_BSTR,
                &encoding,
                UTF8_ACCENTED_STR_WITH_REPLACEMENT_CHARACTER,
            );
        }
    }
    #[test]
    fn test_decode_commit_message_given_valid_latin1() {
        should_decode_into(
            LATIN1_ACCENTED_BSTR,
            &Some(BString::from("iso-8859-1")),
            UTF8_ACCENTED_STR,
        );
    }
    #[test]
    fn test_decode_commit_message_given_malformed_latin1() {
        should_decode_into(
            UTF8_ACCENTED_BSTR,
            &Some(BString::from("iso-8859-1")),
            BROKEN_LATIN1_FROM_UTF8_ACCENTED_STR,
        );
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use core::future::Future;
use std::str;
use std::sync::Arc;

use anyhow::format_err;
use anyhow::Error;
use bytes::Bytes;
use context::CoreContext;
use filestore::StoreRequest;
use futures::stream;
use futures::Stream;
use futures::StreamExt;
use futures::TryStreamExt;
use git_hash::ObjectId;
use http::Uri;
use hyper::body;
use hyper::client::connect::HttpConnector;
use hyper::Client;
use hyper::StatusCode;
use hyper_openssl::HttpsConnector;
use mononoke_types::hash;
use rand::thread_rng;
use rand::Rng;
use slog::error;
use slog::warn;
use tokio::sync::Semaphore;
use tokio::time::sleep;
use tokio::time::Duration;
/// We will not try to parse any file bigger then this.
/// Any valid gitlfs metadata file should be smaller then this.
const MAX_METADATA_LENGTH: usize = 511;

/// Module to be passed into gitimport that defines how LFS files are imported.
/// The default will disable any LFS support (and the metadata of files pointing to LFS files
/// will be imported, this means that the mononoke repo will mirror the git-repo).
/// Autodetect and each file under MAX_METADATA_LENGTH will be scanned, and if it matched git-lfs
/// metadata file, then the configured lfs_server will be used to try and fetch the data.
#[derive(Debug)]
pub struct GitImportLfsInner {
    /// Server information.
    lfs_server: String,
    /// How to deal with the case when the file does not exist on the LFS server.
    /// allow_not_found=false
    ///   A non existing LFS file considerd unrecoverable error and bail out
    /// allow_not_found=true
    ///   put the content of the LFS-metafile in its place, and print a warning.
    allow_not_found: bool,
    /// Retries.
    max_attempts: u32,
    time_ms_between_attempts: u32,
    /// Limit the amount of simultaneous connections.
    conn_limit_sem: Option<Arc<Semaphore>>,
    /// Hyperium client we use to connect with
    client: Client<HttpsConnector<HttpConnector>>,
}

#[derive(Clone, Debug, Default)]
pub struct GitImportLfs {
    inner: Option<Arc<GitImportLfsInner>>,
}

#[derive(Debug)]
pub struct LfsMetaData {
    pub version: String,
    pub sha256: hash::Sha256,
    pub size: u64,
    /// gitblob and gitid, where this metadata comes from. This is useful if we
    /// end up storing the metadata instead of the content (if the content cannot
    /// be found on the LFS server for example).
    pub gitblob: Vec<u8>,
    pub gitid: ObjectId,
}

/// Layout of the metafiles:
/// | version https://git-lfs.github.com/spec/v1
/// | oid sha256:73e2200459562bb068f08e33210ed106014b877f878932b2147991e17a7c089b
/// | size 8423391
fn parse_lfs_metafile(gitblob: &[u8], gitid: ObjectId) -> Option<LfsMetaData> {
    if gitblob.len() > MAX_METADATA_LENGTH {
        return None;
    }

    let mut lines = str::from_utf8(gitblob).ok()?.lines();
    let version = lines.next()?.strip_prefix("version ")?;
    if version != "https://git-lfs.github.com/spec/v1" {
        return None;
    }
    let sha256 = lines
        .next()?
        .strip_prefix("oid sha256:")?
        .parse::<hash::Sha256>()
        .ok()?;
    let size = lines.next()?.strip_prefix("size ")?.parse::<u64>().ok()?;
    // As a precaution. If we have an additional line after this, then we assume its not a valid file.
    if lines.next().is_some() {
        return None;
    }
    Some(LfsMetaData {
        version: version.to_string(),
        sha256,
        size,
        gitblob: gitblob.to_vec(),
        gitid,
    })
}

impl GitImportLfs {
    pub fn new_disabled() -> Self {
        GitImportLfs { inner: None }
    }
    pub fn new(
        lfs_server: String,
        allow_not_found: bool,
        conn_limit: Option<usize>,
    ) -> Result<Self, Error> {
        let connector = HttpsConnector::new().map_err(Error::from)?;
        let client: Client<_, body::Body> = Client::builder().build(connector);
        let inner = GitImportLfsInner {
            lfs_server,
            allow_not_found,
            max_attempts: 30,
            time_ms_between_attempts: 10000,
            conn_limit_sem: conn_limit.map(|x| Arc::new(Semaphore::new(x))),
            client,
        };
        Ok(GitImportLfs {
            inner: Some(Arc::new(inner)),
        })
    }

    pub fn is_lfs_file(&self, gitblob: &[u8], gitid: ObjectId) -> Option<LfsMetaData> {
        if self.inner.is_some() {
            parse_lfs_metafile(gitblob, gitid)
        } else {
            None
        }
    }

    /// Download the LFS file. This works fine with Dewey but should be improved to work
    /// with other backends as well.
    async fn fetch_bytes_internal(
        &self,
        ctx: &CoreContext,
        metadata: &LfsMetaData,
    ) -> Result<
        (
            StoreRequest,
            impl Stream<Item = Result<Bytes, Error>> + Unpin,
        ),
        Error,
    > {
        let inner = self.inner.as_ref().ok_or_else(|| {
            format_err!("GitImportLfs::fetch_bytes_internal called on disabled GitImportLfs")
        })?;

        let uri = [&inner.lfs_server, "/", &metadata.sha256.to_string()]
            .concat()
            .parse::<Uri>()?;
        let resp = inner.client.get(uri.clone()).await?;

        if resp.status().is_success() {
            let bytes = resp.into_body().map_err(Error::from);
            let sr = StoreRequest::with_sha256(metadata.size, metadata.sha256);
            return Ok((sr, bytes.left_stream()));
        }
        if resp.status() == StatusCode::NOT_FOUND && inner.allow_not_found {
            warn!(
                ctx.logger(),
                "{} not found. Using gitlfs metadata as file content instead.", uri,
            );
            let bytes = Bytes::copy_from_slice(&metadata.gitblob);
            let size = metadata.gitblob.len().try_into()?;
            let git_sha1 = hash::RichGitSha1::from_bytes(
                Bytes::copy_from_slice(metadata.gitid.as_bytes()),
                "blob",
                size,
            )?;
            let sr = StoreRequest::with_git_sha1(size, git_sha1);
            return Ok((sr, stream::once(futures::future::ok(bytes)).right_stream()));
        }
        Err(format_err!("{} response {:?}", uri, resp))
    }

    async fn fetch_bytes(
        &self,
        ctx: &CoreContext,
        metadata: &LfsMetaData,
    ) -> Result<(StoreRequest, impl Stream<Item = Result<Bytes, Error>>), Error> {
        let inner = self.inner.as_ref().ok_or_else(|| {
            format_err!("GitImportLfs::fetch_bytes called on disabled GitImportLfs")
        })?;

        let mut attempt: u32 = 0;
        loop {
            let r = self.fetch_bytes_internal(ctx, metadata).await;
            match r {
                Ok(res) => {
                    return Ok(res);
                }
                Err(err) => {
                    if attempt >= inner.max_attempts {
                        return Err(err);
                    }

                    attempt += 1;
                    // Sleep on avarage time_ms_between_attempts between attempts.
                    let sleep_time_ms =
                        thread_rng().gen_range(0..inner.time_ms_between_attempts * 2);
                    error!(
                        ctx.logger(),
                        "{}. Attempt {} of {} - Retrying in {} ms",
                        err,
                        attempt,
                        inner.max_attempts,
                        sleep_time_ms,
                    );
                    sleep(Duration::from_millis(sleep_time_ms.into())).await;
                }
            }
        }
    }

    pub async fn with<F, T, Fut>(
        self,
        ctx: CoreContext,
        metadata: LfsMetaData,
        f: F,
    ) -> Result<T, Error>
    where
        F: FnOnce(
                CoreContext,
                LfsMetaData,
                StoreRequest,
                Box<dyn Stream<Item = Result<Bytes, Error>> + Send + Unpin>,
            ) -> Fut
            + Send
            + 'static,
        T: Send + Sync + 'static,
        Fut: Future<Output = Result<T, Error>> + Send,
    {
        tokio::spawn(async move {
            let inner = self.inner.as_ref().ok_or_else(|| {
                format_err!("GitImportLfs::fetch_bytes_internal called on disabled GitImportLfs")
            })?;

            // If configured a connection limit, grab semaphore lock enforcing it.
            let _slock = if let Some(semaphore) = &inner.conn_limit_sem {
                Some(semaphore.clone().acquire_owned().await?)
            } else {
                None
            };

            let (req, bstream) = self.fetch_bytes(&ctx, &metadata).await?;
            f(ctx, metadata, req, Box::new(bstream)).await
        })
        .await?
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;
use std::future::Future;
use std::path::Path;
use std::process::Stdio;
use std::sync::Arc;
use std::sync::Mutex;

use anyhow::anyhow;
use anyhow::bail;
use anyhow::Context;
use anyhow::Result;
use bytes::Bytes;
use git_hash::ObjectId;
use git_object::Kind;
use git_object::Object;
use git_object::ObjectRef;
use tokio::io::AsyncBufReadExt;
use tokio::io::AsyncReadExt;
use tokio::io::AsyncWriteExt;
use tokio::io::BufReader;
use tokio::process::ChildStdout;
use tokio::process::Command;
use tokio::sync::mpsc;
use tokio::sync::oneshot;

type ObjectSender = oneshot::Sender<Result<ObjectContent>>;

#[derive(Clone, Debug)]
pub struct ObjectContent {
    pub parsed: Object,
    pub raw: Bytes,
}

/// Uses `git-cat-file` to read a git repository's ODB directly
#[derive(Clone)]
pub struct GitRepoReader {
    send_request: mpsc::Sender<ObjectId>,
    outstanding_requests: Arc<Mutex<HashMap<ObjectId, Vec<ObjectSender>>>>,
}

impl GitRepoReader {
    /// Create a new repo reader for the repo at `repo_path`, using `git_command_path`
    /// as `git`
    pub async fn new(git_command_path: &Path, repo_path: &Path) -> Result<Self> {
        let mut batch_cat_file = Command::new(git_command_path)
            .current_dir(repo_path)
            .env_clear()
            // We expect dropping to close stdin, which will cause git cat-file
            // to quit
            .kill_on_drop(false)
            .stdin(Stdio::piped())
            .stdout(Stdio::piped())
            .arg("cat-file")
            .arg("--batch")
            .arg("--unordered")
            .spawn()?;

        let outstanding_requests: Arc<Mutex<HashMap<ObjectId, _>>> =
            Arc::new(Mutex::new(HashMap::new()));

        {
            let outstanding_requests = outstanding_requests.clone();
            tokio::spawn(read_objects_task(
                outstanding_requests,
                batch_cat_file
                    .stdout
                    .take()
                    .context("stdout not set up properly")?,
            ))
        };

        // Channel is used so that we don't have issues around ownership of stdin in `get_object`
        let (send_request, mut recv_request) = mpsc::channel(1);
        {
            let outstanding_requests = outstanding_requests.clone();
            let mut cat_file_stdin = batch_cat_file
                .stdin
                .take()
                .context("stdin not set up properly")?;
            tokio::spawn(async move {
                while let Some(object) = recv_request.recv().await {
                    if cat_file_stdin
                        .write_all(format!("{}\n", object).as_bytes())
                        .await
                        .is_err()
                    {
                        recv_request.close();
                        // stdin is gone - we can't continue
                        let mut outstanding_requests =
                            outstanding_requests.lock().expect("lock poisoned");
                        for (_, queue) in outstanding_requests.drain() {
                            for sender in queue {
                                let _ = sender.send(Err(anyhow!("git cat-file stdin closed")));
                            }
                        }
                    }
                }
            })
        };

        Ok(Self {
            send_request,
            outstanding_requests,
        })
    }

    /// Read `oid` from the git store
    pub fn get_object(&self, oid: &git_hash::oid) -> impl Future<Output = Result<ObjectContent>> {
        let outstanding_requests = self.outstanding_requests.clone();
        let send_request = self.send_request.clone();
        let oid = oid.to_owned();
        async move {
            let permit = send_request
                .reserve()
                .await
                .context("get_object: failed to send request")?;
            let (sender, recv) = oneshot::channel();
            outstanding_requests
                .lock()
                .expect("lock poisoned")
                .entry(oid.to_owned())
                .or_default()
                .push(sender);

            permit.send(oid);

            recv.await.context("get_object: received an error")?
        }
    }
}

fn parse_kind_and_size(header: &str) -> Result<(Kind, usize)> {
    if header == "missing" {
        bail!("Object is missing");
    }
    let (kind, size) = header
        .split_once(' ')
        .context("Expected object kind and size")?;

    Ok((kind_str_to_kind(kind)?, size.trim_end().parse()?))
}

fn kind_str_to_kind(kind: &str) -> Result<Kind> {
    let kind = match kind {
        "tree" => Kind::Tree,
        "blob" => Kind::Blob,
        "commit" => Kind::Commit,
        "tag" => Kind::Tag,
        _ => bail!("Object kind {} unknown", kind),
    };
    Ok(kind)
}

fn parse_cat_header(header: &str) -> Result<(ObjectId, Result<(Kind, usize)>)> {
    let (oid, content_type) = header.split_once(' ').context("No git object id")?;
    Ok((oid.parse()?, parse_kind_and_size(content_type)))
}

fn convert_to_object(kind: Kind, size: usize, bytes: Vec<u8>) -> Result<ObjectContent> {
    let object_ref = ObjectRef::from_bytes(kind, &bytes).with_context(|| {
        format!(
            "Failed to parse:\n```\n{}\n```\ninto object of kind {:?}",
            String::from_utf8_lossy(&bytes),
            kind
        )
    })?;
    let mut raw = format!("{} {}\x00", kind, size).into_bytes();
    raw.append(&mut bytes.clone());

    Ok(ObjectContent {
        parsed: object_ref.into_owned(),
        raw: Bytes::from(raw),
    })
}

async fn read_objects_task(
    outstanding_requests: Arc<Mutex<HashMap<ObjectId, Vec<ObjectSender>>>>,
    cat_file_stdout: ChildStdout,
) -> Result<()> {
    let mut reader = BufReader::new(cat_file_stdout);
    let mut buf = String::new();

    while reader.read_line(&mut buf).await? > 0 {
        // buf contains either "oid missing" for a missing object or "oid type size" for a present object
        // If it doesn't give us an ObjectId, all we can do is loop on until we find a header again.
        if let Ok((oid, maybe_details)) = parse_cat_header(&buf) {
            // If no-one is waiting on this ObjectId, we can't report the object state,
            // but still need to follow protocol to clear it out.
            let maybe_sender = {
                // Take at most one waiter - we send the request once for each reader
                // so we'll get it re-read from disk for the next one in the queue
                let mut outstanding_requests = outstanding_requests.lock().expect("lock poisoned");
                outstanding_requests.remove(&oid).and_then(|mut queue| {
                    let item = queue.pop();
                    if !queue.is_empty() {
                        outstanding_requests.insert(oid.clone(), queue);
                    }
                    item
                })
            };

            // If the header doesn't indicate an object, we send the error onwards and loop again
            let (kind, size) = match maybe_details {
                Ok(d) => d,
                Err(e) => {
                    if let Some(sender) = maybe_sender {
                        let _ = sender.send(
                            Err(e)
                                .with_context(|| format!("read_objects_task failed for {}", &buf)),
                        );
                    }
                    continue;
                }
            };

            // We have an object. Let's try and read it.

            // We need to read size bytes, and then send it on as an object to unblock our listener
            let mut bytes = Vec::new();
            bytes.resize(size, 0u8);
            reader
                .read_exact(&mut bytes)
                .await
                .with_context(|| format!("failed to read exactly {} bytes", size))?;
            if let Some(sender) = maybe_sender {
                let object = convert_to_object(kind, size, bytes);
                let _ = sender.send(object);
            }
            // Finally, there's an empty line after the object, but before the next header. Consume it
            reader
                .read_line(&mut buf)
                .await
                .context("expected an empty line after the object but before the next header")?;
            buf.clear();
        }
    }

    Ok(())
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn test_kind_str_to_kind() {
        assert!(kind_str_to_kind("forest").is_err());
        assert_eq!(kind_str_to_kind("blob").unwrap(), Kind::Blob);
        assert_eq!(kind_str_to_kind("commit").unwrap(), Kind::Commit);
        assert_eq!(kind_str_to_kind("tag").unwrap(), Kind::Tag);
        assert_eq!(kind_str_to_kind("tree").unwrap(), Kind::Tree);
    }

    #[test]
    fn test_parse_kind_and_size() {
        assert!(parse_kind_and_size("missing").is_err());
        assert_eq!(
            parse_kind_and_size("tree 12345").unwrap(),
            (Kind::Tree, 12345)
        );
    }

    #[test]
    fn test_parse_cat_header() {
        assert!(parse_cat_header("I am a fish").is_err());
        let (oid, kind_and_size) =
            parse_cat_header("99cd00206e418c5fb0e9bd885ded84b8781194b7 tag 682\n").unwrap();
        assert_eq!(
            oid,
            ObjectId::from_hex(b"99cd00206e418c5fb0e9bd885ded84b8781194b7").unwrap()
        );
        assert_eq!(kind_and_size.unwrap(), (Kind::Tag, 682));
    }
}