/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::BTreeMap;
use std::sync::Arc;

use anyhow::anyhow;
use anyhow::format_err;
use anyhow::Context;
use anyhow::Error;
use backsyncer::format_counter as format_backsyncer_counter;
use blobrepo::save_bonsai_changesets;
use blobstore::Loadable;
use bookmarks::BookmarkKey;
use bookmarks::BookmarkUpdateLogRef;
use bookmarks::BookmarkUpdateReason;
use bookmarks::BookmarksRef;
use bookmarks::Freshness;
use cached_config::ConfigStore;
use clap_old::App;
use clap_old::Arg;
use clap_old::ArgMatches;
use clap_old::SubCommand;
use cmdlib::args;
use cmdlib::args::MononokeMatches;
use cmdlib::helpers;
use context::CoreContext;
use cross_repo_sync::create_commit_syncer_lease;
use cross_repo_sync::create_commit_syncers;
use cross_repo_sync::types::Large;
use cross_repo_sync::types::Small;
use cross_repo_sync::validation;
use cross_repo_sync::validation::BookmarkDiff;
use cross_repo_sync::CommitSyncContext;
use cross_repo_sync::CommitSyncRepos;
use cross_repo_sync::CommitSyncer;
use cross_repo_sync::PluralCommitSyncOutcome;
use cross_repo_sync::Syncers;
use cross_repo_sync::CHANGE_XREPO_MAPPING_EXTRA;
use fbinit::FacebookInit;
use filestore::FilestoreConfigRef;
use futures::stream;
use futures::try_join;
use futures::TryFutureExt;
use itertools::Itertools;
use live_commit_sync_config::CfgrLiveCommitSyncConfig;
use live_commit_sync_config::LiveCommitSyncConfig;
use maplit::btreemap;
use maplit::hashmap;
use maplit::hashset;
use metaconfig_types::CommitSyncConfig;
use metaconfig_types::CommitSyncConfigVersion;
use metaconfig_types::CommonCommitSyncConfig;
use metaconfig_types::DefaultSmallToLargeCommitSyncPathAction;
use mononoke_api_types::InnerRepo;
use mononoke_types::BonsaiChangesetMut;
use mononoke_types::ChangesetId;
use mononoke_types::DateTime;
use mononoke_types::FileChange;
use mononoke_types::FileType;
use mononoke_types::MPath;
use mononoke_types::RepositoryId;
use mutable_counters::MutableCountersRef;
use pushrebase::do_pushrebase_bonsai;
use pushrebase::FAIL_PUSHREBASE_EXTRA;
use repo_blobstore::RepoBlobstoreRef;
use repo_identity::RepoIdentityRef;
use slog::info;
use slog::warn;
use slog::Logger;
use sorted_vector_map::sorted_vector_map;
use synced_commit_mapping::EquivalentWorkingCopyEntry;
use synced_commit_mapping::SqlSyncedCommitMapping;
use synced_commit_mapping::SyncedCommitMapping;
use synced_commit_mapping::SyncedCommitMappingEntry;

use crate::common::get_source_target_repos_and_mapping;
use crate::error::SubcommandError;

pub const CROSSREPO: &str = "crossrepo";
const AUTHOR_ARG: &str = "author";
const ONCALL_ARG: &str = "oncall";
const DUMP_MAPPING_LARGE_REPO_PATH_ARG: &str = "dump-mapping-large-repo-path";
const MAP_SUBCOMMAND: &str = "map";
const PREPARE_ROLLOUT_SUBCOMMAND: &str = "prepare-rollout";
const PUSHREDIRECTION_SUBCOMMAND: &str = "pushredirection";
const VERIFY_WC_SUBCOMMAND: &str = "verify-wc";
const VERIFY_BOOKMARKS_SUBCOMMAND: &str = "verify-bookmarks";
const HASH_ARG: &str = "HASH";
const LARGE_REPO_HASH_ARG: &str = "large-repo-hash";
const UPDATE_LARGE_REPO_BOOKMARKS: &str = "update-large-repo-bookmarks";
const LARGE_REPO_BOOKMARK_ARG: &str = "large-repo-bookmark";
const CHANGE_MAPPING_VERSION_SUBCOMMAND: &str = "change-mapping-version";
const INSERT_SUBCOMMAND: &str = "insert";
const REWRITTEN_SUBCOMMAND: &str = "rewritten";
const EQUIVALENT_WORKING_COPY_SUBCOMMAND: &str = "equivalent-working-copy";
const NOT_SYNC_CANDIDATE_SUBCOMMAND: &str = "not-sync-candidate";
const SOURCE_HASH_ARG: &str = "source-hash";
const TARGET_HASH_ARG: &str = "target-hash";
const VIA_EXTRAS_ARG: &str = "via-extra";

const SUBCOMMAND_CONFIG: &str = "config";
const SUBCOMMAND_BY_VERSION: &str = "by-version";
const SUBCOMMAND_LIST: &str = "list";
const ARG_VERSION_NAME: &str = "version-name";
const ARG_WITH_CONTENTS: &str = "with-contents";

// TODO: Move to its own repo type to deblobrepoify
type CrossRepo = InnerRepo;

pub async fn subcommand_crossrepo<'a>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'_>,
    sub_m: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
    let config_store = matches.config_store();
    let live_commit_sync_config = CfgrLiveCommitSyncConfig::new(&logger, config_store)?;

    let ctx = CoreContext::new_with_logger(fb, logger.clone());
    match sub_m.subcommand() {
        (MAP_SUBCOMMAND, Some(sub_sub_m)) => {
            let (source_repo, target_repo, mapping) =
                get_source_target_repos_and_mapping::<CrossRepo>(fb, logger, matches).await?;

            let common_config =
                live_commit_sync_config.get_common_config(source_repo.repo_identity().id())?;
            let commit_sync_repos = CommitSyncRepos::new(source_repo, target_repo, &common_config)?;
            let live_commit_sync_config: Arc<dyn LiveCommitSyncConfig> =
                Arc::new(live_commit_sync_config);

            let caching = matches.caching();
            let x_repo_syncer_lease = create_commit_syncer_lease(ctx.fb, caching)?;

            let commit_syncer = CommitSyncer::new(
                &ctx,
                mapping,
                commit_sync_repos,
                live_commit_sync_config,
                x_repo_syncer_lease,
            );
            let hash = sub_sub_m.value_of(HASH_ARG).unwrap().to_owned();
            subcommand_map(ctx, commit_syncer, hash).await
        }
        (VERIFY_WC_SUBCOMMAND, Some(sub_sub_m)) => {
            let (source_repo, target_repo, mapping) =
                get_source_target_repos_and_mapping(fb, logger, matches).await?;

            let live_commit_sync_config: Arc<dyn LiveCommitSyncConfig> =
                Arc::new(live_commit_sync_config);
            let commit_syncer = get_large_to_small_commit_syncer(
                &ctx,
                source_repo,
                target_repo,
                live_commit_sync_config.clone(),
                mapping,
                matches,
            )
            .await?;

            let large_hash = {
                let large_hash = sub_sub_m.value_of(LARGE_REPO_HASH_ARG).unwrap().to_owned();
                let large_repo = commit_syncer.get_large_repo();
                helpers::csid_resolve(&ctx, large_repo, large_hash).await?
            };

            validation::verify_working_copy_fast_path(
                &ctx,
                &commit_syncer,
                large_hash,
                live_commit_sync_config,
            )
            .await
            .map_err(|e| e.into())
        }
        (VERIFY_BOOKMARKS_SUBCOMMAND, Some(sub_sub_m)) => {
            let (source_repo, target_repo, mapping) =
                get_source_target_repos_and_mapping(fb, logger, matches).await?;

            let update_large_repo_bookmarks = sub_sub_m.is_present(UPDATE_LARGE_REPO_BOOKMARKS);

            subcommand_verify_bookmarks(
                ctx,
                source_repo,
                target_repo,
                mapping,
                update_large_repo_bookmarks,
                Arc::new(live_commit_sync_config),
                matches,
            )
            .await
        }
        (SUBCOMMAND_CONFIG, Some(sub_sub_m)) => {
            run_config_sub_subcommand(matches, sub_sub_m, live_commit_sync_config).await
        }
        (PUSHREDIRECTION_SUBCOMMAND, Some(sub_sub_m)) => {
            run_pushredirection_subcommand(
                fb,
                ctx,
                matches,
                sub_sub_m,
                config_store,
                live_commit_sync_config,
            )
            .await
        }
        (INSERT_SUBCOMMAND, Some(sub_sub_m)) => {
            run_insert_subcommand(ctx, matches, sub_sub_m, live_commit_sync_config).await
        }
        _ => Err(SubcommandError::InvalidArgs),
    }
}

async fn run_config_sub_subcommand<'a>(
    matches: &'a MononokeMatches<'_>,
    config_subcommand_matches: &'a ArgMatches<'a>,
    live_commit_sync_config: CfgrLiveCommitSyncConfig,
) -> Result<(), SubcommandError> {
    let config_store = matches.config_store();

    let repo_id = args::not_shardmanager_compatible::get_repo_id(config_store, matches)?;

    match config_subcommand_matches.subcommand() {
        (SUBCOMMAND_BY_VERSION, Some(sub_m)) => {
            let version_name: String = sub_m.value_of(ARG_VERSION_NAME).unwrap().to_string();
            subcommand_by_version(repo_id, live_commit_sync_config, version_name)
                .await
                .map_err(|e| e.into())
        }
        (SUBCOMMAND_LIST, Some(sub_m)) => {
            let with_contents = sub_m.is_present(ARG_WITH_CONTENTS);
            subcommand_list(repo_id, live_commit_sync_config, with_contents)
                .await
                .map_err(|e| e.into())
        }
        _ => Err(SubcommandError::InvalidArgs),
    }
}

async fn run_pushredirection_subcommand<'a>(
    fb: FacebookInit,
    ctx: CoreContext,
    matches: &'a MononokeMatches<'_>,
    config_subcommand_matches: &'a ArgMatches<'a>,
    config_store: &ConfigStore,
    live_commit_sync_config: CfgrLiveCommitSyncConfig,
) -> Result<(), SubcommandError> {
    let (source_repo, target_repo, mapping) =
        get_source_target_repos_and_mapping(fb, ctx.logger().clone(), matches).await?;

    let live_commit_sync_config: Arc<dyn LiveCommitSyncConfig> = Arc::new(live_commit_sync_config);

    match config_subcommand_matches.subcommand() {
        (PREPARE_ROLLOUT_SUBCOMMAND, Some(_sub_m)) => {
            let commit_syncer = get_large_to_small_commit_syncer(
                &ctx,
                source_repo,
                target_repo,
                live_commit_sync_config.clone(),
                mapping,
                matches,
            )
            .await?;

            if live_commit_sync_config.push_redirector_enabled_for_public(
                commit_syncer.get_small_repo().repo_identity().id(),
            ) {
                return Err(format_err!(
                    "not allowed to run {} if pushredirection is enabled",
                    PREPARE_ROLLOUT_SUBCOMMAND
                )
                .into());
            }

            let small_repo = commit_syncer.get_small_repo();
            let large_repo = commit_syncer.get_large_repo();
            let largest_id = large_repo
                .bookmark_update_log()
                .get_largest_log_id(ctx.clone(), Freshness::MostRecent)
                .await?
                .ok_or_else(|| anyhow!("No bookmarks update log entries for large repo"))?;

            let counter = format_backsyncer_counter(&large_repo.repo_identity().id());
            info!(
                ctx.logger(),
                "setting value {} to counter {} for repo {}",
                largest_id,
                counter,
                small_repo.repo_identity().id()
            );
            let res = small_repo
                .mutable_counters()
                .set_counter(
                    &ctx,
                    &counter,
                    largest_id.try_into().unwrap(),
                    None, // prev_value
                )
                .await?;

            if !res {
                return Err(anyhow!("failed to set backsyncer counter").into());
            }
            info!(ctx.logger(), "successfully updated the counter");

            Ok(())
        }
        (CHANGE_MAPPING_VERSION_SUBCOMMAND, Some(sub_m)) => {
            let commit_syncer = get_large_to_small_commit_syncer(
                &ctx,
                source_repo,
                target_repo,
                live_commit_sync_config.clone(),
                mapping,
                matches,
            )
            .await?;

            if sub_m.is_present(VIA_EXTRAS_ARG) {
                change_mapping_via_extras(
                    &ctx,
                    matches,
                    sub_m,
                    &commit_syncer,
                    config_store,
                    &live_commit_sync_config,
                )
                .await?;
                return Ok(());
            }

            if live_commit_sync_config.push_redirector_enabled_for_public(
                commit_syncer.get_small_repo().repo_identity().id(),
            ) {
                return Err(format_err!(
                    "not allowed to run {} if pushredirection is enabled",
                    CHANGE_MAPPING_VERSION_SUBCOMMAND
                )
                .into());
            }

            let large_bookmark = Large(
                sub_m
                    .value_of(LARGE_REPO_BOOKMARK_ARG)
                    .map(BookmarkKey::new)
                    .transpose()?
                    .ok_or_else(|| format_err!("{} is not specified", LARGE_REPO_BOOKMARK_ARG))?,
            );
            let small_bookmark = Small(
                commit_syncer.get_bookmark_renamer().await?(&large_bookmark).ok_or_else(|| {
                    format_err!("{} bookmark doesn't remap to small repo", large_bookmark)
                })?,
            );

            let large_repo = Large(commit_syncer.get_large_repo());
            let small_repo = Small(commit_syncer.get_small_repo());
            let large_bookmark_value =
                Large(get_bookmark_value(&ctx, &large_repo, &large_bookmark).await?);
            let small_bookmark_value =
                Small(get_bookmark_value(&ctx, &small_repo, &small_bookmark).await?);

            let mapping_version = sub_m
                .value_of(ARG_VERSION_NAME)
                .ok_or_else(|| format_err!("{} is not specified", ARG_VERSION_NAME))?;
            let mapping_version = CommitSyncConfigVersion(mapping_version.to_string());
            if !commit_syncer.version_exists(&mapping_version).await? {
                return Err(format_err!("{} version does not exist", mapping_version).into());
            }

            let dump_mapping_file = sub_m
                .value_of(DUMP_MAPPING_LARGE_REPO_PATH_ARG)
                .map(MPath::new)
                .transpose()?;

            let large_cs_id = create_commit_for_mapping_change(
                &ctx,
                sub_m,
                &large_repo,
                &small_repo,
                &large_bookmark_value,
                &mapping_version,
                MappingCommitOptions {
                    add_mapping_change_extra: false,
                    dump_mapping_file,
                },
                &commit_syncer,
                &live_commit_sync_config,
            )
            .await?;

            let maybe_rewritten_small_cs_id = commit_syncer
                .unsafe_always_rewrite_sync_commit(
                    &ctx,
                    large_cs_id.0,
                    Some(hashmap! {
                      large_bookmark_value.0.clone() => small_bookmark_value.0.clone(),
                    }),
                    &mapping_version,
                    CommitSyncContext::AdminChangeMapping,
                )
                .await?;

            let rewritten_small_cs_id = Small(maybe_rewritten_small_cs_id.ok_or_else(|| {
                format_err!("{} was rewritten into non-existent commit", large_cs_id)
            })?);

            let f1 = move_bookmark(
                &ctx,
                &large_repo,
                &large_bookmark,
                *large_bookmark_value,
                *large_cs_id,
            );

            let f2 = move_bookmark(
                &ctx,
                &small_repo,
                &small_bookmark,
                *small_bookmark_value,
                *rewritten_small_cs_id,
            );

            try_join!(f1, f2)?;

            Ok(())
        }
        _ => Err(SubcommandError::InvalidArgs),
    }
}

async fn change_mapping_via_extras<'a>(
    ctx: &CoreContext,
    matches: &'a MononokeMatches<'a>,
    sub_m: &'a ArgMatches<'a>,
    commit_syncer: &'a CommitSyncer<SqlSyncedCommitMapping, CrossRepo>,
    config_store: &ConfigStore,
    live_commit_sync_config: &Arc<dyn LiveCommitSyncConfig>,
) -> Result<(), Error> {
    if !live_commit_sync_config
        .push_redirector_enabled_for_public(commit_syncer.get_small_repo().repo_identity().id())
    {
        return Err(format_err!(
            "not allowed to run {} if pushredirection is not enabled",
            CHANGE_MAPPING_VERSION_SUBCOMMAND
        ));
    }

    let small_repo = commit_syncer.get_small_repo();
    let large_repo = commit_syncer.get_large_repo();

    let (_, repo_config) =
        args::get_config_by_repoid(config_store, matches, large_repo.repo_identity().id())?;

    let large_bookmark = Large(
        sub_m
            .value_of(LARGE_REPO_BOOKMARK_ARG)
            .map(BookmarkKey::new)
            .transpose()?
            .ok_or_else(|| format_err!("{} is not specified", LARGE_REPO_BOOKMARK_ARG))?,
    );
    let large_bookmark_value = Large(get_bookmark_value(ctx, large_repo, &large_bookmark).await?);

    let mapping_version = sub_m
        .value_of(ARG_VERSION_NAME)
        .ok_or_else(|| format_err!("{} is not specified", ARG_VERSION_NAME))?;
    let mapping_version = CommitSyncConfigVersion(mapping_version.to_string());
    if !commit_syncer.version_exists(&mapping_version).await? {
        return Err(format_err!("{} version does not exist", mapping_version));
    }

    let dump_mapping_file = sub_m
        .value_of(DUMP_MAPPING_LARGE_REPO_PATH_ARG)
        .map(MPath::new)
        .transpose()?;
    let large_cs_id = create_commit_for_mapping_change(
        ctx,
        sub_m,
        &Large(large_repo),
        &Small(small_repo),
        &large_bookmark_value,
        &mapping_version,
        MappingCommitOptions {
            add_mapping_change_extra: true,
            dump_mapping_file,
        },
        commit_syncer,
        live_commit_sync_config,
    )
    .await?;

    let pushrebase_flags = &repo_config.pushrebase.flags;
    let pushrebase_hooks = bookmarks_movement::get_pushrebase_hooks(
        ctx,
        large_repo,
        &large_bookmark,
        &repo_config.pushrebase,
    )?;

    let bcs = large_cs_id
        .load(ctx, &large_repo.repo_blobstore().clone())
        .map_err(Error::from)
        .await?;
    let pushrebase_res = do_pushrebase_bonsai(
        ctx,
        large_repo,
        pushrebase_flags,
        &large_bookmark,
        &hashset![bcs],
        &pushrebase_hooks,
    )
    .map_err(Error::from)
    .await?;

    println!("{}", pushrebase_res.head);

    Ok(())
}

async fn run_insert_subcommand<'a>(
    ctx: CoreContext,
    matches: &'a MononokeMatches<'_>,
    insert_subcommand_matches: &'a ArgMatches<'a>,
    live_commit_sync_config: CfgrLiveCommitSyncConfig,
) -> Result<(), SubcommandError> {
    let (source_repo, target_repo, mapping) =
        get_source_target_repos_and_mapping::<CrossRepo>(ctx.fb, ctx.logger().clone(), matches)
            .await?;

    let live_commit_sync_config: Arc<dyn LiveCommitSyncConfig> = Arc::new(live_commit_sync_config);
    let commit_syncer = get_large_to_small_commit_syncer(
        &ctx,
        source_repo.clone(),
        target_repo.clone(),
        live_commit_sync_config.clone(),
        mapping.clone(),
        matches,
    )
    .await?;

    match insert_subcommand_matches.subcommand() {
        (REWRITTEN_SUBCOMMAND, Some(sub_m)) => {
            let (source_cs_id, target_cs_id, mapping_version) =
                get_source_target_cs_ids_and_version(&ctx, sub_m, &commit_syncer).await?;
            let small_repo_id = commit_syncer.get_small_repo().repo_identity().id();
            let large_repo_id = commit_syncer.get_large_repo().repo_identity().id();

            let mapping_entry = if small_repo_id == source_repo.repo_identity().id() {
                SyncedCommitMappingEntry {
                    large_repo_id,
                    small_repo_id,
                    small_bcs_id: source_cs_id,
                    large_bcs_id: target_cs_id,
                    version_name: Some(mapping_version),
                    source_repo: Some(commit_syncer.get_source_repo_type()),
                }
            } else {
                SyncedCommitMappingEntry {
                    large_repo_id,
                    small_repo_id,
                    small_bcs_id: target_cs_id,
                    large_bcs_id: source_cs_id,
                    version_name: Some(mapping_version),
                    source_repo: Some(commit_syncer.get_source_repo_type()),
                }
            };

            let res = mapping.add(&ctx, mapping_entry).await?;
            if res {
                info!(
                    ctx.logger(),
                    "successfully inserted rewritten mapping entry"
                );
                Ok(())
            } else {
                Err(anyhow!("failed to insert entry").into())
            }
        }
        (EQUIVALENT_WORKING_COPY_SUBCOMMAND, Some(sub_m)) => {
            let (source_cs_id, target_cs_id, mapping_version) =
                get_source_target_cs_ids_and_version(&ctx, sub_m, &commit_syncer).await?;
            let small_repo_id = commit_syncer.get_small_repo().repo_identity().id();
            let large_repo_id = commit_syncer.get_large_repo().repo_identity().id();

            let mapping_entry = if small_repo_id == source_repo.repo_identity().id() {
                EquivalentWorkingCopyEntry {
                    large_repo_id,
                    small_repo_id,
                    small_bcs_id: Some(source_cs_id),
                    large_bcs_id: target_cs_id,
                    version_name: Some(mapping_version),
                }
            } else {
                EquivalentWorkingCopyEntry {
                    large_repo_id,
                    small_repo_id,
                    small_bcs_id: Some(target_cs_id),
                    large_bcs_id: source_cs_id,
                    version_name: Some(mapping_version),
                }
            };

            let res = mapping
                .insert_equivalent_working_copy(&ctx, mapping_entry)
                .await?;
            if res {
                info!(
                    ctx.logger(),
                    "successfully inserted equivalent working copy"
                );
                Ok(())
            } else {
                Err(anyhow!("failed to insert entry").into())
            }
        }
        (NOT_SYNC_CANDIDATE_SUBCOMMAND, Some(sub_m)) => {
            let large_repo = commit_syncer.get_large_repo();
            let large_repo_hash = sub_m
                .value_of(LARGE_REPO_HASH_ARG)
                .ok_or_else(|| anyhow!("{} is not specified", LARGE_REPO_HASH_ARG))?;
            let large_repo_cs_id = helpers::csid_resolve(&ctx, large_repo, large_repo_hash).await?;

            let small_repo_id = commit_syncer.get_small_repo().repo_identity().id();
            let large_repo_id = commit_syncer.get_large_repo().repo_identity().id();

            let maybe_mapping_version = sub_m.value_of(ARG_VERSION_NAME);
            let maybe_mapping_version = match maybe_mapping_version {
                Some(mapping_version) => {
                    let mapping_version = CommitSyncConfigVersion(mapping_version.to_string());
                    if !commit_syncer.version_exists(&mapping_version).await? {
                        return Err(
                            format_err!("{} version does not exist", mapping_version).into()
                        );
                    }
                    Some(mapping_version)
                }
                None => None,
            };

            let mapping_entry = EquivalentWorkingCopyEntry {
                large_repo_id,
                small_repo_id,
                small_bcs_id: None,
                large_bcs_id: large_repo_cs_id,
                version_name: maybe_mapping_version,
            };

            let res = mapping
                .insert_equivalent_working_copy(&ctx, mapping_entry)
                .await?;
            if res {
                info!(
                    ctx.logger(),
                    "successfully inserted not sync candidate entry"
                );
                Ok(())
            } else {
                Err(anyhow!("failed to insert entry").into())
            }
        }
        _ => Err(SubcommandError::InvalidArgs),
    }
}

async fn get_source_target_cs_ids_and_version(
    ctx: &CoreContext,
    sub_m: &ArgMatches<'_>,
    commit_syncer: &CommitSyncer<SqlSyncedCommitMapping, CrossRepo>,
) -> Result<(ChangesetId, ChangesetId, CommitSyncConfigVersion), Error> {
    async fn fetch_cs_id(
        ctx: &CoreContext,
        sub_m: &ArgMatches<'_>,
        repo: &CrossRepo,
        arg: &str,
    ) -> Result<ChangesetId, Error> {
        let hash = sub_m
            .value_of(arg)
            .ok_or_else(|| anyhow!("{} is not specified", arg))?;
        helpers::csid_resolve(ctx, repo, hash).await
    }

    let source_cs_id = fetch_cs_id(ctx, sub_m, commit_syncer.get_source_repo(), SOURCE_HASH_ARG);
    let target_cs_id = fetch_cs_id(ctx, sub_m, commit_syncer.get_target_repo(), TARGET_HASH_ARG);

    let (source_cs_id, target_cs_id) = try_join!(source_cs_id, target_cs_id)?;
    let mapping_version = sub_m
        .value_of(ARG_VERSION_NAME)
        .ok_or_else(|| format_err!("{} is not specified", ARG_VERSION_NAME))?;

    let mapping_version = CommitSyncConfigVersion(mapping_version.to_string());
    if !commit_syncer.version_exists(&mapping_version).await? {
        return Err(format_err!("{} version does not exist", mapping_version));
    }

    Ok((source_cs_id, target_cs_id, mapping_version))
}

struct MappingCommitOptions {
    add_mapping_change_extra: bool,
    dump_mapping_file: Option<MPath>,
}

async fn create_commit_for_mapping_change(
    ctx: &CoreContext,
    sub_m: &ArgMatches<'_>,
    large_repo: &Large<&CrossRepo>,
    small_repo: &Small<&CrossRepo>,
    parent: &Large<ChangesetId>,
    mapping_version: &CommitSyncConfigVersion,
    options: MappingCommitOptions,
    commit_syncer: &CommitSyncer<SqlSyncedCommitMapping, CrossRepo>,
    live_commit_sync_config: &Arc<dyn LiveCommitSyncConfig>,
) -> Result<Large<ChangesetId>, Error> {
    let author = sub_m
        .value_of(AUTHOR_ARG)
        .ok_or_else(|| format_err!("{} is not specified", AUTHOR_ARG))?;

    let oncall = sub_m.value_of(ONCALL_ARG);
    let oncall_msg_part = oncall.map(|o| format!("\n\nOncall Short Name: {}\n", o));

    let commit_msg = format!(
        "Changing synced mapping version to {} for {}->{} sync{}",
        mapping_version,
        large_repo.repo_identity().name(),
        small_repo.repo_identity().name(),
        oncall_msg_part.as_deref().unwrap_or("")
    );

    let mut extras = sorted_vector_map! {
        FAIL_PUSHREBASE_EXTRA.to_string() => b"1".to_vec(),
    };
    if options.add_mapping_change_extra {
        extras.insert(
            CHANGE_XREPO_MAPPING_EXTRA.to_string(),
            mapping_version.0.clone().into_bytes(),
        );
    }

    let file_changes = create_file_changes(
        ctx,
        small_repo,
        large_repo,
        mapping_version,
        options,
        commit_syncer,
        live_commit_sync_config,
    )
    .await?;

    // Create an empty commit on top of large bookmark
    let bcs = BonsaiChangesetMut {
        parents: vec![parent.0.clone()],
        author: author.to_string(),
        author_date: DateTime::now(),
        committer: None,
        committer_date: None,
        message: commit_msg,
        hg_extra: extras,
        git_extra_headers: None,
        git_tree_hash: None,
        file_changes: file_changes.into(),
        is_snapshot: false,
        git_annotated_tag: None,
    }
    .freeze()?;

    let large_cs_id = bcs.get_changeset_id();
    save_bonsai_changesets(vec![bcs], ctx.clone(), &large_repo.0).await?;

    Ok(Large(large_cs_id))
}

async fn create_file_changes(
    ctx: &CoreContext,
    small_repo: &Small<&CrossRepo>,
    large_repo: &Large<&CrossRepo>,
    mapping_version: &CommitSyncConfigVersion,
    options: MappingCommitOptions,
    commit_syncer: &CommitSyncer<SqlSyncedCommitMapping, CrossRepo>,
    live_commit_sync_config: &Arc<dyn LiveCommitSyncConfig>,
) -> Result<BTreeMap<MPath, FileChange>, Error> {
    let mut file_changes = btreemap! {};
    if let Some(path) = options.dump_mapping_file {
        // This "dump-mapping-file" is going to be created in the large repo,
        // but this file needs to rewrite to a small repo as well. If it doesn't
        // rewrite to a small repo, then the whole mapping change commit isn't
        // going to exist in the small repo.

        let mover = if commit_syncer.get_source_repo().repo_identity().id()
            == large_repo.repo_identity().id()
        {
            commit_syncer.get_mover_by_version(mapping_version).await?
        } else {
            commit_syncer
                .get_reverse_mover_by_version(mapping_version)
                .await?
        };

        if mover(&path)?.is_none() {
            return Err(anyhow!(
                "cannot dump mapping to a file because path doesn't rewrite to a small repo"
            ));
        }

        // Now get the mapping and create json with it
        let commit_sync_config = live_commit_sync_config
            .get_commit_sync_config_by_version(large_repo.repo_identity().id(), mapping_version)
            .await?;

        let small_repo_sync_config = commit_sync_config
            .small_repos
            .get(&small_repo.repo_identity().id())
            .ok_or_else(|| {
                format_err!(
                    "small repo {} not found in {} mapping",
                    small_repo.repo_identity().id(),
                    mapping_version
                )
            })?;

        let default_prefix = match &small_repo_sync_config.default_action {
            DefaultSmallToLargeCommitSyncPathAction::Preserve => String::new(),
            DefaultSmallToLargeCommitSyncPathAction::PrependPrefix(prefix) => prefix.to_string(),
        };

        let mut map = serde_json::Map::new();
        map.insert("default_prefix".to_string(), default_prefix.into());
        let mut map_overrides = serde_json::Map::new();
        for (key, value) in &small_repo_sync_config.map {
            map_overrides.insert(key.to_string(), value.to_string().into());
        }
        map.insert("overrides".to_string(), map_overrides.into());

        let content = (get_generated_string() + &serde_json::to_string_pretty(&map)?).into_bytes();
        let content = bytes::Bytes::from(content);
        let size = content.len() as u64;
        let content_metadata = filestore::store(
            large_repo.repo_blobstore(),
            *large_repo.filestore_config(),
            ctx,
            &filestore::StoreRequest::new(size),
            stream::once(async move { Ok(content) }),
        )
        .await?;

        let file_change =
            FileChange::tracked(content_metadata.content_id, FileType::Regular, size, None);

        file_changes.insert(path, file_change);
    }

    Ok(file_changes)
}

// Mark content as (at)generated to discourage people from modifying it
// manually.
// However split this so that this source file is not marked as generated
fn get_generated_string() -> String {
    "\x40generated by the megarepo bind, reach out to Source Control @ FB with any questions\n"
        .to_owned()
}

async fn get_bookmark_value(
    ctx: &CoreContext,
    repo: &CrossRepo,
    bookmark: &BookmarkKey,
) -> Result<ChangesetId, Error> {
    let maybe_bookmark_value = repo.bookmarks().get(ctx.clone(), bookmark).await?;

    maybe_bookmark_value.ok_or_else(|| {
        format_err!(
            "{} is not found in {}",
            bookmark,
            repo.repo_identity().name()
        )
    })
}

async fn move_bookmark(
    ctx: &CoreContext,
    repo: &CrossRepo,
    bookmark: &BookmarkKey,
    prev_value: ChangesetId,
    new_value: ChangesetId,
) -> Result<(), Error> {
    let mut book_txn = repo.bookmarks().create_transaction(ctx.clone());

    info!(
        ctx.logger(),
        "moving {} to {} in {}",
        bookmark,
        new_value,
        repo.repo_identity().name()
    );
    book_txn.update(
        bookmark,
        new_value,
        prev_value,
        BookmarkUpdateReason::ManualMove,
    )?;

    let res = book_txn.commit().await?;

    if res {
        Ok(())
    } else {
        Err(format_err!(
            "failed to move bookmark {} in {}",
            bookmark,
            repo.repo_identity().name()
        ))
    }
}

fn print_commit_sync_config(csc: CommitSyncConfig, line_prefix: &str) {
    println!("{}large repo: {}", line_prefix, csc.large_repo_id);
    println!(
        "{}common pushrebase bookmarks: {:?}",
        line_prefix, csc.common_pushrebase_bookmarks
    );
    println!("{}version name: {}", line_prefix, csc.version_name);
    for (small_repo_id, small_repo_config) in csc
        .small_repos
        .into_iter()
        .sorted_by_key(|(small_repo_id, _)| *small_repo_id)
    {
        println!("{}  small repo: {}", line_prefix, small_repo_id);
        println!(
            "{}  default action: {:?}",
            line_prefix, small_repo_config.default_action
        );
        println!("{}  prefix map:", line_prefix);
        for (from, to) in small_repo_config
            .map
            .into_iter()
            .sorted_by_key(|(from, _)| from.clone())
        {
            println!("{}    {}->{}", line_prefix, from, to);
        }
    }
}

async fn subcommand_list<'a, L: LiveCommitSyncConfig>(
    repo_id: RepositoryId,
    live_commit_sync_config: L,
    with_contents: bool,
) -> Result<(), Error> {
    let all = live_commit_sync_config
        .get_all_commit_sync_config_versions(repo_id)
        .await?;
    for (version_name, csc) in all.into_iter().sorted_by_key(|(vn, _)| vn.clone()) {
        if with_contents {
            println!("{}:", version_name);
            print_commit_sync_config(csc, "  ");
            println!("\n");
        } else {
            println!("{}", version_name);
        }
    }

    Ok(())
}

async fn subcommand_by_version<'a, L: LiveCommitSyncConfig>(
    repo_id: RepositoryId,
    live_commit_sync_config: L,
    version_name: String,
) -> Result<(), Error> {
    let csc = live_commit_sync_config
        .get_commit_sync_config_by_version(repo_id, &CommitSyncConfigVersion(version_name))
        .await?;
    print_commit_sync_config(csc, "");
    Ok(())
}

async fn subcommand_map(
    ctx: CoreContext,
    commit_syncer: CommitSyncer<SqlSyncedCommitMapping, CrossRepo>,
    hash: String,
) -> Result<(), SubcommandError> {
    let source_repo = commit_syncer.get_source_repo();
    let source_cs_id = helpers::csid_resolve(&ctx, source_repo, &hash).await?;

    let plural_commit_sync_outcome = commit_syncer
        .get_plural_commit_sync_outcome(&ctx, source_cs_id)
        .await?;
    match plural_commit_sync_outcome {
        Some(plural_commit_sync_outcome) => {
            println!("{:?}", plural_commit_sync_outcome);
        }
        None => {
            println!("{} is not remapped", hash);
        }
    }

    Ok(())
}

async fn subcommand_verify_bookmarks(
    ctx: CoreContext,
    source_repo: CrossRepo,
    target_repo: CrossRepo,
    mapping: SqlSyncedCommitMapping,
    should_update_large_repo_bookmarks: bool,
    live_commit_sync_config: Arc<dyn LiveCommitSyncConfig>,
    matches: &MononokeMatches<'_>,
) -> Result<(), SubcommandError> {
    let common_config =
        live_commit_sync_config.get_common_config(target_repo.repo_identity().id())?;
    let syncers = get_syncers(
        &ctx,
        source_repo.clone(),
        target_repo.clone(),
        live_commit_sync_config,
        mapping.clone(),
        matches,
    )
    .await?;

    let diff = validation::find_bookmark_diff(ctx.clone(), &syncers.large_to_small).await?;

    if diff.is_empty() {
        info!(ctx.logger(), "all is well!");
        Ok(())
    } else if should_update_large_repo_bookmarks {
        update_large_repo_bookmarks(ctx.clone(), &diff, &syncers, &common_config).await?;

        Ok(())
    } else {
        for d in &diff {
            use BookmarkDiff::*;
            match d {
                InconsistentValue {
                    target_bookmark,
                    target_cs_id,
                    source_cs_id,
                } => {
                    warn!(
                        ctx.logger(),
                        "inconsistent value of {}: '{}' has {}, but '{}' bookmark points to {:?}",
                        target_bookmark,
                        target_repo.repo_identity().name(),
                        target_cs_id,
                        source_repo.repo_identity().name(),
                        source_cs_id,
                    );
                }
                MissingInTarget {
                    target_bookmark,
                    source_cs_id,
                } => {
                    warn!(
                        ctx.logger(),
                        "'{}' doesn't have bookmark {} but '{}' has it and it points to {}",
                        target_repo.repo_identity().name(),
                        target_bookmark,
                        source_repo.repo_identity().name(),
                        source_cs_id,
                    );
                }
                NoSyncOutcome { target_bookmark } => {
                    warn!(
                        ctx.logger(),
                        "'{}' has a bookmark {} but it points to a commit that has no \
                         equivalent in '{}'. If it's a shared bookmark (e.g. master) \
                         that might mean that it points to a commit from another repository",
                        target_repo.repo_identity().name(),
                        target_bookmark,
                        source_repo.repo_identity().name(),
                    );
                }
            }
        }
        Err(format_err!("found {} inconsistencies", diff.len()).into())
    }
}

async fn update_large_repo_bookmarks(
    ctx: CoreContext,
    diff: &Vec<BookmarkDiff>,
    syncers: &Syncers<SqlSyncedCommitMapping, CrossRepo>,
    common_commit_sync_config: &CommonCommitSyncConfig,
) -> Result<(), Error> {
    warn!(
        ctx.logger(),
        "found {} inconsistencies, trying to update them...",
        diff.len()
    );
    let large_repo = syncers.small_to_large.get_large_repo();
    let mut book_txn = large_repo.bookmarks().create_transaction(ctx.clone());

    let bookmark_renamer = syncers.small_to_large.get_bookmark_renamer().await?;
    for d in diff {
        if common_commit_sync_config
            .common_pushrebase_bookmarks
            .contains(d.target_bookmark())
        {
            info!(
                ctx.logger(),
                "skipping {} because it's a common bookmark",
                d.target_bookmark()
            );
            continue;
        }

        use validation::BookmarkDiff::*;
        match d {
            InconsistentValue {
                target_bookmark,
                target_cs_id,
                ..
            } => {
                let outcomes = syncers
                    .small_to_large
                    .get_plural_commit_sync_outcome(&ctx, *target_cs_id)
                    .await?
                    .with_context(|| {
                        format!("Missing outcome for {} from small repo", target_cs_id)
                    })?;

                use PluralCommitSyncOutcome::*;
                let new_value = match outcomes {
                    NotSyncCandidate(..) => {
                        warn!(
                            ctx.logger(),
                            "{} from small repo doesn't remap to large repo", target_cs_id,
                        );
                        None
                    }
                    EquivalentWorkingCopyAncestor(large_cs_id, _) => Some(large_cs_id),
                    RewrittenAs(rewritten_commits) if rewritten_commits.len() == 1 => {
                        Some(rewritten_commits.into_iter().next().unwrap().0)
                    }
                    RewrittenAs(rewritten_commits) => {
                        return Err(format_err!(
                            "multiple remappings of {} in {}: {:?}",
                            *target_cs_id,
                            large_repo.repo_identity().name(),
                            rewritten_commits,
                        ));
                    }
                };

                if let Some(large_cs_id) = new_value {
                    let reason = BookmarkUpdateReason::XRepoSync;
                    let large_bookmark = bookmark_renamer(target_bookmark).ok_or_else(|| {
                        format_err!("small bookmark {} remaps to nothing", target_bookmark)
                    })?;

                    info!(ctx.logger(), "setting {} {}", large_bookmark, large_cs_id);
                    book_txn.force_set(&large_bookmark, large_cs_id, reason)?;
                }
            }
            MissingInTarget {
                target_bookmark, ..
            } => {
                warn!(
                    ctx.logger(),
                    "large repo bookmark (renames to {}) not found in small repo", target_bookmark,
                );
                let large_bookmark = bookmark_renamer(target_bookmark).ok_or_else(|| {
                    format_err!("small bookmark {} remaps to nothing", target_bookmark)
                })?;
                let reason = BookmarkUpdateReason::XRepoSync;
                info!(ctx.logger(), "deleting {}", large_bookmark);
                book_txn.force_delete(&large_bookmark, reason)?;
            }
            NoSyncOutcome { target_bookmark } => {
                warn!(
                    ctx.logger(),
                    "Not updating {} because it points to a commit that has no \
                     equivalent in source repo.",
                    target_bookmark,
                );
            }
        }
    }

    book_txn.commit().await?;
    Ok(())
}

pub fn build_subcommand<'a, 'b>() -> App<'a, 'b> {
    let map_subcommand = SubCommand::with_name(MAP_SUBCOMMAND)
        .about("Check cross-repo commit mapping")
        .arg(
            Arg::with_name(HASH_ARG)
                .required(true)
                .help("bonsai changeset hash to map"),
        );

    let verify_wc_subcommand = SubCommand::with_name(VERIFY_WC_SUBCOMMAND)
        .about("verify working copy")
        .arg(
            Arg::with_name(LARGE_REPO_HASH_ARG)
                .required(true)
                .help("bonsai changeset hash from large repo to verify"),
        );

    let verify_bookmarks_subcommand = SubCommand::with_name(VERIFY_BOOKMARKS_SUBCOMMAND).about(
        "verify that bookmarks are the same in small and large repo (subject to bookmark renames)",
    ).arg(
        Arg::with_name(UPDATE_LARGE_REPO_BOOKMARKS)
            .long(UPDATE_LARGE_REPO_BOOKMARKS)
            .required(false)
            .takes_value(false)
            .help("update any inconsistencies between bookmarks (except for the common bookmarks between large and small repo e.g. 'master')"),
    );

    let commit_sync_config_subcommand = {
        let by_version_subcommand = SubCommand::with_name(SUBCOMMAND_BY_VERSION)
            .about("print info about a particular version of CommitSyncConfig")
            .arg(
                Arg::with_name(ARG_VERSION_NAME)
                    .required(true)
                    .takes_value(true)
                    .help("commit sync config version name to query"),
            );

        let list_subcommand = SubCommand::with_name(SUBCOMMAND_LIST)
            .about("list all available CommitSyncConfig versions for repo")
            .arg(
                Arg::with_name(ARG_WITH_CONTENTS)
                    .long(ARG_WITH_CONTENTS)
                    .required(false)
                    .takes_value(false)
                    .help("Do not just print version names, also include config bodies"),
            );

        SubCommand::with_name(SUBCOMMAND_CONFIG)
            .about("query available CommitSyncConfig versions for repo")
            .subcommand(list_subcommand)
            .subcommand(by_version_subcommand)
    };

    let prepare_rollout_subcommand = SubCommand::with_name(PREPARE_ROLLOUT_SUBCOMMAND)
        .about("command to prepare rollout of pushredirection");

    let change_mapping_version = SubCommand::with_name(CHANGE_MAPPING_VERSION_SUBCOMMAND)
        .about(
            "a command to change mapping version for a given bookmark. \
        Note that this command doesn't check that the working copies of source and target repo \
        are equivalent according to the new mapping. This needs to ensured before calling this command",
        )
        .arg(
            Arg::with_name(AUTHOR_ARG)
                .long(AUTHOR_ARG)
                .required(true)
                .takes_value(true)
                .help("Author of the commit that will change the mapping"),
        )
        .arg(
            Arg::with_name(ONCALL_ARG)
                .long(ONCALL_ARG)
                .required(false)
                .takes_value(true)
                .help("Oncall for the commit that will change the mapping"),
        )
        .arg(
            Arg::with_name(LARGE_REPO_BOOKMARK_ARG)
                .long(LARGE_REPO_BOOKMARK_ARG)
                .required(true)
                .takes_value(true)
                .help("bookmark in the large repo"),
        )
        .arg(
            Arg::with_name(ARG_VERSION_NAME)
                .long(ARG_VERSION_NAME)
                .required(true)
                .takes_value(true)
                .help("mapping version to change to"),
        )
        .arg(
            Arg::with_name(VIA_EXTRAS_ARG)
                .long(VIA_EXTRAS_ARG)
                .required(false)
                .takes_value(false)
                .help("change mapping via pushing a commit with a special extra set. \
                This should become a default method, but for now let's hide behind this arg")
        )
        .arg(
            Arg::with_name(DUMP_MAPPING_LARGE_REPO_PATH_ARG)
                .long(DUMP_MAPPING_LARGE_REPO_PATH_ARG)
                .required(false)
                .takes_value(true)
                .help("Path in the repo where new mapping version will be dumped.")
        );

    let pushredirection_subcommand = SubCommand::with_name(PUSHREDIRECTION_SUBCOMMAND)
        .about("helper commands to enable/disable pushredirection")
        .subcommand(prepare_rollout_subcommand)
        .subcommand(change_mapping_version);

    let rewritten_subcommand = SubCommand::with_name(REWRITTEN_SUBCOMMAND)
        .about("mark a pair of commits as rewritten")
        .arg(
            Arg::with_name(SOURCE_HASH_ARG)
                .long(SOURCE_HASH_ARG)
                .required(true)
                .takes_value(true)
                .help("hash in the source repo"),
        )
        .arg(
            Arg::with_name(TARGET_HASH_ARG)
                .long(TARGET_HASH_ARG)
                .required(true)
                .takes_value(true)
                .help("hash in the target repo"),
        )
        .arg(
            Arg::with_name(ARG_VERSION_NAME)
                .long(ARG_VERSION_NAME)
                .required(true)
                .takes_value(true)
                .help("mapping version to write to db"),
        );

    let equivalent_wc_subcommand = SubCommand::with_name(EQUIVALENT_WORKING_COPY_SUBCOMMAND)
        .about("mark a pair of commits as having an equivalent working copy")
        .arg(
            Arg::with_name(SOURCE_HASH_ARG)
                .long(SOURCE_HASH_ARG)
                .required(true)
                .takes_value(true)
                .help("hash in the source repo"),
        )
        .arg(
            Arg::with_name(TARGET_HASH_ARG)
                .long(TARGET_HASH_ARG)
                .required(true)
                .takes_value(true)
                .help("hash in the target repo"),
        )
        .arg(
            Arg::with_name(ARG_VERSION_NAME)
                .long(ARG_VERSION_NAME)
                .required(true)
                .takes_value(true)
                .help("mapping version to write to db"),
        );

    let not_sync_candidate_subcommand = SubCommand::with_name(NOT_SYNC_CANDIDATE_SUBCOMMAND)
        .about("mark a source commit as not having a synced commit in the target repo")
        .arg(
            Arg::with_name(LARGE_REPO_HASH_ARG)
                .long(LARGE_REPO_HASH_ARG)
                .required(true)
                .takes_value(true)
                .help("hash in the source repo"),
        )
        .arg(
            Arg::with_name(ARG_VERSION_NAME)
                .long(ARG_VERSION_NAME)
                .required(false)
                .takes_value(true)
                .help("optional mapping version to write to db"),
        );

    let insert_subcommand = SubCommand::with_name(INSERT_SUBCOMMAND)
        .about("helper commands to insert mappings directly into db")
        .subcommand(equivalent_wc_subcommand)
        .subcommand(rewritten_subcommand)
        .subcommand(not_sync_candidate_subcommand);

    SubCommand::with_name(CROSSREPO)
        .subcommand(map_subcommand)
        .subcommand(verify_wc_subcommand)
        .subcommand(verify_bookmarks_subcommand)
        .subcommand(commit_sync_config_subcommand)
        .subcommand(pushredirection_subcommand)
        .subcommand(insert_subcommand)
}

async fn get_syncers<'a>(
    ctx: &'a CoreContext,
    source_repo: CrossRepo,
    target_repo: CrossRepo,
    live_commit_sync_config: Arc<dyn LiveCommitSyncConfig>,
    mapping: SqlSyncedCommitMapping,
    matches: &'a MononokeMatches<'a>,
) -> Result<Syncers<SqlSyncedCommitMapping, CrossRepo>, Error> {
    let caching = matches.caching();
    let x_repo_sync_lease = create_commit_syncer_lease(ctx.fb, caching)?;

    let common_sync_config =
        live_commit_sync_config.get_common_config(source_repo.repo_identity().id())?;

    let (large_repo, small_repo) = if common_sync_config.large_repo_id
        == source_repo.repo_identity().id()
        && common_sync_config
            .small_repos
            .contains_key(&target_repo.repo_identity().id())
    {
        (source_repo, target_repo)
    } else if common_sync_config.large_repo_id == target_repo.repo_identity().id()
        && common_sync_config
            .small_repos
            .contains_key(&source_repo.repo_identity().id())
    {
        (target_repo, source_repo)
    } else {
        return Err(format_err!(
            "CommitSyncMapping incompatible with source repo {:?} and target repo {:?}",
            source_repo.repo_identity().id(),
            target_repo.repo_identity().id()
        ));
    };

    create_commit_syncers(
        ctx,
        small_repo,
        large_repo,
        mapping,
        live_commit_sync_config,
        x_repo_sync_lease,
    )
}

async fn get_large_to_small_commit_syncer<'a>(
    ctx: &'a CoreContext,
    source_repo: CrossRepo,
    target_repo: CrossRepo,
    live_commit_sync_config: Arc<dyn LiveCommitSyncConfig>,
    mapping: SqlSyncedCommitMapping,
    matches: &'a MononokeMatches<'a>,
) -> Result<CommitSyncer<SqlSyncedCommitMapping, CrossRepo>, Error> {
    Ok(get_syncers(
        ctx,
        source_repo,
        target_repo,
        live_commit_sync_config,
        mapping,
        matches,
    )
    .await?
    .large_to_small)
}

#[cfg(test)]
mod test {
    use std::collections::HashSet;
    use std::sync::Arc;

    use ascii::AsciiString;
    use bookmarks::BookmarkKey;
    use cacheblob::InProcessLease;
    use changeset_fetcher::ChangesetFetcherArc;
    use cross_repo_sync::validation::find_bookmark_diff;
    use fixtures::set_bookmark;
    use fixtures::Linear;
    use fixtures::TestRepoFixture;
    use futures::compat::Stream01CompatExt;
    use futures::TryStreamExt;
    use live_commit_sync_config::TestLiveCommitSyncConfig;
    use maplit::hashmap;
    use maplit::hashset;
    use metaconfig_types::CommitSyncConfig;
    use metaconfig_types::CommitSyncConfigVersion;
    use metaconfig_types::CommonCommitSyncConfig;
    use metaconfig_types::SmallRepoCommitSyncConfig;
    use metaconfig_types::SmallRepoPermanentConfig;
    use mononoke_types::RepositoryId;
    use revset::AncestorsNodeStream;
    use sql_construct::SqlConstruct;
    use synced_commit_mapping::SyncedCommitMappingEntry;
    use synced_commit_mapping::SyncedCommitSourceRepo;

    use super::*;

    #[fbinit::test]
    fn test_bookmark_diff(fb: FacebookInit) -> Result<(), Error> {
        let runtime = tokio::runtime::Runtime::new()?;
        runtime.block_on(test_bookmark_diff_impl(fb))
    }

    async fn test_bookmark_diff_impl(fb: FacebookInit) -> Result<(), Error> {
        let ctx = CoreContext::test_mock(fb);
        let syncers = init_syncers(fb).await?;

        let small_repo = syncers.small_to_large.get_small_repo();
        let large_repo = syncers.small_to_large.get_large_repo();

        let master = BookmarkKey::new("master")?;
        let maybe_master_val = small_repo.bookmarks().get(ctx.clone(), &master).await?;
        let master_val = maybe_master_val.ok_or_else(|| Error::msg("master not found"))?;

        // Everything is identical - no diff at all
        {
            let diff = find_bookmark_diff(ctx.clone(), &syncers.large_to_small).await?;

            assert!(diff.is_empty());
        }

        // Move bookmark to another changeset
        let another_hash = "607314ef579bd2407752361ba1b0c1729d08b281";
        set_bookmark(fb, &small_repo, another_hash, master.clone()).await;
        let another_bcs_id =
            helpers::csid_resolve(&ctx, small_repo, another_hash.to_string()).await?;

        let actual_diff = find_bookmark_diff(ctx.clone(), &syncers.large_to_small).await?;

        let mut expected_diff = hashset! {
            BookmarkDiff::InconsistentValue {
                target_bookmark: master.clone(),
                target_cs_id: another_bcs_id,
                source_cs_id: Some(master_val),
            }
        };
        assert!(!actual_diff.is_empty());
        assert_eq!(
            actual_diff.into_iter().collect::<HashSet<_>>(),
            expected_diff,
        );

        // Create another bookmark
        let another_book = BookmarkKey::new("newbook")?;
        set_bookmark(fb, &small_repo, another_hash, another_book.clone()).await;

        let actual_diff = find_bookmark_diff(ctx.clone(), &syncers.large_to_small).await?;

        expected_diff.insert(BookmarkDiff::InconsistentValue {
            target_bookmark: another_book,
            target_cs_id: another_bcs_id,
            source_cs_id: None,
        });
        assert_eq!(
            actual_diff.clone().into_iter().collect::<HashSet<_>>(),
            expected_diff
        );

        // Update the bookmarks
        {
            let mut common_config = CommonCommitSyncConfig {
                common_pushrebase_bookmarks: vec![master.clone()],
                small_repos: hashmap! {
                    small_repo.repo_identity().id() => SmallRepoPermanentConfig {
                        bookmark_prefix: Default::default()
                    },
                },
                large_repo_id: large_repo.repo_identity().id(),
            };

            update_large_repo_bookmarks(ctx.clone(), &actual_diff, &syncers, &common_config)
                .await?;

            let actual_diff = find_bookmark_diff(ctx.clone(), &syncers.large_to_small).await?;

            // Master bookmark hasn't been updated because it's a common pushrebase bookmark
            let expected_diff = hashset! {
                BookmarkDiff::InconsistentValue {
                    target_bookmark: master.clone(),
                    target_cs_id: another_bcs_id,
                    source_cs_id: Some(master_val),
                }
            };
            assert_eq!(
                actual_diff.clone().into_iter().collect::<HashSet<_>>(),
                expected_diff,
            );

            // Now remove master bookmark from common_pushrebase_bookmarks and update large repo
            // bookmarks again
            common_config.common_pushrebase_bookmarks = vec![];

            update_large_repo_bookmarks(ctx.clone(), &actual_diff, &syncers, &common_config)
                .await?;
            let actual_diff = find_bookmark_diff(ctx.clone(), &syncers.large_to_small).await?;
            assert!(actual_diff.is_empty());
        }
        Ok(())
    }

    async fn init_syncers(
        fb: FacebookInit,
    ) -> Result<Syncers<SqlSyncedCommitMapping, CrossRepo>, Error> {
        let ctx = CoreContext::test_mock(fb);
        let small_repo = Linear::get_inner_repo_with_id(fb, RepositoryId::new(0)).await;
        let large_repo = Linear::get_inner_repo_with_id(fb, RepositoryId::new(1)).await;

        let master = BookmarkKey::new("master")?;
        let maybe_master_val = small_repo.bookmarks().get(ctx.clone(), &master).await?;

        let master_val = maybe_master_val.ok_or_else(|| Error::msg("master not found"))?;
        let changesets: Vec<_> =
            AncestorsNodeStream::new(ctx.clone(), &small_repo.changeset_fetcher_arc(), master_val)
                .compat()
                .try_collect()
                .await?;

        let current_version = CommitSyncConfigVersion("TEST_VERSION_NAME".to_string());

        let mapping = SqlSyncedCommitMapping::with_sqlite_in_memory()?;
        for cs_id in changesets {
            mapping
                .add(
                    &ctx,
                    SyncedCommitMappingEntry {
                        large_repo_id: large_repo.repo_identity().id(),
                        small_repo_id: small_repo.repo_identity().id(),
                        small_bcs_id: cs_id,
                        large_bcs_id: cs_id,
                        version_name: Some(current_version.clone()),
                        source_repo: Some(SyncedCommitSourceRepo::Large),
                    },
                )
                .await?;
        }

        let (lv_cfg, lv_cfg_src) = TestLiveCommitSyncConfig::new_with_source();

        let common_config = CommonCommitSyncConfig {
            common_pushrebase_bookmarks: vec![BookmarkKey::new("master")?],
            small_repos: hashmap! {
                small_repo.repo_identity().id() => SmallRepoPermanentConfig {
                    bookmark_prefix: AsciiString::new(),
                }
            },
            large_repo_id: large_repo.repo_identity().id(),
        };

        let current_version_config = CommitSyncConfig {
            large_repo_id: large_repo.repo_identity().id(),
            common_pushrebase_bookmarks: vec![BookmarkKey::new("master")?],
            small_repos: hashmap! {
                small_repo.repo_identity().id() => SmallRepoCommitSyncConfig {
                    default_action: DefaultSmallToLargeCommitSyncPathAction::Preserve,
                    map: hashmap! { },
                },
            },
            version_name: current_version.clone(),
        };

        lv_cfg_src.add_common_config(common_config);
        lv_cfg_src.add_config(current_version_config);
        let x_repo_sync_lease = Arc::new(InProcessLease::new());
        let live_commit_sync_config = Arc::new(lv_cfg);

        create_commit_syncers(
            &ctx,
            small_repo,
            large_repo,
            mapping,
            live_commit_sync_config,
            x_repo_sync_lease,
        )
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;
use std::collections::HashSet;
use std::fmt;
use std::sync::Arc;

use anyhow::anyhow;
use anyhow::Error;
use blobrepo::BlobRepo;
use blobrepo_override::DangerousOverride;
use blobstore::Loadable;
use bonsai_hg_mapping::BonsaiHgMappingArc;
use bonsai_hg_mapping::BonsaiHgMappingRef;
use bookmarks::BookmarksRef;
use cacheblob::dummy::DummyLease;
use cacheblob::LeaseOps;
use changesets::ChangesetsArc;
use changesets::ChangesetsRef;
use clap_old::App;
use clap_old::Arg;
use clap_old::ArgMatches;
use clap_old::SubCommand;
use cmdlib::args;
use cmdlib::args::MononokeMatches;
use cmdlib::helpers;
use cmdlib::helpers::csid_resolve;
use commit_graph::CommitGraphArc;
use commit_graph::CommitGraphRef;
use context::CoreContext;
use context::SessionClass;
use derived_data::BonsaiDerived;
use derived_data_manager::BonsaiDerivable;
use derived_data_utils::derived_data_utils;
use derived_data_utils::derived_data_utils_for_config;
use derived_data_utils::DEFAULT_BACKFILLING_CONFIG_NAME;
use derived_data_utils::POSSIBLE_DERIVED_TYPES;
use fbinit::FacebookInit;
use filenodes::FilenodesArc;
use filenodes::FilenodesRef;
use fsnodes::RootFsnodeId;
use futures::future::try_join_all;
use futures::future::FutureExt as PreviewFutureExt;
use futures::stream;
use futures::StreamExt;
use futures::TryStreamExt;
use futures_stats::TimedFutureExt;
use manifest::ManifestOps;
use mercurial_derivation::DeriveHgChangeset;
use mercurial_derivation::MappedHgChangesetId;
use mononoke_types::ChangesetId;
use mononoke_types::ContentId;
use mononoke_types::FileType;
use mononoke_types::MPath;
use repo_blobstore::RepoBlobstoreRef;
use repo_derived_data::RepoDerivedDataArc;
use repo_identity::RepoIdentityRef;
use skeleton_manifest::RootSkeletonManifestId;
use slog::info;
use slog::Logger;
use unodes::RootUnodeManifestId;

use crate::error::SubcommandError;

pub const DERIVED_DATA: &str = "derived-data";
const SUBCOMMAND_DERIVE: &str = "derive";
const SUBCOMMAND_EXISTS: &str = "exists";
const SUBCOMMAND_COUNT_UNDERIVED: &str = "count-underived";
const SUBCOMMAND_VERIFY_MANIFESTS: &str = "verify-manifests";

const ARG_CHANGESET: &str = "changeset";
const ARG_HASH_OR_BOOKMARK: &str = "hash-or-bookmark";
const ARG_TYPE: &str = "type";
const ARG_INPUT_FILE: &str = "input-file";
const ARG_IF_DERIVED: &str = "if-derived";
const ARG_BACKFILL: &str = "backfill";
const ARG_BACKFILL_CONFIG_NAME: &str = "backfill-config-name";
const ARG_REDERIVE: &str = "rederive";

const MANIFEST_DERIVED_DATA_TYPES: &[&str] = &[
    RootFsnodeId::NAME,
    MappedHgChangesetId::NAME,
    RootUnodeManifestId::NAME,
    RootSkeletonManifestId::NAME,
];

pub fn build_subcommand<'a, 'b>() -> App<'a, 'b> {
    SubCommand::with_name(DERIVED_DATA)
        .about("request information about derived data")
        .subcommand(
            SubCommand::with_name(SUBCOMMAND_EXISTS)
                .about("check if derived data has been generated")
                .arg(
                    Arg::with_name(ARG_BACKFILL)
                        .long("backfill")
                        .help("use backfilling config rather than enabled config"),
                )
                .arg(
                    Arg::with_name(ARG_TYPE)
                        .help("type of derived data")
                        .takes_value(true)
                        .possible_values(POSSIBLE_DERIVED_TYPES)
                        .required(true),
                )
                .arg(
                    Arg::with_name(ARG_HASH_OR_BOOKMARK)
                        .help("(hg|bonsai) commit hash or bookmark")
                        .takes_value(true)
                        .multiple(true)
                        .required(true),
                )
                .arg(
                    Arg::with_name(ARG_BACKFILL_CONFIG_NAME)
                        .long(ARG_BACKFILL_CONFIG_NAME)
                        .help("sets the name for backfilling derived data types config")
                        .takes_value(true),
                ),
        )
        .subcommand(
            SubCommand::with_name(SUBCOMMAND_COUNT_UNDERIVED)
                .about("count how many ancestors of a given commit weren't derived")
                .arg(
                    Arg::with_name(ARG_BACKFILL)
                        .long("backfill")
                        .help("use backfilling config rather than enabled config"),
                )
                .arg(
                    Arg::with_name(ARG_TYPE)
                        .help("type of derived data")
                        .takes_value(true)
                        .possible_values(POSSIBLE_DERIVED_TYPES)
                        .required(true),
                )
                .arg(
                    Arg::with_name(ARG_HASH_OR_BOOKMARK)
                        .help("(hg|bonsai) commit hash or bookmark")
                        .takes_value(true)
                        .multiple(true)
                        .required(true),
                )
                .arg(
                    Arg::with_name(ARG_BACKFILL_CONFIG_NAME)
                        .long(ARG_BACKFILL_CONFIG_NAME)
                        .help("sets the name for backfilling derived data types config")
                        .takes_value(true),
                ),
        )
        .subcommand(
            SubCommand::with_name(SUBCOMMAND_VERIFY_MANIFESTS)
                .about("compare check if derived data has been generated")
                .arg(
                    Arg::with_name(ARG_TYPE)
                        .help("types of derived data representing a manifest")
                        .long(ARG_TYPE)
                        .takes_value(true)
                        .multiple(true)
                        .possible_values(MANIFEST_DERIVED_DATA_TYPES),
                )
                .arg(
                    Arg::with_name(ARG_HASH_OR_BOOKMARK)
                        .help("(hg|bonsai) commit hash or bookmark")
                        .takes_value(true)
                        .required(true),
                )
                .arg(
                    Arg::with_name(ARG_IF_DERIVED)
                        .help("only verify the manifests if they are already derived")
                        .long(ARG_IF_DERIVED),
                ),
        )
        .subcommand(
            SubCommand::with_name(SUBCOMMAND_DERIVE)
                .about("actually derive data")
                .arg(
                    Arg::with_name(ARG_TYPE)
                        .help("type of derived data")
                        .takes_value(true)
                        .possible_values(POSSIBLE_DERIVED_TYPES)
                        .required(true),
                )
                .arg(
                    Arg::with_name(ARG_CHANGESET)
                        .long(ARG_CHANGESET)
                        .help("(hg|bonsai) commit hash or bookmark that needs to be derived")
                        .takes_value(true)
                        .required(true)
                        .conflicts_with(ARG_INPUT_FILE),
                )
                .arg(
                    Arg::with_name(ARG_INPUT_FILE)
                        .long(ARG_INPUT_FILE)
                        .takes_value(true)
                        .required(false)
                        .help("File with a list of changeset hashes {hd|bonsai} or bookmarks that need to be derived"),
                )
                .arg(
                    Arg::with_name(ARG_REDERIVE)
                        .long(ARG_REDERIVE)
                        .help("whether the changesets need to be rederived or not")
                        .takes_value(false)
                        .required(false),
                )
        )
}

pub async fn subcommand_derived_data<'a>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'_>,
    sub_m: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
    let mut ctx = CoreContext::new_with_logger(fb, logger.clone());

    match sub_m.subcommand() {
        (SUBCOMMAND_EXISTS, Some(arg_matches)) => {
            let repo = args::not_shardmanager_compatible::open_repo(fb, &logger, matches).await?;
            let hashes_or_bookmarks: Vec<_> = arg_matches
                .values_of(ARG_HASH_OR_BOOKMARK)
                .map(|matches| matches.map(|cs| cs.to_string()).collect())
                .unwrap();

            let derived_data_type = arg_matches
                .value_of(ARG_TYPE)
                .map(|m| m.to_string())
                .unwrap();

            let backfill = arg_matches.is_present(ARG_BACKFILL);

            let backfill_config_name = sub_m
                .value_of(ARG_BACKFILL_CONFIG_NAME)
                .unwrap_or(DEFAULT_BACKFILLING_CONFIG_NAME);

            check_derived_data_exists(
                ctx,
                repo,
                derived_data_type,
                hashes_or_bookmarks,
                backfill,
                backfill_config_name,
            )
            .await
        }
        (SUBCOMMAND_COUNT_UNDERIVED, Some(arg_matches)) => {
            let repo: BlobRepo =
                args::not_shardmanager_compatible::open_repo(fb, &logger, matches).await?;
            let hashes_or_bookmarks: Vec<_> = arg_matches
                .values_of(ARG_HASH_OR_BOOKMARK)
                .map(|matches| matches.map(|cs| cs.to_string()).collect())
                .unwrap();

            let derived_data_type = arg_matches
                .value_of(ARG_TYPE)
                .map(|m| m.to_string())
                .unwrap();

            let backfill = arg_matches.is_present(ARG_BACKFILL);

            let backfill_config_name = sub_m
                .value_of(ARG_BACKFILL_CONFIG_NAME)
                .unwrap_or(DEFAULT_BACKFILLING_CONFIG_NAME);

            count_underived(
                ctx,
                repo,
                derived_data_type,
                hashes_or_bookmarks,
                backfill,
                backfill_config_name,
            )
            .await
        }
        (SUBCOMMAND_VERIFY_MANIFESTS, Some(arg_matches)) => {
            let repo = args::not_shardmanager_compatible::open_repo(fb, &logger, matches).await?;
            let hash_or_bookmark = arg_matches
                .value_of(ARG_HASH_OR_BOOKMARK)
                .map(|m| m.to_string())
                .unwrap();

            let derived_data_types = arg_matches.values_of(ARG_TYPE).map_or_else(
                || {
                    MANIFEST_DERIVED_DATA_TYPES
                        .iter()
                        .map(|s| String::from(*s))
                        .collect::<Vec<_>>()
                },
                |matches| matches.map(|cs| cs.to_string()).collect(),
            );

            let fetch_derived = arg_matches.is_present(ARG_IF_DERIVED);

            verify_manifests(
                ctx,
                repo,
                derived_data_types,
                hash_or_bookmark,
                fetch_derived,
            )
            .await
        }
        (SUBCOMMAND_DERIVE, Some(arg_matches)) => {
            let mut repo_factory = args::get_repo_factory(matches)?;

            if arg_matches.is_present(ARG_REDERIVE) {
                repo_factory.with_bonsai_hg_mapping_override();
            }

            let repo: BlobRepo = args::not_shardmanager_compatible::open_repo_with_factory(
                fb,
                &logger,
                matches,
                repo_factory,
            )
            .await?;
            let repo = repo.dangerous_override(|_| Arc::new(DummyLease {}) as Arc<dyn LeaseOps>);
            let ty = arg_matches
                .value_of(ARG_TYPE)
                .ok_or_else(|| anyhow!("{} is not set", ARG_TYPE))?;
            let derived_utils = derived_data_utils(ctx.fb, &repo, ty)?;

            let mut csids = vec![];
            if let Some(hash_or_bookmark) = arg_matches.value_of(ARG_CHANGESET) {
                let csid = helpers::csid_resolve(&ctx, repo.clone(), hash_or_bookmark).await?;
                csids.push(csid);
            }

            if let Some(input_file) = arg_matches.value_of(ARG_INPUT_FILE) {
                let new_csids =
                    helpers::csids_resolve_from_file(&ctx, repo.clone(), input_file).await?;
                csids.extend(new_csids);
            }

            if arg_matches.is_present(ARG_REDERIVE) {
                info!(ctx.logger(), "about to rederive {} commits", csids.len());
                derived_utils.regenerate(&csids);
                // Force this binary to write to all blobstores
                ctx.session_mut()
                    .override_session_class(SessionClass::Background);
            } else {
                info!(ctx.logger(), "about to derive {} commits", csids.len());
            };

            for csid in csids {
                info!(ctx.logger(), "deriving {}", csid);
                let (stats, res) = derived_utils
                    .derive(ctx.clone(), repo.repo_derived_data_arc(), csid)
                    .timed()
                    .await;
                info!(
                    ctx.logger(),
                    "derived {} in {}ms, {:?}",
                    csid,
                    stats.completion_time.as_millis(),
                    res,
                );
            }

            Ok(())
        }
        _ => Err(SubcommandError::InvalidArgs),
    }
}

async fn check_derived_data_exists(
    ctx: CoreContext,
    repo: BlobRepo,
    derived_data_type: String,
    hashes_or_bookmarks: Vec<String>,
    backfill: bool,
    backfill_config_name: &str,
) -> Result<(), SubcommandError> {
    let derived_utils = if backfill {
        derived_data_utils_for_config(ctx.fb, &repo, derived_data_type, backfill_config_name)?
    } else {
        derived_data_utils(ctx.fb, &repo, derived_data_type)?
    };

    let cs_id_futs: Vec<_> = hashes_or_bookmarks
        .into_iter()
        .map(|hash_or_bm| csid_resolve(&ctx, repo.clone(), hash_or_bm))
        .collect();

    let cs_ids = try_join_all(cs_id_futs).await?;

    let pending = derived_utils
        .pending(ctx.clone(), repo.repo_derived_data_arc(), cs_ids.clone())
        .await?;

    for cs_id in cs_ids {
        if pending.contains(&cs_id) {
            println!("Not Derived: {}", cs_id);
        } else {
            println!("Derived: {}", cs_id);
        }
    }

    Ok(())
}

async fn count_underived(
    ctx: CoreContext,
    repo: impl RepoDerivedDataArc
    + RepoBlobstoreRef
    + FilenodesArc
    + FilenodesRef
    + RepoIdentityRef
    + BonsaiHgMappingArc
    + BonsaiHgMappingRef
    + BookmarksRef
    + ChangesetsArc
    + ChangesetsRef
    + CommitGraphArc
    + CommitGraphRef
    + Clone,
    derived_data_type: String,
    hashes_or_bookmarks: Vec<String>,
    backfill: bool,
    backfill_config_name: &str,
) -> Result<(), SubcommandError> {
    let derived_utils = if backfill {
        derived_data_utils_for_config(ctx.fb, &repo, derived_data_type, backfill_config_name)?
    } else {
        derived_data_utils(ctx.fb, &repo, derived_data_type)?
    };

    let cs_id_futs: Vec<_> = hashes_or_bookmarks
        .into_iter()
        .map(|hash_or_bm| csid_resolve(&ctx, repo.clone(), hash_or_bm))
        .collect();

    let cs_ids = try_join_all(cs_id_futs).await?;

    let ctx = &ctx;
    let repo = &repo;
    let derived_utils = &derived_utils;
    let res = stream::iter(cs_ids)
        .map(|cs_id| async move {
            let underived = derived_utils
                .count_underived(ctx, repo.repo_derived_data(), cs_id)
                .await?;
            Result::<_, Error>::Ok((cs_id, underived))
        })
        .buffer_unordered(10)
        .try_collect::<Vec<_>>()
        .await?;

    for (cs_id, underived) in res {
        println!("{}: {}", cs_id, underived);
    }

    Ok(())
}

async fn verify_manifests(
    ctx: CoreContext,
    repo: BlobRepo,
    derived_data_types: Vec<String>,
    hash_or_bookmark: String,
    fetch_derived: bool,
) -> Result<(), SubcommandError> {
    let cs_id = csid_resolve(&ctx, repo.clone(), hash_or_bookmark).await?;
    let mut manifests = HashSet::new();
    let mut futs = vec![];
    for ty in derived_data_types {
        if ty == RootFsnodeId::NAME {
            manifests.insert(ManifestType::Fsnodes);
            futs.push(list_fsnodes(&ctx, &repo, cs_id, fetch_derived).boxed());
        } else if ty == RootUnodeManifestId::NAME {
            manifests.insert(ManifestType::Unodes);
            futs.push(list_unodes(&ctx, &repo, cs_id, fetch_derived).boxed());
        } else if ty == MappedHgChangesetId::NAME {
            manifests.insert(ManifestType::Hg);
            futs.push(list_hg_manifest(&ctx, &repo, cs_id).boxed());
        } else if ty == RootSkeletonManifestId::NAME {
            manifests.insert(ManifestType::Skeleton);
            futs.push(list_skeleton_manifest(&ctx, &repo, cs_id, fetch_derived).boxed());
        } else {
            return Err(anyhow!("unknown derived data manifest type").into());
        }
    }
    let mut combined: HashMap<MPath, FileContentValue> = HashMap::new();
    let contents = try_join_all(futs).await?;
    info!(ctx.logger(), "Combining {} manifests", contents.len());
    for (mf_type, map) in contents {
        for (path, new_val) in map {
            combined
                .entry(path)
                .or_insert_with(FileContentValue::new)
                .update(new_val.clone());
        }
        info!(ctx.logger(), "Completed {} manifest", mf_type);
    }

    info!(ctx.logger(), "Checking {} paths", combined.len());
    let mut invalid_count = 0u64;
    for (path, val) in combined {
        if !val.is_valid(&manifests) {
            println!("Invalid!\nPath: {}", path);
            println!("{}\n", val);
            invalid_count += 1;
        }
    }
    if invalid_count == 0 {
        info!(ctx.logger(), "Check complete");
    } else {
        info!(ctx.logger(), "Found {} invalid paths", invalid_count);
    }

    Ok(())
}

#[derive(Clone, Default)]
struct FileContentValue {
    values: Vec<ManifestData>,
}

impl fmt::Display for FileContentValue {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        for (i, value) in self.values.iter().enumerate() {
            if i > 0 {
                write!(f, " ")?;
            }
            write!(f, "({})", value)?;
        }
        Ok(())
    }
}

impl FileContentValue {
    pub fn new() -> Self {
        Self { values: vec![] }
    }

    pub fn update(&mut self, val: ManifestData) {
        self.values.push(val);
    }

    pub fn is_valid(&self, expected_manifests: &HashSet<ManifestType>) -> bool {
        if self.values.is_empty() {
            return false;
        }

        let manifest_types: HashSet<_> = self
            .values
            .iter()
            .map(ManifestData::manifest_type)
            .collect();
        if &manifest_types != expected_manifests {
            return false;
        }
        let contents: HashSet<_> = self
            .values
            .iter()
            .filter_map(ManifestData::content)
            .collect();
        // Skeleton manifests have no content, so 0 is valid for them.
        // Otherwise, we should have exactly one.
        contents.len() <= 1
    }
}

#[derive(Clone, Hash, Eq, PartialEq)]
enum ManifestType {
    Fsnodes,
    Hg,
    Unodes,
    Skeleton,
}

#[derive(Clone, Hash, Eq, PartialEq)]
enum ManifestData {
    Fsnodes(FileType, ContentId),
    Hg(FileType, ContentId),
    Unodes(FileType, ContentId),
    Skeleton,
}

impl fmt::Display for ManifestType {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        use ManifestType::*;

        match &self {
            Fsnodes => write!(f, "Fsnodes"),
            Hg => write!(f, "Hg"),
            Unodes => write!(f, "Unodes"),
            Skeleton => write!(f, "Skeleton"),
        }
    }
}

impl ManifestData {
    fn manifest_type(&self) -> ManifestType {
        use ManifestData::*;

        match self {
            Fsnodes(..) => ManifestType::Fsnodes,
            Hg(..) => ManifestType::Hg,
            Unodes(..) => ManifestType::Unodes,
            Skeleton => ManifestType::Skeleton,
        }
    }

    fn content(&self) -> Option<(FileType, ContentId)> {
        use ManifestData::*;

        match self {
            Fsnodes(ty, id) | Hg(ty, id) | Unodes(ty, id) => Some((*ty, *id)),
            Skeleton => None,
        }
    }
}

impl fmt::Display for ManifestData {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        use ManifestData::*;
        match &self {
            Fsnodes(ty, id) | Hg(ty, id) | Unodes(ty, id) => {
                write!(f, "{}: {}, {}", self.manifest_type(), ty, id)
            }
            Skeleton => write!(f, "{}: present", self.manifest_type()),
        }
    }
}

pub(crate) async fn derive_or_fetch<T: BonsaiDerived>(
    ctx: &CoreContext,
    repo: &BlobRepo,
    csid: ChangesetId,
    fetch_derived: bool,
) -> Result<T, Error> {
    if fetch_derived {
        let value = T::fetch_derived(ctx, repo, &csid).await?;
        value.ok_or_else(|| anyhow!("{} are not derived for {}", T::DERIVABLE_NAME, csid))
    } else {
        Ok(T::derive(ctx, repo, csid).await?)
    }
}

async fn list_hg_manifest(
    ctx: &CoreContext,
    repo: &BlobRepo,
    cs_id: ChangesetId,
) -> Result<(ManifestType, HashMap<MPath, ManifestData>), Error> {
    let hg_cs_id = repo.derive_hg_changeset(ctx, cs_id).await?;

    let hg_cs = hg_cs_id.load(ctx, repo.repo_blobstore()).await?;
    let mfid = hg_cs.manifestid();

    let map: HashMap<_, _> = mfid
        .list_leaf_entries(ctx.clone(), repo.repo_blobstore().clone())
        .map_ok(|(path, (ty, filenode_id))| async move {
            let filenode = filenode_id.load(ctx, repo.repo_blobstore()).await?;
            let content_id = filenode.content_id();
            let val = ManifestData::Hg(ty, content_id);
            Ok((path, val))
        })
        .try_buffer_unordered(100)
        .try_collect()
        .await?;
    info!(ctx.logger(), "Loaded hg manifests for {} paths", map.len());
    Ok((ManifestType::Hg, map))
}

async fn list_skeleton_manifest(
    ctx: &CoreContext,
    repo: &BlobRepo,
    cs_id: ChangesetId,
    fetch_derived: bool,
) -> Result<(ManifestType, HashMap<MPath, ManifestData>), Error> {
    let root_skeleton_id =
        derive_or_fetch::<RootSkeletonManifestId>(ctx, repo, cs_id, fetch_derived).await?;

    let skeleton_id = root_skeleton_id.skeleton_manifest_id();
    let map: HashMap<_, _> = skeleton_id
        .list_leaf_entries(ctx.clone(), repo.repo_blobstore().clone())
        .map_ok(|(path, ())| (path, ManifestData::Skeleton))
        .try_collect()
        .await?;
    info!(
        ctx.logger(),
        "Loaded skeleton manifests for {} paths",
        map.len()
    );
    Ok((ManifestType::Skeleton, map))
}

async fn list_fsnodes(
    ctx: &CoreContext,
    repo: &BlobRepo,
    cs_id: ChangesetId,
    fetch_derived: bool,
) -> Result<(ManifestType, HashMap<MPath, ManifestData>), Error> {
    let root_fsnode_id = derive_or_fetch::<RootFsnodeId>(ctx, repo, cs_id, fetch_derived).await?;

    let fsnode_id = root_fsnode_id.fsnode_id();
    let map: HashMap<_, _> = fsnode_id
        .list_leaf_entries(ctx.clone(), repo.repo_blobstore().clone())
        .map_ok(|(path, fsnode)| {
            let (content_id, ty): (ContentId, FileType) = fsnode.into();
            let val = ManifestData::Fsnodes(ty, content_id);
            (path, val)
        })
        .try_collect()
        .await?;
    info!(ctx.logger(), "Loaded fsnodes for {} paths", map.len());
    Ok((ManifestType::Fsnodes, map))
}

async fn list_unodes(
    ctx: &CoreContext,
    repo: &BlobRepo,
    cs_id: ChangesetId,
    fetch_derived: bool,
) -> Result<(ManifestType, HashMap<MPath, ManifestData>), Error> {
    let root_unode_id =
        derive_or_fetch::<RootUnodeManifestId>(ctx, repo, cs_id, fetch_derived).await?;

    let unode_id = root_unode_id.manifest_unode_id();
    let map: HashMap<_, _> = unode_id
        .list_leaf_entries(ctx.clone(), repo.repo_blobstore().clone())
        .map_ok(|(path, unode_id)| async move {
            let unode = unode_id.load(ctx, repo.repo_blobstore()).await?;
            let val = ManifestData::Unodes(*unode.file_type(), *unode.content_id());
            Ok((path, val))
        })
        .try_buffer_unordered(100)
        .try_collect()
        .await?;
    info!(ctx.logger(), "Loaded unodes for {} paths", map.len());
    Ok((ManifestType::Unodes, map))
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use anyhow::Error;
use thiserror::Error;

#[derive(Debug, Error)]
pub enum SubcommandError {
    #[error("SubcommandError::InvalidArgs")]
    InvalidArgs,
    #[error("SubcommandError::Error")]
    Error(#[from] Error),
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use anyhow::anyhow;
use anyhow::format_err;
use anyhow::Error;
use blobrepo::BlobRepo;
use blobrepo_hg::BlobRepoHg;
use blobstore::Loadable;
use clap_old::App;
use clap_old::Arg;
use clap_old::ArgMatches;
use clap_old::SubCommand;
use cloned::cloned;
use cmdlib::args;
use cmdlib::args::MononokeMatches;
use cmdlib::helpers;
use context::CoreContext;
use fbinit::FacebookInit;
use filenodes::FilenodeInfo;
use filenodes::FilenodeRange;
use filenodes::FilenodesRef;
use futures::future::try_join_all;
use futures::TryStreamExt;
use futures_stats::TimedFutureExt;
use manifest::Entry;
use manifest::ManifestOps;
use mercurial_derivation::DeriveHgChangeset;
use mercurial_types::HgFileEnvelope;
use mercurial_types::HgFileNodeId;
use mercurial_types::MPath;
use mononoke_types::RepoPath;
use repo_blobstore::RepoBlobstoreRef;
use slog::debug;
use slog::info;
use slog::Logger;

use crate::common::get_file_nodes;
use crate::error::SubcommandError;

pub const FILENODES: &str = "filenodes";
const COMMAND_ID: &str = "by-id";
const COMMAND_REVISION: &str = "by-revision";
const COMMAND_VALIDATE: &str = "validate";
const COMMAND_ALL_FILENODES: &str = "all-filenodes";

const ARG_ENVELOPE: &str = "envelope";
const ARG_IS_TREE: &str = "is-tree";

const ARG_REVISION: &str = "changeset-id";
const ARG_PATHS: &str = "paths";

const ARG_ID: &str = "id";
const ARG_PATH: &str = "path";

pub fn build_subcommand<'a, 'b>() -> App<'a, 'b> {
    SubCommand::with_name(FILENODES)
        .about("fetches hg filenodes information for a commit and one or more paths")
        .arg(
            Arg::with_name(ARG_ENVELOPE)
                .long(ARG_ENVELOPE)
                .required(false)
                .takes_value(false)
                .help("whether to show the envelope as well"),
        )
        .subcommand(
            SubCommand::with_name(COMMAND_REVISION)
                .arg(
                    Arg::with_name(ARG_REVISION)
                        .required(true)
                        .takes_value(true)
                        .help("hg/bonsai changeset id or bookmark to lookup filenodes for"),
                )
                .arg(
                    Arg::with_name(ARG_PATHS)
                        .required(true)
                        .multiple(true)
                        .takes_value(true)
                        .help("a list of file paths to lookup filenodes for"),
                ),
        )
        .subcommand(
            SubCommand::with_name(COMMAND_ID)
                .arg(
                    Arg::with_name(ARG_PATH)
                        .required(true)
                        .takes_value(true)
                        .help("path to lookup filenode for (use trailing / for directories)"),
                )
                .arg(
                    Arg::with_name(ARG_ID)
                        .required(true)
                        .takes_value(true)
                        .help("filenode ID"),
                ),
        )
        .subcommand(
            SubCommand::with_name(COMMAND_VALIDATE)
                .about("for a public changeset validates that all files and trees exist")
                .arg(
                    Arg::with_name(ARG_REVISION)
                        .required(true)
                        .takes_value(true)
                        .help("hg/bonsai changeset id or bookmark"),
                ),
        )
        .subcommand(
            SubCommand::with_name(COMMAND_ALL_FILENODES)
                .about("print all filenodes for a path")
                .arg(
                    Arg::with_name(ARG_PATH)
                        .long(ARG_PATH)
                        .required(true)
                        .takes_value(true)
                        .help("path of the filenode"),
                )
                .arg(
                    Arg::with_name(ARG_IS_TREE)
                        .long(ARG_IS_TREE)
                        .required(false)
                        .takes_value(false)
                        .help("whether it's a tree"),
                ),
        )
}

fn extract_path(path: &str) -> Result<MPath, Error> {
    MPath::new(path).map_err(|err| format_err!("Could not parse path {}: {:?}", path, err))
}

fn log_filenode(
    logger: &Logger,
    path: &RepoPath,
    filenode: &FilenodeInfo,
    envelope: Option<&HgFileEnvelope>,
) -> Option<()> {
    let FilenodeInfo {
        filenode,
        p1,
        p2,
        copyfrom,
        linknode,
    } = filenode;

    info!(logger, "Filenode: {:?}", filenode);
    info!(logger, "-- path: {:?}", path);
    info!(logger, "-- p1: {:?}", p1);
    info!(logger, "-- p2: {:?}", p2);
    info!(logger, "-- copyfrom: {:?}", copyfrom);
    info!(logger, "-- linknode: {:?}", linknode);
    info!(logger, "-- content id: {:?}", envelope?.content_id());
    info!(logger, "-- content size: {:?}", envelope?.content_size());
    info!(logger, "-- metadata: {:?}", envelope?.metadata());
    Some(())
}

async fn handle_filenodes_at_revision(
    ctx: CoreContext,
    blobrepo: BlobRepo,
    revision: &str,
    paths: Vec<MPath>,
    log_envelope: bool,
) -> Result<(), Error> {
    let cs_id = helpers::csid_resolve(&ctx, &blobrepo, revision.to_string()).await?;
    let cs_id = blobrepo.derive_hg_changeset(&ctx, cs_id).await?;
    let filenode_ids = get_file_nodes(
        ctx.clone(),
        ctx.logger().clone(),
        &blobrepo,
        cs_id,
        paths.clone(),
    )
    .await?;
    let logger = ctx.logger().clone();
    let filenodes_futs =
        paths
            .into_iter()
            .zip(filenode_ids.into_iter())
            .map(move |(path, filenode_id)| {
                cloned!(ctx, blobrepo);
                async move {
                    let path = RepoPath::FilePath(path);

                    let filenode_fut = async {
                        blobrepo
                            .get_filenode(ctx.clone(), &path, filenode_id)
                            .await?
                            .do_not_handle_disabled_filenodes()
                    };
                    let envelope_fut = async {
                        if log_envelope {
                            Ok(Some(
                                filenode_id.load(&ctx, blobrepo.repo_blobstore()).await?,
                            ))
                        } else {
                            Ok(None)
                        }
                    };
                    let (filenode, envelope) = futures::try_join!(filenode_fut, envelope_fut)?;

                    Ok::<_, Error>((path, filenode, envelope))
                }
            });
    let filenodes = try_join_all(filenodes_futs).await?;
    for (path, filenode, envelope) in filenodes {
        log_filenode(&logger, &path, &filenode, envelope.as_ref());
    }
    Ok(())
}

pub async fn subcommand_filenodes<'a>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'_>,
    sub_m: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
    let ctx = CoreContext::new_with_logger(fb, logger.clone());
    let repo = args::not_shardmanager_compatible::open_repo(fb, ctx.logger(), matches).await?;
    let log_envelope = sub_m.is_present(ARG_ENVELOPE);

    match sub_m.subcommand() {
        (COMMAND_REVISION, Some(matches)) => {
            let rev = matches.value_of(ARG_REVISION).unwrap().to_string();

            let paths: Result<Vec<_>, Error> = matches
                .values_of(ARG_PATHS)
                .unwrap()
                .map(extract_path)
                .collect();
            let paths = paths?;

            handle_filenodes_at_revision(ctx, repo, &rev, paths, log_envelope)
                .await
                .map_err(SubcommandError::Error)
        }
        (COMMAND_ID, Some(matches)) => {
            let path = matches.value_of(ARG_PATH).unwrap();
            let path = match path.chars().last() {
                Some('/') => extract_path(path).map(RepoPath::DirectoryPath),
                Some(_) => extract_path(path).map(RepoPath::FilePath),
                None => Ok(RepoPath::RootPath),
            }?;

            let id = matches.value_of(ARG_ID).unwrap().parse::<HgFileNodeId>()?;

            let filenode = repo
                .get_filenode(ctx.clone(), &path, id)
                .await?
                .do_not_handle_disabled_filenodes()?;
            let envelope = if log_envelope {
                let res = filenode
                    .filenode
                    .load(&ctx, repo.repo_blobstore())
                    .await
                    .map_err(Error::from)?;
                Some(res)
            } else {
                None
            };
            log_filenode(ctx.logger(), &path, &filenode, envelope.as_ref());
            Ok(())
        }
        (COMMAND_VALIDATE, Some(matches)) => {
            let rev = matches.value_of(ARG_REVISION).unwrap().to_string();
            let ctx = CoreContext::new_with_logger(fb, logger.clone());

            let mf_id = helpers::get_root_manifest_id(&ctx, repo.clone(), rev).await?;
            mf_id
                .list_all_entries(ctx.clone(), repo.repo_blobstore().clone())
                .map_ok(move |(path, entry)| {
                    cloned!(ctx, repo);
                    async move {
                        let (repo_path, filenode_id) = match entry {
                            Entry::Leaf((_, filenode_id)) => (
                                RepoPath::FilePath(path.expect("unexpected empty file path")),
                                filenode_id,
                            ),
                            Entry::Tree(mf_id) => {
                                let filenode_id = HgFileNodeId::new(mf_id.into_nodehash());
                                match path {
                                    Some(path) => (RepoPath::DirectoryPath(path), filenode_id),
                                    None => (RepoPath::RootPath, filenode_id),
                                }
                            }
                        };
                        let maybe_filenode = repo
                            .get_filenode_opt(ctx, &repo_path, filenode_id)
                            .await?
                            .do_not_handle_disabled_filenodes()?;
                        if maybe_filenode.is_some() {
                            Ok(())
                        } else {
                            Err(format_err!("not found filenode for {}", repo_path))
                        }
                    }
                })
                .try_buffer_unordered(100)
                .try_for_each(|_| async { Ok(()) })
                .await?;
            Ok(())
        }
        (COMMAND_ALL_FILENODES, Some(matches)) => {
            let maybe_mpath = MPath::new_opt(matches.value_of(ARG_PATH).unwrap())?;
            let is_tree = matches.is_present(ARG_IS_TREE);
            let path = match (maybe_mpath, is_tree) {
                (Some(path), true) => RepoPath::DirectoryPath(path),
                (Some(path), false) => RepoPath::FilePath(path),
                (None, true) => RepoPath::RootPath,
                (None, false) => {
                    return Err(
                        anyhow!("you need to provide a non-empty path or pass --is-tree").into(),
                    );
                }
            };

            let (stats, res) = repo
                .filenodes()
                .get_all_filenodes_maybe_stale(&ctx, &path, None)
                .timed()
                .await;

            debug!(ctx.logger(), "took {:?}", stats.completion_time);
            let filenodes = match res?.do_not_handle_disabled_filenodes()? {
                FilenodeRange::Filenodes(filenodes) => filenodes,
                FilenodeRange::TooBig => {
                    return Err(anyhow!("unexpected failure: history is too long?").into());
                }
            };
            for filenode in filenodes {
                log_filenode(ctx.logger(), &path, &filenode, None);
            }
            Ok(())
        }
        _ => Err(SubcommandError::InvalidArgs),
    }
}