/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::str::FromStr;

use anyhow::anyhow;
use anyhow::Error;
use blobrepo::BlobRepo;
use bonsai_git_mapping::BonsaiGitMappingRef;
use bonsai_hg_mapping::BonsaiHgMappingRef;
use clap_old::App;
use clap_old::Arg;
use clap_old::ArgMatches;
use clap_old::SubCommand;
use cmdlib::args;
use cmdlib::args::MononokeMatches;
use context::CoreContext;
use fbinit::FacebookInit;
use mercurial_derivation::DeriveHgChangeset;
use mercurial_types::HgChangesetId;
use mononoke_types::hash::GitSha1;
use mononoke_types::ChangesetId;
use slog::Logger;

use crate::error::SubcommandError;

pub const HASH_CONVERT: &str = "convert";
const ARG_FROM: &str = "from";
const ARG_TO: &str = "to";

pub fn build_subcommand<'a, 'b>() -> App<'a, 'b> {
    SubCommand::with_name(HASH_CONVERT)
        .about("convert between bonsai and hg changeset hashes")
        .arg(
            Arg::with_name(ARG_FROM)
                .long(ARG_FROM)
                .short("f")
                .required(true)
                .takes_value(true)
                .possible_values(&["hg", "bonsai", "git"])
                .help("Source hash type"),
        )
        .arg(
            Arg::with_name(ARG_TO)
                .long(ARG_TO)
                .short("t")
                .required(true)
                .takes_value(true)
                .possible_values(&["hg", "bonsai", "git"])
                .help("Target hash type"),
        )
        .args_from_usage("<HASH>  'source hash'")
}

pub async fn subcommand_hash_convert<'a>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'_>,
    sub_m: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
    let source_hash = sub_m.value_of("HASH").unwrap().to_string();
    let source = sub_m.value_of("from").unwrap().to_string();
    let target = sub_m.value_of("to").unwrap();
    // Check that source and target are different types.
    if source == target {
        return Err(anyhow!("source and target should be different").into());
    }
    let ctx = CoreContext::new_with_logger(fb, logger.clone());
    let repo: BlobRepo = args::not_shardmanager_compatible::open_repo(fb, &logger, matches).await?;

    let cs_id = convert_to_bonsai(&ctx, &repo, &source, &source_hash).await?;
    println!("{}", convert_from_bonsai(&ctx, &repo, cs_id, target).await?);

    Ok(())
}

async fn convert_to_bonsai(
    ctx: &CoreContext,
    repo: &BlobRepo,
    from: &str,
    hash: &str,
) -> Result<ChangesetId, Error> {
    if from == "hg" {
        let maybebonsai = repo
            .bonsai_hg_mapping()
            .get_bonsai_from_hg(
                ctx,
                HgChangesetId::from_str(hash).expect("source hash is not valid hg changeset id"),
            )
            .await?;

        maybebonsai.ok_or_else(|| anyhow!("bonsai not found for {}", hash))
    } else if from == "git" {
        let maybebonsai = repo
            .bonsai_git_mapping()
            .get_bonsai_from_git_sha1(ctx, GitSha1::from_str(hash)?)
            .await?;

        maybebonsai.ok_or_else(|| anyhow!("git not found for {}", hash))
    } else if from == "bonsai" {
        ChangesetId::from_str(hash)
    } else {
        Err(anyhow!("unknown source {}", from))
    }
}

async fn convert_from_bonsai(
    ctx: &CoreContext,
    repo: &BlobRepo,
    cs_id: ChangesetId,
    to: &str,
) -> Result<String, Error> {
    if to == "hg" {
        let hg = repo.derive_hg_changeset(ctx, cs_id).await?;
        Ok(format!("{}", hg))
    } else if to == "git" {
        let maybegit = repo
            .bonsai_git_mapping()
            .get_git_sha1_from_bonsai(ctx, cs_id)
            .await?;

        let git = maybegit.ok_or_else(|| anyhow!("git not found for {}", cs_id))?;
        Ok(format!("{}", git))
    } else if to == "bonsai" {
        Ok(format!("{}", cs_id))
    } else {
        Err(anyhow!("unknown target {}", to))
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::borrow::Borrow;
use std::collections::BTreeMap;
use std::io;
use std::str::FromStr;

use anyhow::format_err;
use anyhow::Error;
use blobrepo::BlobRepo;
use blobstore::Loadable;
use bonsai_hg_mapping::BonsaiHgMappingRef;
use clap_old::App;
use clap_old::ArgMatches;
use clap_old::SubCommand;
use cmdlib::args;
use cmdlib::args::MononokeMatches;
use commit_graph::CommitGraphRef;
use context::CoreContext;
use fbinit::FacebookInit;
use futures::StreamExt;
use futures::TryStreamExt;
use manifest::bonsai_diff;
use manifest::BonsaiDiffFileChange;
use mercurial_derivation::DeriveHgChangeset;
use mercurial_types::HgChangesetId;
use mercurial_types::HgManifestId;
use mercurial_types::MPath;
use repo_blobstore::RepoBlobstoreRef;
use serde_derive::Serialize;
use slog::Logger;

use crate::error::SubcommandError;

pub const HG_CHANGESET: &str = "hg-changeset";
const HG_CHANGESET_DIFF: &str = "diff";
const HG_CHANGESET_RANGE: &str = "range";

pub fn build_subcommand<'a, 'b>() -> App<'a, 'b> {
    SubCommand::with_name(HG_CHANGESET)
        .about("mercural changeset level queries")
        .subcommand(
            SubCommand::with_name(HG_CHANGESET_DIFF)
                .about("compare two changeset (used by pushrebase replayer)")
                .args_from_usage(
                    "<LEFT_CS>  'left changeset id'
                     <RIGHT_CS> 'right changeset id'",
                ),
        )
        .subcommand(
            SubCommand::with_name(HG_CHANGESET_RANGE)
                .about("returns `x::y` revset")
                .args_from_usage(
                    "<START_CS> 'start changeset id'
                     <STOP_CS>  'stop changeset id'",
                ),
        )
}

pub async fn subcommand_hg_changeset<'a>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'_>,
    sub_m: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
    let ctx = CoreContext::new_with_logger(fb, logger.clone());

    match sub_m.subcommand() {
        (HG_CHANGESET_DIFF, Some(sub_m)) => {
            let left_cs = sub_m
                .value_of("LEFT_CS")
                .ok_or_else(|| format_err!("LEFT_CS argument expected"))
                .and_then(HgChangesetId::from_str)?;
            let right_cs = sub_m
                .value_of("RIGHT_CS")
                .ok_or_else(|| format_err!("RIGHT_CS argument expected"))
                .and_then(HgChangesetId::from_str)?;

            let repo = args::not_shardmanager_compatible::open_repo(fb, &logger, matches).await?;
            let diff = hg_changeset_diff(ctx, repo, left_cs, right_cs).await?;
            serde_json::to_writer(io::stdout(), &diff).map_err(Error::from)?;
            Ok(())
        }
        (HG_CHANGESET_RANGE, Some(sub_m)) => {
            let start_cs = sub_m
                .value_of("START_CS")
                .ok_or_else(|| format_err!("START_CS argument expected"))
                .and_then(HgChangesetId::from_str)?;
            let stop_cs = sub_m
                .value_of("STOP_CS")
                .ok_or_else(|| format_err!("STOP_CS argument expected"))
                .and_then(HgChangesetId::from_str)?;

            let repo: BlobRepo =
                args::not_shardmanager_compatible::open_repo(fb, &logger, matches).await?;
            let (start_cs_opt, stop_cs_opt) = futures::try_join!(
                repo.bonsai_hg_mapping().get_bonsai_from_hg(&ctx, start_cs),
                repo.bonsai_hg_mapping().get_bonsai_from_hg(&ctx, stop_cs),
            )?;
            let start_cs = start_cs_opt.ok_or_else(|| Error::msg("failed to resolve changeset"))?;
            let stop_cs = stop_cs_opt.ok_or_else(|| Error::msg("failed to resovle changeset"))?;
            let css: Vec<_> = repo
                .commit_graph()
                .range_stream(&ctx, start_cs, stop_cs)
                .await?
                .map(|cs| repo.derive_hg_changeset(&ctx, cs))
                .buffer_unordered(100)
                .map_ok(|cs| cs.to_hex().to_string())
                .try_collect()
                .await?;
            serde_json::to_writer(io::stdout(), &css).map_err(Error::from)?;
            Ok(())
        }
        _ => Err(SubcommandError::InvalidArgs),
    }
}

async fn hg_changeset_diff(
    ctx: CoreContext,
    repo: BlobRepo,
    left_id: HgChangesetId,
    right_id: HgChangesetId,
) -> Result<ChangesetDiff, Error> {
    let (left, right) = futures::try_join!(
        left_id.load(&ctx, repo.repo_blobstore()),
        right_id.load(&ctx, repo.repo_blobstore()),
    )?;

    let mut diff = ChangesetDiff {
        left: left_id,
        right: right_id,
        diff: Vec::new(),
    };

    if left.user() != right.user() {
        diff.diff.push(ChangesetAttrDiff::User(
            slice_to_str(left.user()),
            slice_to_str(right.user()),
        ));
    }

    if left.message() != right.message() {
        diff.diff.push(ChangesetAttrDiff::Comments(
            slice_to_str(left.message()),
            slice_to_str(right.message()),
        ))
    }

    if left.files() != right.files() {
        diff.diff.push(ChangesetAttrDiff::Files(
            left.files().iter().map(mpath_to_str).collect(),
            right.files().iter().map(mpath_to_str).collect(),
        ))
    }

    if left.extra() != right.extra() {
        diff.diff.push(ChangesetAttrDiff::Extra(
            left.extra()
                .iter()
                .map(|(k, v)| (slice_to_str(k), slice_to_str(v)))
                .collect(),
            right
                .extra()
                .iter()
                .map(|(k, v)| (slice_to_str(k), slice_to_str(v)))
                .collect(),
        ))
    }

    let mdiff = hg_manifest_diff(ctx, repo, left.manifestid(), right.manifestid()).await?;
    diff.diff.extend(mdiff);
    Ok(diff)
}

async fn hg_manifest_diff(
    ctx: CoreContext,
    repo: BlobRepo,
    left: HgManifestId,
    right: HgManifestId,
) -> Result<Option<ChangesetAttrDiff>, Error> {
    let diffs: Vec<_> = bonsai_diff(
        ctx,
        repo.repo_blobstore().clone(),
        left,
        Some(right).into_iter().collect(),
    )
    .try_collect()
    .await?;

    let diff = diffs.into_iter().fold(
        ManifestDiff {
            modified: Vec::new(),
            deleted: Vec::new(),
        },
        |mut mdiff, diff| {
            match diff {
                BonsaiDiffFileChange::Changed(path, ..)
                | BonsaiDiffFileChange::ChangedReusedId(path, ..) => {
                    mdiff.modified.push(mpath_to_str(path))
                }
                BonsaiDiffFileChange::Deleted(path) => mdiff.deleted.push(mpath_to_str(path)),
            };
            mdiff
        },
    );
    if diff.modified.is_empty() && diff.deleted.is_empty() {
        Ok(None)
    } else {
        Ok(Some(ChangesetAttrDiff::Manifest(diff)))
    }
}

fn slice_to_str(slice: &[u8]) -> String {
    String::from_utf8_lossy(slice).into_owned()
}

fn mpath_to_str<P: Borrow<MPath>>(mpath: P) -> String {
    let bytes = mpath.borrow().to_vec();
    String::from_utf8_lossy(bytes.as_ref()).into_owned()
}

#[derive(Serialize)]
struct ChangesetDiff {
    left: HgChangesetId,
    right: HgChangesetId,
    diff: Vec<ChangesetAttrDiff>,
}

#[derive(Serialize)]
enum ChangesetAttrDiff {
    #[serde(rename = "user")]
    User(String, String),
    #[serde(rename = "comments")]
    Comments(String, String),
    #[serde(rename = "manifest")]
    Manifest(ManifestDiff),
    #[serde(rename = "files")]
    Files(Vec<String>, Vec<String>),
    #[serde(rename = "extra")]
    Extra(BTreeMap<String, String>, BTreeMap<String, String>),
}

#[derive(Serialize)]
struct ManifestDiff {
    modified: Vec<String>,
    deleted: Vec<String>,
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use anyhow::anyhow;
use blobrepo::BlobRepo;
use changeset_fetcher::ChangesetFetcherArc;
use clap_old::App;
use clap_old::Arg;
use clap_old::ArgMatches;
use clap_old::SubCommand;
use cmdlib::args;
use cmdlib::args::MononokeMatches;
use cmdlib::helpers;
use context::CoreContext;
use fbinit::FacebookInit;
use futures::compat::Stream01CompatExt;
use futures::StreamExt;
use futures::TryStreamExt;
use revset::AncestorsNodeStream;
use slog::Logger;

use crate::error::SubcommandError;

pub const LIST_ANCESTORS: &str = "list-ancestors";
const ARG_CHANGESET: &str = "changeset-id";
const ARG_LIMIT: &str = "limit";

pub fn build_subcommand<'a, 'b>() -> App<'a, 'b> {
    SubCommand::with_name(LIST_ANCESTORS)
        .about("list ancestors of a commit")
        .arg(
            Arg::with_name(ARG_CHANGESET)
                .required(true)
                .takes_value(true)
                .help("hg/bonsai changeset id or bookmark to start listing ancestors from"),
        )
        .arg(
            Arg::with_name(ARG_LIMIT)
                .long(ARG_LIMIT)
                .short("l")
                .takes_value(true)
                .required(false)
                .help("Imposes the limit on number of log records in output."),
        )
}

pub async fn list_ancestors<'a>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'_>,
    sub_m: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
    let ctx = CoreContext::new_with_logger(fb, logger.clone());
    let repo: BlobRepo =
        args::not_shardmanager_compatible::open_repo(fb, ctx.logger(), matches).await?;
    let rev = sub_m
        .value_of(ARG_CHANGESET)
        .ok_or_else(|| anyhow!("{} is not set", ARG_CHANGESET))?;
    let limit = args::get_usize(sub_m, ARG_LIMIT, 10);

    let cs_id = helpers::csid_resolve(&ctx, &repo, rev).await?;

    let ancestors = AncestorsNodeStream::new(ctx, &repo.changeset_fetcher_arc(), cs_id)
        .compat()
        .take(limit)
        .try_collect::<Vec<_>>()
        .await?;

    for cs_id in ancestors {
        println!("{}", cs_id);
    }
    Ok(())
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

#![type_length_limit = "8000000"]
#![feature(btree_drain_filter)]

use std::process::ExitCode;

use blobstore::PutBehaviour;
use cmdlib::args;
use cmdlib::args::ArgType;
use cmdlib::args::MononokeClapApp;
use context::CoreContext;
use fbinit::FacebookInit;
use slog::error;

use crate::blobstore_fetch::subcommand_blobstore_fetch;
use crate::blobstore_unlink::subcommand_blobstore_unlink;
use crate::blobstore_upload::subcommand_blobstore_upload;
use crate::bonsai_fetch::subcommand_bonsai_fetch;
use crate::content_fetch::subcommand_content_fetch;
use crate::crossrepo::subcommand_crossrepo;
use crate::error::SubcommandError;
use crate::filenodes::subcommand_filenodes;
use crate::hash_convert::subcommand_hash_convert;
use crate::hg_changeset::subcommand_hg_changeset;
use crate::redaction::subcommand_redaction;

mod blobstore_fetch;
mod blobstore_unlink;
mod blobstore_upload;
mod bonsai_fetch;
mod bookmarks_manager;
mod common;
mod content_fetch;
mod crossrepo;
mod derived_data;
mod error;
mod filenodes;
mod hash_convert;
mod hg_changeset;
mod redaction;
mod rsync;
mod subcommand_blame;
mod subcommand_deleted_manifest;
mod subcommand_fsnodes;
mod subcommand_phases;
mod subcommand_skeleton_manifests;
mod subcommand_unodes;
mod truncate_segmented_changelog;

fn setup_app<'a, 'b>() -> MononokeClapApp<'a, 'b> {
    args::MononokeAppBuilder::new("Mononoke admin command line tool")
        .with_arg_types(vec![ArgType::Scrub])
        .with_advanced_args_hidden()
        .with_source_and_target_repos()
        .with_special_put_behaviour(PutBehaviour::Overwrite)
        .build()
        .about("Poke at mononoke internals for debugging and investigating data structures.")
        .subcommand(blobstore_fetch::build_subcommand())
        .subcommand(blobstore_unlink::build_subcommand())
        .subcommand(blobstore_upload::build_subcommand())
        .subcommand(bonsai_fetch::build_subcommand())
        .subcommand(content_fetch::build_subcommand())
        .subcommand(bookmarks_manager::build_subcommand())
        .subcommand(hg_changeset::build_subcommand())
        .subcommand(hash_convert::build_subcommand())
        .subcommand(redaction::build_subcommand())
        .subcommand(filenodes::build_subcommand())
        .subcommand(subcommand_phases::build_subcommand())
        .subcommand(subcommand_unodes::build_subcommand())
        .subcommand(subcommand_fsnodes::build_subcommand())
        .subcommand(crossrepo::build_subcommand())
        .subcommand(subcommand_blame::build_subcommand())
        .subcommand(subcommand_deleted_manifest::build_subcommand())
        .subcommand(derived_data::build_subcommand())
        .subcommand(rsync::build_subcommand())
        .subcommand(subcommand_skeleton_manifests::build_subcommand())
        .subcommand(truncate_segmented_changelog::build_subcommand())
}

#[fbinit::main]
fn main(fb: FacebookInit) -> ExitCode {
    let (matches, runtime) = setup_app()
        .get_matches(fb)
        .expect("Failed to start Mononoke");

    let logger = matches.logger().clone();
    let error_logger = logger.clone();

    let debug = matches.is_present("debug");

    let res = runtime.block_on(async {
        match matches.subcommand() {
            (blobstore_fetch::BLOBSTORE_FETCH, Some(sub_m)) => {
                subcommand_blobstore_fetch(fb, logger, &matches, sub_m).await
            }
            (blobstore_unlink::BLOBSTORE_UNLINK, Some(sub_m)) => {
                subcommand_blobstore_unlink(fb, logger, &matches, sub_m).await
            }
            (blobstore_upload::BLOBSTORE_UPLOAD, Some(sub_m)) => {
                subcommand_blobstore_upload(fb, logger, &matches, sub_m).await
            }
            (bonsai_fetch::BONSAI_FETCH, Some(sub_m)) => {
                subcommand_bonsai_fetch(fb, logger, &matches, sub_m).await
            }
            (content_fetch::CONTENT_FETCH, Some(sub_m)) => {
                subcommand_content_fetch(fb, logger, &matches, sub_m).await
            }
            (bookmarks_manager::BOOKMARKS, Some(sub_m)) => {
                let ctx = CoreContext::new_with_logger(fb, logger.clone());
                let repo =
                    args::not_shardmanager_compatible::open_repo(fb, &logger, &matches).await?;
                bookmarks_manager::handle_command(ctx, repo, sub_m, logger.clone()).await
            }
            (hg_changeset::HG_CHANGESET, Some(sub_m)) => {
                subcommand_hg_changeset(fb, logger, &matches, sub_m).await
            }
            (hash_convert::HASH_CONVERT, Some(sub_m)) => {
                subcommand_hash_convert(fb, logger, &matches, sub_m).await
            }
            (redaction::REDACTION, Some(sub_m)) => {
                subcommand_redaction(fb, logger, &matches, sub_m).await
            }
            (filenodes::FILENODES, Some(sub_m)) => {
                subcommand_filenodes(fb, logger, &matches, sub_m).await
            }
            (subcommand_phases::PHASES, Some(sub_m)) => {
                subcommand_phases::subcommand_phases(fb, logger, &matches, sub_m).await
            }
            (subcommand_unodes::UNODES, Some(sub_m)) => {
                subcommand_unodes::subcommand_unodes(fb, logger, &matches, sub_m).await
            }
            (subcommand_fsnodes::FSNODES, Some(sub_m)) => {
                subcommand_fsnodes::subcommand_fsnodes(fb, logger, &matches, sub_m).await
            }
            (crossrepo::CROSSREPO, Some(sub_m)) => {
                subcommand_crossrepo(fb, logger, &matches, sub_m).await
            }
            (subcommand_blame::BLAME, Some(sub_m)) => {
                subcommand_blame::subcommand_blame(fb, logger, &matches, sub_m).await
            }
            (subcommand_deleted_manifest::DELETED_MANIFEST, Some(sub_m)) => {
                subcommand_deleted_manifest::subcommand_deleted_manifest(
                    fb, logger, &matches, sub_m,
                )
                .await
            }
            (derived_data::DERIVED_DATA, Some(sub_m)) => {
                derived_data::subcommand_derived_data(fb, logger, &matches, sub_m).await
            }
            (rsync::RSYNC, Some(sub_m)) => {
                rsync::subcommand_rsync(fb, logger, &matches, sub_m).await
            }
            (subcommand_skeleton_manifests::SKELETON_MANIFESTS, Some(sub_m)) => {
                subcommand_skeleton_manifests::subcommand_skeleton_manifests(
                    fb, logger, &matches, sub_m,
                )
                .await
            }
            (truncate_segmented_changelog::TRUNCATE_SEGMENTED_CHANGELOG, Some(sub_m)) => {
                truncate_segmented_changelog::subcommand_truncate_segmented_changelog(
                    fb, logger, &matches, sub_m,
                )
                .await
            }
            _ => Err(SubcommandError::InvalidArgs),
        }
    });

    match res {
        Ok(_) => ExitCode::SUCCESS,
        Err(SubcommandError::Error(err)) => {
            if debug {
                error!(error_logger, "{:?}", err);
                error!(error_logger, "\n============ DEBUG ERROR ============");
                error!(error_logger, "{:#?}", err);
            } else {
                let mut err_string = format!("{:?}", err);
                if let Some(pos) = err_string.find("\n\nStack backtrace:") {
                    err_string.truncate(pos);
                }
                error!(error_logger, "{}", err_string);
            }
            ExitCode::FAILURE
        }
        Err(SubcommandError::InvalidArgs) => {
            eprintln!("{}", matches.usage());
            ExitCode::FAILURE
        }
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashSet;
use std::fs::File;
use std::io::BufRead;
use std::io::BufReader;
use std::sync::Arc;

use anyhow::anyhow;
use anyhow::format_err;
use anyhow::Context;
use anyhow::Error;
use blobrepo::BlobRepo;
use blobstore::Loadable;
use blobstore::Storable;
use clap_old::App;
use clap_old::Arg;
use clap_old::ArgGroup;
use clap_old::ArgMatches;
use clap_old::SubCommand;
use cloned::cloned;
use cmdlib::args;
use cmdlib::args::MononokeMatches;
use cmdlib::helpers;
use context::CoreContext;
use fbinit::FacebookInit;
use futures::future::try_join_all;
use futures::future::TryFutureExt;
use futures::stream::StreamExt;
use futures::stream::TryStreamExt;
use manifest::ManifestOps;
use mercurial_derivation::DeriveHgChangeset;
use mercurial_types::blobs::HgBlobChangeset;
use mercurial_types::HgChangesetId;
use mercurial_types::MPath;
use mononoke_types::blob::BlobstoreValue;
use mononoke_types::typed_hash::BlobstoreKey;
use mononoke_types::ContentId;
use mononoke_types::RedactionKeyList;
use mononoke_types::Timestamp;
use redactedblobstore::SqlRedactedContentStore;
use repo_blobstore::RepoBlobstoreRef;
use repo_factory::RepoFactory;
use slog::error;
use slog::info;
use slog::Logger;

use crate::common::get_file_nodes;
use crate::error::SubcommandError;

pub const REDACTION: &str = "redaction";
const REDACTION_ADD: &str = "add";
const REDACTION_CREATE_KEY_LIST: &str = "create-key-list";
const REDACTION_CREATE_KEY_LIST_RAW: &str = "create-key-list-from-ids";
const REDACTION_REMOVE: &str = "remove";
const REDACTION_LIST: &str = "list";
const ARG_KEYS: &str = "keys";
const ARG_HASH: &str = "hash";
const ARG_TASK: &str = "task";
const ARG_LOG_ONLY: &str = "log-only";
const ARG_FORCE: &str = "force";
const ARG_INPUT_FILE: &str = "input-file";
const ARG_MAIN_BOOKMARK: &str = "main-bookmark";
const DEFAULT_MAIN_BOOKMARK: &str = "master";

pub fn build_subcommand<'a, 'b>() -> App<'a, 'b> {
    SubCommand::with_name(REDACTION)
        .about("handle file redaction")
        .subcommand(add_path_parameters(
            SubCommand::with_name(REDACTION_CREATE_KEY_LIST)
                .about("Add a new redaction key list to the blobstore, to be later configured via configerator.")
                .arg(
                    Arg::with_name(ARG_HASH)
                        .help("hg id, bonsai id or bookmark")
                        .takes_value(true)
                        .required(true),
                )
                .arg(
                    Arg::with_name(ARG_MAIN_BOOKMARK)
                        .long(ARG_MAIN_BOOKMARK)
                        .takes_value(true)
                        .required(false)
                        .default_value(DEFAULT_MAIN_BOOKMARK)
                        .help("Redaction fails if any of the content to be redacted is present in --main-bookmark unless --force is set.")
                )
                .arg(
                    Arg::with_name(ARG_FORCE)
                        .long(ARG_FORCE)
                        .takes_value(false)
                        .help("by default redaction fails if any of the redacted files is in main-bookmark. This flag overrides it.")
                )
        ))
        .subcommand(
            SubCommand::with_name(REDACTION_CREATE_KEY_LIST_RAW)
                .about("Add a new redaction set to the blobstore, to be later configured via configerator. This is given hashes directly.")
                .arg(
                    Arg::with_name(ARG_KEYS)
                        .long(ARG_KEYS)
                        .multiple(true)
                        .takes_value(true)
                        .help("List of keys")
                )
        )
        .subcommand(add_path_parameters(
            SubCommand::with_name(REDACTION_ADD)
                .about("add a new redacted file at a given commit")
                .arg(
                    Arg::with_name(ARG_TASK)
                        .help("Task tracking the redaction request")
                        .takes_value(true)
                        .required(true),
                )
                .arg(
                    Arg::with_name(ARG_HASH)
                        .help("hg commit hash")
                        .takes_value(true)
                        .required(true),
                )
                .arg(
                    Arg::with_name(ARG_MAIN_BOOKMARK)
                        .long(ARG_MAIN_BOOKMARK)
                        .takes_value(true)
                        .required(false)
                        .default_value(DEFAULT_MAIN_BOOKMARK)
                        .help("Redaction fails if any of the content to be redacted is present in --main-bookmark unless --force is set.")
                )
                .arg(
                    Arg::with_name(ARG_FORCE)
                        .long(ARG_FORCE)
                        .takes_value(false)
                        .help("by default redaction fails if any of the redacted files is in main-bookmark. This flag overrides it.")
                )
                .arg(
                    Arg::with_name(ARG_LOG_ONLY)
                        .long(ARG_LOG_ONLY)
                        .takes_value(false)
                        .help("redact file in log-only mode. All accesses to this file will be allowed, but they will all be logged")
                )
        ))
        .subcommand(add_path_parameters(
            SubCommand::with_name(REDACTION_REMOVE)
                .about("remove a file from the redaction")
                .arg(
                    Arg::with_name(ARG_HASH)
                        .help("hg commit hash")
                        .takes_value(true)
                        .required(true),
                ),
        ))
        .subcommand(
            SubCommand::with_name(REDACTION_LIST)
                .about("list all redacted file for a given commit")
                .arg(
                    Arg::with_name(ARG_HASH)
                        .help("hg commit hash or a bookmark")
                        .takes_value(true)
                        .required(true),
                ),
        )
}

pub fn add_path_parameters<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> {
    app.arg(
        Arg::with_name(ARG_INPUT_FILE)
            .long(ARG_INPUT_FILE)
            .help("file with a list of filenames to redact")
            .takes_value(true)
            .required(false),
    )
    .args_from_usage(
        r#"
                [FILES_LIST]...                             'list of files to be be redacted'
                "#,
    )
    .group(
        ArgGroup::with_name("input_files")
            .args(&["FILES_LIST", ARG_INPUT_FILE])
            .required(true),
    )
}

async fn find_files_with_given_content_id_blobstore_keys<'a>(
    ctx: &'a CoreContext,
    repo: &'a BlobRepo,
    cs: HgBlobChangeset,
    keys: HashSet<&String>,
) -> Result<Vec<(MPath, ContentId)>, Error> {
    let manifest_id = cs.manifestid();
    let mut s = manifest_id
        .list_leaf_entries(ctx.clone(), repo.repo_blobstore().clone())
        .map_ok(|(full_path, (_, filenode_id))| async move {
            let env = filenode_id.load(ctx, repo.repo_blobstore()).await?;
            Result::<_, Error>::Ok((env.content_id(), full_path))
        })
        .try_buffer_unordered(100);

    let mut paths_and_content_ids = vec![];
    let mut processed_files_count = 0usize;
    while let Some(key_and_path) = s.next().await {
        let (key, full_path) = key_and_path?;
        processed_files_count += 1;
        if processed_files_count % 100_000 == 0 {
            info!(ctx.logger(), "Processed files: {}", processed_files_count);
        }

        if keys.contains(&key.blobstore_key()) {
            paths_and_content_ids.push((full_path, key));
        }
    }
    Ok(paths_and_content_ids)
}

/// Entrypoint for redaction subcommand handling
pub async fn subcommand_redaction<'a>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'_>,
    sub_m: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
    match sub_m.subcommand() {
        (REDACTION_CREATE_KEY_LIST, Some(sub_sub_m)) => {
            redaction_create_key_list(fb, logger, matches, sub_sub_m).await
        }
        (REDACTION_CREATE_KEY_LIST_RAW, Some(sub_sub_m)) => {
            redaction_create_key_list_raw(fb, logger, matches, sub_sub_m).await
        }
        (REDACTION_ADD, Some(sub_sub_m)) => redaction_add(fb, logger, matches, sub_sub_m).await,
        (REDACTION_REMOVE, Some(sub_sub_m)) => {
            redaction_remove(fb, logger, matches, sub_sub_m).await
        }
        (REDACTION_LIST, Some(sub_sub_m)) => redaction_list(fb, logger, matches, sub_sub_m).await,
        _ => {
            eprintln!("{}", matches.usage());
            ::std::process::exit(1);
        }
    }
}

/// Fetch the file list from the subcommand cli matches
fn paths_parser(sub_m: &ArgMatches<'_>) -> Result<Vec<MPath>, Error> {
    match sub_m.values_of("FILES_LIST") {
        Some(values) => values.map(|s| s.to_string()).map(MPath::new).collect(),
        None => match sub_m.value_of(ARG_INPUT_FILE) {
            Some(inputfile) => {
                let inputfile = File::open(inputfile)?;
                let input_file = BufReader::new(&inputfile);
                let mut files = vec![];
                for line in input_file.lines() {
                    let line = line?;
                    files.push(MPath::new(line)?);
                }

                Ok(files)
            }
            None => Err(format_err!("file list is not specified")),
        },
    }
}

fn task_parser(sub_m: &ArgMatches<'_>) -> Result<String, Error> {
    match sub_m.value_of(ARG_TASK) {
        Some(task) => Ok(task.to_string()),
        None => Err(format_err!("Task is needed")),
    }
}

/// Fetch the task id and the file list from the subcommand cli matches
fn task_and_paths_parser(sub_m: &ArgMatches<'_>) -> Result<(String, Vec<MPath>), Error> {
    Ok((task_parser(sub_m)?, paths_parser(sub_m)?))
}

/// Boilerplate to prepare a bunch of prerequisites for the rest of blaclisting operations
async fn get_ctx_blobrepo_cs_id<'a>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'_>,
    sub_m: &'a ArgMatches<'_>,
) -> Result<(CoreContext, BlobRepo, HgChangesetId), SubcommandError> {
    let rev = match sub_m.value_of(ARG_HASH) {
        Some(rev) => rev.to_string(),
        None => return Err(SubcommandError::InvalidArgs),
    };

    let blobrepo: BlobRepo =
        args::not_shardmanager_compatible::open_repo(fb, &logger, matches).await?;

    let ctx = CoreContext::new_with_logger(fb, logger);

    let cs_id = helpers::csid_resolve(&ctx, blobrepo.clone(), rev.to_string()).await?;
    let hg_cs_id = blobrepo.derive_hg_changeset(&ctx, cs_id).await?;

    Ok((ctx, blobrepo, hg_cs_id))
}

/// Fetch a vector of `ContentId`s for a vector of `MPath`s
async fn content_ids_for_paths(
    ctx: CoreContext,
    logger: Logger,
    blobrepo: BlobRepo,
    cs_id: HgChangesetId,
    paths: Vec<MPath>,
) -> Result<Vec<ContentId>, Error> {
    let hg_node_ids = get_file_nodes(ctx.clone(), logger, &blobrepo, cs_id, paths).await?;
    let content_ids = hg_node_ids.into_iter().map(|hg_node_id| {
        cloned!(ctx, blobrepo);
        async move {
            let env = hg_node_id.load(&ctx, blobrepo.repo_blobstore()).await?;
            Ok(env.content_id())
        }
    });
    try_join_all(content_ids).await
}

async fn redaction_create_key_list_impl<'a, 'b>(
    _fb: FacebookInit,
    _logger: Logger,
    matches: &'a MononokeMatches<'b>,
    ctx: &'a CoreContext,
    blobstore_keys: Vec<String>,
) -> Result<(), SubcommandError> {
    let common_config = args::load_common_config(matches.config_store(), matches)?;
    let factory = RepoFactory::new(matches.environment().clone());

    let blobstore = factory
        .redaction_config_blobstore(&Arc::new(common_config.clone()))
        .await?;
    let darkstorm_blobstore = factory
        .redaction_config_blobstore_from_config(
            &common_config
                .redaction_config
                .darkstorm_blobstore
                .ok_or_else(|| {
                    anyhow!("Admin tier config must have darkstorm_blobstore field set")
                })?,
        )
        .await?;

    let store_keys = |blobstore| {
        cloned!(blobstore_keys);
        async move {
            RedactionKeyList {
                keys: blobstore_keys,
            }
            .into_blob()
            .store(ctx, &blobstore)
            .await
        }
    };

    let (id, id2) = futures::try_join!(store_keys(blobstore), store_keys(darkstorm_blobstore))?;

    if id != id2 {
        Err(format_err!(
            "Id mismatch on darkstorm and non-darkstorm blobstores: {} vs {}",
            id,
            id2
        ))?
    }

    println!("Redaction saved as: {}", id);
    println!(
        "To finish the redaction process, you need to commit this id to \
        scm/mononoke/redaction/redaction_sets.cconf on configerator"
    );
    Ok(())
}

async fn redaction_create_key_list<'a, 'b>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'b>,
    sub_m: &'a ArgMatches<'b>,
) -> Result<(), SubcommandError> {
    let paths = paths_parser(sub_m)?;
    let (ctx, blobrepo, cs_id) = get_ctx_blobrepo_cs_id(fb, logger.clone(), matches, sub_m).await?;
    let content_ids =
        content_ids_for_paths(ctx.clone(), logger.clone(), blobrepo.clone(), cs_id, paths).await?;
    let blobstore_keys: Vec<_> = content_ids
        .iter()
        .map(|content_id| content_id.blobstore_key())
        .collect();

    let force = sub_m.is_present(ARG_FORCE);
    if !force {
        let main_bookmark = sub_m
            .value_of(ARG_MAIN_BOOKMARK)
            .unwrap_or(DEFAULT_MAIN_BOOKMARK);
        check_if_content_is_reachable_from_bookmark(
            &ctx,
            &blobrepo,
            blobstore_keys.iter().collect(),
            main_bookmark,
        )
        .await?;
    }

    redaction_create_key_list_impl(fb, logger, matches, &ctx, blobstore_keys).await
}

async fn redaction_create_key_list_raw<'a, 'b>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'b>,
    sub_m: &'a ArgMatches<'b>,
) -> Result<(), SubcommandError> {
    let ctx = CoreContext::new_with_logger(fb, logger.clone());

    let keys = sub_m
        .values_of(ARG_KEYS)
        .ok_or(SubcommandError::InvalidArgs)?
        .map(|key| key.to_string())
        .collect::<Vec<_>>();

    redaction_create_key_list_impl(fb, logger, matches, &ctx, keys).await
}

async fn redaction_add<'a, 'b>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'b>,
    sub_m: &'a ArgMatches<'b>,
) -> Result<(), SubcommandError> {
    let (task, paths) = task_and_paths_parser(sub_m)?;
    let (ctx, blobrepo, cs_id) = get_ctx_blobrepo_cs_id(fb, logger.clone(), matches, sub_m).await?;
    let redacted_blobs = args::not_shardmanager_compatible::open_sql::<SqlRedactedContentStore>(
        fb,
        matches.config_store(),
        matches,
    )
    .context("While opening SqlRedactedContentStore")?;

    let content_ids =
        content_ids_for_paths(ctx.clone(), logger.clone(), blobrepo.clone(), cs_id, paths).await?;

    let blobstore_keys: Vec<_> = content_ids
        .iter()
        .map(|content_id| content_id.blobstore_key())
        .collect();

    let force = sub_m.is_present(ARG_FORCE);
    let log_only = sub_m.is_present(ARG_LOG_ONLY);

    if !force {
        let main_bookmark = sub_m
            .value_of(ARG_MAIN_BOOKMARK)
            .unwrap_or(DEFAULT_MAIN_BOOKMARK);
        check_if_content_is_reachable_from_bookmark(
            &ctx,
            &blobrepo,
            blobstore_keys.iter().collect(),
            main_bookmark,
        )
        .await?;
    }

    let timestamp = Timestamp::now();
    redacted_blobs
        .insert_redacted_blobs(&blobstore_keys, &task, &timestamp, log_only)
        .await?;

    Ok(())
}

async fn redaction_list<'a>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'_>,
    sub_m: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
    let (ctx, blobrepo, cs_id) = get_ctx_blobrepo_cs_id(fb, logger.clone(), matches, sub_m).await?;
    let redacted_blobs = args::not_shardmanager_compatible::open_sql::<SqlRedactedContentStore>(
        fb,
        matches.config_store(),
        matches,
    )
    .context("While opening SqlRedactedContentStore")?;
    info!(
        logger,
        "Listing redacted files for ChangesetId: {:?}", cs_id
    );
    info!(logger, "Please be patient.");
    let (redacted_blobs, hg_cs) = futures::try_join!(
        redacted_blobs.get_all_redacted_blobs(),
        cs_id
            .load(&ctx, blobrepo.repo_blobstore())
            .map_err(Error::from),
    )?;
    let redacted_map = redacted_blobs.redacted();
    let redacted_keys = redacted_map.iter().map(|(key, _)| key).collect();
    let path_keys =
        find_files_with_given_content_id_blobstore_keys(&ctx, &blobrepo, hg_cs, redacted_keys)
            .await?;
    let mut res = path_keys
        .into_iter()
        .filter_map(move |(path, key)| {
            redacted_map
                .get(&key.blobstore_key())
                .cloned()
                .map(|redacted_meta| (redacted_meta.task, path, redacted_meta.log_only))
        })
        .collect::<Vec<_>>();
    if res.is_empty() {
        info!(logger, "No files are redacted at this commit");
    } else {
        res.sort();
        res.into_iter().for_each(|(task_id, file_path, log_only)| {
            let log_only_msg = if log_only { " (log only)" } else { "" };
            info!(logger, "{:20}: {}{}", task_id, file_path, log_only_msg);
        })
    }
    Ok(())
}

async fn redaction_remove<'a>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'_>,
    sub_m: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
    let paths = paths_parser(sub_m)?;
    let (ctx, blobrepo, cs_id) = get_ctx_blobrepo_cs_id(fb, logger.clone(), matches, sub_m).await?;
    let redacted_blobs = args::not_shardmanager_compatible::open_sql::<SqlRedactedContentStore>(
        fb,
        matches.config_store(),
        matches,
    )
    .context("While opening SqlRedactedContentStore")?;
    let content_ids = content_ids_for_paths(ctx, logger, blobrepo, cs_id, paths).await?;
    let blobstore_keys: Vec<_> = content_ids
        .into_iter()
        .map(|content_id| content_id.blobstore_key())
        .collect();
    redacted_blobs
        .delete_redacted_blobs(&blobstore_keys)
        .await
        .map_err(SubcommandError::Error)
}

async fn check_if_content_is_reachable_from_bookmark(
    ctx: &CoreContext,
    blobrepo: &BlobRepo,
    keys_to_redact: HashSet<&String>,
    main_bookmark: &str,
) -> Result<(), Error> {
    info!(
        ctx.logger(),
        "Checking if redacted content exist in '{}' bookmark...", main_bookmark
    );
    let csid = helpers::csid_resolve(ctx, blobrepo, main_bookmark).await?;
    let hg_cs_id = blobrepo.derive_hg_changeset(ctx, csid).await?;

    let hg_cs = hg_cs_id.load(ctx, blobrepo.repo_blobstore()).await?;

    let redacted_files =
        find_files_with_given_content_id_blobstore_keys(ctx, blobrepo, hg_cs, keys_to_redact)
            .await?;
    let redacted_files_len = redacted_files.len();
    if redacted_files_len > 0 {
        for (path, content_id) in redacted_files {
            error!(
                ctx.logger(),
                "Redacted in {}: {} {}",
                main_bookmark,
                path,
                content_id.blobstore_key()
            );
        }
        return Err(anyhow!(
            "{} files will be redacted in {}. \
            That means that checking it out will be impossible!",
            redacted_files_len,
            main_bookmark,
        ));
    }

    Ok(())
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::num::NonZeroU64;

use anyhow::anyhow;
use anyhow::Context;
use anyhow::Error;
use blobrepo::BlobRepo;
use clap_old::App;
use clap_old::Arg;
use clap_old::ArgMatches;
use clap_old::SubCommand;
use cmdlib::args;
use cmdlib::args::MononokeMatches;
use cmdlib::helpers;
use context::CoreContext;
use copy_utils::copy;
use copy_utils::remove_excessive_files;
use copy_utils::Limits;
use copy_utils::Options;
use fbinit::FacebookInit;
use futures::future::try_join;
use mononoke_types::ChangesetId;
use mononoke_types::MPath;
use regex::Regex;
use slog::warn;
use slog::Logger;

use crate::common::get_source_target_repos_and_mapping;
use crate::error::SubcommandError;

pub const ARG_COMMIT_AUTHOR: &str = "commit-author";
pub const ARG_COMMIT_MESSAGE: &str = "commit-message";
pub const ARG_SOURCE_CSID: &str = "source-csid";
pub const ARG_TARGET_CSID: &str = "target-csid";
pub const ARG_EXCLUDE_FILE_REGEX: &str = "exclude-file-regex";
pub const ARG_TOTAL_FILE_NUM_LIMIT: &str = "total-file-num-limit";
pub const ARG_TOTAL_SIZE_LIMIT: &str = "total-size-limit";
pub const ARG_FROM_DIR: &str = "from-dir";
pub const ARG_FROM_TO_DIRS: &str = "from-to-dirs";
pub const ARG_LFS_THRESHOLD: &str = "lfs-threshold";
pub const ARG_OVERWRITE: &str = "overwrite";
pub const ARG_TO_DIR: &str = "to-dir";
pub const RSYNC: &str = "rsync";
pub const SUBCOMMAND_COPY: &str = "copy";
pub const SUBCOMMAND_REMOVE_EXCESSIVE_FILES: &str = "remove-excessive-files";

pub fn build_subcommand<'a, 'b>() -> App<'a, 'b> {
    SubCommand::with_name(RSYNC)
        .subcommand(
            add_common_args(SubCommand::with_name(SUBCOMMAND_COPY)
            .about("creates commits that copy content of one directory into another")
            .arg(
                Arg::with_name(ARG_EXCLUDE_FILE_REGEX)
                    .long(ARG_EXCLUDE_FILE_REGEX)
                    .help("exclude files that should not be copied")
                    .takes_value(true)
                    .required(false),
            )
            .arg(
                Arg::with_name(ARG_TOTAL_SIZE_LIMIT)
                    .long(ARG_TOTAL_SIZE_LIMIT)
                    .help("total size of all files in all commits")
                    .takes_value(true)
                    .required(false),
            )
            .arg(
                Arg::with_name(ARG_LFS_THRESHOLD)
                    .long(ARG_LFS_THRESHOLD)
                    .help(
                        "lfs threshold - files with size above that are excluded from file size limit",
                    )
                    .takes_value(true)
                    .required(false),
            )
            .arg(
                Arg::with_name(ARG_OVERWRITE)
                    .long(ARG_OVERWRITE)
                    .help("overwrite a file if it exists in the destination directory")
                    .takes_value(false)
                    .required(false),
            )
        ))
        .subcommand(
            add_common_args(SubCommand::with_name(SUBCOMMAND_REMOVE_EXCESSIVE_FILES)
                .about("remove files from --to directory that are not present in --from directory"))
        )
}

pub fn add_common_args<'a, 'b>(sub_m: App<'a, 'b>) -> App<'a, 'b> {
    sub_m
        .arg(
            Arg::with_name(ARG_FROM_DIR)
                .long(ARG_FROM_DIR)
                .takes_value(true)
                .required(false)
                .help(
                    "name of the directory to copy from. \
                       Error is return if this path doesn't exist or if it's a file",
                ),
        )
        .arg(
            Arg::with_name(ARG_TO_DIR)
                .long(ARG_TO_DIR)
                .takes_value(true)
                .required(false)
                .help(
                    "name of the directory to copy to. \
                       Error is return if this path is a file",
                ),
        )
        .arg(
            Arg::with_name(ARG_FROM_TO_DIRS)
                .long(ARG_FROM_TO_DIRS)
                .multiple(true)
                .takes_value(true)
                .required(false)
                .help("'from_dir=to_dir' directories that needs copying")
                .conflicts_with_all(&[ARG_FROM_DIR, ARG_TO_DIR]),
        )
        .arg(
            Arg::with_name(ARG_COMMIT_MESSAGE)
                .long(ARG_COMMIT_MESSAGE)
                .help("commit message to use")
                .takes_value(true)
                .required(true),
        )
        .arg(
            Arg::with_name(ARG_COMMIT_AUTHOR)
                .long(ARG_COMMIT_AUTHOR)
                .help("commit author to use")
                .takes_value(true)
                .required(true),
        )
        .arg(
            Arg::with_name(ARG_TOTAL_FILE_NUM_LIMIT)
                .long(ARG_TOTAL_FILE_NUM_LIMIT)
                .help("limit the number of files moved in all commits")
                .takes_value(true)
                .required(false),
        )
        .arg(
            Arg::with_name(ARG_SOURCE_CSID)
                .long(ARG_SOURCE_CSID)
                .takes_value(true)
                .required(true)
                .help("source {hg|bonsai} changeset id or bookmark name"),
        )
        .arg(
            Arg::with_name(ARG_TARGET_CSID)
                .long(ARG_TARGET_CSID)
                .takes_value(true)
                .required(true)
                .help("target {hg|bonsai} changeset id or bookmark name"),
        )
}

/// Get source_cs_id, target_cs_id, from_dir, to_dir, author and commit_msg
/// from subcommand matches
async fn parse_common_args<'a>(
    ctx: &'a CoreContext,
    matches: &'a ArgMatches<'_>,
    source_repo: &'a BlobRepo,
    target_repo: &'a BlobRepo,
) -> Result<
    (
        ChangesetId,
        ChangesetId,
        Vec<(MPath, MPath)>,
        String,
        String,
    ),
    Error,
> {
    let source_cs_id = matches
        .value_of(ARG_SOURCE_CSID)
        .ok_or_else(|| anyhow!("{} arg is not specified", ARG_SOURCE_CSID))?;

    let target_cs_id = matches
        .value_of(ARG_TARGET_CSID)
        .ok_or_else(|| anyhow!("{} arg is not specified", ARG_TARGET_CSID))?;

    let (source_cs_id, target_cs_id) = try_join(
        async {
            helpers::csid_resolve(ctx, source_repo, source_cs_id)
                .await
                .context("failed resolving source_cs_id")
        },
        async {
            helpers::csid_resolve(ctx, target_repo, target_cs_id)
                .await
                .context("failed resolving target_cs_id")
        },
    )
    .await?;

    let from_to_dirs = if let Some(from_to_dirs) = matches.value_of(ARG_FROM_TO_DIRS) {
        let mut res = vec![];
        for from_to in from_to_dirs.split(',') {
            let dirs = from_to.split('=').collect::<Vec<_>>();
            if dirs.len() != 2 {
                return Err(anyhow!("invalid format of {}", ARG_FROM_TO_DIRS));
            }
            res.push((MPath::new(dirs[0])?, MPath::new(dirs[1])?));
        }

        res
    } else {
        let from_dir = matches
            .value_of(ARG_FROM_DIR)
            .ok_or_else(|| anyhow!("{} arg is not specified", ARG_FROM_DIR))?;
        let from_dir = MPath::new(from_dir)?;

        let to_dir = matches
            .value_of(ARG_TO_DIR)
            .ok_or_else(|| anyhow!("{} arg is not specified", ARG_TO_DIR))?;
        let to_dir = MPath::new(to_dir)?;

        vec![(from_dir, to_dir)]
    };

    let author = matches
        .value_of(ARG_COMMIT_AUTHOR)
        .ok_or_else(|| anyhow!("{} arg is not specified", ARG_COMMIT_AUTHOR))?;

    let msg = matches
        .value_of(ARG_COMMIT_MESSAGE)
        .ok_or_else(|| anyhow!("{} arg is not specified", ARG_COMMIT_MESSAGE))?;

    Ok((
        source_cs_id,
        target_cs_id,
        from_to_dirs,
        author.to_string(),
        msg.to_string(),
    ))
}

pub async fn subcommand_rsync<'a>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'_>,
    sub_matches: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
    let ctx = CoreContext::new_with_logger(fb, logger.clone());
    let (source_repo, target_repo, _) =
        get_source_target_repos_and_mapping(fb, logger, matches).await?;

    match sub_matches.subcommand() {
        (SUBCOMMAND_COPY, Some(sub_matches)) => {
            let (source_cs_id, target_cs_id, from_to_dirs, author, msg) =
                parse_common_args(&ctx, sub_matches, &source_repo, &target_repo).await?;
            let cs_ids = copy(
                &ctx,
                &source_repo,
                &target_repo,
                source_cs_id,
                target_cs_id,
                from_to_dirs,
                author,
                msg,
                limits_from_matches(sub_matches),
                options_from_matches(sub_matches)?,
                Some(
                    &(|ctx: &CoreContext, path| {
                        warn!(
                            ctx.logger(),
                            "skipping {} because it already exists in the destination, \
                            use --overwrite to override this behaviour",
                            path
                        );
                    }),
                ),
            )
            .await?;

            let result_cs_id = cs_ids
                .last()
                .copied()
                .ok_or_else(|| anyhow!("nothing to move!"))?;

            println!("{}", result_cs_id);
        }
        (SUBCOMMAND_REMOVE_EXCESSIVE_FILES, Some(sub_matches)) => {
            let (source_cs_id, target_cs_id, from_to_dirs, author, msg) =
                parse_common_args(&ctx, sub_matches, &source_repo, &target_repo).await?;

            let maybe_total_file_num_limit: Option<NonZeroU64> =
                args::get_and_parse_opt(sub_matches, ARG_TOTAL_FILE_NUM_LIMIT);

            let result_cs_id = remove_excessive_files(
                &ctx,
                &source_repo,
                &target_repo,
                source_cs_id,
                target_cs_id,
                from_to_dirs,
                author,
                msg,
                maybe_total_file_num_limit,
            )
            .await?;

            println!("{}", result_cs_id);
        }
        _ => return Err(SubcommandError::InvalidArgs),
    }

    Ok(())
}

fn limits_from_matches(sub_m: &ArgMatches<'_>) -> Limits {
    let maybe_total_file_num_limit: Option<NonZeroU64> =
        args::get_and_parse_opt(sub_m, ARG_TOTAL_FILE_NUM_LIMIT);
    let maybe_total_size_limit: Option<NonZeroU64> =
        args::get_and_parse_opt(sub_m, ARG_TOTAL_SIZE_LIMIT);
    let maybe_lfs_threshold: Option<NonZeroU64> = args::get_and_parse_opt(sub_m, ARG_LFS_THRESHOLD);

    Limits {
        total_file_num_limit: maybe_total_file_num_limit,
        total_size_limit: maybe_total_size_limit,
        lfs_threshold: maybe_lfs_threshold,
    }
}

fn options_from_matches(sub_m: &ArgMatches<'_>) -> Result<Options, Error> {
    let maybe_exclude_file_regex = sub_m.value_of(ARG_EXCLUDE_FILE_REGEX);
    let maybe_exclude_file_regex = maybe_exclude_file_regex
        .map(Regex::new)
        .transpose()
        .map_err(Error::from)?;

    let overwrite = sub_m.is_present(ARG_OVERWRITE);

    Ok(Options {
        maybe_exclude_file_regex,
        overwrite,
    })
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;
use std::fmt::Write;

use anyhow::format_err;
use anyhow::Error;
use blame::fetch_blame_v2;
use blame::fetch_content_for_blame;
use blame::FetchOutcome;
use blobrepo::BlobRepo;
use blobrepo_hg::BlobRepoHg;
use blobstore::Loadable;
use bounded_traversal::bounded_traversal_dag;
use bounded_traversal::Iter;
use clap_old::App;
use clap_old::Arg;
use clap_old::ArgMatches;
use clap_old::SubCommand;
use cloned::cloned;
use cmdlib::args;
use cmdlib::args::MononokeMatches;
use cmdlib::helpers;
use context::CoreContext;
use derived_data::BonsaiDerived;
use fbinit::FacebookInit;
use futures::future::try_join;
use futures::future::try_join_all;
use futures::Future;
use futures::FutureExt;
use futures::StreamExt;
use futures::TryFutureExt;
use futures::TryStreamExt;
use manifest::ManifestOps;
use mononoke_types::blame_v2::BlameParent;
use mononoke_types::blame_v2::BlameRejected;
use mononoke_types::blame_v2::BlameV2;
use mononoke_types::BlameV2Id;
use mononoke_types::ChangesetId;
use mononoke_types::FileChange;
use mononoke_types::FileUnodeId;
use mononoke_types::MPath;
use repo_blobstore::RepoBlobstoreArc;
use repo_blobstore::RepoBlobstoreRef;
use slog::Logger;
use unodes::RootUnodeManifestId;

use crate::error::SubcommandError;

pub const BLAME: &str = "blame";
const COMMAND_DERIVE: &str = "derive";
const COMMAND_COMPUTE: &str = "compute";
const COMMAND_DIFF: &str = "diff";
const COMMAND_FIND_REJECTED: &str = "find-rejected";

const ARG_CSID: &str = "csid";
const ARG_PATH: &str = "path";
const ARG_PRINT_ERRORS: &str = "print-errors";
const ARG_LINE: &str = "line";

pub fn build_subcommand<'a, 'b>() -> App<'a, 'b> {
    let csid_arg = Arg::with_name(ARG_CSID)
        .help("{hg|bonsai} changeset id or bookmark name")
        .index(1)
        .required(true);
    let path_arg = Arg::with_name(ARG_PATH)
        .help("path")
        .index(2)
        .required(true);
    let line_number_arg = Arg::with_name(ARG_LINE)
        .help("show line number at the first appearance")
        .short("l")
        .long("line-number")
        .takes_value(false)
        .required(false);

    SubCommand::with_name(BLAME)
        .about("fetch/derive blame for specified changeset and path")
        .subcommand(
            SubCommand::with_name(COMMAND_DERIVE)
                .arg(line_number_arg.clone())
                .arg(csid_arg.clone())
                .arg(path_arg.clone()),
        )
        .subcommand(
            SubCommand::with_name(COMMAND_DIFF)
                .arg(csid_arg.clone())
                .arg(path_arg.clone()),
        )
        .subcommand(
            SubCommand::with_name(COMMAND_COMPUTE)
                .arg(line_number_arg.clone())
                .arg(csid_arg.clone())
                .arg(path_arg.clone()),
        )
        .subcommand(
            SubCommand::with_name(COMMAND_FIND_REJECTED)
                .arg(csid_arg.clone())
                .arg(
                    Arg::with_name(ARG_PRINT_ERRORS)
                        .help("print why the file is rejected")
                        .long(ARG_PRINT_ERRORS)
                        .takes_value(false)
                        .required(false),
                ),
        )
}

pub async fn subcommand_blame<'a>(
    fb: FacebookInit,
    logger: Logger,
    toplevel_matches: &'a MononokeMatches<'_>,
    sub_matches: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
    let ctx = CoreContext::new_with_logger(fb, logger.clone());

    match sub_matches.subcommand() {
        (COMMAND_DERIVE, Some(matches)) => {
            let repo =
                args::not_shardmanager_compatible::open_repo(fb, &logger, toplevel_matches).await?;
            let line_number = matches.is_present(ARG_LINE);
            with_changeset_and_path(ctx, repo, matches, move |ctx, repo, csid, path| {
                subcommand_show_blame(ctx, repo, csid, path, line_number)
            })
            .await
        }
        (COMMAND_DIFF, Some(matches)) => {
            let repo =
                args::not_shardmanager_compatible::open_repo(fb, &logger, toplevel_matches).await?;
            with_changeset_and_path(ctx, repo, matches, subcommand_show_diffs).await
        }
        (COMMAND_COMPUTE, Some(matches)) => {
            let repo =
                args::not_shardmanager_compatible::open_repo(fb, &logger, toplevel_matches).await?;
            let line_number = matches.is_present(ARG_LINE);
            with_changeset_and_path(ctx, repo, matches, move |ctx, repo, csid, path| {
                subcommand_compute_blame(ctx, repo, csid, path, line_number)
            })
            .await
        }
        (COMMAND_FIND_REJECTED, Some(matches)) => {
            let print_errors = matches.is_present(ARG_PRINT_ERRORS);
            let hash_or_bookmark = String::from(matches.value_of(ARG_CSID).unwrap());
            let repo: BlobRepo =
                args::not_shardmanager_compatible::open_repo(fb, &logger, toplevel_matches).await?;
            let cs_id = helpers::csid_resolve(&ctx, repo.clone(), hash_or_bookmark).await?;

            let derived_unode = RootUnodeManifestId::derive(&ctx, &repo, cs_id)
                .map_err(Error::from)
                .await?;

            let mut paths = derived_unode
                .manifest_unode_id()
                .list_leaf_entries(ctx.clone(), repo.repo_blobstore().clone())
                .map_ok(|(path, file_unode_id)| {
                    let id = BlameV2Id::from(file_unode_id);
                    cloned!(ctx, repo);
                    async move { id.load(&ctx, repo.repo_blobstore()).await }
                        .map_ok(move |blame| (path, blame))
                        .map_err(Error::from)
                })
                .try_buffer_unordered(100)
                .try_filter_map(|(path, blame_maybe_rejected)| async move {
                    match blame_maybe_rejected {
                        BlameV2::Rejected(rejected) => Ok(Some((path, rejected))),
                        BlameV2::Blame(_) => Ok(None),
                    }
                })
                .boxed();

            while let Some((p, rejected)) = paths.try_next().await? {
                if print_errors {
                    println!("{} {}", p, rejected);
                } else {
                    println!("{}", p);
                }
            }
            Ok(())
        }
        _ => Err(SubcommandError::InvalidArgs),
    }
}

async fn with_changeset_and_path<F, FOut>(
    ctx: CoreContext,
    repo: BlobRepo,
    matches: &ArgMatches<'_>,
    fun: F,
) -> Result<(), SubcommandError>
where
    F: FnOnce(CoreContext, BlobRepo, ChangesetId, MPath) -> FOut + Send + 'static,
    FOut: Future<Output = Result<(), Error>> + Send + 'static,
{
    let hash_or_bookmark = String::from(matches.value_of(ARG_CSID).unwrap());
    let path = MPath::new(matches.value_of(ARG_PATH).unwrap())?;
    let csid = helpers::csid_resolve(&ctx, &repo, hash_or_bookmark).await?;
    fun(ctx, repo, csid, path).await?;
    Ok(())
}

async fn subcommand_show_blame(
    ctx: CoreContext,
    repo: BlobRepo,
    csid: ChangesetId,
    path: MPath,
    line_number: bool,
) -> Result<(), Error> {
    let (blame, unode_id) = fetch_blame_v2(&ctx, &repo, csid, path.clone()).await?;
    let content = fetch_content_for_blame(&ctx, &repo, unode_id)
        .await?
        .into_bytes()?;
    let annotate = blame_hg_annotate(ctx, repo, content, blame, line_number).await?;
    println!("{}", annotate);
    Ok(())
}

/// Finds a leaf that should exist.  Returns an error if the path is not
/// a file in this changeset.
async fn find_leaf(
    ctx: CoreContext,
    repo: BlobRepo,
    csid: ChangesetId,
    path: MPath,
) -> Result<FileUnodeId, Error> {
    let mf_root = RootUnodeManifestId::derive(&ctx, &repo, csid).await?;
    let entry_opt = mf_root
        .manifest_unode_id()
        .clone()
        .find_entry(ctx, repo.repo_blobstore().clone(), Some(path.clone()))
        .await?;
    let entry = entry_opt.ok_or_else(|| format_err!("No such path: {}", path))?;
    match entry.into_leaf() {
        None => Err(format_err!(
            "Blame is not available for directories: {}",
            path
        )),
        Some(file_unode_id) => Ok(file_unode_id),
    }
}

/// Attempts to find a leaf, but returns `None` if the path is not a file.
async fn try_find_leaf(
    ctx: CoreContext,
    repo: BlobRepo,
    csid: ChangesetId,
    path: MPath,
) -> Result<Option<FileUnodeId>, Error> {
    let mf_root = RootUnodeManifestId::derive(&ctx, &repo, csid).await?;
    let entry_opt = mf_root
        .manifest_unode_id()
        .clone()
        .find_entry(ctx, repo.repo_blobstore().clone(), Some(path.clone()))
        .await?;
    Ok(entry_opt.and_then(|entry| entry.into_leaf()))
}

async fn subcommand_show_diffs(
    ctx: CoreContext,
    repo: BlobRepo,
    csid: ChangesetId,
    path: MPath,
) -> Result<(), Error> {
    let file_unode_id = find_leaf(ctx.clone(), repo.clone(), csid, path).await?;
    let file_unode = file_unode_id.load(&ctx, repo.repo_blobstore()).await?;
    let diffs = file_unode
        .parents()
        .iter()
        .map(|parent| diff(ctx.clone(), repo.clone(), file_unode_id, *parent));
    let diffs = try_join_all(diffs).await?;
    for diff in diffs {
        println!("{}", diff);
    }
    Ok(())
}

async fn diff(
    ctx: CoreContext,
    repo: BlobRepo,
    new: FileUnodeId,
    old: FileUnodeId,
) -> Result<String, Error> {
    let f1 = fetch_content_for_blame(&ctx, &repo, new);
    let f2 = fetch_content_for_blame(&ctx, &repo, old);
    let (new, old) = try_join(f1, f2).await?;
    let new = xdiff::DiffFile {
        path: "new",
        contents: xdiff::FileContent::Inline(new.into_bytes()?),
        file_type: xdiff::FileType::Regular,
    };
    let old = xdiff::DiffFile {
        path: "old",
        contents: xdiff::FileContent::Inline(old.into_bytes()?),
        file_type: xdiff::FileType::Regular,
    };
    let diff = xdiff::diff_unified(
        Some(old),
        Some(new),
        xdiff::DiffOpts {
            context: 3,
            copy_info: xdiff::CopyInfo::None,
        },
    );
    Ok(String::from_utf8_lossy(&diff).into_owned())
}

/// Recalculate blame by going through whole history of a file
async fn subcommand_compute_blame(
    ctx: CoreContext,
    repo: BlobRepo,
    csid: ChangesetId,
    path: MPath,
    line_number: bool,
) -> Result<(), Error> {
    let blobstore = repo.repo_blobstore_arc();
    let file_unode_id = find_leaf(ctx.clone(), repo.clone(), csid, path.clone()).await?;
    let (_, _, content, blame) = bounded_traversal_dag(
        256,
        (None, path.clone(), file_unode_id),
        {
            // unfold operator traverses all parents of a given unode, accounting for
            // renames and treating them as another parent.
            |(parent_index, path, file_unode_id)| {
                cloned!(ctx, repo, blobstore);
                async move {
                    let file_unode = file_unode_id.load(&ctx, &blobstore).await?;
                    let csid = *file_unode.linknode();
                    let bonsai = csid.load(&ctx, &blobstore).await?;
                    let mut parents = Vec::new();
                    if bonsai.parents().count() == 1 {
                        // The bonsai changeset only has a single parent, so
                        // we can assume that is where the file came from.
                        for parent_unode_id in file_unode.parents().iter() {
                            parents.push((Some(0), path.clone(), *parent_unode_id));
                        }
                    } else {
                        // We must work out which is the first changeset parent that the
                        // parent unodes came from.
                        let mut parent_indexes = HashMap::new();
                        for (parent_index, parent_csid) in bonsai.parents().enumerate() {
                            if let Some(parent_file_unode_id) =
                                try_find_leaf(ctx.clone(), repo.clone(), parent_csid, path.clone())
                                    .await?
                            {
                                parent_indexes.insert(parent_file_unode_id, parent_index);
                            }
                        }
                        for parent_unode_id in file_unode.parents().iter() {
                            parents.push((
                                parent_indexes.get(parent_unode_id).copied(),
                                path.clone(),
                                *parent_unode_id,
                            ));
                        }
                    }
                    let copy_from = bonsai
                        .file_changes_map()
                        .get(&path)
                        .and_then(|file_change| match file_change {
                            FileChange::Change(tc) => Some(tc.copy_from().clone()?),
                            FileChange::Deletion
                            | FileChange::UntrackedDeletion
                            | FileChange::UntrackedChange(_) => None,
                        });
                    if let Some((r_path, r_csid)) = copy_from {
                        let r_parent_index = bonsai
                            .parents()
                            .position(|csid| csid == *r_csid)
                            .ok_or_else(|| {
                                format_err!(
                                    "commit {} path {} has copy-from with invalid parent {}",
                                    csid,
                                    path,
                                    r_csid,
                                )
                            })?;
                        let r_unode_id =
                            find_leaf(ctx.clone(), repo, *r_csid, r_path.clone()).await?;
                        parents.push((Some(r_parent_index), r_path.clone(), r_unode_id))
                    };
                    Ok::<_, Error>(((csid, parent_index, path, file_unode_id), parents))
                }
                .boxed()
            }
        },
        {
            |(csid, parent_index, path, file_unode_id),
             parents: Iter<
                Result<(Option<usize>, MPath, bytes::Bytes, BlameV2), BlameRejected>,
            >| {
                cloned!(ctx, repo);
                async move {
                    match fetch_content_for_blame(&ctx, &repo, file_unode_id).await? {
                        FetchOutcome::Rejected(rejected) => Ok(Err(rejected)),
                        FetchOutcome::Fetched(content) => {
                            let parents = parents
                                .into_iter()
                                .filter_map(|parent| match parent {
                                    Ok((Some(parent_index), parent_path, content, blame)) => {
                                        Some(BlameParent::new(
                                            parent_index,
                                            parent_path,
                                            content,
                                            blame,
                                        ))
                                    }
                                    _ => None,
                                })
                                .collect();
                            let blame = BlameV2::new(csid, path.clone(), content.clone(), parents)?;
                            Ok(Ok((parent_index, path, content, blame)))
                        }
                    }
                }
                .boxed()
            }
        },
    )
    .await?
    .ok_or_else(|| Error::msg("cycle found"))??;
    let annotate = blame_hg_annotate(ctx, repo, content, blame, line_number).await?;
    println!("{}", annotate);
    Ok(())
}

/// Format blame the same way `hg blame` does
async fn blame_hg_annotate<C: AsRef<[u8]> + 'static + Send>(
    ctx: CoreContext,
    repo: BlobRepo,
    content: C,
    blame: BlameV2,
    show_line_number: bool,
) -> Result<String, Error> {
    if content.as_ref().is_empty() {
        return Ok(String::new());
    }
    let content = String::from_utf8_lossy(content.as_ref());
    let mut result = String::new();
    let csids: Vec<_> = blame.changeset_ids()?.map(|(csid, _)| csid).collect();
    let mapping = repo.get_hg_bonsai_mapping(ctx, csids).await?;
    let mapping: HashMap<_, _> = mapping.into_iter().map(|(k, v)| (v, k)).collect();

    for (line, blame_line) in content.lines().zip(blame.lines()?) {
        let hg_csid = mapping
            .get(blame_line.changeset_id)
            .ok_or_else(|| format_err!("unresolved bonsai csid: {}", blame_line.changeset_id))?;
        write!(
            result,
            "{:>5} ",
            format!("#{}", blame_line.changeset_index + 1)
        )?;
        result.push_str(&hg_csid.to_string()[..12]);
        result.push(':');
        if show_line_number {
            write!(&mut result, "{:>4}:", blame_line.origin_offset + 1)?;
        }
        result.push(' ');
        result.push_str(line);
        result.push('\n');
    }

    Ok(result)
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::BTreeSet;
use std::str::FromStr;

use anyhow::format_err;
use anyhow::Error;
use blobrepo::BlobRepo;
use blobrepo_hg::BlobRepoHg;
use blobstore::Loadable;
use changeset_fetcher::ChangesetFetcherArc;
use clap_old::App;
use clap_old::Arg;
use clap_old::ArgMatches;
use clap_old::SubCommand;
use cloned::cloned;
use cmdlib::args;
use cmdlib::args::MononokeMatches;
use cmdlib::helpers;
use context::CoreContext;
use deleted_manifest::DeletedManifestOps;
use deleted_manifest::RootDeletedManifestV2Id;
use derived_data::BonsaiDerived;
use fbinit::FacebookInit;
use futures::compat::Stream01CompatExt;
use futures::future;
use futures::StreamExt;
use futures::TryStreamExt;
use manifest::get_implicit_deletes;
use manifest::PathOrPrefix;
use mercurial_derivation::DeriveHgChangeset;
use mononoke_types::ChangesetId;
use mononoke_types::DeletedManifestV2Id;
use mononoke_types::MPath;
use repo_blobstore::RepoBlobstoreRef;
use revset::AncestorsNodeStream;
use slog::debug;
use slog::Logger;

use crate::error::SubcommandError;

pub const DELETED_MANIFEST: &str = "deleted-manifest";
const COMMAND_MANIFEST: &str = "manifest";
const COMMAND_VERIFY: &str = "verify";
const COMMAND_FETCH: &str = "fetch";
const ARG_CSID: &str = "csid";
const ARG_ID: &str = "id";
const ARG_LIMIT: &str = "limit";
const ARG_PATH: &str = "path";

pub fn build_subcommand<'a, 'b>() -> App<'a, 'b> {
    let csid_arg = Arg::with_name(ARG_CSID)
        .help("{hg|bonsai} changeset id or bookmark name")
        .index(1)
        .required(true);

    let path_arg = Arg::with_name(ARG_PATH)
        .help("path")
        .index(2)
        .default_value("");

    SubCommand::with_name(DELETED_MANIFEST)
        .about("derive, inspect and verify deleted manifest")
        .subcommand(
            SubCommand::with_name(COMMAND_MANIFEST)
                .about("recursively list all deleted manifest entries under the given path")
                .arg(csid_arg.clone())
                .arg(path_arg.clone()),
        )
        .subcommand(
            SubCommand::with_name(COMMAND_VERIFY)
                .about("verify deleted manifest against actual paths deleted in commits")
                .arg(csid_arg.clone())
                .arg(
                    Arg::with_name(ARG_LIMIT)
                        .help("number of commits to be verified")
                        .takes_value(true)
                        .required(true),
                ),
        )
        .subcommand(
            SubCommand::with_name(COMMAND_FETCH)
                .about("fetch and print deleted manifest entry by id")
                .arg(
                    Arg::with_name(ARG_ID)
                        .help("deleted file manifest id to fetch")
                        .takes_value(true)
                        .required(true),
                ),
        )
}

pub async fn subcommand_deleted_manifest<'a>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'_>,
    sub_matches: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
    let repo: BlobRepo = args::not_shardmanager_compatible::open_repo(fb, &logger, matches).await?;
    let ctx = CoreContext::new_with_logger(fb, logger.clone());

    match sub_matches.subcommand() {
        (COMMAND_MANIFEST, Some(matches)) => {
            let hash_or_bookmark = String::from(matches.value_of(ARG_CSID).unwrap());
            let path = match matches.value_of(ARG_PATH).unwrap() {
                "" => None,
                p => MPath::new(p).map(Some)?,
            };
            let cs_id = helpers::csid_resolve(&ctx, repo.clone(), hash_or_bookmark).await?;
            subcommand_manifest(ctx, repo, cs_id, path).await?;
            Ok(())
        }
        (COMMAND_VERIFY, Some(matches)) => {
            let hash_or_bookmark = String::from(matches.value_of(ARG_CSID).unwrap());
            let limit = matches
                .value_of(ARG_LIMIT)
                .unwrap()
                .parse::<u64>()
                .expect("limit must be an integer");
            let cs_id = helpers::csid_resolve(&ctx, repo.clone(), hash_or_bookmark).await?;
            subcommand_verify(ctx, repo, cs_id, limit).await?;
            Ok(())
        }
        (COMMAND_FETCH, Some(matches)) => {
            let mf_id = DeletedManifestV2Id::from_str(
                matches
                    .value_of(ARG_ID)
                    .ok_or_else(|| format_err!("{} not set", ARG_ID))?,
            )?;
            let mf = mf_id
                .load(&ctx, repo.repo_blobstore())
                .await
                .map_err(Error::from)?;
            println!("{:?}", mf);
            Ok(())
        }
        _ => Err(SubcommandError::InvalidArgs),
    }
}

async fn subcommand_manifest(
    ctx: CoreContext,
    repo: BlobRepo,
    cs_id: ChangesetId,
    prefix: Option<MPath>,
) -> Result<(), Error> {
    let root_manifest = RootDeletedManifestV2Id::derive(&ctx, &repo, cs_id).await?;
    debug!(ctx.logger(), "ROOT Deleted Manifest V2 {:?}", root_manifest,);
    let mut entries: Vec<_> = root_manifest
        .find_entries(
            &ctx,
            repo.repo_blobstore(),
            Some(PathOrPrefix::Prefix(prefix)),
        )
        .try_collect()
        .await?;
    entries.sort_by_key(|(path, _)| path.clone());
    for (path, mf_id) in entries {
        println!("{}/ {:?}", MPath::display_opt(path.as_ref()), mf_id);
    }
    Ok(())
}

async fn subcommand_verify(
    ctx: CoreContext,
    repo: BlobRepo,
    cs_id: ChangesetId,
    limit: u64,
) -> Result<(), Error> {
    let mut csids = AncestorsNodeStream::new(ctx.clone(), &repo.changeset_fetcher_arc(), cs_id)
        .compat()
        .take(limit as usize);
    while let Some(cs_id) = csids.try_next().await? {
        verify_single_commit(ctx.clone(), repo.clone(), cs_id).await?
    }
    Ok(())
}

async fn get_file_changes(
    ctx: CoreContext,
    repo: BlobRepo,
    cs_id: ChangesetId,
) -> Result<(Vec<MPath>, Vec<MPath>), Error> {
    let paths_added_fut = async {
        let bonsai = cs_id.load(&ctx, repo.repo_blobstore()).await?;
        let paths = bonsai
            .into_mut()
            .file_changes
            .into_iter()
            .filter_map(|(path, change)| change.is_changed().then_some(path))
            .collect::<Vec<_>>();
        Ok::<_, Error>(paths)
    };

    let parent_manifests_fut = async {
        let hg_cs_id = repo.derive_hg_changeset(&ctx, cs_id).await?;
        let parents = repo.get_hg_changeset_parents(ctx.clone(), hg_cs_id).await?;
        let parents_futs = parents.into_iter().map(|csid| {
            cloned!(ctx, repo);
            async move {
                let blob = csid.load(&ctx, repo.repo_blobstore()).await?;
                Ok(blob.manifestid())
            }
        });
        future::try_join_all(parents_futs).await
    };

    let (paths_added, parent_manifests) =
        futures::try_join!(paths_added_fut, parent_manifests_fut)?;
    let paths_deleted = get_implicit_deletes(
        &ctx,
        repo.repo_blobstore().clone(),
        paths_added.clone(),
        parent_manifests,
    )
    .try_collect()
    .await?;
    Ok((paths_added, paths_deleted))
}

async fn verify_single_commit(
    ctx: CoreContext,
    repo: BlobRepo,
    cs_id: ChangesetId,
) -> Result<(), Error> {
    let file_changes = get_file_changes(ctx.clone(), repo.clone(), cs_id.clone());
    let deleted_manifest_paths = async move {
        let root_manifest = RootDeletedManifestV2Id::derive(&ctx, &repo, cs_id).await?;
        let entries: BTreeSet<_> = root_manifest
            .list_all_entries(&ctx, repo.repo_blobstore())
            .try_filter_map(|(path_opt, ..)| async move { Ok(path_opt) })
            .try_collect()
            .await?;
        Ok(entries)
    };
    let ((paths_added, paths_deleted), deleted_manifest_paths) =
        futures::try_join!(file_changes, deleted_manifest_paths)?;
    for path in paths_added {
        // check that changed files are alive
        if deleted_manifest_paths.contains(&path) {
            println!("Path {} is alive in changeset {:?}", path, cs_id);
            return Err(format_err!("Path {} is alive", path));
        }
    }
    for path in paths_deleted {
        // check that deleted files are in the manifest
        if !deleted_manifest_paths.contains(&path) {
            println!("Path {} was deleted in changeset {:?}", path, cs_id);
            return Err(format_err!("Path {} is deleted", path));
        }
    }
    Ok(())
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use anyhow::Error;
use blobrepo::BlobRepo;
use clap_old::App;
use clap_old::Arg;
use clap_old::ArgMatches;
use clap_old::SubCommand;
use cmdlib::args;
use cmdlib::args::MononokeMatches;
use cmdlib::helpers;
use context::CoreContext;
use derived_data::BonsaiDerived;
use fbinit::FacebookInit;
use fsnodes::RootFsnodeId;
use futures::stream::StreamExt;
use manifest::Entry;
use manifest::ManifestOps;
use manifest::PathOrPrefix;
use mononoke_types::ChangesetId;
use mononoke_types::MPath;
use repo_blobstore::RepoBlobstoreRef;
use slog::info;
use slog::Logger;

use crate::error::SubcommandError;

pub const FSNODES: &str = "fsnodes";
const COMMAND_TREE: &str = "tree";
const ARG_CSID: &str = "csid";
const ARG_PATH: &str = "path";

pub fn build_subcommand<'a, 'b>() -> App<'a, 'b> {
    SubCommand::with_name(FSNODES)
        .about("inspect fsnodes")
        .subcommand(
            SubCommand::with_name(COMMAND_TREE)
                .about("recursively list all fsnode entries starting with prefix")
                .arg(
                    Arg::with_name(ARG_CSID)
                        .help("{hg|bonsai} changeset id or bookmark name")
                        .required(true),
                )
                .arg(Arg::with_name(ARG_PATH).help("path")),
        )
}

pub async fn subcommand_fsnodes<'a>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'_>,
    sub_matches: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
    let repo: BlobRepo = args::not_shardmanager_compatible::open_repo(fb, &logger, matches).await?;
    let ctx = CoreContext::new_with_logger(fb, logger.clone());

    match sub_matches.subcommand() {
        (COMMAND_TREE, Some(matches)) => {
            let hash_or_bookmark = String::from(matches.value_of(ARG_CSID).unwrap());
            let path = matches.value_of(ARG_PATH).map(MPath::new).transpose()?;

            let csid = helpers::csid_resolve(&ctx, repo.clone(), hash_or_bookmark).await?;
            subcommand_tree(&ctx, &repo, csid, path).await?;
            Ok(())
        }
        _ => Err(SubcommandError::InvalidArgs),
    }
}

async fn subcommand_tree(
    ctx: &CoreContext,
    repo: &BlobRepo,
    csid: ChangesetId,
    path: Option<MPath>,
) -> Result<(), Error> {
    let root = RootFsnodeId::derive(ctx, repo, csid).await?;

    info!(ctx.logger(), "ROOT: {:?}", root);
    info!(ctx.logger(), "PATH: {:?}", path);

    let mut stream = root.fsnode_id().find_entries(
        ctx.clone(),
        repo.repo_blobstore().clone(),
        vec![PathOrPrefix::Prefix(path)],
    );

    while let Some((path, entry)) = stream.next().await.transpose()? {
        match entry {
            Entry::Tree(..) => {}
            Entry::Leaf(file) => {
                println!(
                    "{}\t{}\t{}\t{}",
                    MPath::display_opt(path.as_ref()),
                    file.content_id(),
                    file.file_type(),
                    file.size(),
                );
            }
        };
    }

    Ok(())
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::fs::File;
use std::io::BufRead;
use std::io::BufReader;
use std::io::Write;
use std::str::FromStr;
use std::time::Duration;

use anyhow::bail;
use anyhow::format_err;
use anyhow::Error;
use blobrepo::BlobRepo;
use blobrepo_hg::BlobRepoHg;
use bonsai_hg_mapping::BonsaiHgMappingRef;
use clap_old::App;
use clap_old::Arg;
use clap_old::ArgMatches;
use clap_old::SubCommand;
use cmdlib::args;
use cmdlib::args::MononokeMatches;
use context::CoreContext;
use fbinit::FacebookInit;
use futures::stream;
use futures::StreamExt;
use futures::TryStreamExt;
use mercurial_types::HgChangesetId;
use mononoke_types::ChangesetId;
use phases::PhasesRef;
use slog::info;
use slog::Logger;

use crate::error::SubcommandError;

pub const PHASES: &str = "phases";
const ADD_PUBLIC_PHASES: &str = "add-public";
const FETCH_PHASE: &str = "fetch";
const LIST_PUBLIC: &str = "list-public";

pub fn build_subcommand<'a, 'b>() -> App<'a, 'b> {
    SubCommand::with_name(PHASES)
        .about("commands to work with phases")
        .subcommand(
            SubCommand::with_name(ADD_PUBLIC_PHASES)
                .about("mark mercurial commits as public from provided new-line separated list")
                .arg(
                    Arg::with_name("input-file")
                        .help("new-line separated mercurial public commits")
                        .required(true)
                        .index(1),
                )
                .arg(
                    Arg::with_name("chunk-size")
                        .help("partition input file to chunks of specified size")
                        .long("chunk-size")
                        .takes_value(true),
                ),
        )
        .subcommand(
            SubCommand::with_name(FETCH_PHASE)
                .about("fetch phase of a commit")
                .arg(
                    Arg::with_name("changeset-type")
                        .long("changeset-type")
                        .short("c")
                        .takes_value(true)
                        .possible_values(&["bonsai", "hg"])
                        .required(false)
                        .help(
                            "What changeset type to return, either bonsai or hg. Defaults to hg.",
                        ),
                )
                .arg(
                    Arg::with_name("hash")
                        .help("changeset hash")
                        .takes_value(true),
                ),
        )
        .subcommand(
            SubCommand::with_name(LIST_PUBLIC)
                .arg(
                    Arg::with_name("changeset-type")
                        .long("changeset-type")
                        .short("c")
                        .takes_value(true)
                        .possible_values(&["bonsai", "hg"])
                        .required(false)
                        .help(
                            "What changeset type to return, either bonsai or hg. Defaults to hg.",
                        ),
                )
                .about("List all public commits"),
        )
}

pub async fn subcommand_phases<'a>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'a>,
    sub_m: &'a ArgMatches<'a>,
) -> Result<(), SubcommandError> {
    let repo = args::not_shardmanager_compatible::open_repo(fb, &logger, matches).await?;
    let ctx = CoreContext::new_with_logger(fb, logger.clone());

    match sub_m.subcommand() {
        (FETCH_PHASE, Some(sub_m)) => {
            let ty = sub_m.value_of("changeset-type").unwrap_or("hg").to_string();
            let hash = sub_m
                .value_of("hash")
                .map(|s| s.to_string())
                .ok_or_else(|| Error::msg("changeset hash is not specified"));

            subcommand_fetch_phase_impl(fb, repo, hash, ty)
                .await
                .map_err(SubcommandError::Error)
        }
        (ADD_PUBLIC_PHASES, Some(sub_m)) => {
            let path = String::from(sub_m.value_of("input-file").unwrap());
            let chunk_size = sub_m
                .value_of("chunk-size")
                .and_then(|chunk_size| chunk_size.parse::<usize>().ok())
                .unwrap_or(16384);

            add_public_phases(ctx, repo, logger, path, chunk_size)
                .await
                .map_err(SubcommandError::Error)
        }
        (LIST_PUBLIC, Some(sub_m)) => {
            let ty = sub_m.value_of("changeset-type").unwrap_or("hg").to_string();

            subcommand_list_public_impl(ctx, ty, repo)
                .await
                .map_err(SubcommandError::Error)
        }
        _ => Err(SubcommandError::InvalidArgs),
    }
}

async fn add_public_phases(
    ctx: CoreContext,
    repo: BlobRepo,
    logger: Logger,
    path: impl AsRef<str>,
    chunk_size: usize,
) -> Result<(), Error> {
    let phases = repo.phases();
    let file = File::open(path.as_ref()).map_err(Error::from)?;
    let hg_changesets = BufReader::new(file)
        .lines()
        .filter_map(|id_str| {
            id_str
                .map_err(Error::from)
                .and_then(|v| HgChangesetId::from_str(&v))
                .ok()
        })
        .map(Ok);
    let mut entries_processed: usize = 0;
    info!(logger, "start processing hashes");
    let mut chunks = stream::iter(hg_changesets)
        .chunks(chunk_size)
        .map(|chunk| chunk.into_iter().collect::<Result<Vec<_>, Error>>());
    while let Some(chunk) = chunks.try_next().await? {
        let count = chunk.len();
        let changesets = repo.get_hg_bonsai_mapping(ctx.clone(), chunk).await?;
        let csids = changesets.into_iter().map(|(_, csid)| csid).collect();
        phases
            .add_public_with_known_public_ancestors(&ctx, csids)
            .await?;
        entries_processed += count;
        print!("\x1b[Khashes processed: {}\r", entries_processed);
        std::io::stdout().flush().expect("flush on stdout failed");
        tokio::time::sleep(Duration::from_secs(5)).await;
    }
    Ok(())
}

async fn subcommand_list_public_impl(
    ctx: CoreContext,
    ty: String,
    repo: BlobRepo,
) -> Result<(), Error> {
    let public = repo.phases().list_all_public(&ctx).await?;
    if ty == "bonsai" {
        for p in public {
            println!("{}", p);
        }
    } else {
        for chunk in public.chunks(1000) {
            let bonsais: Vec<_> = chunk.to_vec();
            let hg_bonsais = repo.get_hg_bonsai_mapping(ctx.clone(), bonsais).await?;
            let hg_css: Vec<HgChangesetId> = hg_bonsais
                .clone()
                .into_iter()
                .map(|(hg_cs_id, _)| hg_cs_id)
                .collect();

            for hg_cs in hg_css {
                println!("{}", hg_cs);
            }
        }
    }
    Ok(())
}

pub async fn subcommand_fetch_phase_impl<'a>(
    fb: FacebookInit,
    repo: BlobRepo,
    hash: Result<String, Error>,
    ty: String,
) -> Result<(), Error> {
    let ctx = CoreContext::test_mock(fb);
    let hash = hash?;
    let phases = repo.phases();

    let bcs_id = if ty == "bonsai" {
        ChangesetId::from_str(&hash)?
    } else if ty == "hg" {
        let maybe_bonsai = repo
            .bonsai_hg_mapping()
            .get_bonsai_from_hg(&ctx, HgChangesetId::from_str(&hash)?)
            .await?;
        maybe_bonsai.ok_or_else(|| format_err!("bonsai not found for {}", hash))?
    } else {
        bail!("unknown hash type: {}", ty);
    };

    let public_phases = phases.get_public(&ctx, vec![bcs_id], false).await?;

    if public_phases.contains(&bcs_id) {
        println!("public");
    } else {
        println!("draft");
    }

    Ok(())
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use anyhow::Error;
use blobrepo::BlobRepo;
use blobstore::Loadable;
use clap_old::App;
use clap_old::Arg;
use clap_old::ArgMatches;
use clap_old::SubCommand;
use cmdlib::args;
use cmdlib::args::MononokeMatches;
use cmdlib::helpers;
use context::CoreContext;
use fbinit::FacebookInit;
use futures::stream::StreamExt;
use manifest::Entry;
use manifest::ManifestOps;
use manifest::ManifestOrderedOps;
use manifest::PathOrPrefix;
use mononoke_types::skeleton_manifest::SkeletonManifestEntry;
use mononoke_types::ChangesetId;
use mononoke_types::MPath;
use repo_blobstore::RepoBlobstoreRef;
use skeleton_manifest::RootSkeletonManifestId;
use slog::info;
use slog::Logger;

use crate::derived_data::derive_or_fetch;
use crate::error::SubcommandError;

pub const SKELETON_MANIFESTS: &str = "skeleton-manifests";
const COMMAND_TREE: &str = "tree";
const COMMAND_LIST: &str = "list";
const ARG_CSID: &str = "csid";
const ARG_PATH: &str = "path";
const ARG_IF_DERIVED: &str = "if-derived";
const ARG_ORDERED: &str = "ordered";

pub fn build_subcommand<'a, 'b>() -> App<'a, 'b> {
    SubCommand::with_name(SKELETON_MANIFESTS)
        .about("inspect skeleton manifests")
        .subcommand(
            SubCommand::with_name(COMMAND_TREE)
                .about("recursively list all skeleton manifest entries starting with prefix")
                .arg(
                    Arg::with_name(ARG_CSID)
                        .help("{hg|bonsai} changeset id or bookmark name")
                        .required(true),
                )
                .arg(
                    Arg::with_name(ARG_IF_DERIVED)
                        .help("only list the manifests if they are already derived")
                        .long(ARG_IF_DERIVED),
                )
                .arg(
                    Arg::with_name(ARG_ORDERED)
                        .help("list the manifest in order")
                        .long(ARG_ORDERED),
                )
                .arg(Arg::with_name(ARG_PATH).help("path")),
        )
        .subcommand(
            SubCommand::with_name(COMMAND_LIST)
                .about("list all skeleton manifest entries in a directory")
                .arg(
                    Arg::with_name(ARG_CSID)
                        .help("{hg|bonsai} changeset id or bookmark name")
                        .required(true),
                )
                .arg(
                    Arg::with_name(ARG_IF_DERIVED)
                        .help("only list the manifests if they are already derived")
                        .long(ARG_IF_DERIVED),
                )
                .arg(Arg::with_name(ARG_PATH).help("path")),
        )
}

pub async fn subcommand_skeleton_manifests<'a>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'_>,
    sub_matches: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
    let repo: BlobRepo = args::not_shardmanager_compatible::open_repo(fb, &logger, matches).await?;
    let ctx = CoreContext::new_with_logger(fb, logger.clone());

    match sub_matches.subcommand() {
        (COMMAND_TREE, Some(matches)) => {
            let hash_or_bookmark = String::from(matches.value_of(ARG_CSID).unwrap());
            let path = matches.value_of(ARG_PATH).map(MPath::new).transpose()?;

            let csid = helpers::csid_resolve(&ctx, repo.clone(), hash_or_bookmark).await?;
            let fetch_derived = matches.is_present(ARG_IF_DERIVED);
            let ordered = matches.is_present(ARG_ORDERED);
            subcommand_tree(&ctx, &repo, csid, path, fetch_derived, ordered).await?;
            Ok(())
        }
        (COMMAND_LIST, Some(matches)) => {
            let hash_or_bookmark = String::from(matches.value_of(ARG_CSID).unwrap());
            let path = matches.value_of(ARG_PATH).map(MPath::new).transpose()?;

            let csid = helpers::csid_resolve(&ctx, repo.clone(), hash_or_bookmark).await?;
            let fetch_derived = matches.is_present(ARG_IF_DERIVED);
            subcommand_list(&ctx, &repo, csid, path, fetch_derived).await?;
            Ok(())
        }
        _ => Err(SubcommandError::InvalidArgs),
    }
}

async fn subcommand_list(
    ctx: &CoreContext,
    repo: &BlobRepo,
    csid: ChangesetId,
    path: Option<MPath>,
    fetch_derived: bool,
) -> Result<(), Error> {
    let root = derive_or_fetch::<RootSkeletonManifestId>(ctx, repo, csid, fetch_derived).await?;

    info!(ctx.logger(), "ROOT: {:?}", root);
    info!(ctx.logger(), "PATH: {:?}", path);

    match root
        .skeleton_manifest_id()
        .find_entry(ctx.clone(), repo.repo_blobstore().clone(), path.clone())
        .await?
    {
        Some(Entry::Tree(skeleton_id)) => {
            for (elem, entry) in skeleton_id.load(ctx, repo.repo_blobstore()).await?.list() {
                match entry {
                    SkeletonManifestEntry::Directory(..) => {
                        println!("{}/", MPath::join_opt_element(path.as_ref(), elem));
                    }
                    SkeletonManifestEntry::File => {
                        println!("{}", MPath::join_opt_element(path.as_ref(), elem));
                    }
                }
            }
        }
        Some(Entry::Leaf(())) => println!("{}", MPath::display_opt(path.as_ref())),
        None => {}
    }

    Ok(())
}

async fn subcommand_tree(
    ctx: &CoreContext,
    repo: &BlobRepo,
    csid: ChangesetId,
    path: Option<MPath>,
    fetch_derived: bool,
    ordered: bool,
) -> Result<(), Error> {
    let root = derive_or_fetch::<RootSkeletonManifestId>(ctx, repo, csid, fetch_derived).await?;

    info!(ctx.logger(), "ROOT: {:?}", root);
    info!(ctx.logger(), "PATH: {:?}", path);

    let mut stream = if ordered {
        root.skeleton_manifest_id()
            .find_entries_ordered(
                ctx.clone(),
                repo.repo_blobstore().clone(),
                vec![PathOrPrefix::Prefix(path)],
                None,
            )
            .left_stream()
    } else {
        root.skeleton_manifest_id()
            .find_entries(
                ctx.clone(),
                repo.repo_blobstore().clone(),
                vec![PathOrPrefix::Prefix(path)],
            )
            .right_stream()
    };

    while let Some((path, entry)) = stream.next().await.transpose()? {
        match entry {
            Entry::Tree(..) => {}
            Entry::Leaf(()) => {
                println!("{}", MPath::display_opt(path.as_ref()),);
            }
        };
    }

    Ok(())
}