/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;
use std::fs;
use std::ops::Add;
use std::ops::Sub;
use std::path::Path;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use std::time::Duration;

use anyhow::Context;
use anyhow::Error;
use async_trait::async_trait;
use blobrepo::ChangesetFetcher;
use blobrepo_hg::BlobRepoHg;
use blobstore::Blobstore;
use blobstore::Loadable;
use bonsai_hg_mapping::BonsaiHgMapping;
use bookmarks::BookmarkKey;
use bookmarks::Bookmarks;
use bytes::Bytes;
use changesets::deserialize_cs_entries;
use changesets::ChangesetEntry;
use changesets::Changesets;
use clap::Parser;
use context::CoreContext;
use executor_lib::RepoShardedProcess;
use executor_lib::RepoShardedProcessExecutor;
use executor_lib::ShardedProcessExecutor;
use fbinit::FacebookInit;
use filestore::FilestoreConfig;
use futures::compat::Stream01CompatExt;
use futures::future::FutureExt;
use futures::stream;
use futures::stream::Stream;
use futures::stream::StreamExt;
use futures::stream::TryStreamExt;
use futures::try_join;
use futures_ext::BoxStream;
use futures_ext::StreamExt as OldStreamExt;
use manifest::Diff;
use manifest::Entry;
use manifest::ManifestOps;
use mercurial_derivation::derive_hg_changeset;
use mercurial_types::FileBytes;
use mercurial_types::HgChangesetId;
use mercurial_types::HgFileNodeId;
use mercurial_types::HgManifestId;
use mononoke_app::args::OptRepoArgs;
use mononoke_app::fb303::AliveService;
use mononoke_app::fb303::Fb303AppExtension;
use mononoke_app::MononokeApp;
use mononoke_app::MononokeAppBuilder;
use mononoke_types::FileType;
use mononoke_types::RepositoryId;
use once_cell::sync::OnceCell;
use redactedblobstore::ErrorKind as RedactedBlobstoreError;
use repo_blobstore::RepoBlobstore;
use repo_blobstore::RepoBlobstoreRef;
use repo_derived_data::RepoDerivedData;
use repo_derived_data::RepoDerivedDataRef;
use scuba_ext::MononokeScubaSampleBuilder;
use sharding_ext::RepoShard;
use slog::error;
use slog::info;
use slog::Logger;
use stats::prelude::*;
use tokio::time::sleep;

const SM_SERVICE_SCOPE: &str = "global";
const SM_CLEANUP_TIMEOUT_SECS: u64 = 120;

#[facet::container]
pub struct Repo {
    #[facet]
    repo_blobstore: RepoBlobstore,
    #[facet]
    repo_derived_data: RepoDerivedData,
    #[facet]
    changesets: dyn Changesets,
    #[facet]
    bookmarks: dyn Bookmarks,
    #[facet]
    bonsai_hg_mapping: dyn BonsaiHgMapping,
    #[facet]
    changeset_fetcher: dyn ChangesetFetcher,
    #[facet]
    filestore_config: FilestoreConfig,
}

/// Tool to calculate repo statistic
#[derive(Parser)]
#[clap(about = "Tool to calculate repo statistic.")]
struct RepoStatisticsArgs {
    /// The repo against which the repo statistic command needs to be executed
    #[clap(flatten)]
    repo: OptRepoArgs,
    /// Bookmark from which we get statistics. Default is master.
    #[clap(long, default_value = "master")]
    bookmark: String,
    /// If set, then statistics are logged to scuba
    #[clap(long)]
    log_to_scuba: bool,
    /// A file with a list of bonsai changesets to calculate stats for. If this
    /// argument is provided, then the statistics will be calculated for file based commit only.
    #[clap(long)]
    in_filename: Option<String>,
    /// The name of ShardManager service to be used when running statistics collector in sharded setting.
    #[clap(long, conflicts_with_all = &["repo-name", "repo-id"])]
    pub sharded_service_name: Option<String>,
}

/// Struct representing the Statistics Collector process.
pub struct StatisticsCollectorProcess {
    app: MononokeApp,
    args: Arc<RepoStatisticsArgs>,
}

impl StatisticsCollectorProcess {
    pub fn new(app: MononokeApp) -> anyhow::Result<Self> {
        let args: Arc<RepoStatisticsArgs> = Arc::new(app.args()?);
        Ok(Self { app, args })
    }
}

#[async_trait]
impl RepoShardedProcess for StatisticsCollectorProcess {
    async fn setup(&self, repo: &RepoShard) -> anyhow::Result<Arc<dyn RepoShardedProcessExecutor>> {
        let logger = self.app.repo_logger(&repo.to_string());
        info!(
            &logger,
            "Setting up statistics collector for repo {}",
            repo.to_string()
        );
        let statistics_collector =
            StatisticsCollector::from_process(self, repo.repo_name.to_string(), logger).await?;
        Ok(Arc::new(statistics_collector))
    }
}

/// Struct representing the Statistics Collector process over the context of Repo.
struct StatisticsCollector {
    repo: Arc<Repo>,
    args: Arc<RepoStatisticsArgs>,
    ctx: CoreContext,
    logger: Logger,
    scuba_logger: MononokeScubaSampleBuilder,
    repo_name: String,
    bookmark: BookmarkKey,
    cancellation_requested: Arc<AtomicBool>,
}

impl StatisticsCollector {
    async fn from_process(
        process: &StatisticsCollectorProcess,
        repo_name: String,
        logger: Logger,
    ) -> anyhow::Result<Self> {
        let ctx = CoreContext::new_with_logger(process.app.fb, logger.clone());
        let repos = process
            .app
            .open_named_managed_repos(Some(repo_name.to_string()), None)
            .await?;
        let repo = repos
            .repos()
            .get_by_name(&repo_name)
            .ok_or_else(|| anyhow::anyhow!("Repo {} is not loaded on the server", repo_name))?;
        let args = process.args.clone();
        let bookmark = BookmarkKey::new(&process.args.bookmark)?;
        let scuba_logger = if process.args.log_to_scuba {
            MononokeScubaSampleBuilder::new(process.app.fb, SCUBA_DATASET_NAME)?
        } else {
            MononokeScubaSampleBuilder::with_discard()
        };
        Ok(Self {
            repo,
            args,
            ctx,
            scuba_logger,
            repo_name,
            bookmark,
            logger,
            cancellation_requested: Arc::new(AtomicBool::new(false)),
        })
    }
}

#[async_trait]
impl RepoShardedProcessExecutor for StatisticsCollector {
    async fn execute(&self) -> anyhow::Result<()> {
        info!(
            self.logger,
            "Initiating statistics collector execution for repo {}", &self.repo_name,
        );

        let val = run_statistics(
            self.repo.clone(),
            self.args.clone(),
            self.ctx.clone(),
            self.scuba_logger.clone(),
            self.repo_name.clone(),
            self.bookmark.clone(),
            self.cancellation_requested.clone(),
        )
        .await;
        if let Err(ref e) = val {
            error!(
                self.logger,
                "Statistics Collector failure in repo {}. Terminating execution. Cause: {:?}",
                &self.repo_name,
                e
            );
        };
        Ok(())
    }

    async fn stop(&self) -> anyhow::Result<()> {
        info!(
            self.logger,
            "Terminating statistics collector execution for repo {}", &self.repo_name,
        );
        self.cancellation_requested.store(true, Ordering::Relaxed);
        Ok(())
    }
}

define_stats! {
    prefix = "mononoke.statistics_collector";
    calculated_changesets: timeseries(Rate, Sum),
}

const SCUBA_DATASET_NAME: &str = "mononoke_repository_statistics";
// Tool doesn't count number of lines from files with size greater than 10MB
const BIG_FILE_THRESHOLD: u64 = 10000000;

#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct RepoStatistics {
    num_files: i64,
    total_file_size: i64,
    num_lines: i64,
}

impl RepoStatistics {
    pub fn new(num_files: i64, total_file_size: i64, num_lines: i64) -> Self {
        Self {
            num_files,
            total_file_size,
            num_lines,
        }
    }
}

impl Add for RepoStatistics {
    type Output = RepoStatistics;

    fn add(self, other: Self) -> Self {
        Self {
            num_files: self.num_files + other.num_files,
            total_file_size: self.total_file_size + other.total_file_size,
            num_lines: self.num_lines + other.num_lines,
        }
    }
}

impl Sub for RepoStatistics {
    type Output = RepoStatistics;

    fn sub(self, other: Self) -> Self {
        Self {
            num_files: self.num_files - other.num_files,
            total_file_size: self.total_file_size - other.total_file_size,
            num_lines: self.num_lines - other.num_lines,
        }
    }
}

pub async fn number_of_lines(
    bytes_stream: impl Stream<Item = Result<FileBytes, Error>>,
) -> Result<i64, Error> {
    bytes_stream
        .map_ok(|bytes| {
            bytes.into_iter().fold(0, |num_lines, byte| {
                if byte == b'\n' {
                    num_lines + 1
                } else {
                    num_lines
                }
            })
        })
        .try_fold(0, |result, num_lines| async move {
            Ok::<_, Error>(result + num_lines)
        })
        .await
}

pub async fn number_of_lines_unless_redacted(
    bytes_stream: impl Stream<Item = Result<FileBytes, Error>>,
) -> Result<i64, Error> {
    match number_of_lines(bytes_stream).await {
        Ok(lines) => Ok(lines),
        Err(e) => match e.downcast_ref::<RedactedBlobstoreError>() {
            Some(RedactedBlobstoreError::Censored(..)) => Ok(0),
            _ => Err(e),
        },
    }
}

pub async fn get_manifest_from_changeset(
    ctx: &CoreContext,
    repo: &Repo,
    hg_cs_id: &HgChangesetId,
) -> Result<HgManifestId, Error> {
    let changeset = hg_cs_id.load(ctx, repo.repo_blobstore()).await?;
    Ok(changeset.manifestid())
}

pub async fn get_changeset_timestamp_from_changeset(
    ctx: &CoreContext,
    repo: &Repo,
    hg_cs_id: &HgChangesetId,
) -> Result<i64, Error> {
    let changeset = hg_cs_id.load(ctx, repo.repo_blobstore()).await?;
    Ok(changeset.time().timestamp_secs())
}

// Calculates number of lines only for regular-type file
pub async fn get_statistics_from_entry(
    ctx: &CoreContext,
    repo: &Repo,
    entry: Entry<HgManifestId, (FileType, HgFileNodeId)>,
) -> Result<RepoStatistics, Error> {
    match entry {
        Entry::Leaf((file_type, filenode_id)) => {
            let envelope = filenode_id
                .load(ctx, repo.repo_blobstore())
                .await
                .context("Failed to load envelope")?;
            let size = envelope.content_size();
            let content_id = envelope.content_id();
            let lines = if FileType::Regular == file_type && size < BIG_FILE_THRESHOLD {
                let content =
                    filestore::fetch_stream(repo.repo_blobstore(), ctx.clone(), content_id)
                        .map_ok(FileBytes);
                number_of_lines_unless_redacted(content)
                    .await
                    .context("Failed to compute number of lines")?
            } else {
                0
            };
            Ok(RepoStatistics::new(1, size as i64, lines))
        }
        Entry::Tree(_) => Ok(RepoStatistics::default()),
    }
}

pub async fn get_statistics_from_changeset(
    ctx: &CoreContext,
    repo: &Repo,
    blobstore: &(impl Blobstore + Clone + 'static),
    hg_cs_id: &HgChangesetId,
) -> Result<RepoStatistics, Error> {
    info!(
        ctx.logger(),
        "Started calculating statistics for changeset {}", hg_cs_id
    );

    let manifest_id = get_manifest_from_changeset(ctx, repo, hg_cs_id).await?;
    let statistics = manifest_id
        .list_leaf_entries(ctx.clone(), blobstore.clone())
        .map(move |result| match result {
            Ok((_, leaf)) => get_statistics_from_entry(ctx, repo, Entry::Leaf(leaf)).boxed(),
            Err(e) => async move { Err(e) }.boxed(),
        })
        .buffered(100usize)
        .try_fold(
            RepoStatistics::default(),
            |statistics, new_stat| async move { Ok::<_, Error>(statistics + new_stat) },
        )
        .await?;

    info!(
        ctx.logger(),
        "Finished calculating statistics for changeset {}", hg_cs_id
    );

    Ok(statistics)
}

pub async fn update_statistics<'a>(
    ctx: &'a CoreContext,
    repo: &'a Repo,
    statistics: RepoStatistics,
    diff: BoxStream<Diff<Entry<HgManifestId, (FileType, HgFileNodeId)>>, Error>,
) -> Result<RepoStatistics, Error> {
    diff.compat()
        .map(move |result| async move {
            let diff = result?;
            match diff {
                Diff::Added(_, entry) => {
                    let stat = get_statistics_from_entry(ctx, repo, entry).await?;
                    Ok((stat, Operation::Add))
                }
                Diff::Removed(_, entry) => {
                    let stat = get_statistics_from_entry(ctx, repo, entry).await?;
                    Ok((stat, Operation::Sub))
                }
                Diff::Changed(_, old_entry, new_entry) => {
                    let (old_stats, new_stats) = try_join!(
                        get_statistics_from_entry(ctx, repo, old_entry),
                        get_statistics_from_entry(ctx, repo, new_entry),
                    )?;
                    let stat = new_stats - old_stats;
                    Ok((stat, Operation::Add))
                }
            }
        })
        .buffered(100usize)
        .try_fold(
            statistics,
            |statistics, (file_stats, operation)| async move {
                match operation {
                    Operation::Add => Ok::<_, Error>(statistics + file_stats),
                    Operation::Sub => Ok::<_, Error>(statistics - file_stats),
                }
            },
        )
        .await
}

pub fn log_statistics(
    ctx: &CoreContext,
    mut scuba_logger: MononokeScubaSampleBuilder,
    cs_timestamp: i64,
    repo_name: &str,
    hg_cs_id: &HgChangesetId,
    statistics: &RepoStatistics,
) {
    info!(
        ctx.logger(),
        "Statistics for changeset {}\nCs timestamp: {}\nNumber of files {}\nTotal file size {}\nNumber of lines {}",
        hg_cs_id,
        cs_timestamp,
        statistics.num_files,
        statistics.total_file_size,
        statistics.num_lines
    );
    scuba_logger
        .add("repo_name", repo_name.to_owned())
        .add("num_files", statistics.num_files)
        .add("total_file_size", statistics.total_file_size)
        .add("num_lines", statistics.num_lines)
        .add("changeset", hg_cs_id.to_hex().to_string())
        .log_with_time(cs_timestamp as u64);
}

fn parse_serialized_commits<P: AsRef<Path>>(file: P) -> Result<Vec<ChangesetEntry>, Error> {
    let data = fs::read(file).map_err(Error::from)?;
    deserialize_cs_entries(&Bytes::from(data))
}

pub async fn generate_statistics_from_file<P: AsRef<Path>>(
    ctx: &CoreContext,
    repo: &Repo,
    in_path: &P,
) -> Result<(), Error> {
    // 1 day in seconds
    const REQUIRED_COMMITS_DISTANCE: i64 = 60 * 60 * 24;
    let blobstore = Arc::new(repo.repo_blobstore().clone());
    // TODO(dgrzegorzewski): T55705023 consider creating csv file here and save statistics using
    // e.g. serde deserialize. To avoid saving fields separately it may be necessary to add new
    // fields to RepoStatistics struct, like cs_timestamp, hg_cs_id, repo_id and refactor code.
    println!("repo_id,hg_cs_id,cs_timestamp,num_files,total_file_size,num_lines");
    let changesets = parse_serialized_commits(in_path)?;
    info!(ctx.logger(), "Started calculating changesets timestamps");

    let mut changeset_info_vec = stream::iter(changesets)
        .map({
            move |changeset| async move {
                let ChangesetEntry { repo_id, cs_id, .. } = changeset;
                let hg_cs_id = derive_hg_changeset(ctx, repo.repo_derived_data(), cs_id).await?;
                let cs_timestamp =
                    get_changeset_timestamp_from_changeset(ctx, repo, &hg_cs_id).await?;
                // the error type annotation in principle should be inferred,
                // but the compiler currently needs it. See https://fburl.com/n1s2ujjb
                Ok::<(HgChangesetId, i64, RepositoryId), Error>((hg_cs_id, cs_timestamp, repo_id))
            }
        })
        .buffered(100)
        .try_collect::<Vec<_>>()
        .await?;

    info!(
        ctx.logger(),
        "Timestamps calculated, sorting them and starting calculating statistics"
    );
    changeset_info_vec.sort_by_key(|(_, cs_timestamp, _)| cs_timestamp.clone());

    // accumulate stats into a map
    let mut repo_stats_map = HashMap::<RepositoryId, (i64, HgChangesetId, RepoStatistics)>::new();
    for (hg_cs_id, cs_timestamp, repo_id) in changeset_info_vec {
        match repo_stats_map.get(&repo_id).cloned() {
            Some((old_cs_timestamp, old_hg_cs_id, old_stats)) => {
                // Calculate statistics for changeset only if changeset
                // was created at least REQUIRED_COMMITS_DISTANCE seconds after
                // changeset we used previously to calculate statistics.
                if cs_timestamp - old_cs_timestamp <= REQUIRED_COMMITS_DISTANCE {
                    continue;
                }
                info!(
                    ctx.logger(),
                    "Changeset {} with timestamp {} was created more than {} seconds after previous, calculating statistics for it",
                    hg_cs_id,
                    cs_timestamp,
                    REQUIRED_COMMITS_DISTANCE
                );
                let (old_manifest, manifest) = try_join!(
                    get_manifest_from_changeset(ctx, repo, &old_hg_cs_id,),
                    get_manifest_from_changeset(ctx, repo, &hg_cs_id),
                )?;
                let statistics = update_statistics(
                    ctx,
                    repo,
                    old_stats,
                    old_manifest
                        .diff(ctx.clone(), blobstore.clone(), manifest.clone())
                        .compat()
                        .boxify(),
                )
                .await?;

                info!(
                    ctx.logger(),
                    "Statistics for changeset {} calculated", hg_cs_id
                );
                println!(
                    "{},{},{},{},{},{}",
                    repo_id.id(),
                    hg_cs_id.to_hex(),
                    cs_timestamp,
                    statistics.num_files,
                    statistics.total_file_size,
                    statistics.num_lines
                );
                repo_stats_map.insert(repo_id, (cs_timestamp, hg_cs_id, statistics));
            }
            None => {
                info!(
                    ctx.logger(),
                    "Found first changeset for repo_id {}",
                    repo_id.id()
                );
                let statistics =
                    get_statistics_from_changeset(ctx, repo, &blobstore, &hg_cs_id).await?;

                info!(
                    ctx.logger(),
                    "First changeset for repo_id {} calculated",
                    repo_id.id()
                );
                println!(
                    "{},{},{},{},{},{}",
                    repo_id.id(),
                    hg_cs_id.to_hex(),
                    cs_timestamp,
                    statistics.num_files,
                    statistics.total_file_size,
                    statistics.num_lines
                );
                repo_stats_map.insert(repo_id, (cs_timestamp, hg_cs_id, statistics));
            }
        }
    }
    Ok(())
}

enum Operation {
    Add,
    Sub,
}

#[allow(unreachable_code)]
async fn run_statistics(
    repo: Arc<Repo>,
    args: Arc<RepoStatisticsArgs>,
    ctx: CoreContext,
    scuba_logger: MononokeScubaSampleBuilder,
    repo_name: String,
    bookmark: BookmarkKey,
    cancellation_requested: Arc<AtomicBool>,
) -> Result<(), Error> {
    if let Some(in_filename) = args.in_filename.as_ref() {
        return generate_statistics_from_file(&ctx, &repo, in_filename).await;
    }

    let blobstore = Arc::new(repo.repo_blobstore().clone());
    let mut changeset = repo
        .get_bookmark_hg(ctx.clone(), &bookmark)
        .await?
        .ok_or_else(|| Error::msg("cannot load bookmark"))?;

    // initialize the loop

    let mut statistics = get_statistics_from_changeset(&ctx, &repo, &blobstore, &changeset).await?;

    let cs_timestamp = get_changeset_timestamp_from_changeset(&ctx, &repo, &changeset).await?;

    log_statistics(
        &ctx,
        scuba_logger.clone(),
        cs_timestamp,
        &repo_name,
        &changeset,
        &statistics,
    );

    STATS::calculated_changesets.add_value(1);

    // run the loop
    loop {
        if cancellation_requested.load(Ordering::Relaxed) {
            info!(
                ctx.logger(),
                "Cancellation requested for statistics collector. Exiting"
            );
            return Ok(());
        }
        let prev_changeset = changeset;
        changeset = repo
            .get_bookmark_hg(ctx.clone(), &bookmark)
            .await?
            .ok_or_else(|| Error::msg("cannot load bookmark"))?;

        if prev_changeset == changeset {
            let duration = Duration::from_millis(1000);
            info!(
                ctx.logger(),
                "Changeset hasn't changed, sleeping {:?}", duration
            );

            sleep(duration).await;
        } else {
            info!(
                ctx.logger(),
                "Found new changeset: {}, updating statistics", changeset
            );

            let (prev_manifest_id, cur_manifest_id) = try_join!(
                get_manifest_from_changeset(&ctx, &repo, &prev_changeset),
                get_manifest_from_changeset(&ctx, &repo, &changeset),
            )?;

            statistics = update_statistics(
                &ctx,
                &repo,
                statistics,
                prev_manifest_id
                    .diff(ctx.clone(), blobstore.clone(), cur_manifest_id.clone())
                    .compat()
                    .boxify(),
            )
            .await?;

            info!(ctx.logger(), "Statistics for new changeset updated.");

            let cs_timestamp =
                get_changeset_timestamp_from_changeset(&ctx, &repo, &changeset).await?;

            log_statistics(
                &ctx,
                scuba_logger.clone(),
                cs_timestamp,
                &repo_name,
                &changeset,
                &statistics,
            );
            STATS::calculated_changesets.add_value(1);
        }
    }

    // unreachable, but needed so that the future has type Result
    // which lets us propagate Errors to main.
    Ok(())
}

async fn async_main(app: MononokeApp) -> Result<(), Error> {
    let args: RepoStatisticsArgs = app.args()?;
    let process = StatisticsCollectorProcess::new(app)?;
    match args.sharded_service_name {
        None => {
            let maybe_repo_arg = args.repo.as_repo_arg();
            let (repo_name, _) = match maybe_repo_arg {
                Some(ref repo_arg) => process.app.repo_config(repo_arg)?,
                None => anyhow::bail!(
                    "Repo name or ID not provided. Either sharded-service-name or repo id/name should be provided."
                ),
            };
            let statistics_collector = process
                .setup(&RepoShard::with_repo_name(repo_name.as_ref()))
                .await?;
            statistics_collector.execute().await
        }
        Some(name) => {
            let logger = process.app.logger().clone();
            // The service name needs to be 'static to satisfy SM contract
            static SM_SERVICE_NAME: OnceCell<String> = OnceCell::new();
            let mut executor = ShardedProcessExecutor::new(
                process.app.fb,
                process.app.runtime().clone(),
                &logger,
                SM_SERVICE_NAME.get_or_init(|| name),
                SM_SERVICE_SCOPE,
                SM_CLEANUP_TIMEOUT_SECS,
                Arc::new(process),
                true, // enable shard (repo) level healing
            )?;
            executor
                .block_and_execute(&logger, Arc::new(AtomicBool::new(false)))
                .await
        }
    }
}

#[fbinit::main]
fn main(fb: FacebookInit) -> Result<(), Error> {
    let app = MononokeAppBuilder::new(fb)
        .with_app_extension(Fb303AppExtension {})
        .build::<RepoStatisticsArgs>()?;
    app.run_with_monitoring_and_logging(async_main, "repo_statistics_collector", AliveService)
}

#[cfg(test)]
mod tests {
    use std::str::FromStr;

    use bonsai_hg_mapping::BonsaiHgMappingRef;
    use borrowed::borrowed;
    use bytes::Bytes;
    use fixtures::Linear;
    use fixtures::TestRepoFixture;
    use futures::future;
    use futures::stream;
    use maplit::btreemap;
    use tests_utils::create_commit;
    use tests_utils::store_files;
    use tokio::runtime::Runtime;

    use super::Repo as MyRepo;
    use super::*;

    #[test]
    fn test_number_of_lines_empty_stream() -> Result<(), Error> {
        let rt = Runtime::new().unwrap();

        let stream = stream::once(async { Ok(FileBytes(Bytes::from(&b""[..]))) });
        let result = rt.block_on(number_of_lines(stream))?;
        assert_eq!(result, 0);
        Ok(())
    }

    #[test]
    fn test_number_of_lines_one_line() -> Result<(), Error> {
        let rt = Runtime::new().unwrap();

        let stream = stream::once(async { Ok(FileBytes(Bytes::from(&b"First line\n"[..]))) });
        let result = rt.block_on(number_of_lines(stream))?;
        assert_eq!(result, 1);
        Ok(())
    }

    #[test]
    fn test_number_of_lines_many_lines() -> Result<(), Error> {
        let rt = Runtime::new().unwrap();

        let stream = stream::once(async {
            Ok(FileBytes(Bytes::from(
                &b"First line\nSecond line\nThird line\n"[..],
            )))
        });
        let result = rt.block_on(number_of_lines(stream))?;
        assert_eq!(result, 3);
        Ok(())
    }

    #[test]
    fn test_number_of_lines_many_items() -> Result<(), Error> {
        let rt = Runtime::new().unwrap();

        let vec = vec![
            FileBytes(Bytes::from(&b"First line\n"[..])),
            FileBytes(Bytes::from(&b""[..])),
            FileBytes(Bytes::from(&b"First line\nSecond line\nThird line\n"[..])),
        ];
        let stream = stream::iter(vec.into_iter().map(Ok));
        let result = rt.block_on(number_of_lines(stream))?;
        assert_eq!(result, 4);
        Ok(())
    }

    #[fbinit::test]
    fn linear_test_get_statistics_from_changeset(fb: FacebookInit) {
        let runtime = Runtime::new().unwrap();
        runtime.block_on(async move {
            let repo: MyRepo = Linear::get_custom_test_repo(fb).await;

            let ctx = CoreContext::test_mock(fb);
            let blobstore = repo.repo_blobstore().clone();
            borrowed!(ctx, blobstore, repo);

            // Commit consists two files (name => content):
            //     "1" => "1\n"
            //     "files" => "1\n"
            // */
            let root = HgChangesetId::from_str("2d7d4ba9ce0a6ffd222de7785b249ead9c51c536").unwrap();
            let p = repo
                .bonsai_hg_mapping()
                .get_bonsai_from_hg(ctx, root)
                .await
                .unwrap()
                .unwrap();
            let parents = vec![p];

            let bcs_id = create_commit(
                ctx.clone(),
                repo,
                parents,
                store_files(
                    ctx,
                    btreemap! {
                        "dir1/dir2/file1" => Some("first line\nsecond line\n"),
                        "dir1/dir3/file2" => Some("first line\n"),
                    },
                    repo,
                )
                .await,
            )
            .await;

            let hg_cs_id = derive_hg_changeset(ctx, repo.repo_derived_data(), bcs_id)
                .await
                .unwrap();

            let stats = get_statistics_from_changeset(ctx, repo, blobstore, &hg_cs_id)
                .await
                .unwrap();

            // (num_files, total_file_size, num_lines)
            assert_eq!(stats, RepoStatistics::new(4, 38, 5));
        });
    }

    #[fbinit::test]
    fn linear_test_get_statistics_from_entry_tree(fb: FacebookInit) {
        let runtime = Runtime::new().unwrap();
        runtime.block_on(async move {
            let repo: MyRepo = Linear::get_custom_test_repo(fb).await;

            let ctx = CoreContext::test_mock(fb);
            let blobstore = repo.repo_blobstore().clone();
            borrowed!(ctx, blobstore, repo);

            // Commit consists two files (name => content):
            //     "1" => "1\n"
            //     "files" => "1\n"
            // */
            let root = HgChangesetId::from_str("2d7d4ba9ce0a6ffd222de7785b249ead9c51c536").unwrap();
            let p = repo
                .bonsai_hg_mapping()
                .get_bonsai_from_hg(ctx, root)
                .await
                .unwrap()
                .unwrap();
            let parents = vec![p];

            let bcs_id = create_commit(
                ctx.clone(),
                repo,
                parents,
                store_files(
                    ctx,
                    btreemap! {
                        "dir1/dir2/file1" => Some("first line\nsecond line\n"),
                        "dir1/dir3/file2" => Some("first line\n"),
                    },
                    repo,
                )
                .await,
            )
            .await;

            let hg_cs_id = derive_hg_changeset(ctx, repo.repo_derived_data(), bcs_id)
                .await
                .unwrap();

            let manifest = get_manifest_from_changeset(ctx, repo, &hg_cs_id)
                .await
                .unwrap();

            let mut tree_entries = manifest
                .list_all_entries(ctx.clone(), blobstore.clone())
                .try_filter_map(|(_, entry)| match entry {
                    Entry::Tree(_) => future::ok(Some(entry)),
                    _ => future::ok(None),
                })
                .try_collect::<Vec<_>>()
                .await
                .unwrap();

            let stats = get_statistics_from_entry(ctx, repo, tree_entries.pop().unwrap())
                .await
                .unwrap();

            // For Entry::Tree we expect repository with all statistics equal 0
            // (num_files, total_file_size, num_lines)
            assert_eq!(stats, RepoStatistics::default());
        });
    }

    #[fbinit::test]
    fn linear_test_update_statistics(fb: FacebookInit) {
        let runtime = Runtime::new().unwrap();
        runtime.block_on(async move {
            let repo: MyRepo = Linear::get_custom_test_repo(fb).await;

            let ctx = CoreContext::test_mock(fb);
            let blobstore = repo.repo_blobstore().clone();
            borrowed!(ctx, blobstore, repo);

            /*
            Commit consists two files (name => content):
                "1" => "1\n"
                "files" => "1\n"
            */
            let prev_hg_cs_id =
                HgChangesetId::from_str("2d7d4ba9ce0a6ffd222de7785b249ead9c51c536").unwrap();
            /*
            Commit consists two files (name => content):
                "2" => "2\n"
                "files" => "1\n2\n"
            */
            let cur_hg_cs_id =
                HgChangesetId::from_str("3e0e761030db6e479a7fb58b12881883f9f8c63f").unwrap();

            let stats = get_statistics_from_changeset(ctx, repo, blobstore, &prev_hg_cs_id)
                .await
                .unwrap();

            let (prev_manifest, cur_manifest) = try_join!(
                get_manifest_from_changeset(ctx, repo, &prev_hg_cs_id),
                get_manifest_from_changeset(ctx, repo, &cur_hg_cs_id),
            )
            .unwrap();

            let new_stats = update_statistics(
                ctx,
                repo,
                stats,
                prev_manifest
                    .diff(ctx.clone(), blobstore.clone(), cur_manifest.clone())
                    .compat()
                    .boxify(),
            )
            .await
            .unwrap();

            // (num_files, total_file_size, num_lines)
            assert_eq!(new_stats, RepoStatistics::new(3, 8, 4));
        });
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::ffi::OsStr;
use std::fmt;
use std::sync::Arc;

use anyhow::format_err;
use anyhow::Error;
use anyhow::Result;
use blobstore::Blobstore;
use blobstore::BlobstoreGetData;
use blobstore_factory::make_blobstore;
use blobstore_factory::BlobstoreOptions;
use blobstore_factory::ReadOnlyStorage;
use cacheblob::new_memcache_blobstore;
use cacheblob::CacheBlobstoreExt;
use cached_config::ConfigStore;
use clap_old::App;
use clap_old::Arg;
use clap_old::ArgMatches;
use clap_old::SubCommand;
use cmdlib::args;
use cmdlib::args::MononokeMatches;
use context::CoreContext;
use fbinit::FacebookInit;
use futures::future::try_join;
use git_types::Tree as GitTree;
use mercurial_types::HgChangesetEnvelope;
use mercurial_types::HgFileEnvelope;
use mercurial_types::HgManifestEnvelope;
use metaconfig_types::BlobConfig;
use metaconfig_types::BlobstoreId;
use metaconfig_types::Redaction;
use metaconfig_types::StorageConfig;
use mononoke_types::FileContents;
use mononoke_types::RepositoryId;
use prefixblob::PrefixBlobstore;
use redactedblobstore::RedactedBlobs;
use redactedblobstore::RedactedBlobstore;
use redactedblobstore::RedactedBlobstoreConfig;
use redactedblobstore::RedactionConfigBlobstore;
use redactedblobstore::SqlRedactedContentStore;
use scuba_ext::MononokeScubaSampleBuilder;
use slog::info;
use slog::warn;
use slog::Logger;
use sql_ext::facebook::MysqlOptions;
use tokio::fs::File;
use tokio::io::AsyncWriteExt;
use tunables::tunables;

use crate::error::SubcommandError;

pub const BLOBSTORE_FETCH: &str = "blobstore-fetch";
const RAW_FILE_NAME_ARG: &str = "raw-blob";

pub fn build_subcommand<'a, 'b>() -> App<'a, 'b> {
    SubCommand::with_name(BLOBSTORE_FETCH)
        .about("fetches blobs from manifold")
        .args_from_usage("[KEY]    'key of the blob to be fetched with no repo prefex'")
        .arg(
            Arg::with_name("decode-as")
                .long("decode-as")
                .short("d")
                .takes_value(true)
                .possible_values(&["auto", "changeset", "manifest", "file", "contents", "git-tree"])
                .required(false)
                .help("if provided decode the value"),
        )
        .arg(
            Arg::with_name("use-memcache")
                .long("use-memcache")
                .short("m")
                .takes_value(true)
                .possible_values(&["cache-only", "no-fill", "fill-mc"])
                .required(false)
                .help("Use memcache to cache access to the blob store"),
        )
        .arg(
            Arg::with_name("no-prefix")
                .long("no-prefix")
                .short("P")
                .takes_value(false)
                .required(false)
                .help("Don't prepend a prefix based on the repo id to the key"),
        )
        .arg(
            Arg::with_name("inner-blobstore-id")
                .long("inner-blobstore-id")
                .takes_value(true)
                .required(false)
                .help("If main blobstore in the storage config is a multiplexed one, use inner blobstore with this id")
        )
        .arg(
            Arg::with_name(RAW_FILE_NAME_ARG)
                .long(RAW_FILE_NAME_ARG)
                .takes_value(true)
                .required(false)
                .help("Write the raw blob bytes to the given filename instead of printing to stdout."),
        )
}

fn get_blobconfig(blob_config: BlobConfig, inner_blobstore_id: Option<u64>) -> Result<BlobConfig> {
    match inner_blobstore_id {
        None => Ok(blob_config),
        Some(inner_blobstore_id) => match blob_config {
            BlobConfig::MultiplexedWal { blobstores, .. } => {
                let seeked_id = BlobstoreId::new(inner_blobstore_id);
                blobstores
                    .into_iter()
                    .find_map(|(blobstore_id, _, blobstore)| {
                        if blobstore_id == seeked_id {
                            Some(blobstore)
                        } else {
                            None
                        }
                    })
                    .ok_or_else(|| {
                        format_err!("could not find a blobstore with id {}", inner_blobstore_id)
                    })
            }
            _ => Err(format_err!(
                "inner-blobstore-id supplied but blobstore is not multiplexed"
            )),
        },
    }
}

async fn get_blobstore<'a>(
    fb: FacebookInit,
    storage_config: StorageConfig,
    inner_blobstore_id: Option<u64>,
    mysql_options: &'a MysqlOptions,
    logger: Logger,
    readonly_storage: ReadOnlyStorage,
    blobstore_options: &'a BlobstoreOptions,
    config_store: &'a ConfigStore,
) -> Result<Arc<dyn Blobstore>, Error> {
    let blobconfig = get_blobconfig(storage_config.blobstore, inner_blobstore_id)?;

    make_blobstore(
        fb,
        blobconfig,
        mysql_options,
        readonly_storage,
        blobstore_options,
        &logger,
        config_store,
        &blobstore_factory::default_scrub_handler(),
        None,
    )
    .await
}

pub async fn subcommand_blobstore_fetch<'a>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'a>,
    sub_m: &'a ArgMatches<'a>,
) -> Result<(), SubcommandError> {
    let config_store = matches.config_store();
    let repo_id = args::not_shardmanager_compatible::get_repo_id(config_store, matches)?;
    let (_, config) = args::not_shardmanager_compatible::get_config(config_store, matches)?;
    let redaction = config.redaction;
    let storage_config = config.storage_config;
    let inner_blobstore_id = args::get_u64_opt(&sub_m, "inner-blobstore-id");
    let mysql_options = matches.mysql_options();
    let blobstore_options = matches.blobstore_options();
    let readonly_storage = matches.readonly_storage();
    let blobstore_fut = get_blobstore(
        fb,
        storage_config,
        inner_blobstore_id,
        mysql_options,
        logger.clone(),
        *readonly_storage,
        blobstore_options,
        config_store,
    );

    let common_config = args::load_common_config(config_store, matches)?;
    let censored_scuba_params = common_config.censored_scuba_params;
    let mut scuba_redaction_builder =
        MononokeScubaSampleBuilder::with_opt_table(fb, censored_scuba_params.table)?;

    if let Some(scuba_log_file) = censored_scuba_params.local_path {
        scuba_redaction_builder = scuba_redaction_builder
            .with_log_file(scuba_log_file)
            .map_err(Error::from)?;
    }

    let ctx = CoreContext::new_with_logger(fb, logger.clone());
    let key = sub_m.value_of("KEY").unwrap().to_string();
    let decode_as = sub_m.value_of("decode-as").map(|val| val.to_string());
    let use_memcache = sub_m.value_of("use-memcache").map(|val| val.to_string());
    let no_prefix = sub_m.is_present("no-prefix");
    let maybe_output_file = sub_m.value_of_os(RAW_FILE_NAME_ARG);

    let maybe_redacted_blobs_fut = async {
        match redaction {
            Redaction::Enabled => {
                let redacted_blobs = if tunables().redaction_config_from_xdb().unwrap_or_default() {
                    let redacted_blobs_db = args::not_shardmanager_compatible::open_sql::<
                        SqlRedactedContentStore,
                    >(fb, config_store, matches)?;
                    redacted_blobs_db.get_all_redacted_blobs().await?
                } else {
                    let redaction_config_blobstore = Arc::new(RedactionConfigBlobstore::new(
                        make_blobstore(
                            fb,
                            common_config.redaction_config.blobstore.clone(),
                            mysql_options,
                            *readonly_storage,
                            blobstore_options,
                            &logger,
                            config_store,
                            &blobstore_factory::default_scrub_handler(),
                            None,
                        )
                        .await?,
                    ));
                    RedactedBlobs::from_configerator(
                        config_store,
                        &common_config.redaction_config.redaction_sets_location,
                        ctx.clone(),
                        redaction_config_blobstore,
                    )
                    .await?
                };
                Ok(Some(Arc::new(redacted_blobs)))
            }
            Redaction::Disabled => Ok(None),
        }
    };

    let (blobstore, maybe_redacted_blobs) =
        try_join(blobstore_fut, maybe_redacted_blobs_fut).await?;

    info!(logger, "using blobstore: {:?}", blobstore);
    let value = get_from_sources(
        fb,
        use_memcache,
        blobstore,
        no_prefix,
        key.clone(),
        ctx,
        maybe_redacted_blobs,
        scuba_redaction_builder,
        repo_id,
    )
    .await?;

    output_blob(maybe_output_file, value.as_ref()).await?;
    if let Some(value) = value {
        let decode_as = decode_as.as_ref().and_then(|val| {
            let val = val.as_str();
            if val == "auto" {
                detect_decode(&key, &logger)
            } else {
                Some(val)
            }
        });

        match decode_as {
            Some("changeset") => display(&HgChangesetEnvelope::from_blob(value.into())),
            Some("manifest") => display(&HgManifestEnvelope::from_blob(value.into())),
            Some("file") => display(&HgFileEnvelope::from_blob(value.into())),
            // TODO: (rain1) T30974137 add a better way to print out file contents
            Some("contents") => println!(
                "{:?}",
                FileContents::from_encoded_bytes(value.into_raw_bytes())
            ),
            Some("git-tree") => display::<GitTree>(&value.try_into()),
            _ => {}
        }
    }

    Ok(())
}

async fn output_blob(path: Option<&OsStr>, value: Option<&BlobstoreGetData>) -> Result<(), Error> {
    match (path, value) {
        (None, _) | (_, None) => {
            println!("{:?}", value);
        }
        (Some(path), Some(value)) => {
            let mut file = File::create(path).await?;
            file.write_all(value.as_raw_bytes()).await?;
            file.flush().await?;
        }
    }
    Ok(())
}

async fn get_from_sources<T: Blobstore + Clone>(
    fb: FacebookInit,
    use_memcache: Option<String>,
    blobstore: T,
    no_prefix: bool,
    key: String,
    ctx: CoreContext,
    redacted_blobs: Option<Arc<RedactedBlobs>>,
    scuba_redaction_builder: MononokeScubaSampleBuilder,
    repo_id: RepositoryId,
) -> Result<Option<BlobstoreGetData>, Error> {
    let empty_prefix = "".to_string();

    match use_memcache {
        Some(mode) => {
            let blobstore = new_memcache_blobstore(fb, blobstore, "multiplexed", "").unwrap();
            let blobstore = match no_prefix {
                false => PrefixBlobstore::new(blobstore, repo_id.prefix()),
                true => PrefixBlobstore::new(blobstore, empty_prefix),
            };
            let blobstore = RedactedBlobstore::new(
                blobstore,
                RedactedBlobstoreConfig::new(redacted_blobs, scuba_redaction_builder),
            );
            get_cache(&ctx, &blobstore, &key, mode)
                .await
                .map(|opt_blob| opt_blob.map(Into::into))
        }
        None => {
            let blobstore = match no_prefix {
                false => PrefixBlobstore::new(blobstore, repo_id.prefix()),
                true => PrefixBlobstore::new(blobstore, empty_prefix),
            };
            let blobstore = RedactedBlobstore::new(
                blobstore,
                RedactedBlobstoreConfig::new(redacted_blobs, scuba_redaction_builder),
            );
            blobstore.get(&ctx, &key).await
        }
    }
}

fn display<T>(res: &Result<T>)
where
    T: fmt::Display + fmt::Debug,
{
    match res {
        Ok(val) => println!("---\n{}---", val),
        err => println!("{:?}", err),
    }
}

fn detect_decode(key: &str, logger: &Logger) -> Option<&'static str> {
    // Use a simple heuristic to figure out how to decode this key.
    if key.contains("hgchangeset.") {
        info!(logger, "Detected changeset key");
        Some("changeset")
    } else if key.contains("hgmanifest.") {
        info!(logger, "Detected manifest key");
        Some("manifest")
    } else if key.contains("hgfilenode.") {
        info!(logger, "Detected file key");
        Some("file")
    } else if key.contains("content.") {
        info!(logger, "Detected content key");
        Some("contents")
    } else if key.contains("git.tree.") {
        info!(logger, "Detected git-tree key");
        Some("git-tree")
    } else {
        warn!(
            logger,
            "Unable to detect how to decode this blob based on key";
            "key" => key,
        );
        None
    }
}

async fn get_cache<B: CacheBlobstoreExt>(
    ctx: &CoreContext,
    blobstore: &B,
    key: &str,
    mode: String,
) -> Result<Option<BlobstoreGetData>, Error> {
    if mode == "cache-only" {
        blobstore.get_cache_only(ctx, key).await
    } else if mode == "no-fill" {
        blobstore.get_no_cache_fill(ctx, key).await
    } else {
        blobstore.get(ctx, key).await
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::sync::Arc;

use anyhow::format_err;
use anyhow::Error;
use anyhow::Result;
use blobstore::BlobstoreUnlinkOps;
use blobstore_factory::make_sql_blobstore;
use blobstore_factory::BlobstoreOptions;
use blobstore_factory::ReadOnlyStorage;
use cached_config::ConfigStore;
use clap_old::App;
use clap_old::Arg;
use clap_old::ArgMatches;
use clap_old::SubCommand;
use cmdlib::args;
use cmdlib::args::MononokeMatches;
use context::CoreContext;
use fbinit::FacebookInit;
use metaconfig_types::BlobConfig;
use metaconfig_types::BlobstoreId;
use metaconfig_types::StorageConfig;
use slog::info;
use slog::Logger;

use crate::error::SubcommandError;

pub const BLOBSTORE_UNLINK: &str = "blobstore-unlink";

pub fn build_subcommand<'a, 'b>() -> App<'a, 'b> {
    SubCommand::with_name(BLOBSTORE_UNLINK)
        .about("unlinks blobs in the blobstore")
        .args_from_usage("[KEY]    'key of the blob to be unlinked'")
        .arg(
            Arg::with_name("inner-blobstore-id")
                .long("inner-blobstore-id")
                .takes_value(true)
                .required(false)
                .help("If main blobstore in the storage config is a multiplexed one, use inner blobstore with this id")
        )
}

fn remove_wrapper_blobconfigs(mut blob_config: BlobConfig) -> BlobConfig {
    // Pack is a wrapper store - remove it
    while let BlobConfig::Pack { ref blobconfig, .. } = blob_config {
        blob_config = BlobConfig::clone(blobconfig);
    }
    blob_config
}

fn get_blobconfig(blob_config: BlobConfig, inner_blobstore_id: Option<u64>) -> Result<BlobConfig> {
    match inner_blobstore_id {
        None => Ok(blob_config),
        Some(inner_blobstore_id) => match blob_config {
            BlobConfig::MultiplexedWal { blobstores, .. } => {
                let seeked_id = BlobstoreId::new(inner_blobstore_id);
                blobstores
                    .into_iter()
                    .find_map(|(blobstore_id, _, blobstore)| {
                        if blobstore_id == seeked_id {
                            Some(remove_wrapper_blobconfigs(blobstore))
                        } else {
                            None
                        }
                    })
                    .ok_or_else(|| {
                        format_err!("could not find a blobstore with id {}", inner_blobstore_id)
                    })
            }
            _ => Err(format_err!(
                "inner-blobstore-id supplied but blobstore is not multiplexed"
            )),
        },
    }
}

async fn get_blobstore(
    fb: FacebookInit,
    storage_config: StorageConfig,
    inner_blobstore_id: Option<u64>,
    readonly_storage: ReadOnlyStorage,
    blobstore_options: &BlobstoreOptions,
    config_store: &ConfigStore,
) -> Result<Arc<dyn BlobstoreUnlinkOps>, Error> {
    let blobconfig = get_blobconfig(storage_config.blobstore, inner_blobstore_id)?;

    // TODO: Do this for all blobstores that can support unlink, not just SQLBlob
    let sql_blob = make_sql_blobstore(
        fb,
        blobconfig,
        readonly_storage,
        blobstore_options,
        config_store,
    )
    .await?;

    Ok(Arc::new(sql_blob) as Arc<dyn BlobstoreUnlinkOps>)
}

pub async fn subcommand_blobstore_unlink<'a>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'a>,
    sub_m: &'a ArgMatches<'a>,
) -> Result<(), SubcommandError> {
    let config_store = matches.config_store();
    let (_, config) = args::not_shardmanager_compatible::get_config(config_store, matches)?;
    let storage_config = config.storage_config;
    let inner_blobstore_id = args::get_u64_opt(&sub_m, "inner-blobstore-id");
    let blobstore_options = matches.blobstore_options();

    let readonly_storage = matches.readonly_storage();
    let blobstore = get_blobstore(
        fb,
        storage_config,
        inner_blobstore_id,
        *readonly_storage,
        blobstore_options,
        config_store,
    )
    .await?;

    let ctx = CoreContext::new_with_logger(fb, logger.clone());
    let key = sub_m.value_of("KEY").unwrap();

    info!(logger, "using blobstore: {:?}", blobstore);

    blobstore.unlink(&ctx, key).await?;

    Ok(())
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use anyhow::anyhow;
use anyhow::Error;
use blobrepo::BlobRepo;
use blobstore::Blobstore;
use blobstore::BlobstoreBytes;
use clap_old::App;
use clap_old::Arg;
use clap_old::ArgMatches;
use clap_old::SubCommand;
use cmdlib::args;
use cmdlib::args::MononokeMatches;
use context::CoreContext;
use fbinit::FacebookInit;
use futures::TryFutureExt;
use repo_blobstore::RepoBlobstoreRef;
use slog::info;
use slog::Logger;

use crate::error::SubcommandError;

pub const KEY_ARG: &str = "key";
pub const VALUE_FILE_ARG: &str = "value-file";
pub const BLOBSTORE_UPLOAD: &str = "blobstore-upload";

pub fn build_subcommand<'a, 'b>() -> App<'a, 'b> {
    SubCommand::with_name(BLOBSTORE_UPLOAD)
        .about("uploads blobs to blobstore")
        .arg(
            Arg::with_name(KEY_ARG)
                .long(KEY_ARG)
                .takes_value(true)
                .required(true)
                .help("blobstore key to upload to"),
        )
        .arg(
            Arg::with_name(VALUE_FILE_ARG)
                .long(VALUE_FILE_ARG)
                .takes_value(true)
                .required(true)
                .help("file with value to write to blobstore"),
        )
}

pub async fn subcommand_blobstore_upload<'a>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'a>,
    sub_m: &'a ArgMatches<'a>,
) -> Result<(), SubcommandError> {
    let ctx = CoreContext::new_with_logger(fb, logger.clone());
    let repo: BlobRepo = args::not_shardmanager_compatible::open_repo(fb, &logger, matches).await?;

    let key = sub_m
        .value_of(KEY_ARG)
        .ok_or_else(|| anyhow!("{} is not set", KEY_ARG))?;
    let value_file = sub_m
        .value_of(VALUE_FILE_ARG)
        .ok_or_else(|| anyhow!("{} is not set", VALUE_FILE_ARG))?;
    let data = tokio::fs::read(value_file).map_err(Error::from).await?;
    info!(ctx.logger(), "writing {} bytes to blobstore", data.len());

    repo.repo_blobstore()
        .put(&ctx, key.to_string(), BlobstoreBytes::from_bytes(data))
        .await?;

    Ok(())
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::BTreeMap;
use std::str::FromStr;

use anyhow::Context;
use bonsai_hg_mapping::BonsaiHgMapping;
use bookmarks::Bookmarks;
use clap_old::App;
use clap_old::ArgMatches;
use clap_old::SubCommand;
use cmdlib::args;
use cmdlib::args::MononokeMatches;
use context::CoreContext;
use ephemeral_blobstore::BubbleId;
use ephemeral_blobstore::RepoEphemeralStore;
use fbinit::FacebookInit;
use mononoke_types::hash::GitSha1;
use mononoke_types::BonsaiChangeset;
use mononoke_types::ChangesetId;
use mononoke_types::DateTime;
use mononoke_types::FileChange;
use repo_blobstore::RepoBlobstore;
use repo_identity::RepoIdentity;
use serde_derive::Serialize;
use slog::Logger;

use crate::common::fetch_bonsai_changeset;
use crate::common::print_bonsai_changeset;
use crate::error::SubcommandError;

pub const BONSAI_FETCH: &str = "bonsai-fetch";

pub fn build_subcommand<'a, 'b>() -> App<'a, 'b> {
    SubCommand::with_name(BONSAI_FETCH)
        .about("fetches bonsai changeset information")
        .args_from_usage(
            r#"<CHANGESET_ID>    'bonsai id to fetch'
                          -b --bubble [ID]       'if provided, will check bubble'
                          --json            'if provided json will be returned'"#,
        )
}

pub async fn subcommand_bonsai_fetch<'a>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'_>,
    sub_m: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
    let rev = sub_m.value_of("CHANGESET_ID").unwrap().to_string();

    let ctx = CoreContext::new_with_logger(fb, logger.clone());
    let json_flag = sub_m.is_present("json");
    let bubble = sub_m
        .value_of("bubble")
        .map(std::num::NonZeroU64::from_str)
        .transpose()
        .context("parsing --bubble")?
        .map(BubbleId::new);

    #[facet::container]
    struct BonsaiFetchContainer {
        #[facet]
        id: RepoIdentity,
        #[facet]
        hg_mapping: dyn BonsaiHgMapping,
        #[facet]
        bookmarks: dyn Bookmarks,
        #[facet]
        blobstore: RepoBlobstore,
        #[facet]
        ephemeral_blobstore: RepoEphemeralStore,
    }
    let container: BonsaiFetchContainer =
        args::not_shardmanager_compatible::open_repo(fb, &logger, matches).await?;
    let blobstore = if let Some(bubble_id) = bubble {
        let bubble = container.ephemeral_blobstore.open_bubble(bubble_id).await?;
        bubble.wrap_repo_blobstore((*container.blobstore).clone())
    } else {
        (*container.blobstore).clone()
    };
    let bcs = fetch_bonsai_changeset(ctx, &rev, container, &blobstore).await?;
    if json_flag {
        match serde_json::to_string(&SerializableBonsaiChangeset::from(bcs)) {
            Ok(json) => println!("{}", json),
            Err(e) => println!("{}", e),
        }
    } else {
        print_bonsai_changeset(&bcs);
    }
    Ok(())
}

#[derive(Serialize)]
pub struct SerializableBonsaiChangeset {
    pub parents: Vec<ChangesetId>,
    pub author: String,
    pub author_date: DateTime,
    pub committer: Option<String>,
    // XXX should committer date always be recorded? If so, it should probably be a
    // monotonically increasing value:
    // max(author date, max(committer date of parents) + epsilon)
    pub committer_date: Option<DateTime>,
    pub message: String,
    pub hg_extra: BTreeMap<String, Vec<u8>>,
    pub git_extra_headers: Option<BTreeMap<Vec<u8>, Vec<u8>>>,
    pub file_changes: BTreeMap<String, FileChange>,
    // SHA1 hash representing a git tree object. If this changeset
    // corresponds to a Git tree object, then this field will have
    // value, otherwise it would be omitted.
    pub git_tree_hash: Option<GitSha1>,
}

impl From<BonsaiChangeset> for SerializableBonsaiChangeset {
    fn from(bonsai: BonsaiChangeset) -> Self {
        let mut parents = Vec::new();
        parents.extend(bonsai.parents());

        let author = bonsai.author().to_string();
        let author_date = bonsai.author_date().clone();

        let committer = bonsai.committer().map(|s| s.to_string());
        let committer_date = bonsai.committer_date().cloned();

        let message = bonsai.message().to_string();

        let hg_extra = bonsai
            .hg_extra()
            .map(|(k, v)| (k.to_string(), v.to_vec()))
            .collect();

        let git_extra_headers = bonsai
            .git_extra_headers()
            .map(|extra| extra.map(|(k, v)| (k.to_vec(), v.to_vec())).collect());
        let file_changes = bonsai
            .file_changes_map()
            .iter()
            .map(|(k, v)| {
                (
                    String::from_utf8(k.to_vec()).expect("Found invalid UTF-8"),
                    v.clone(),
                )
            })
            .collect();
        let git_tree_hash = bonsai.git_tree_hash().cloned();

        SerializableBonsaiChangeset {
            parents,
            author,
            author_date,
            committer,
            committer_date,
            message,
            hg_extra,
            git_extra_headers,
            file_changes,
            git_tree_hash,
        }
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use anyhow::format_err;
use anyhow::Error;
use blobrepo::BlobRepo;
use blobrepo_hg::BlobRepoHg;
use bookmarks::BookmarkKey;
use bookmarks::BookmarkUpdateLogRef;
use bookmarks::BookmarkUpdateReason;
use bookmarks::BookmarksMaybeStaleExt;
use bookmarks::BookmarksRef;
use bookmarks::Freshness;
use clap_old::App;
use clap_old::Arg;
use clap_old::ArgMatches;
use clap_old::SubCommand;
use cloned::cloned;
use context::CoreContext;
use futures::TryStreamExt;
use humantime::parse_duration;
use mercurial_derivation::DeriveHgChangeset;
use mononoke_types::Timestamp;
use repo_blobstore::RepoBlobstoreRef;
use serde_json::json;
use serde_json::to_string_pretty;
use slog::info;
use slog::Logger;

use crate::common::fetch_bonsai_changeset;
use crate::common::format_bookmark_log_entry;
use crate::error::SubcommandError;

pub const BOOKMARKS: &str = "bookmarks";
const SET_CMD: &str = "set";
const GET_CMD: &str = "get";
const LOG_CMD: &str = "log";
const LIST_CMD: &str = "list";
const DEL_CMD: &str = "delete";

const ARG_CHANGESET_TYPE: &str = "changeset-type";
const ARG_LIMIT: &str = "limit";
const ARG_START_TIME: &str = "start-time";
const ARG_END_TIME: &str = "end-time";
const ARG_KIND: &str = "kind";

pub fn build_subcommand<'a, 'b>() -> App<'a, 'b> {
    let parent_subcommand = SubCommand::with_name(BOOKMARKS);
    let set = SubCommand::with_name(SET_CMD)
        .about(
            "sets a bookmark to a specific hg changeset, if the bookmark does not exist it will
                be created",
        )
        .args_from_usage(
            "<BOOKMARK_NAME>        'bookmark to target'
             <HG_CHANGESET_ID>      'revision to which the bookmark should point to'",
        );

    let get = SubCommand::with_name(GET_CMD)
        .about("gets the changeset of a specific bookmark")
        .args_from_usage(
            r#"
            <BOOKMARK_NAME>        'bookmark to target'
            --json                 'if provided json will be returned'
            "#,
        )
        .arg(
            Arg::with_name(ARG_CHANGESET_TYPE)
                .long(ARG_CHANGESET_TYPE)
                .short("cs")
                .takes_value(true)
                .possible_values(&["bonsai", "hg"])
                .required(false)
                .help("What changeset type to return, either bonsai or hg. Defaults to hg."),
        );

    let log = SubCommand::with_name(LOG_CMD)
        .about("gets the log of changesets for a specific bookmark")
        .args_from_usage(
            r#"
            <BOOKMARK_NAME>        'bookmark to target'
            --json                 'if provided json will be returned'
            "#,
        )
        .arg(
            Arg::with_name(ARG_CHANGESET_TYPE)
                .long(ARG_CHANGESET_TYPE)
                .short("cs")
                .takes_value(true)
                .possible_values(&["bonsai", "hg"])
                .required(false)
                .help("What changeset type to return, either bonsai or hg. Defaults to hg."),
        )
        .arg(
            Arg::with_name(ARG_LIMIT)
                .long(ARG_LIMIT)
                .short("l")
                .takes_value(true)
                .required(false)
                .help("Imposes the limit on number of log records in output."),
        )
        .arg(
            Arg::with_name(ARG_START_TIME)
                .long(ARG_START_TIME)
                .short("s")
                .takes_value(true)
                .required(false)
                .help(
                    "Filter log records by timestamp lower bound. \
                    Takes time difference in free form e.g. 1h, 10m 30s, etc.",
                ),
        )
        .arg(
            Arg::with_name(ARG_END_TIME)
                .long(ARG_END_TIME)
                .short("e")
                .takes_value(true)
                .required(false)
                .requires(ARG_START_TIME)
                .help(
                    "Filter log records by timestamp upper bound. \
                    Takes time difference in free form e.g. 1h, 10m 30s, etc.",
                ),
        );

    let list = SubCommand::with_name(LIST_CMD).about("list bookmarks").arg(
        Arg::with_name(ARG_KIND)
            .long(ARG_KIND)
            .takes_value(true)
            .possible_values(&["publishing"])
            .required(true)
            .help("What set of bookmarks to list"),
    );

    let del = SubCommand::with_name(DEL_CMD)
        .about("delete bookmark")
        .args_from_usage(
            r#"
            <BOOKMARK_NAME>        'bookmark to delete'
            "#,
        );

    parent_subcommand
        .about("set of commands to manipulate bookmarks")
        .subcommand(set)
        .subcommand(get)
        .subcommand(log)
        .subcommand(list)
        .subcommand(del)
}

pub async fn handle_command(
    ctx: CoreContext,
    repo: BlobRepo,
    matches: &ArgMatches<'_>,
    _logger: Logger,
) -> Result<(), SubcommandError> {
    match matches.subcommand() {
        (GET_CMD, Some(sub_m)) => handle_get(sub_m, ctx, repo).await?,
        (SET_CMD, Some(sub_m)) => handle_set(sub_m, ctx, repo).await?,
        (LOG_CMD, Some(sub_m)) => handle_log(sub_m, ctx, repo).await?,
        (LIST_CMD, Some(sub_m)) => handle_list(sub_m, ctx, repo).await?,
        (DEL_CMD, Some(sub_m)) => handle_delete(sub_m, ctx, repo).await?,
        _ => return Err(SubcommandError::InvalidArgs),
    }
    Ok(())
}

fn format_output(json_flag: bool, changeset_id: String, changeset_type: &str) -> String {
    if json_flag {
        let answer = json!({
            "changeset_type": changeset_type,
            "changeset_id": changeset_id
        });
        to_string_pretty(&answer).unwrap()
    } else {
        format!("({}) {}", changeset_type.to_uppercase(), changeset_id)
    }
}

async fn handle_get(args: &ArgMatches<'_>, ctx: CoreContext, repo: BlobRepo) -> Result<(), Error> {
    let bookmark_name = args.value_of("BOOKMARK_NAME").unwrap().to_string();
    let bookmark = BookmarkKey::new(bookmark_name).unwrap();
    let changeset_type = args.value_of(ARG_CHANGESET_TYPE).unwrap_or("hg");
    let json_flag = args.is_present("json");

    match changeset_type {
        "hg" => {
            let cs = repo.get_bookmark_hg(ctx, &bookmark).await?;
            let changeset_id_str = cs.expect("bookmark could not be found").to_string();
            let output = format_output(json_flag, changeset_id_str, "hg");
            println!("{}", output);
            Ok(())
        }
        "bonsai" => {
            let bonsai_cs = fetch_bonsai_changeset(
                ctx,
                bookmark.to_string().as_str(),
                &repo,
                repo.repo_blobstore(),
            )
            .await?;
            let changeset_id_str = bonsai_cs.get_changeset_id().to_string();
            let output = format_output(json_flag, changeset_id_str, "bonsai");
            println!("{}", output);
            Ok(())
        }
        _ => Err(format_err!("Unknown changeset-type supplied")),
    }
}

async fn handle_log(args: &ArgMatches<'_>, ctx: CoreContext, repo: BlobRepo) -> Result<(), Error> {
    let bookmark_name = args.value_of("BOOKMARK_NAME").unwrap().to_string();
    let bookmark = BookmarkKey::new(bookmark_name).unwrap();
    let changeset_type = args.value_of(ARG_CHANGESET_TYPE).unwrap_or("hg");
    let json_flag = args.is_present("json");
    let output_limit_as_string = args.value_of(ARG_LIMIT).unwrap_or("25");
    let max_rec = match output_limit_as_string.parse::<u32>() {
        Ok(n) => n,
        Err(e) => {
            return Err(format_err!(
                "Bad limit value supplied: \"{}\" - {}",
                output_limit_as_string,
                e
            ));
        }
    };

    let filter_by_ts_range = args.is_present(ARG_START_TIME) || args.is_present(ARG_END_TIME);
    let entries = if filter_by_ts_range {
        let min_ts_diff_ns = parse_duration(args.value_of(ARG_START_TIME).ok_or_else(|| {
            format_err!(
                "{} is required if {} is present",
                ARG_START_TIME,
                ARG_END_TIME
            )
        })?)?
        .as_nanos() as i64;
        let max_ts_diff_ns =
            parse_duration(args.value_of(ARG_END_TIME).unwrap_or("0h"))?.as_nanos() as i64;
        if min_ts_diff_ns < max_ts_diff_ns {
            return Err(format_err!("Start time should be earlier than end time"));
        }
        let current_timestamp_ns = Timestamp::now().timestamp_nanos();
        repo.bookmark_update_log()
            .list_bookmark_log_entries_ts_in_range(
                ctx.clone(),
                bookmark.clone(),
                max_rec,
                Timestamp::from_timestamp_nanos(current_timestamp_ns - min_ts_diff_ns),
                Timestamp::from_timestamp_nanos(current_timestamp_ns - max_ts_diff_ns),
            )
    } else {
        repo.bookmark_update_log().list_bookmark_log_entries(
            ctx.clone(),
            bookmark.clone(),
            max_rec,
            None,
            Freshness::MostRecent,
        )
    };

    match changeset_type {
        "hg" => {
            entries
                .map_ok({
                    cloned!(ctx, repo);
                    move |(entry_id, cs_id, rs, ts)| {
                        cloned!(ctx, repo);
                        async move {
                            match cs_id {
                                Some(cs_id) => {
                                    let cs = repo.derive_hg_changeset(&ctx, cs_id).await?;
                                    Ok((entry_id, Some(cs), rs, ts))
                                }
                                None => Ok((entry_id, None, rs, ts)),
                            }
                        }
                    }
                })
                .try_buffered(100)
                .map_ok(move |rows| {
                    let (entry_id, cs_id, reason, timestamp) = rows;
                    let cs_id_str = match cs_id {
                        None => String::new(),
                        Some(x) => x.to_string(),
                    };
                    let output = format_bookmark_log_entry(
                        json_flag,
                        cs_id_str,
                        reason,
                        timestamp,
                        "hg",
                        bookmark.clone(),
                        Some(entry_id),
                    );
                    println!("{}", output);
                })
                .try_for_each(|_| async { Ok(()) })
                .await
        }
        "bonsai" => {
            entries
                .map_ok(move |rows| {
                    let (entry_id, cs_id, reason, timestamp) = rows;
                    let cs_id_str = match cs_id {
                        None => String::new(),
                        Some(x) => x.to_string(),
                    };
                    let output = format_bookmark_log_entry(
                        json_flag,
                        cs_id_str,
                        reason,
                        timestamp,
                        "bonsai",
                        bookmark.clone(),
                        Some(entry_id),
                    );
                    println!("{}", output);
                })
                .try_for_each(|_| async { Ok(()) })
                .await
        }
        _ => Err(format_err!("Unknown changeset-type supplied")),
    }
}

async fn handle_list(args: &ArgMatches<'_>, ctx: CoreContext, repo: BlobRepo) -> Result<(), Error> {
    match args.value_of(ARG_KIND) {
        Some("publishing") => {
            repo.bookmarks()
                .get_publishing_bookmarks_maybe_stale(ctx.clone())
                .try_for_each_concurrent(100, {
                    cloned!(repo, ctx);
                    move |(bookmark, bonsai_cs_id)| {
                        cloned!(ctx, repo);
                        async move {
                            let hg_cs_id = repo.derive_hg_changeset(&ctx, bonsai_cs_id).await?;
                            println!("{}\t{}\t{}", bookmark.into_key(), bonsai_cs_id, hg_cs_id);
                            Ok(())
                        }
                    }
                })
                .await
        }
        kind => Err(format_err!("Invalid kind {:?}", kind)),
    }
}

async fn handle_set(args: &ArgMatches<'_>, ctx: CoreContext, repo: BlobRepo) -> Result<(), Error> {
    let bookmark_name = args.value_of("BOOKMARK_NAME").unwrap().to_string();
    let rev = args.value_of("HG_CHANGESET_ID").unwrap().to_string();
    let bookmark = BookmarkKey::new(bookmark_name).unwrap();
    let new_bcs = fetch_bonsai_changeset(ctx.clone(), &rev, &repo, repo.repo_blobstore()).await?;
    let maybe_old_bcs_id = repo.bookmarks().get(ctx.clone(), &bookmark).await?;
    info!(
        ctx.logger(),
        "Current position of {:?} is {:?}", bookmark, maybe_old_bcs_id
    );
    let mut transaction = repo.bookmarks().create_transaction(ctx);
    match maybe_old_bcs_id {
        Some(old_bcs_id) => {
            transaction.update(
                &bookmark,
                new_bcs.get_changeset_id(),
                old_bcs_id,
                BookmarkUpdateReason::ManualMove,
            )?;
        }
        None => {
            transaction.create(
                &bookmark,
                new_bcs.get_changeset_id(),
                BookmarkUpdateReason::ManualMove,
            )?;
        }
    }
    transaction.commit().await?;
    Ok(())
}

async fn handle_delete(
    args: &ArgMatches<'_>,
    ctx: CoreContext,
    repo: BlobRepo,
) -> Result<(), Error> {
    let bookmark_name = args.value_of("BOOKMARK_NAME").unwrap().to_string();
    let bookmark = BookmarkKey::new(bookmark_name).unwrap();
    let maybe_bcs_id = repo.bookmarks().get(ctx.clone(), &bookmark).await?;
    info!(
        ctx.logger(),
        "Current position of {:?} is {:?}", bookmark, maybe_bcs_id
    );
    match maybe_bcs_id {
        Some(bcs_id) => {
            let mut transaction = repo.bookmarks().create_transaction(ctx);
            transaction.delete(&bookmark, bcs_id, BookmarkUpdateReason::ManualMove)?;
            transaction.commit().await?;
            Ok(())
        }
        None => Err(format_err!("Cannot delete missing bookmark")),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn json_output_format() {
        let expected_answer = json!({
            "changeset_type": "hg",
            "changeset_id": "123"
        });
        assert_eq!(
            format_output(true, "123".to_string(), "hg"),
            to_string_pretty(&expected_answer).unwrap()
        );
    }

    #[test]
    fn plain_output_format() {
        assert_eq!(format_output(false, "123".to_string(), "hg"), "(HG) 123");
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;

use anyhow::format_err;
use anyhow::Error;
use blobrepo::BlobRepo;
use blobstore::Loadable;
use bonsai_hg_mapping::BonsaiHgMappingRef;
use bookmarks::BookmarkKey;
use bookmarks::BookmarkUpdateReason;
use bookmarks::BookmarksRef;
use cmdlib::args;
use cmdlib::helpers;
use context::CoreContext;
use facet::AsyncBuildable;
use fbinit::FacebookInit;
use futures::try_join;
use futures::TryStreamExt;
use manifest::ManifestOps;
use mercurial_types::HgChangesetId;
use mercurial_types::HgFileNodeId;
use mercurial_types::MPath;
use mononoke_types::BonsaiChangeset;
use mononoke_types::DateTime;
use mononoke_types::FileChange;
use mononoke_types::Timestamp;
use repo_blobstore::RepoBlobstore;
use repo_blobstore::RepoBlobstoreRef;
use repo_factory::RepoFactoryBuilder;
use repo_identity::RepoIdentityRef;
use serde_json::json;
use serde_json::to_string_pretty;
use slog::debug;
use slog::Logger;
use synced_commit_mapping::SqlSyncedCommitMapping;

pub async fn fetch_bonsai_changeset(
    ctx: CoreContext,
    rev: &str,
    repo: impl RepoIdentityRef + BonsaiHgMappingRef + BookmarksRef,
    blobstore: &RepoBlobstore,
) -> Result<BonsaiChangeset, Error> {
    let csid = helpers::csid_resolve(&ctx, repo, rev.to_string()).await?;
    let cs = csid.load(&ctx, blobstore).await?;
    Ok(cs)
}

pub fn print_bonsai_changeset(bcs: &BonsaiChangeset) {
    println!(
        "BonsaiChangesetId: {} \n\
                     Author: {} \n\
                     Message: {} \n\
                     FileChanges:",
        bcs.get_changeset_id(),
        bcs.author(),
        bcs.message().lines().next().unwrap_or("")
    );

    for (path, file_change) in bcs.file_changes() {
        match file_change {
            FileChange::Change(file_change) => match file_change.copy_from() {
                Some(_) => println!("\t COPY/MOVE: {} {}", path, file_change.content_id()),
                None => println!("\t ADDED/MODIFIED: {} {}", path, file_change.content_id()),
            },
            FileChange::Deletion => println!("\t REMOVED: {}", path),
            FileChange::UntrackedChange(fc) => {
                println!("\t UNTRACKED ADD/MODIFY: {} {}", path, fc.content_id())
            }
            FileChange::UntrackedDeletion => println!("\t MISSING: {}", path),
        }
    }
}

pub fn format_bookmark_log_entry(
    json_flag: bool,
    changeset_id: String,
    reason: BookmarkUpdateReason,
    timestamp: Timestamp,
    changeset_type: &str,
    bookmark: BookmarkKey,
    bundle_id: Option<u64>,
) -> String {
    let reason_str = reason.to_string();
    if json_flag {
        let answer = json!({
            "changeset_type": changeset_type,
            "changeset_id": changeset_id,
            "reason": reason_str,
            "timestamp_sec": timestamp.timestamp_seconds(),
            "bundle_id": bundle_id,
        });
        to_string_pretty(&answer).unwrap()
    } else {
        let dt: DateTime = timestamp.into();
        let dts = dt.as_chrono().format("%b %e %T %Y");
        match bundle_id {
            Some(bundle_id) => format!(
                "{} ({}) {} {} {}",
                bundle_id, bookmark, changeset_id, reason, dts
            ),
            None => format!("({}) {} {} {}", bookmark, changeset_id, reason, dts),
        }
    }
}

// The function retrieves the HgFileNodeId of a file, based on path and rev.
// If the path is not valid an error is expected.
pub async fn get_file_nodes(
    ctx: CoreContext,
    logger: Logger,
    repo: &BlobRepo,
    cs_id: HgChangesetId,
    paths: Vec<MPath>,
) -> Result<Vec<HgFileNodeId>, Error> {
    let cs = cs_id.load(&ctx, repo.repo_blobstore()).await?;
    let root_mf_id = cs.manifestid().clone();
    let manifest_entries: HashMap<_, _> = root_mf_id
        .find_entries(ctx, repo.repo_blobstore().clone(), paths.clone())
        .try_filter_map(|(path, entry)| async move {
            let result =
                path.and_then(move |path| entry.into_leaf().map(move |leaf| (path, leaf.1)));
            Ok(result)
        })
        .try_collect()
        .await?;

    let mut existing_hg_nodes = Vec::new();
    let mut non_existing_paths = Vec::new();
    for path in paths.iter() {
        match manifest_entries.get(path) {
            Some(hg_node) => existing_hg_nodes.push(*hg_node),
            None => non_existing_paths.push(path.clone()),
        };
    }
    match non_existing_paths.len() {
        0 => {
            debug!(logger, "All the file paths are valid");
            Ok(existing_hg_nodes)
        }
        _ => Err(format_err!(
            "failed to identify the files associated with the file paths {:?}",
            non_existing_paths
        )),
    }
}

pub async fn get_source_target_repos_and_mapping<'a, R>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a args::MononokeMatches<'_>,
) -> Result<(R, R, SqlSyncedCommitMapping), Error>
where
    for<'builder> R: AsyncBuildable<'builder, RepoFactoryBuilder<'builder>>,
{
    let config_store = matches.config_store();

    let source_repo_id =
        args::not_shardmanager_compatible::get_source_repo_id(config_store, matches)?;
    let target_repo_id =
        args::not_shardmanager_compatible::get_target_repo_id(config_store, matches)?;

    let source_repo = args::open_repo_with_repo_id(fb, &logger, source_repo_id, matches);
    let target_repo = args::open_repo_with_repo_id(fb, &logger, target_repo_id, matches);
    // TODO(stash): in reality both source and target should point to the same mapping
    // It'll be nice to verify it
    let (source_repo, target_repo) = try_join!(source_repo, target_repo)?;

    let mapping = args::not_shardmanager_compatible::open_source_sql::<SqlSyncedCommitMapping>(
        fb,
        config_store,
        matches,
    )?;

    Ok((source_repo, target_repo, mapping))
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use anyhow::format_err;
use anyhow::Error;
use blobrepo::BlobRepo;
use blobstore::Loadable;
use clap_old::App;
use clap_old::ArgMatches;
use clap_old::SubCommand;
use cmdlib::args;
use cmdlib::args::MononokeMatches;
use cmdlib::helpers;
use context::CoreContext;
use fbinit::FacebookInit;
use manifest::Entry;
use manifest::Manifest;
use manifest::ManifestOps;
use mercurial_derivation::DeriveHgChangeset;
use mercurial_types::HgFileNodeId;
use mercurial_types::HgManifestId;
use mercurial_types::MPath;
use mononoke_types::FileType;
use repo_blobstore::RepoBlobstoreRef;
use slog::Logger;

use crate::error::SubcommandError;

pub const CONTENT_FETCH: &str = "content-fetch";

pub fn build_subcommand<'a, 'b>() -> App<'a, 'b> {
    SubCommand::with_name(CONTENT_FETCH)
        .about("fetches content of the file or manifest from blobrepo")
        .args_from_usage(
            "<CHANGESET_ID>    'hg/bonsai id or bookmark to fetch file from'
             <PATH>            'path to fetch'",
        )
}

pub async fn subcommand_content_fetch<'a>(
    fb: FacebookInit,
    logger: Logger,
    matches: &'a MononokeMatches<'_>,
    sub_m: &'a ArgMatches<'_>,
) -> Result<(), SubcommandError> {
    let rev = sub_m.value_of("CHANGESET_ID").unwrap().to_string();
    let path = sub_m.value_of("PATH").unwrap().to_string();

    let ctx = CoreContext::new_with_logger(fb, logger.clone());

    let repo = args::not_shardmanager_compatible::open_repo(fb, &logger, matches).await?;
    let entry = fetch_entry(&ctx, &repo, &rev, &path).await?;

    match entry {
        Entry::Leaf((FileType::Executable, id)) => {
            let envelope = id
                .load(&ctx, repo.repo_blobstore())
                .await
                .map_err(Error::from)?;
            let content_id = envelope.content_id();
            let maybe_metadata =
                filestore::get_metadata(repo.repo_blobstore(), &ctx, &content_id.into()).await?;
            let metadata = maybe_metadata.ok_or_else(|| {
                format_err!(
                    "Corruption: content id {} for executable file {} in {} not found!",
                    id,
                    path,
                    rev
                )
            })?;

            println!(
                "Binary file. Size: {}\nContent id: {}\nSha1: {}\nSha256: {}\nGit sha1: {}",
                metadata.total_size,
                metadata.content_id,
                metadata.sha1,
                metadata.sha256,
                metadata.git_sha1
            );
        }
        Entry::Leaf((FileType::Symlink, id)) | Entry::Leaf((FileType::Regular, id)) => {
            let envelope = id
                .load(&ctx, repo.repo_blobstore())
                .await
                .map_err(Error::from)?;
            let bytes = filestore::fetch_concat(
                &repo.repo_blobstore().clone(),
                &ctx,
                envelope.content_id(),
            )
            .await?;
            let content = String::from_utf8(bytes.to_vec()).expect("non-utf8 file content");
            println!("{}", content);
        }
        Entry::Tree(id) => {
            let manifest = id
                .load(&ctx, repo.repo_blobstore())
                .await
                .map_err(Error::from)?;

            let entries: Vec<_> = manifest.list().collect();
            let mut longest_len = 0;
            for (name, _) in entries.iter() {
                let basename_len = name.len();
                if basename_len > longest_len {
                    longest_len = basename_len;
                }
            }

            for (name, entry) in entries {
                let mut name = String::from_utf8_lossy(name.as_ref()).to_string();
                for _ in name.len()..longest_len {
                    name.push(' ');
                }

                let (t, h) = match entry {
                    Entry::Leaf((t, id)) => (t.to_string(), id.to_string()),
                    Entry::Tree(id) => ("tree".to_string(), id.to_string()),
                };

                println!("{} {} {}", name, h, t);
            }
        }
    };

    Ok(())
}

async fn fetch_entry(
    ctx: &CoreContext,
    repo: &BlobRepo,
    rev: &str,
    path: &str,
) -> Result<Entry<HgManifestId, (FileType, HgFileNodeId)>, Error> {
    let mpath = MPath::new(path)?;

    let bcs_id = helpers::csid_resolve(ctx, repo.clone(), rev.to_string()).await?;
    let hg_cs_id = repo.derive_hg_changeset(ctx, bcs_id).await?;
    let hg_cs = hg_cs_id.load(ctx, repo.repo_blobstore()).await?;

    let ret = hg_cs
        .manifestid()
        .find_entry(ctx.clone(), repo.repo_blobstore().clone(), Some(mpath))
        .await?
        .ok_or_else(|| format_err!("Path does not exist: {}", path))?;

    Ok(ret)
}