/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

mod app;
mod cache;
mod matches;

use std::borrow::Borrow;
use std::collections::HashSet;
use std::future::Future;
use std::path::PathBuf;
use std::str::FromStr;
use std::time::Duration;

use anyhow::bail;
use anyhow::format_err;
use anyhow::Error;
use anyhow::Result;
use cached_config::ConfigStore;
use clap_old::ArgMatches;
use fbinit::FacebookInit;
pub use metaconfig_parser::RepoConfigs;
pub use metaconfig_parser::StorageConfigs;
use metaconfig_types::BlobConfig;
use metaconfig_types::CommonConfig;
use metaconfig_types::Redaction;
use metaconfig_types::RepoConfig;
use mononoke_types::RepositoryId;
use repo_factory::RepoFactory;
use repo_factory::RepoFactoryBuilder;
use scribe_ext::Scribe;
use slog::info;
use slog::warn;
use slog::Logger;
use sql_construct::SqlConstructFromMetadataDatabaseConfig;

pub use self::app::ArgType;
pub use self::app::MononokeAppBuilder;
pub use self::app::MononokeClapApp;
pub use self::app::RepoRequirement;
use self::app::CONFIG_PATH;
use self::app::REPO_ID;
use self::app::REPO_NAME;
use self::app::SCRIBE_LOGGING_DIRECTORY;
use self::app::SOURCE_REPO_ID;
use self::app::SOURCE_REPO_NAME;
use self::app::TARGET_REPO_ID;
use self::app::TARGET_REPO_NAME;
pub use self::cache::CachelibSettings;
pub use self::matches::MononokeMatches;
use crate::helpers::setup_repo_dir;
use crate::helpers::CreateStorage;

pub struct ResolvedRepo {
    pub id: RepositoryId,
    pub name: String,
    pub config: RepoConfig,
}

pub fn resolve_repo_by_name<'a>(
    config_store: &ConfigStore,
    matches: &'a MononokeMatches<'a>,
    repo_name: &str,
) -> Result<ResolvedRepo> {
    let configs = load_repo_configs(config_store, matches)?;
    resolve_repo_given_name(repo_name, &configs)
}

pub fn resolve_repo_by_id<'a>(
    config_store: &ConfigStore,
    matches: &'a MononokeMatches<'a>,
    repo_id: i32,
) -> Result<ResolvedRepo> {
    let configs = load_repo_configs(config_store, matches)?;
    resolve_repo_given_id(RepositoryId::new(repo_id), &configs)
}

/// The following functions assume a repo is provided locally via CLI args, and thus
/// is not compatible with prod jobs that use SM. Moving them to a separate module
/// to make usage clearer.
pub mod not_shardmanager_compatible {
    use super::*;

    fn get_repo_id_and_name_from_values<'a>(
        config_store: &ConfigStore,
        matches: &'a MononokeMatches<'a>,
        option_repo_name: &str,
        option_repo_id: &str,
    ) -> Result<(RepositoryId, String)> {
        let resolved = resolve_repo(config_store, matches, option_repo_name, option_repo_id)?;
        Ok((resolved.id, resolved.name))
    }

    pub fn resolve_repo<'a>(
        config_store: &ConfigStore,
        matches: &'a MononokeMatches<'a>,
        option_repo_name: &str,
        option_repo_id: &str,
    ) -> Result<ResolvedRepo> {
        let repo_name = matches.value_of(option_repo_name);
        let repo_id = matches.value_of(option_repo_id);
        let configs = load_repo_configs(config_store, matches)?;
        match (repo_name, repo_id) {
            (Some(_), Some(_)) => bail!("both repo-name and repo-id parameters set"),
            (None, None) => bail!("neither repo-name nor repo-id parameter set"),
            (None, Some(repo_id)) => {
                resolve_repo_given_id(RepositoryId::from_str(repo_id)?, &configs)
            }
            (Some(repo_name), None) => resolve_repo_given_name(repo_name, &configs),
        }
    }

    pub fn resolve_repos<'a>(
        config_store: &ConfigStore,
        matches: &'a MononokeMatches<'a>,
    ) -> Result<Vec<ResolvedRepo>> {
        resolve_repos_from_args(config_store, matches, REPO_NAME, REPO_ID)
    }

    fn resolve_repos_from_args<'a>(
        config_store: &ConfigStore,
        matches: &'a MononokeMatches<'a>,
        option_repo_name: &str,
        option_repo_id: &str,
    ) -> Result<Vec<ResolvedRepo>> {
        if matches.app_data().repo_required == Some(RepoRequirement::ExactlyOne) {
            return resolve_repo(config_store, matches, option_repo_name, option_repo_id)
                .map(|r| vec![r]);
        }

        let repo_names = matches.values_of(option_repo_name);
        let repo_ids = matches.values_of(option_repo_id);
        let configs = load_repo_configs(config_store, matches)?;

        let mut repos = Vec::new();
        let mut names = HashSet::new();
        if let Some(repo_ids) = repo_ids {
            for i in repo_ids {
                let resolved = resolve_repo_given_id(RepositoryId::from_str(i)?, &configs)?;
                if names.insert(resolved.name.clone()) {
                    repos.push(resolved);
                }
            }
        }
        if let Some(repo_names) = repo_names {
            for n in repo_names {
                let resolved = resolve_repo_given_name(n, &configs)?;
                if names.insert(n.to_string()) {
                    repos.push(resolved)
                }
            }
        }
        if repos.is_empty() {
            bail!("neither repo-name nor repo-id parameters set");
        }
        Ok(repos)
    }

    pub fn get_repo_id<'a>(
        config_store: &ConfigStore,
        matches: &'a MononokeMatches<'a>,
    ) -> Result<RepositoryId> {
        let (repo_id, _) =
            get_repo_id_and_name_from_values(config_store, matches, REPO_NAME, REPO_ID)?;
        Ok(repo_id)
    }

    pub fn get_repo_name<'a>(
        config_store: &ConfigStore,
        matches: &'a MononokeMatches<'a>,
    ) -> Result<String> {
        let (_, repo_name) =
            get_repo_id_and_name_from_values(config_store, matches, REPO_NAME, REPO_ID)?;
        Ok(repo_name)
    }

    pub fn get_source_repo_id<'a>(
        config_store: &ConfigStore,
        matches: &'a MononokeMatches<'a>,
    ) -> Result<RepositoryId> {
        let (repo_id, _) = get_repo_id_and_name_from_values(
            config_store,
            matches,
            SOURCE_REPO_NAME,
            SOURCE_REPO_ID,
        )?;
        Ok(repo_id)
    }

    pub fn get_source_repo_id_opt<'a>(
        config_store: &ConfigStore,
        matches: &'a MononokeMatches<'a>,
    ) -> Result<Option<RepositoryId>> {
        if matches.is_present(SOURCE_REPO_NAME) || matches.is_present(SOURCE_REPO_ID) {
            let (repo_id, _) = get_repo_id_and_name_from_values(
                config_store,
                matches,
                SOURCE_REPO_NAME,
                SOURCE_REPO_ID,
            )?;
            Ok(Some(repo_id))
        } else {
            Ok(None)
        }
    }

    pub fn get_target_repo_id<'a>(
        config_store: &ConfigStore,
        matches: &'a MononokeMatches<'a>,
    ) -> Result<RepositoryId> {
        let (repo_id, _) = get_repo_id_and_name_from_values(
            config_store,
            matches,
            TARGET_REPO_NAME,
            TARGET_REPO_ID,
        )?;
        Ok(repo_id)
    }

    pub fn get_repo_id_from_value<'a>(
        config_store: &ConfigStore,
        matches: &'a MononokeMatches<'a>,
        repo_id_arg: &str,
        repo_name_arg: &str,
    ) -> Result<RepositoryId> {
        let (repo_id, _) =
            get_repo_id_and_name_from_values(config_store, matches, repo_name_arg, repo_id_arg)?;
        Ok(repo_id)
    }

    pub fn open_source_sql<'a, T>(
        fb: FacebookInit,
        config_store: &ConfigStore,
        matches: &'a MononokeMatches<'a>,
    ) -> Result<T, Error>
    where
        T: SqlConstructFromMetadataDatabaseConfig,
    {
        let source_repo_id = get_source_repo_id(config_store, matches)?;
        let (_, config) = get_config_by_repoid(config_store, matches, source_repo_id)?;
        T::with_metadata_database_config(
            fb,
            &config.storage_config.metadata,
            matches.mysql_options(),
            matches.readonly_storage().0,
        )
    }

    pub async fn open_source_repo<'a, R: 'a>(
        _: FacebookInit,
        logger: &'a Logger,
        matches: &'a MononokeMatches<'a>,
    ) -> Result<R, Error>
    where
        R: for<'builder> facet::AsyncBuildable<'builder, RepoFactoryBuilder<'builder>>,
    {
        let config_store = matches.config_store();
        let source_repo_id = get_source_repo_id(config_store, matches)?;
        let repo_factory = get_repo_factory(matches)?;

        open_repo_internal_with_repo_id(
            logger,
            RepoIdentifier::Id(source_repo_id),
            matches,
            false, // use CreateStorage::ExistingOnly when creating blobstore
            None,  // do not override redaction config
            repo_factory,
        )
        .await
    }

    pub async fn open_target_repo<'a, R: 'a>(
        _: FacebookInit,
        logger: &'a Logger,
        matches: &'a MononokeMatches<'a>,
    ) -> Result<R, Error>
    where
        R: for<'builder> facet::AsyncBuildable<'builder, RepoFactoryBuilder<'builder>>,
    {
        let config_store = matches.config_store();
        let source_repo_id = get_target_repo_id(config_store, matches)?;
        let repo_factory = get_repo_factory(matches)?;

        open_repo_internal_with_repo_id(
            logger,
            RepoIdentifier::Id(source_repo_id),
            matches,
            false, // use CreateStorage::ExistingOnly when creating blobstore
            None,  // do not override redaction config
            repo_factory,
        )
        .await
    }

    pub fn get_config<'a>(
        config_store: &ConfigStore,
        matches: &'a MononokeMatches<'a>,
    ) -> Result<(String, RepoConfig)> {
        let repo_id = get_repo_id(config_store, matches)?;
        get_config_by_repoid(config_store, matches, repo_id)
    }

    async fn open_repo_internal<R>(
        _: FacebookInit,
        logger: &Logger,
        matches: &MononokeMatches<'_>,
        create: bool,
        redaction_override: Option<Redaction>,
        maybe_repo_factory: Option<RepoFactory>,
    ) -> Result<R, Error>
    where
        R: for<'builder> facet::AsyncBuildable<'builder, RepoFactoryBuilder<'builder>>,
    {
        let config_store = matches.config_store();
        let repo_id = get_repo_id(config_store, matches)?;

        let repo_factory = match maybe_repo_factory {
            Some(repo_factory) => repo_factory,
            None => get_repo_factory(matches)?,
        };

        open_repo_internal_with_repo_id(
            logger,
            RepoIdentifier::Id(repo_id),
            matches,
            create,
            redaction_override,
            repo_factory,
        )
        .await
    }

    pub fn open_sql<'a, T>(
        fb: FacebookInit,
        config_store: &ConfigStore,
        matches: &'a MononokeMatches<'a>,
    ) -> Result<T, Error>
    where
        T: SqlConstructFromMetadataDatabaseConfig,
    {
        let (_, config) = get_config(config_store, matches)?;
        T::with_metadata_database_config(
            fb,
            &config.storage_config.metadata,
            matches.mysql_options(),
            matches.readonly_storage().0,
        )
    }

    /// Create a new repo object -- for local instances, expect its contents to be empty.
    #[inline]
    pub fn create_repo<'a, R: 'a>(
        fb: FacebookInit,
        logger: &'a Logger,
        matches: &'a MononokeMatches<'a>,
    ) -> impl Future<Output = Result<R, Error>> + 'a
    where
        R: for<'builder> facet::AsyncBuildable<'builder, RepoFactoryBuilder<'builder>>,
    {
        open_repo_internal(fb, logger, matches, true, None, None)
    }

    /// Create a new repo object -- for local instances, expect its contents to be empty.
    /// Make sure that the opened repo has redaction disabled
    #[inline]
    pub fn create_repo_unredacted<'a, R: 'a>(
        fb: FacebookInit,
        logger: &'a Logger,
        matches: &'a MononokeMatches<'a>,
    ) -> impl Future<Output = Result<R, Error>> + 'a
    where
        R: for<'builder> facet::AsyncBuildable<'builder, RepoFactoryBuilder<'builder>>,
    {
        open_repo_internal(fb, logger, matches, true, Some(Redaction::Disabled), None)
    }

    /// Open an existing repo object -- for local instances, expect contents to already be there.
    #[inline]
    pub fn open_repo<'a, R: 'a>(
        fb: FacebookInit,
        logger: &'a Logger,
        matches: &'a MononokeMatches<'a>,
    ) -> impl Future<Output = Result<R, Error>> + 'a
    where
        R: for<'builder> facet::AsyncBuildable<'builder, RepoFactoryBuilder<'builder>>,
    {
        open_repo_internal(fb, logger, matches, false, None, None)
    }

    #[inline]
    pub fn open_repo_with_factory<'a, R: 'a>(
        fb: FacebookInit,
        logger: &'a Logger,
        matches: &'a MononokeMatches<'a>,
        repo_factory: RepoFactory,
    ) -> impl Future<Output = Result<R, Error>> + 'a
    where
        R: for<'builder> facet::AsyncBuildable<'builder, RepoFactoryBuilder<'builder>>,
    {
        open_repo_internal(fb, logger, matches, false, None, Some(repo_factory))
    }

    /// Open an existing repo object -- for local instances, expect contents to already be there.
    /// Make sure that the opened repo has redaction disabled
    #[inline]
    pub fn open_repo_unredacted<'a, R: 'a>(
        fb: FacebookInit,
        logger: &'a Logger,
        matches: &'a MononokeMatches<'a>,
    ) -> impl Future<Output = Result<R, Error>> + 'a
    where
        R: for<'builder> facet::AsyncBuildable<'builder, RepoFactoryBuilder<'builder>>,
    {
        open_repo_internal(fb, logger, matches, false, Some(Redaction::Disabled), None)
    }
}

fn resolve_repo_given_id(id: RepositoryId, configs: &RepoConfigs) -> Result<ResolvedRepo> {
    let config = configs
        .repos
        .iter()
        .filter(|(_, c)| c.repoid == id)
        .enumerate()
        .last();
    if let Some((count, (name, config))) = config {
        if count > 1 {
            Err(format_err!("multiple configs defined for repo-id {:?}", id))
        } else {
            Ok(ResolvedRepo {
                id,
                name: name.to_string(),
                config: config.clone(),
            })
        }
    } else {
        Err(format_err!("unknown config for repo-id {:?}", id))
    }
}

fn resolve_repo_given_name(name: &str, configs: &RepoConfigs) -> Result<ResolvedRepo> {
    let config = configs.repos.get(name);
    if let Some(config) = config {
        Ok(ResolvedRepo {
            id: config.repoid,
            name: name.to_string(),
            config: config.clone(),
        })
    } else {
        Err(format_err!("unknown repo-name {:?}", name))
    }
}

pub fn open_sql_with_config<'a, T>(
    fb: FacebookInit,
    matches: &'a MononokeMatches<'a>,
    repo_config: &RepoConfig,
) -> Result<T, Error>
where
    T: SqlConstructFromMetadataDatabaseConfig,
{
    T::with_metadata_database_config(
        fb,
        &repo_config.storage_config.metadata,
        matches.mysql_options(),
        matches.readonly_storage().0,
    )
}

#[inline]
pub fn open_repo_by_name<'a, R: 'a>(
    fb: FacebookInit,
    logger: &'a Logger,
    matches: &'a MononokeMatches<'a>,
    repo_name: String,
) -> impl Future<Output = Result<R, Error>> + 'a
where
    R: for<'builder> facet::AsyncBuildable<'builder, RepoFactoryBuilder<'builder>>,
{
    open_repo_by_name_internal(fb, logger, matches, false, None, None, repo_name)
}

/// Open the repo corresponding to the provided repo-name.
/// Make sure that the opened repo has redaction disabled
#[inline]
pub fn open_repo_by_name_unredacted<'a, R: 'a>(
    fb: FacebookInit,
    logger: &'a Logger,
    matches: &'a MononokeMatches<'a>,
    repo_name: String,
) -> impl Future<Output = Result<R, Error>> + 'a
where
    R: for<'builder> facet::AsyncBuildable<'builder, RepoFactoryBuilder<'builder>>,
{
    open_repo_by_name_internal(
        fb,
        logger,
        matches,
        false,
        Some(Redaction::Disabled),
        None,
        repo_name,
    )
}

pub fn get_repo_factory<'a>(matches: &'a MononokeMatches<'a>) -> Result<RepoFactory, Error> {
    Ok(RepoFactory::new(matches.environment().clone()))
}

/// Open an existing repo object by ID -- for local instances, expect contents to already be there.
/// It useful when we need to open more than 1 mononoke repo based on command line arguments
#[inline]
pub async fn open_repo_by_id<'a, R: 'a>(
    _: FacebookInit,
    logger: &'a Logger,
    matches: &'a MononokeMatches<'a>,
    repo_id: RepositoryId,
) -> Result<R, Error>
where
    R: for<'builder> facet::AsyncBuildable<'builder, RepoFactoryBuilder<'builder>>,
{
    let repo_factory = get_repo_factory(matches)?;
    open_repo_internal_with_repo_id(
        logger,
        RepoIdentifier::Id(repo_id),
        matches,
        false, // use CreateStorage::ExistingOnly when creating blobstore
        None,  // do not override redaction config
        repo_factory,
    )
    .await
}

#[inline]
pub async fn open_repo_by_id_unredacted<'a, R: 'a>(
    _: FacebookInit,
    logger: &'a Logger,
    matches: &'a MononokeMatches<'a>,
    repo_id: RepositoryId,
) -> Result<R, Error>
where
    R: for<'builder> facet::AsyncBuildable<'builder, RepoFactoryBuilder<'builder>>,
{
    let repo_factory = get_repo_factory(matches)?;
    open_repo_internal_with_repo_id(
        logger,
        RepoIdentifier::Id(repo_id),
        matches,
        false,
        Some(Redaction::Disabled), // make sure it's unredacted
        repo_factory,
    )
    .await
}

pub fn get_shutdown_grace_period<'a>(matches: &MononokeMatches<'a>) -> Result<Duration> {
    let seconds = matches
        .value_of("shutdown-grace-period")
        .ok_or_else(|| Error::msg("shutdown-grace-period must be specified"))?
        .parse()
        .map_err(Error::from)?;
    Ok(Duration::from_secs(seconds))
}

pub fn get_shutdown_timeout<'a>(matches: &MononokeMatches<'a>) -> Result<Duration> {
    let seconds = matches
        .value_of("shutdown-timeout")
        .ok_or_else(|| Error::msg("shutdown-timeout must be specified"))?
        .parse()
        .map_err(Error::from)?;
    Ok(Duration::from_secs(seconds))
}

pub fn get_scribe<'a>(fb: FacebookInit, matches: &MononokeMatches<'a>) -> Result<Scribe> {
    match matches.value_of(SCRIBE_LOGGING_DIRECTORY) {
        Some(dir) => Ok(Scribe::new_to_file(PathBuf::from(dir))),
        None => Ok(Scribe::new(fb)),
    }
}

pub fn get_config_path<'a>(matches: &'a MononokeMatches<'a>) -> Result<&'a str> {
    matches
        .value_of(CONFIG_PATH)
        .ok_or_else(|| Error::msg(format!("{} must be specified", CONFIG_PATH)))
}

pub fn load_repo_configs<'a>(
    config_store: &ConfigStore,
    matches: &'a MononokeMatches<'a>,
) -> Result<RepoConfigs> {
    metaconfig_parser::load_repo_configs(get_config_path(matches)?, config_store)
}

pub fn load_common_config<'a>(
    config_store: &ConfigStore,
    matches: &'a MononokeMatches<'a>,
) -> Result<CommonConfig> {
    metaconfig_parser::load_common_config(get_config_path(matches)?, config_store)
}

pub fn load_storage_configs<'a>(
    config_store: &ConfigStore,
    matches: &'a MononokeMatches<'a>,
) -> Result<StorageConfigs> {
    metaconfig_parser::load_storage_configs(get_config_path(matches)?, config_store)
}

pub fn get_config_by_repoid<'a>(
    config_store: &ConfigStore,
    matches: &'a MononokeMatches<'a>,
    repo_id: RepositoryId,
) -> Result<(String, RepoConfig)> {
    let configs = load_repo_configs(config_store, matches)?;
    configs
        .get_repo_config(repo_id)
        .ok_or_else(|| format_err!("unknown repoid {:?}", repo_id))
        .map(|(name, config)| (name.clone(), config.clone()))
}

pub fn get_config_by_name<'a>(
    config_store: &ConfigStore,
    matches: &'a MononokeMatches<'a>,
    repo_name: String,
) -> Result<RepoConfig> {
    let configs = load_repo_configs(config_store, matches)?;
    configs
        .repos
        .get(&repo_name)
        .cloned()
        .ok_or_else(|| format_err!("unknown reponame {:?}", repo_name))
}

enum RepoIdentifier {
    Id(RepositoryId),
    Name(String),
}

async fn open_repo_by_name_internal<R>(
    _: FacebookInit,
    logger: &Logger,
    matches: &MononokeMatches<'_>,
    create: bool,
    redaction_override: Option<Redaction>,
    maybe_repo_factory: Option<RepoFactory>,
    repo_name: String,
) -> Result<R, Error>
where
    R: for<'builder> facet::AsyncBuildable<'builder, RepoFactoryBuilder<'builder>>,
{
    let repo_factory = match maybe_repo_factory {
        Some(repo_factory) => repo_factory,
        None => get_repo_factory(matches)?,
    };

    open_repo_internal_with_repo_id(
        logger,
        RepoIdentifier::Name(repo_name),
        matches,
        create,
        redaction_override,
        repo_factory,
    )
    .await
}

async fn open_repo_internal_with_repo_id<R>(
    logger: &Logger,
    repo_id: RepoIdentifier,
    matches: &MononokeMatches<'_>,
    create: bool,
    redaction_override: Option<Redaction>,
    repo_factory: RepoFactory,
) -> Result<R, Error>
where
    R: for<'builder> facet::AsyncBuildable<'builder, RepoFactoryBuilder<'builder>>,
{
    let config_store = matches.config_store();
    let common_config = load_common_config(config_store, matches)?;
    let (reponame, repo_id, mut config) = match repo_id {
        RepoIdentifier::Id(repo_id) => {
            let (reponame, config) = get_config_by_repoid(config_store, matches, repo_id)?;
            (reponame, repo_id, config)
        }
        RepoIdentifier::Name(name) => {
            let config = get_config_by_name(config_store, matches, name.clone())?;
            (name, config.repoid, config)
        }
    };

    info!(logger, "using repo \"{}\" repoid {:?}", reponame, repo_id);
    match &config.storage_config.blobstore {
        BlobConfig::Files { path } | BlobConfig::Sqlite { path } => {
            let create = if create {
                // Many path repos can share one blobstore, so allow store to exist or create it.
                CreateStorage::ExistingOrCreate
            } else {
                CreateStorage::ExistingOnly
            };
            setup_repo_dir(path, create)?;
        }
        _ => {}
    };

    if let Some(redaction_override) = redaction_override {
        config.redaction = redaction_override;
    }

    let repo = repo_factory.build(reponame, config, common_config).await?;

    Ok(repo)
}

pub async fn open_repo_with_repo_id<'a, R: 'a>(
    _: FacebookInit,
    logger: &Logger,
    repo_id: RepositoryId,
    matches: &'a MononokeMatches<'a>,
) -> Result<R, Error>
where
    R: for<'builder> facet::AsyncBuildable<'builder, RepoFactoryBuilder<'builder>>,
{
    let repo_factory = get_repo_factory(matches)?;

    open_repo_internal_with_repo_id(
        logger,
        RepoIdentifier::Id(repo_id),
        matches,
        false,
        None,
        repo_factory,
    )
    .await
}

pub async fn open_repo_with_repo_name<'a, R: 'a>(
    _: FacebookInit,
    logger: &Logger,
    repo_name: String,
    matches: &'a MononokeMatches<'a>,
) -> Result<R, Error>
where
    R: for<'builder> facet::AsyncBuildable<'builder, RepoFactoryBuilder<'builder>>,
{
    let repo_factory = get_repo_factory(matches)?;

    open_repo_internal_with_repo_id(
        logger,
        RepoIdentifier::Name(repo_name),
        matches,
        false,
        None,
        repo_factory,
    )
    .await
}

pub fn get_usize_opt<'a>(matches: &impl Borrow<ArgMatches<'a>>, key: &str) -> Option<usize> {
    matches.borrow().value_of(key).map(|val| {
        val.parse::<usize>()
            .unwrap_or_else(|_| panic!("{} must be integer", key))
    })
}

#[inline]
pub fn get_usize<'a>(matches: &impl Borrow<ArgMatches<'a>>, key: &str, default: usize) -> usize {
    get_usize_opt(matches, key).unwrap_or(default)
}

#[inline]
pub fn get_u64<'a>(matches: &impl Borrow<ArgMatches<'a>>, key: &str, default: u64) -> u64 {
    get_u64_opt(matches, key).unwrap_or(default)
}

#[inline]
pub fn get_and_parse_opt<'a, T: ::std::str::FromStr, M: Borrow<ArgMatches<'a>>>(
    matches: &M,
    key: &str,
) -> Option<T>
where
    <T as std::str::FromStr>::Err: std::fmt::Debug,
{
    matches.borrow().value_of(key).map(|val| {
        val.parse::<T>()
            .unwrap_or_else(|_| panic!("{} - invalid value", key))
    })
}

#[inline]
pub fn get_and_parse<'a, T: ::std::str::FromStr, M: Borrow<ArgMatches<'a>>>(
    matches: &M,
    key: &str,
    default: T,
) -> T
where
    <T as std::str::FromStr>::Err: std::fmt::Debug,
{
    get_and_parse_opt(matches, key).unwrap_or(default)
}

#[inline]
pub fn get_u64_opt<'a>(matches: &impl Borrow<ArgMatches<'a>>, key: &str) -> Option<u64> {
    matches.borrow().value_of(key).map(|val| {
        val.parse::<u64>()
            .unwrap_or_else(|_| panic!("{} must be integer", key))
    })
}

#[inline]
pub fn get_i32_opt<'a>(matches: &impl Borrow<ArgMatches<'a>>, key: &str) -> Option<i32> {
    matches.borrow().value_of(key).map(|val| {
        val.parse::<i32>()
            .unwrap_or_else(|_| panic!("{} must be integer", key))
    })
}

#[inline]
pub fn get_i32<'a>(matches: &impl Borrow<ArgMatches<'a>>, key: &str, default: i32) -> i32 {
    get_i32_opt(matches, key).unwrap_or(default)
}

#[inline]
pub fn get_i64_opt<'a>(matches: &impl Borrow<ArgMatches<'a>>, key: &str) -> Option<i64> {
    matches.borrow().value_of(key).map(|val| {
        val.parse::<i64>()
            .unwrap_or_else(|_| panic!("{} must be integer", key))
    })
}

pub fn get_bool_opt<'a>(matches: &impl Borrow<ArgMatches<'a>>, key: &str) -> Option<bool> {
    matches.borrow().value_of(key).map(|val| {
        val.parse::<bool>()
            .unwrap_or_else(|_| panic!("{} must be bool", key))
    })
}

pub fn parse_disabled_hooks_no_repo_prefix<'a>(
    matches: &'a MononokeMatches<'a>,
    logger: &Logger,
) -> HashSet<String> {
    let disabled_hooks: HashSet<String> = matches
        .values_of("disabled-hooks")
        .map(Vec::from_iter)
        .unwrap_or_default()
        .into_iter()
        .map(|s| s.to_string())
        .collect();

    if !disabled_hooks.is_empty() {
        warn!(
            logger,
            "The following Hooks were disabled: {:?}", disabled_hooks
        );
    }

    disabled_hooks
}

/// Fxed macro from clap2 so it refers to clap_old
#[macro_export]
macro_rules! value_t {
    ($m:ident, $v:expr, $t:ty) => {
        value_t!($m.value_of($v), $t)
    };
    ($m:ident.value_of($v:expr), $t:ty) => {
        if let Some(v) = $m.value_of($v) {
            match v.parse::<$t>() {
                Ok(val) => Ok(val),
                Err(_) => Err(::clap_old::Error::value_validation_auto(format!(
                    "The argument '{}' isn't a valid value",
                    v
                ))),
            }
        } else {
            Err(::clap_old::Error::argument_not_found_auto($v))
        }
    };
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

//! Scaffolding that's generally useful to build CLI tools on top of Mononoke.

#![feature(trait_alias)]

use std::sync::Arc;

use anyhow::bail;
use anyhow::Error;
use cacheblob::LeaseOps;
use cmdlib::args;
use cmdlib::args::MononokeMatches;
use context::CoreContext;
use cross_repo_sync::create_commit_syncer_lease;
use cross_repo_sync::create_commit_syncers;
use cross_repo_sync::types::Source;
use cross_repo_sync::types::Target;
use cross_repo_sync::CommitSyncRepos;
use cross_repo_sync::CommitSyncer;
use cross_repo_sync::Syncers;
use futures_util::try_join;
use live_commit_sync_config::CfgrLiveCommitSyncConfig;
use live_commit_sync_config::LiveCommitSyncConfig;
use mononoke_types::RepositoryId;
use sql_construct::SqlConstructFromMetadataDatabaseConfig;
use synced_commit_mapping::SqlSyncedCommitMapping;

pub trait Repo =
    cross_repo_sync::Repo + for<'b> facet::AsyncBuildable<'b, repo_factory::RepoFactoryBuilder<'b>>;

/// Instantiate the `Syncers` struct by parsing `matches`
pub async fn create_commit_syncers_from_matches<R: Repo>(
    ctx: &CoreContext,
    matches: &MononokeMatches<'_>,
    repo_pair: Option<(RepositoryId, RepositoryId)>,
) -> Result<Syncers<SqlSyncedCommitMapping, R>, Error> {
    let (source_repo, target_repo, mapping, live_commit_sync_config) =
        get_things_from_matches::<R>(ctx, matches, repo_pair).await?;

    let common_config =
        live_commit_sync_config.get_common_config(source_repo.0.repo_identity().id())?;

    let caching = matches.caching();
    let x_repo_syncer_lease = create_commit_syncer_lease(ctx.fb, caching)?;

    let large_repo_id = common_config.large_repo_id;
    let source_repo_id = source_repo.0.repo_identity().id();
    let target_repo_id = target_repo.0.repo_identity().id();
    let (small_repo, large_repo) = if large_repo_id == source_repo_id {
        (target_repo.0, source_repo.0)
    } else if large_repo_id == target_repo_id {
        (source_repo.0, target_repo.0)
    } else {
        bail!(
            "Unexpectedly CommitSyncConfig {:?} has neither of {}, {} as a large repo",
            common_config,
            source_repo_id,
            target_repo_id
        );
    };

    create_commit_syncers(
        ctx,
        small_repo,
        large_repo,
        mapping,
        live_commit_sync_config,
        x_repo_syncer_lease,
    )
}

/// Instantiate the source-target `CommitSyncer` struct by parsing `matches`
pub async fn create_commit_syncer_from_matches<R: Repo>(
    ctx: &CoreContext,
    matches: &MononokeMatches<'_>,
    repo_pair: Option<(RepositoryId, RepositoryId)>,
) -> Result<CommitSyncer<SqlSyncedCommitMapping, R>, Error> {
    create_commit_syncer_from_matches_impl(ctx, matches, false /* reverse */, repo_pair).await
}

/// Instantiate the target-source `CommitSyncer` struct by parsing `matches`
pub async fn create_reverse_commit_syncer_from_matches<R: Repo>(
    ctx: &CoreContext,
    matches: &MononokeMatches<'_>,
    repo_pair: Option<(RepositoryId, RepositoryId)>,
) -> Result<CommitSyncer<SqlSyncedCommitMapping, R>, Error> {
    create_commit_syncer_from_matches_impl(ctx, matches, true /* reverse */, repo_pair).await
}

/// Instantiate some auxiliary things from `matches`
/// Naming is hard.
async fn get_things_from_matches<R: Repo>(
    ctx: &CoreContext,
    matches: &MononokeMatches<'_>,
    repo_pair: Option<(RepositoryId, RepositoryId)>,
) -> Result<
    (
        Source<R>,
        Target<R>,
        SqlSyncedCommitMapping,
        Arc<dyn LiveCommitSyncConfig>,
    ),
    Error,
> {
    let fb = ctx.fb;
    let logger = ctx.logger();

    let config_store = matches.config_store();
    let (source_repo_id, target_repo_id) = match repo_pair {
        Some((source_repo_id, target_repo_id)) => (source_repo_id, target_repo_id),
        None => (
            args::not_shardmanager_compatible::get_source_repo_id(config_store, matches)?,
            args::not_shardmanager_compatible::get_target_repo_id(config_store, matches)?,
        ),
    };

    let (_, source_repo_config) =
        args::get_config_by_repoid(config_store, matches, source_repo_id)?;
    let (_, target_repo_config) =
        args::get_config_by_repoid(config_store, matches, target_repo_id)?;

    if source_repo_config.storage_config.metadata != target_repo_config.storage_config.metadata {
        return Err(Error::msg(
            "source repo and target repo have different metadata database configs!",
        ));
    }

    let mysql_options = matches.mysql_options();
    let readonly_storage = matches.readonly_storage();

    let mapping = SqlSyncedCommitMapping::with_metadata_database_config(
        ctx.fb,
        &source_repo_config.storage_config.metadata,
        mysql_options,
        readonly_storage.0,
    )?;

    let source_repo_fut = args::open_repo_with_repo_id(fb, logger, source_repo_id, matches);
    let target_repo_fut = args::open_repo_with_repo_id(fb, logger, target_repo_id, matches);

    let (source_repo, target_repo) = try_join!(source_repo_fut, target_repo_fut)?;

    let live_commit_sync_config: Arc<dyn LiveCommitSyncConfig> =
        Arc::new(CfgrLiveCommitSyncConfig::new(ctx.logger(), config_store)?);

    Ok((
        Source(source_repo),
        Target(target_repo),
        mapping,
        live_commit_sync_config,
    ))
}

fn flip_direction<T>(source_item: Source<T>, target_item: Target<T>) -> (Source<T>, Target<T>) {
    (Source(target_item.0), Target(source_item.0))
}

async fn create_commit_syncer_from_matches_impl<R: Repo>(
    ctx: &CoreContext,
    matches: &MononokeMatches<'_>,
    reverse: bool,
    repo_pair: Option<(RepositoryId, RepositoryId)>,
) -> Result<CommitSyncer<SqlSyncedCommitMapping, R>, Error> {
    let (source_repo, target_repo, mapping, live_commit_sync_config) =
        get_things_from_matches(ctx, matches, repo_pair).await?;

    let (source_repo, target_repo) = if reverse {
        flip_direction(source_repo, target_repo)
    } else {
        (source_repo, target_repo)
    };

    let caching = matches.caching();
    let x_repo_syncer_lease = create_commit_syncer_lease(ctx.fb, caching)?;

    create_commit_syncer(
        ctx,
        source_repo,
        target_repo,
        mapping,
        live_commit_sync_config,
        x_repo_syncer_lease,
    )
    .await
}

async fn create_commit_syncer<'a, R: Repo>(
    ctx: &'a CoreContext,
    source_repo: Source<R>,
    target_repo: Target<R>,
    mapping: SqlSyncedCommitMapping,
    live_commit_sync_config: Arc<dyn LiveCommitSyncConfig>,
    x_repo_syncer_lease: Arc<dyn LeaseOps>,
) -> Result<CommitSyncer<SqlSyncedCommitMapping, R>, Error> {
    let common_config =
        live_commit_sync_config.get_common_config(source_repo.0.repo_identity().id())?;

    let repos = CommitSyncRepos::new(source_repo.0, target_repo.0, &common_config)?;
    let commit_syncer = CommitSyncer::new(
        ctx,
        mapping,
        repos,
        live_commit_sync_config,
        x_repo_syncer_lease,
    );
    Ok(commit_syncer)
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::cmp;
use std::collections::HashSet;
use std::sync::atomic::AtomicBool;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering;
use std::sync::Arc;

use anyhow::bail;
use anyhow::format_err;
use anyhow::Context;
use anyhow::Error;
use anyhow::Result;
use async_trait::async_trait;
use blobstore::Loadable;
use blobstore::Storable;
use bytes::Bytes;
use changesets::Changesets;
use changesets::ChangesetsRef;
use clap::Parser;
use clap::ValueEnum;
use context::CoreContext;
use executor_lib::RepoShardedProcess;
use executor_lib::RepoShardedProcessExecutor;
use executor_lib::ShardedProcessExecutor;
use fbinit::FacebookInit;
use filestore::hash_bytes;
use filestore::Alias;
use filestore::AliasBlob;
use filestore::Blake3IncrementalHasher;
use filestore::FetchKey;
use filestore::GitSha1IncrementalHasher;
use filestore::Sha1IncrementalHasher;
use filestore::Sha256IncrementalHasher;
use futures::future::TryFutureExt;
use futures::stream;
use futures::stream::StreamExt;
use futures::stream::TryStreamExt;
use mercurial_types::FileBytes;
use metaconfig_types::ShardedService;
use mononoke_app::args::OptRepoArgs;
use mononoke_app::fb303::AliveService;
use mononoke_app::fb303::Fb303AppExtension;
use mononoke_app::MononokeApp;
use mononoke_app::MononokeAppBuilder;
use mononoke_app::MononokeReposManager;
use mononoke_repos::MononokeRepos;
use mononoke_types::BlobstoreKey;
use mononoke_types::ChangesetId;
use mononoke_types::ContentAlias;
use mononoke_types::ContentId;
use mononoke_types::FileChange;
use mutable_counters::MutableCounters;
use mutable_counters::MutableCountersRef;
use once_cell::sync::OnceCell;
use repo_blobstore::RepoBlobstore;
use repo_blobstore::RepoBlobstoreRef;
use repo_identity::RepoIdentity;
use sharding_ext::RepoShard;
use slog::debug;
use slog::error;
use slog::info;
use slog::warn;
use slog::Logger;

const LIMIT: usize = 1000;
const SM_SERVICE_SCOPE: &str = "global";
const SM_CLEANUP_TIMEOUT_SECS: u64 = 120;

#[facet::container]
pub struct Repo {
    #[facet]
    repo_identity: RepoIdentity,
    #[facet]
    repo_blobstore: RepoBlobstore,
    #[facet]
    mutable_counters: dyn MutableCounters,
    #[facet]
    changesets: dyn Changesets,
}

#[derive(Debug, Clone, Copy, ValueEnum)]
enum Mode {
    /// Mode to verify if the alias exists, and if it doesn't, report the error
    Verify,
    /// Mode to verify if the alias exists, and if it doesn't then generate it.
    Generate,
    /// Mode to generate aliases (along with metadata) for large collection of files.
    /// Can be used for backfilling repos with metadata and new aliases. In this mode,
    /// min_cs_db_id is ignored and {repo_name}_alias_backfill_counter mutable counter
    /// is used to determine the starting changeset for backfilling. If the mutable counter
    /// doesn't exist, the backfilling starts from cs_id 0.
    Backfill,
}

#[derive(Debug)]
enum RunStatus {
    InProgress,
    Finished,
}

#[derive(Debug, Clone, Copy, ValueEnum)]
enum AliasType {
    Sha256,
    SeededBlake3,
    Sha1,
    GitSha1,
}

impl AliasType {
    fn get_alias(&self, content: &Bytes) -> Alias {
        match self {
            AliasType::GitSha1 => {
                Alias::GitSha1(hash_bytes(GitSha1IncrementalHasher::new(content), content).sha1())
            }
            AliasType::SeededBlake3 => {
                Alias::SeededBlake3(hash_bytes(Blake3IncrementalHasher::new_seeded(), content))
            }
            AliasType::Sha1 => Alias::Sha1(hash_bytes(Sha1IncrementalHasher::new(), content)),
            AliasType::Sha256 => Alias::Sha256(hash_bytes(Sha256IncrementalHasher::new(), content)),
        }
    }
}

/// Struct representing the Alias Verify process.
pub struct AliasVerifyProcess {
    app: MononokeApp,
    args: Arc<AliasVerifyArgs>,
    repos_mgr: MononokeReposManager<Repo>,
}

impl AliasVerifyProcess {
    pub fn new(app: MononokeApp, repos_mgr: MononokeReposManager<Repo>) -> Result<Self> {
        let args: Arc<AliasVerifyArgs> = Arc::new(app.args()?);
        Ok(Self {
            app,
            args,
            repos_mgr,
        })
    }
}

#[async_trait]
impl RepoShardedProcess for AliasVerifyProcess {
    async fn setup(&self, repo: &RepoShard) -> anyhow::Result<Arc<dyn RepoShardedProcessExecutor>> {
        let logger = self.app.repo_logger(&repo.to_string());
        let repo_name = repo.repo_name.as_str();
        let start = repo.chunk_id.unwrap_or(0) as u64;
        let total = repo.total_chunks.unwrap_or(1) as u64;
        info!(&logger, "Setting up alias verify for repo {}", repo_name);
        let ctx = self.app.new_basic_context();
        self.repos_mgr
            .add_repo(repo_name.as_ref())
            .await
            .with_context(|| format!("Failure in opening repo {}", repo_name))?;
        let db_config = self
            .app
            .repo_config_by_name(repo_name.as_ref())?
            .storage_config
            .metadata;
        let common_config = Arc::new(self.app.repo_configs().common.clone());
        let redacted_blobs = self
            .app
            .repo_factory()
            .redacted_blobs(ctx.clone(), &db_config, &common_config)
            .await?;
        let redacted_content_ids = redacted_blobs.redacted().keys().cloned().collect();
        Ok(Arc::new(AliasVerification::new(
            logger.clone(),
            repo_name.to_string(),
            self.repos_mgr.repos().clone(),
            self.args.clone(),
            Arc::new(AtomicBool::new(false)),
            ctx,
            start,
            total,
            redacted_content_ids,
        )))
    }
}

/// Verify and reload all the alias blobs
#[derive(Parser)]
#[clap(about = "Verify and reload all the alias blobs into Mononoke blobstore.")]
struct AliasVerifyArgs {
    /// The type of alias to verify or generate (in case of missing alias)
    #[clap(long, value_enum, default_value_t = AliasType::Sha256)]
    alias_type: AliasType,
    /// Mode for missing blobs
    #[clap(long, value_enum, default_value_t = Mode::Verify)]
    mode: Mode,
    /// Number of commit ids to process at a time
    #[clap(long, default_value_t = 5000)]
    step: u64,
    /// Changeset to start verification from. Id from changeset table. Not connected to hash
    #[clap(long, default_value_t = 0)]
    min_cs_db_id: u64,
    /// Concurrency limit defining how many commits to be processed in parallel
    #[clap(long, default_value_t = LIMIT)]
    concurrency: usize,
    /// The repo against which the alias verify command needs to be executed
    #[clap(flatten)]
    repo: OptRepoArgs,
    /// The name of ShardManager service to be used when running alias verify in sharded setting.
    #[clap(long, conflicts_with_all = &["repo-name", "repo-id"])]
    pub sharded_service_name: Option<String>,
}

/// Struct representing the Alias Verify process over the context of Repo.
struct AliasVerification {
    logger: Logger,
    repos: Arc<MononokeRepos<Repo>>,
    repo_name: String,
    args: Arc<AliasVerifyArgs>,
    ctx: CoreContext,
    err_cnt: Arc<AtomicUsize>,
    cs_processed: Arc<AtomicUsize>,
    cancellation_requested: Arc<AtomicBool>,
    start: u64,
    total: u64,
    redacted_content_ids: HashSet<String>,
}

#[async_trait]
impl RepoShardedProcessExecutor for AliasVerification {
    async fn execute(&self) -> anyhow::Result<()> {
        info!(
            self.logger,
            "Initiating alias verify execution for repo {}", &self.repo_name,
        );

        let val = self.verify_all().await;
        match val {
            Err(ref e) => {
                error!(
                    self.logger,
                    "Alias Verify Failure in repo {}. Terminating execution. Cause: {:?}",
                    &self.repo_name,
                    e
                );
                val
            }
            v => v,
        }
    }

    async fn stop(&self) -> anyhow::Result<()> {
        info!(
            self.logger,
            "Terminating alias verify execution for repo {}", &self.repo_name,
        );
        self.cancellation_requested.store(true, Ordering::Relaxed);
        Ok(())
    }
}

impl AliasVerification {
    pub fn new(
        logger: Logger,
        repo_name: String,
        repos: Arc<MononokeRepos<Repo>>,
        args: Arc<AliasVerifyArgs>,
        cancellation_requested: Arc<AtomicBool>,
        ctx: CoreContext,
        start: u64,
        total: u64,
        redacted_content_ids: HashSet<String>,
    ) -> Self {
        Self {
            logger,
            repo_name,
            repos,
            args,
            cancellation_requested,
            ctx,
            start,
            total,
            redacted_content_ids,
            err_cnt: Arc::new(AtomicUsize::new(0)),
            cs_processed: Arc::new(AtomicUsize::new(0)),
        }
    }

    fn repo(&self) -> Result<Arc<Repo>> {
        self.repos
            .get_by_name(self.repo_name.as_ref())
            .ok_or_else(|| {
                anyhow::anyhow!(
                    "Requested repo {} does not exist on this server",
                    self.repo_name
                )
            })
    }

    async fn get_file_changes_vector(&self, bcs_id: ChangesetId) -> Result<Vec<FileChange>, Error> {
        let cs_cnt = self.cs_processed.fetch_add(1, Ordering::Relaxed);
        if cs_cnt % 1000 == 0 {
            info!(self.logger, "Commit processed {:?}", cs_cnt);
        }
        let bcs = bcs_id
            .load(&self.ctx, self.repo()?.repo_blobstore())
            .await
            .with_context(|| {
                format!(
                    "Failure in fetching changeset for changeset ID {:?}",
                    bcs_id
                )
            })?;
        let file_changes: Vec<_> = bcs
            .file_changes_map()
            .iter()
            .map(|(_path, fc)| fc.clone())
            .collect();
        Ok(file_changes)
    }

    async fn check_alias_blob(
        &self,
        alias: &Alias,
        expected_content_id: ContentId,
        content_id: ContentId,
    ) -> Result<(), Error> {
        if content_id == expected_content_id {
            // Everything is good
            Ok(())
        } else {
            panic!(
                "Collision: Wrong content_id by alias for {:?},
                ContentId in the blobstore {:?},
                Expected ContentId {:?}",
                alias, content_id, expected_content_id
            );
        }
    }

    async fn process_missing_alias_blob(
        &self,
        ctx: &CoreContext,
        alias: &Alias,
        content_id: ContentId,
    ) -> Result<(), Error> {
        self.err_cnt.fetch_add(1, Ordering::Relaxed);
        debug!(
            self.logger,
            "Missing alias blob: alias {}, content_id {:?}", alias, content_id
        );
        match self.args.mode {
            Mode::Verify => Ok(()),
            Mode::Generate | Mode::Backfill => {
                let blobstore = self.repo()?.repo_blobstore().clone();
                let maybe_meta =
                    filestore::get_metadata(&blobstore, ctx, &FetchKey::Canonical(content_id))
                        .await?;
                let meta = maybe_meta.ok_or_else(|| {
                    format_err!("Missing content {:?} for alias {:?}", content_id, alias)
                })?;
                let is_valid_match = match *alias {
                    Alias::Sha256(hash_val) => meta.sha256 == hash_val,
                    Alias::GitSha1(hash_val) => meta.git_sha1.sha1() == hash_val,
                    Alias::SeededBlake3(hash_val) => meta.seeded_blake3 == hash_val,
                    Alias::Sha1(hash_val) => meta.sha1 == hash_val,
                };
                if is_valid_match {
                    AliasBlob(alias.clone(), ContentAlias::from_content_id(content_id))
                        .store(ctx, &blobstore)
                        .await
                } else {
                    Err(format_err!(
                        "Inconsistent hashes for {:?}, got {:?}, metadata hashes are (Sha1: {:?}, Sha256: {:?}, GitSha1: {:?}, SeededBlake3: {:?})",
                        content_id,
                        alias,
                        meta.sha1,
                        meta.sha256,
                        meta.git_sha1.sha1(),
                        meta.seeded_blake3,
                    ))
                }
            }
        }
    }

    async fn process_alias(
        &self,
        ctx: &CoreContext,
        alias: &Alias,
        content_id: ContentId,
    ) -> Result<(), Error> {
        let result = FetchKey::from(alias.clone())
            .load(ctx, self.repo()?.repo_blobstore())
            .await;
        match result {
            Ok(content_id_from_blobstore) => {
                self.check_alias_blob(alias, content_id, content_id_from_blobstore)
                    .await
            }
            Err(_) => {
                // the blob with alias is not found
                self.process_missing_alias_blob(ctx, alias, content_id)
                    .await
            }
        }
    }

    pub async fn process_file_content(&self, content_id: ContentId) -> Result<(), Error> {
        let ctx = &self.ctx;
        let alias = filestore::fetch_concat(self.repo()?.repo_blobstore(), ctx, content_id)
            .map_ok(FileBytes)
            .map_ok(|content| self.args.alias_type.get_alias(&content.into_bytes()))
            .await
            .with_context(|| {
                format!("Failure in fetching content at content ID {:?}", content_id)
            })?;
        self.process_alias(ctx, &alias, content_id).await
    }

    fn print_report(&self, status: RunStatus) {
        let resolution = match status {
            RunStatus::InProgress => "continues",
            RunStatus::Finished => "finished",
        };
        info!(
            self.logger,
            "Alias Verification {}: {:?} errors found",
            resolution,
            self.err_cnt.load(Ordering::Relaxed)
        );
    }

    fn counter_name(&self) -> String {
        let counter_prefix = if self.start == 0 {
            self.repo_name.to_string()
        } else {
            format!("{}_{}_{}", self.repo_name, self.start, self.total)
        };
        format!("{}_alias_backfill_counter", counter_prefix)
    }

    async fn update_overall_progress(&self, completed: usize, max_id: u64) -> Result<()> {
        let repo = self.repo()?;
        info!(self.logger, "Processed {} changesets", completed);
        if let Mode::Backfill = self.args.mode {
            info!(
                self.logger,
                "Completed processing till changeset ID {:?}", max_id
            );
            let counter_name = self.counter_name();
            repo.mutable_counters()
                .set_counter(&self.ctx, &counter_name, max_id as i64, None)
                .await
                .with_context(|| {
                    format!(
                        "Failed to set {} for {} to {}",
                        counter_name, self.repo_name, max_id
                    )
                })?;
            let mut updated = false;
            if self.args.sharded_service_name.is_some() && completed > 0 {
                // Counter to keep track of overall number of commits for the repo.
                let overall_counter_name =
                    format!("overall_{}_alias_backfill_counter", self.repo_name);
                while !updated {
                    let maybe_counter_val = repo
                        .mutable_counters()
                        .get_counter(&self.ctx, &overall_counter_name)
                        .await
                        .with_context(|| {
                            format!(
                                "Error while getting mutable counter {}",
                                overall_counter_name
                            )
                        })?;
                    let new_counter_val =
                        maybe_counter_val.clone().unwrap_or(0) + (completed as i64);
                    updated = repo
                        .mutable_counters()
                        .set_counter(
                            &self.ctx,
                            &overall_counter_name,
                            new_counter_val,
                            maybe_counter_val,
                        )
                        .await
                        .with_context(|| {
                            format!(
                                "Failed to set {} for {} to {}",
                                overall_counter_name, self.repo_name, new_counter_val
                            )
                        })?;
                    info!(
                        self.logger,
                        "Updated total commits processed for {} to {}",
                        self.repo_name,
                        new_counter_val
                    );
                }
            }
        }
        Ok(())
    }

    async fn get_bounded(&self, min_id: u64, max_id: u64) -> Result<(), Error> {
        if self.cancellation_requested.load(Ordering::Relaxed) {
            return Ok(());
        }
        info!(
            self.logger,
            "Process Changesets with ids: [{:?}, {:?})", min_id, max_id
        );
        let repo = self.repo()?;
        let bcs_ids = repo
            .changesets()
            .list_enumeration_range(&self.ctx, min_id, max_id, None, true);
        let count = AtomicUsize::new(0);
        let rcount = &count;
        let file_changes_stream = bcs_ids
            .and_then(move |(bcs_id, _)| async move {
                let file_changes_vec = self.get_file_changes_vector(bcs_id).await?;
                Ok(stream::iter(file_changes_vec).map(anyhow::Ok))
            })
            .try_flatten()
            .boxed();

        file_changes_stream
            .try_for_each_concurrent(self.args.concurrency, move |file_change| async move {
                rcount.fetch_add(1, Ordering::Relaxed);
                match file_change.simplify() {
                    Some(tc) => {
                        if self
                            .redacted_content_ids
                            .contains(&tc.content_id().blobstore_key())
                        {
                            warn!(
                                self.logger,
                                "Skipping content id {:?} since it is part of the redaction list",
                                tc.content_id()
                            );
                            Ok(())
                        } else {
                            self.process_file_content(tc.content_id().clone()).await
                        }
                    }
                    None => Ok(()),
                }
            })
            .await?;
        self.update_overall_progress(rcount.load(Ordering::Relaxed), max_id)
            .await?;
        self.print_report(RunStatus::InProgress);
        Ok(())
    }

    async fn start_and_end_id(&self, min_id: u64, max_id: u64) -> Result<(u64, u64)> {
        let counter_name = self.counter_name();
        let counter_val = self
            .repo()?
            .mutable_counters()
            .get_counter(&self.ctx, &counter_name)
            .await
            .with_context(|| format!("Error while getting mutable counter {}", counter_name))?;
        // No chunking if no start or end provided.
        if self.start == 0 {
            if self.args.min_cs_db_id != 0 {
                return Ok((self.args.min_cs_db_id, max_id));
            } else {
                return Ok((
                    counter_val.unwrap_or(self.args.min_cs_db_id as i64) as u64,
                    max_id,
                ));
            }
        }
        match self.args.mode {
            // Chunking only in backfilling mode
            Mode::Backfill => {
                let factor = (max_id - min_id) / self.total;
                let prev_chunk_id = std::cmp::max(self.start, 1) - 1;
                let default_start_point = (min_id + factor * prev_chunk_id) as i64;
                // If the mutable counter "counter_val" doesn't have a value, then use the
                // default_start_point as the starting point of the chunk. If the counter has
                // a value, then use that unless the value is less than default_start_point.
                let start_point = counter_val.map_or(default_start_point, |counter| {
                    std::cmp::max(counter, default_start_point)
                }) as u64;
                let end_point = min_id + factor * self.start;
                Ok((start_point, end_point))
            }
            _ => Ok((self.args.min_cs_db_id, max_id)),
        }
    }

    pub async fn verify_all(&self) -> Result<(), Error> {
        let (ctx, step, repo) = (&self.ctx, self.args.step, self.repo()?);
        let (min_id, max_id) = repo
            .changesets()
            .enumeration_bounds(ctx, true, vec![])
            .await?
            .unwrap();
        let (init_changeset_id, max_id) = self.start_and_end_id(min_id, max_id).await?;
        let mut bounds = vec![];
        let mut cur_id = cmp::max(min_id, init_changeset_id);
        info!(
            self.logger,
            "Initiating aliasverify in {:?} mode with input init changesetid {} and actual init changesetid {}. Max changesetid {}",
            self.args.mode,
            init_changeset_id,
            cur_id,
            max_id,
        );
        let max_id = max_id + 1;
        while cur_id < max_id {
            let max = cmp::min(max_id, cur_id + step);
            bounds.push((cur_id, max));
            cur_id += step;
        }
        stream::iter(bounds)
            .map(Ok)
            .try_for_each(move |(min_val, max_val)| self.get_bounded(min_val, max_val))
            .await?;
        self.print_report(RunStatus::Finished);
        if self.args.sharded_service_name.is_some() {
            let finished_counter_name = format!("finished_{}", self.counter_name());
            repo.mutable_counters()
                .set_counter(&self.ctx, &finished_counter_name, 0, None)
                .await
                .with_context(|| {
                    format!(
                        "Failed to set {} for {}",
                        finished_counter_name, self.repo_name
                    )
                })?;
        }
        Ok(())
    }
}

async fn async_main(app: MononokeApp) -> Result<(), Error> {
    let args: AliasVerifyArgs = app.args()?;
    let repo_mgr = app
        .open_managed_repos(Some(ShardedService::AliasVerify))
        .await?;
    let process = AliasVerifyProcess::new(app, repo_mgr)?;
    match args.sharded_service_name {
        None => {
            let maybe_repo_arg = args.repo.as_repo_arg();
            let (repo_name, _) = match maybe_repo_arg {
                Some(ref repo_arg) => process.app.repo_config(repo_arg)?,
                None => bail!(
                    "Repo name or ID not provided. Either sharded-service-name or repo id/name should be provided."
                ),
            };
            let alias_verify = process
                .setup(&RepoShard::with_repo_name(repo_name.as_ref()))
                .await?;
            alias_verify.execute().await
        }
        Some(name) => {
            let logger = process.app.logger().clone();
            // The service name needs to be 'static to satisfy SM contract
            static SM_SERVICE_NAME: OnceCell<String> = OnceCell::new();
            let mut executor = ShardedProcessExecutor::new(
                process.app.fb,
                process.app.runtime().clone(),
                &logger,
                SM_SERVICE_NAME.get_or_init(|| name),
                SM_SERVICE_SCOPE,
                SM_CLEANUP_TIMEOUT_SECS,
                Arc::new(process),
                true, // enable shard (repo) level healing
            )?;
            executor
                .block_and_execute(&logger, Arc::new(AtomicBool::new(false)))
                .await
        }
    }
}

#[fbinit::main]
fn main(fb: FacebookInit) -> Result<()> {
    let app = MononokeAppBuilder::new(fb)
        .with_app_extension(Fb303AppExtension {})
        .build::<AliasVerifyArgs>()?;
    app.run_with_monitoring_and_logging(async_main, "aliasverify", AliveService)
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::fs;
use std::io;
use std::io::BufRead;
use std::path::Path;

use anyhow::anyhow;
use anyhow::Context;
use anyhow::Error;
use ascii::AsciiStr;
use blobrepo::BlobRepo;
use blobstore::Loadable;
use bonsai_git_mapping::BonsaiGitMappingRef;
use bonsai_hg_mapping::BonsaiHgMappingRef;
use bonsai_svnrev_mapping::BonsaiSvnrevMappingRef;
use clap::ArgGroup;
use clap::Parser;
use cloned::cloned;
use context::CoreContext;
use fbinit::FacebookInit;
use futures::stream;
use futures_util::future::TryFutureExt;
use futures_util::stream::StreamExt;
use futures_util::stream::TryStreamExt;
use mercurial_types::HgChangesetId;
use mononoke_app::args::RepoArgs;
use mononoke_app::MononokeAppBuilder;
use repo_blobstore::RepoBlobstoreRef;

#[derive(Parser)]
#[clap(about = "Tool to backfill git mappings for given commits")]
#[clap(group(
    ArgGroup::new("mode")
        .required(true)
        .args(&["git", "svnrev"]),
))]

struct BackFillArgs {
    #[clap(flatten)]
    repo: RepoArgs,
    #[clap(long, action)]
    git: bool,
    #[clap(long, action)]
    svnrev: bool,
    #[clap(
        value_parser,
        help = " file with hg changeset ids (separated by newlines) "
    )]
    in_filename: Option<String>,
}

fn parse_input<P: AsRef<Path>>(
    file: P,
) -> Result<impl Iterator<Item = Result<HgChangesetId, Error>>, Error> {
    let file = fs::File::open(file)?;
    let iter = io::BufReader::new(file)
        .lines()
        .map(|line| HgChangesetId::from_ascii_str(AsciiStr::from_ascii(&line?)?));
    Ok(iter)
}

#[derive(Debug, Copy, Clone)]
pub enum BackfillMode {
    Git,
    Svnrev,
}

pub async fn backfill<P: AsRef<Path>>(
    ctx: CoreContext,
    repo: BlobRepo,
    in_path: P,
    mode: BackfillMode,
) -> Result<(), Error> {
    let chunk_size = 1000;
    let ids = parse_input(in_path)?;
    stream::iter(ids)
        .and_then(|hg_cs_id| {
            cloned!(ctx, repo);
            async move {
                let id = repo
                    .bonsai_hg_mapping()
                    .get_bonsai_from_hg(&ctx, hg_cs_id)
                    .await?
                    .ok_or_else(|| anyhow!("hg commit {} is missing", hg_cs_id))?;
                Ok(id)
            }
        })
        .map_ok({
            cloned!(ctx, repo);
            move |id| {
                cloned!(ctx, repo);
                async move { id.load(&ctx, repo.repo_blobstore()).await }
                    .map_err(anyhow::Error::from)
            }
        })
        .try_buffer_unordered(chunk_size)
        // TryStreamExt doesn't have the try_chunks method yet so we have to do it by folding.
        .chunks(chunk_size)
        .map(|chunk| chunk.into_iter().collect::<Result<Vec<_>, _>>())
        .try_for_each(|chunk| {
            cloned!(ctx, repo);
            async move {
                match mode {
                    BackfillMode::Git => {
                        repo.bonsai_git_mapping()
                            .bulk_import_from_bonsai(&ctx, &chunk)
                            .await
                    }
                    BackfillMode::Svnrev => {
                        repo.bonsai_svnrev_mapping()
                            .bulk_import_from_bonsai(&ctx, &chunk)
                            .await
                    }
                }
            }
        })
        .await
}

#[fbinit::main]
fn main(fb: FacebookInit) -> Result<(), Error> {
    let app = MononokeAppBuilder::new(fb).build::<BackFillArgs>()?;

    let args: BackFillArgs = app.args()?;

    let logger = app.logger();
    let runtime = app.runtime();

    let ctx = CoreContext::new_with_logger(fb, logger.clone());

    let mode = if args.git {
        BackfillMode::Git
    } else if args.svnrev {
        BackfillMode::Svnrev
    } else {
        panic!("backfill mode not specified");
    };

    let run = async {
        let repo = app
            .open_repo(&args.repo)
            .await
            .context("Failed to open repo")?;
        let in_filename = args.in_filename.unwrap();
        backfill(ctx, repo, in_filename, mode).await
    };

    runtime.block_on(run)?;
    Ok(())
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

#![cfg_attr(not(fbcode_build), allow(unused_crate_dependencies))]

use std::collections::HashMap;
use std::fs::read;
use std::path::Path;
use std::str::FromStr;
use std::sync::Arc;

use anyhow::bail;
use anyhow::format_err;
use anyhow::Context;
use anyhow::Error;
use anyhow::Result;
use ascii::AsciiString;
use blobrepo::BlobRepo;
use bonsai_globalrev_mapping::SqlBonsaiGlobalrevMappingBuilder;
use changeset_fetcher::ChangesetFetcherRef;
use clap_old::Arg;
use clap_old::ArgGroup;
use cmdlib::args;
use cmdlib::args::MononokeClapApp;
use cmdlib::args::MononokeMatches;
use cmdlib::args::RepoRequirement;
use cmdlib::helpers::block_execute;
use context::CoreContext;
use context::SessionContainer;
use derived_data_manager::BonsaiDerivable;
use derived_data_utils::POSSIBLE_DERIVED_TYPES;
use failure_ext::SlogKVError;
use fbinit::FacebookInit;
use filenodes_derivation::FilenodesOnlyPublic;
use futures::future::try_join;
use futures::future::TryFutureExt;
#[cfg(fbcode_build)]
use mercurial_revlog::revlog::RevIdx;
use mercurial_types::HgChangesetId;
use mercurial_types::HgNodeHash;
use mononoke_types::ChangesetId;
use mutable_counters::MutableCountersRef;
use repo_identity::RepoIdentityRef;
use slog::error;
use slog::info;
use slog::warn;
use slog::Logger;
use synced_commit_mapping::SqlSyncedCommitMapping;
use wireproto_handler::BackupSourceRepo;

const ARG_DERIVED_DATA_TYPE: &str = "derived-data-type";
const ARG_EXCLUDE_DERIVED_DATA_TYPE: &str = "exclude-derived-data-type";
const ARG_FIND_ALREADY_IMPORTED_REV_ONLY: &str = "find-already-imported-rev-only";
const BACKUP_REPO_GROUP: &str = "backup-from-repo";
const BACKUP_FROM_REPO_ID: &str = "backup-from-repo-id";
const BACKUP_FROM_REPO_NAME: &str = "backup-from-repo-name";

fn setup_app<'a, 'b>() -> MononokeClapApp<'a, 'b> {
    args::MononokeAppBuilder::new("revlog to blob importer")
        .with_repo_required(RepoRequirement::ExactlyOne)
        .with_source_repos()
        .with_scuba_logging_args()
        .with_fb303_args()
        .build()
        .about("Import a revlog-backed Mercurial repo into Mononoke blobstore.")
        .args_from_usage(
            r#"
            <INPUT>                          'input revlog repo'
            --changeset [HASH]               'if provided, the only changeset to be imported'
            --no-bookmark                    'if provided won't update bookmarks'
            --prefix-bookmark [PREFIX]       'if provided will update bookmarks, but prefix them with PREFIX'
            --no-create                      'if provided won't create a new repo (only meaningful for local)'
            --lfs-helper [LFS_HELPER]        'if provided, path to an executable that accepts OID SIZE and returns a LFS blob to stdout'
            --concurrent-changesets [LIMIT]  'if provided, max number of changesets to upload concurrently'
            --concurrent-blobs [LIMIT]       'if provided, max number of blobs to process concurrently'
            --concurrent-lfs-imports [LIMIT] 'if provided, max number of LFS files to import concurrently'
            --has-globalrev                  'if provided will update globalrev'
            --manifold-next-rev-to-import [KEY] 'if provided then this manifold key will be updated with the next revision to import'
            --manifold-bucket [BUCKET]        'can only be used if --manifold-next-rev-to-import is set'
        "#,
        )
        .arg(
            Arg::from_usage("--skip [SKIP]  'skips commits from the beginning'")
                .conflicts_with("changeset"),
        )
        .arg(
            Arg::from_usage(
                "--commits-limit [LIMIT] 'import only LIMIT first commits from revlog repo'",
            )
            .conflicts_with("changeset"),
        )
        .arg(
            Arg::with_name("fix-parent-order")
                .long("fix-parent-order")
                .value_name("FILE")
                .takes_value(true)
                .required(false)
                .help(
                    "file which fixes order or parents for commits in format 'HG_CS_ID P1_CS_ID [P2_CS_ID]'\
                     This is useful in case of merge commits - mercurial ignores order of parents of the merge commit \
                     while Mononoke doesn't ignore it. That might result in different bonsai hashes for the same \
                     Mercurial commit. Using --fix-parent-order allows to fix order of the parents."
                 )
        )
        .arg(
            Arg::with_name(ARG_DERIVED_DATA_TYPE)
                .long(ARG_DERIVED_DATA_TYPE)
                .takes_value(true)
                .multiple(true)
                .required(false)
                .possible_values(POSSIBLE_DERIVED_TYPES)
                .help("Derived data type to be backfilled. Note - 'filenodes' will always be derived unless excluded")
        )
        .arg(
            Arg::with_name(ARG_EXCLUDE_DERIVED_DATA_TYPE)
                .long(ARG_EXCLUDE_DERIVED_DATA_TYPE)
                .takes_value(true)
                .multiple(true)
                .required(false)
                .possible_values(POSSIBLE_DERIVED_TYPES)
                .help("Exclude derived data types explicitly")
        )
        .arg(
            Arg::with_name(ARG_FIND_ALREADY_IMPORTED_REV_ONLY)
                .long(ARG_FIND_ALREADY_IMPORTED_REV_ONLY)
                .takes_value(false)
                .multiple(false)
                .required(false)
                .help("Does not do any import. Just finds the rev that was already imported rev and \
                      updates manifold-next-rev-to-import if it's set. Note that we might have \
                      a situation where revision i is imported, i+1 is not and i+2 is imported. \
                      In that case this function would return i.")
        )
        .arg(
            Arg::with_name(BACKUP_FROM_REPO_ID)
            .long(BACKUP_FROM_REPO_ID)
            .value_name("ID")
            .help("numeric ID of backup source of truth mononoke repository (used only for backup jobs to sync bonsai changesets)"),
        )
        .arg(
            Arg::with_name(BACKUP_FROM_REPO_NAME)
            .long(BACKUP_FROM_REPO_NAME)
            .value_name("NAME")
            .help("Name of backup source of truth mononoke repository (used only for backup jobs to sync bonsai changesets)"),
        )
        .group(
            ArgGroup::with_name(BACKUP_REPO_GROUP)
                .args(&[BACKUP_FROM_REPO_ID, BACKUP_FROM_REPO_NAME])
        )
}

fn parse_fixed_parent_order<P: AsRef<Path>>(
    logger: &Logger,
    p: P,
) -> Result<HashMap<HgChangesetId, Vec<HgChangesetId>>> {
    let content = read(p)?;
    let mut res = HashMap::new();

    for line in String::from_utf8(content).map_err(Error::from)?.split('\n') {
        if line.is_empty() {
            continue;
        }
        let mut iter = line.split(' ').map(HgChangesetId::from_str).fuse();
        let maybe_hg_cs_id = iter.next();
        let hg_cs_id = match maybe_hg_cs_id {
            Some(hg_cs_id) => hg_cs_id?,
            None => {
                continue;
            }
        };

        let parents = match (iter.next(), iter.next()) {
            (Some(p1), Some(p2)) => vec![p1?, p2?],
            (Some(p), None) => {
                warn!(
                    logger,
                    "{}: parent order is fixed for a single parent, most likely won't have any effect",
                    hg_cs_id,
                );
                vec![p?]
            }
            (None, None) => {
                warn!(
                    logger,
                    "{}: parent order is fixed for a commit with no parents, most likely won't have any effect",
                    hg_cs_id,
                );
                vec![]
            }
            (None, Some(_)) => unreachable!(),
        };
        if iter.next().is_some() {
            bail!("got 3 parents, but mercurial supports at most 2!");
        }

        if res.insert(hg_cs_id, parents).is_some() {
            warn!(logger, "order is fixed twice for {}!", hg_cs_id);
        }
    }
    Ok(res)
}

#[cfg(fbcode_build)]
mod facebook {
    use manifold_client::cpp_client::ClientOptionsBuilder;
    use manifold_client::cpp_client::ManifoldCppClient;
    use manifold_client::write::WriteRequestOptionsBuilder;
    use manifold_client::ManifoldClient;

    use super::*;

    pub async fn update_manifold_key(
        fb: FacebookInit,
        latest_imported_rev: RevIdx,
        manifold_key: &str,
        manifold_bucket: &str,
    ) -> Result<()> {
        let opts = ClientOptionsBuilder::default()
            .build()
            .map_err(|e| format_err!("Cannot build Manifold options: {}", e))?;

        let client = ManifoldCppClient::from_options(fb, manifold_bucket, &opts)
            .context("Cannot build ManifoldCppClient")?;

        let opts = WriteRequestOptionsBuilder::default()
            .with_allow_overwrite_predicate(true)
            .build()
            .map_err(|e| format_err!("Cannot build Write options: {}", e))?;

        let mut req = client
            .create_write_request(&opts)
            .context("Cannot build write request")?;

        let next_revision_to_import = latest_imported_rev.as_u32() + 1;
        let payload = format!("{}", next_revision_to_import);
        req.write(manifold_key, payload.into()).await?;

        Ok(())
    }

    #[cfg(test)]
    mod test {
        use super::*;

        #[fbinit::test]
        async fn test_manifold_write(fb: FacebookInit) -> Result<(), Error> {
            update_manifold_key(
                fb,
                RevIdx::from(0u32),
                "flat/test_manifold_write",
                "mononoke_test",
            )
            .await?;

            update_manifold_key(
                fb,
                RevIdx::from(1u32),
                "flat/test_manifold_write",
                "mononoke_test",
            )
            .await?;

            Ok(())
        }
    }
}

async fn run_blobimport<'a>(
    fb: FacebookInit,
    ctx: &CoreContext,
    logger: &Logger,
    matches: &'a MononokeMatches<'a>,
) -> Result<()> {
    let config_store = matches.config_store();

    let revlogrepo_path = matches
        .value_of("INPUT")
        .expect("input is not specified")
        .into();

    let changeset = match matches.value_of("changeset") {
        None => None,
        Some(hash) => Some(HgNodeHash::from_str(hash)?),
    };

    let skip = if !matches.is_present("skip") {
        None
    } else {
        Some(args::get_usize(matches, "skip", 0))
    };

    let commits_limit = if !matches.is_present("commits-limit") {
        None
    } else {
        Some(args::get_usize(matches, "commits-limit", 0))
    };

    let manifold_key = matches
        .value_of("manifold-next-rev-to-import")
        .map(|s| s.to_string());

    let manifold_bucket = matches.value_of("manifold-bucket").map(|s| s.to_string());

    let manifold_key_bucket = match (manifold_key, manifold_bucket) {
        (Some(key), Some(bucket)) => Some((key, bucket)),
        (None, None) => None,
        _ => {
            return Err(format_err!(
                "invalid manifold parameters: bucket and key should either both be specified or none"
            ));
        }
    };

    let no_bookmark = matches.is_present("no-bookmark");
    let prefix_bookmark = matches.value_of("prefix-bookmark");
    if no_bookmark && prefix_bookmark.is_some() {
        return Err(format_err!(
            "--no-bookmark is incompatible with --prefix-bookmark"
        ));
    }

    let bookmark_import_policy = if no_bookmark {
        blobimport_lib::BookmarkImportPolicy::Ignore
    } else {
        let prefix = match prefix_bookmark {
            Some(prefix) => AsciiString::from_ascii(prefix).unwrap(),
            None => AsciiString::new(),
        };
        blobimport_lib::BookmarkImportPolicy::Prefix(prefix)
    };

    let lfs_helper = matches.value_of("lfs-helper").map(|l| l.to_string());

    let concurrent_changesets = args::get_usize(matches, "concurrent-changesets", 100);
    let concurrent_blobs = args::get_usize(matches, "concurrent-blobs", 100);
    let concurrent_lfs_imports = args::get_usize(matches, "concurrent-lfs-imports", 10);

    let fixed_parent_order = if let Some(path) = matches.value_of("fix-parent-order") {
        parse_fixed_parent_order(logger, path)
            .context("while parsing file with fixed parent order")?
    } else {
        HashMap::new()
    };

    let mut derived_data_types = matches
        .values_of(ARG_DERIVED_DATA_TYPE)
        .map_or(vec![], |v| v.map(|d| d.to_string()).collect());

    let excluded_derived_data_types = matches
        .values_of(ARG_EXCLUDE_DERIVED_DATA_TYPE)
        .map_or(vec![], |v| v.map(|d| d.to_string()).collect());

    for v in &excluded_derived_data_types {
        if derived_data_types.contains(v) {
            return Err(format_err!("Unexpected exclusion of requested {}", v));
        }
    }

    // Make sure filenodes derived unless specifically excluded since public hg changesets must have filenodes derived
    let filenodes_derived_name = FilenodesOnlyPublic::NAME.to_string();
    if !derived_data_types.contains(&filenodes_derived_name)
        && !excluded_derived_data_types.contains(&filenodes_derived_name)
    {
        derived_data_types.push(filenodes_derived_name);
    }

    let has_globalrev = matches.is_present("has-globalrev");

    let (_repo_name, repo_config) =
        args::not_shardmanager_compatible::get_config(config_store, matches)?;
    let populate_git_mapping = repo_config.pushrebase.populate_git_mapping.clone();

    let small_repo_id =
        args::not_shardmanager_compatible::get_source_repo_id_opt(config_store, matches)?;

    let globalrevs_store_builder = args::not_shardmanager_compatible::open_sql::<
        SqlBonsaiGlobalrevMappingBuilder,
    >(fb, config_store, matches)?;
    let synced_commit_mapping = args::not_shardmanager_compatible::open_sql::<
        SqlSyncedCommitMapping,
    >(fb, config_store, matches)?;

    let blobrepo: BlobRepo = if matches.is_present("no-create") {
        args::not_shardmanager_compatible::open_repo_unredacted(fb, ctx.logger(), matches).await?
    } else {
        args::not_shardmanager_compatible::create_repo_unredacted(fb, ctx.logger(), matches).await?
    };

    let origin_repo: Option<BlobRepo> =
        if matches.is_present(BACKUP_FROM_REPO_ID) || matches.is_present(BACKUP_FROM_REPO_NAME) {
            let repo_id = args::not_shardmanager_compatible::get_repo_id_from_value(
                config_store,
                matches,
                BACKUP_FROM_REPO_ID,
                BACKUP_FROM_REPO_NAME,
            )?;
            Some(args::open_repo_with_repo_id(fb, logger, repo_id, matches).await?)
        } else {
            None
        };

    let globalrevs_store = Arc::new(globalrevs_store_builder.build(blobrepo.repo_identity().id()));
    let synced_commit_mapping = Arc::new(synced_commit_mapping);

    let find_latest_imported_rev_only = matches.is_present(ARG_FIND_ALREADY_IMPORTED_REV_ONLY);
    async move {
        let blobimport = blobimport_lib::Blobimport {
            ctx,
            blobrepo: blobrepo.clone(),
            revlogrepo_path,
            changeset,
            skip,
            commits_limit,
            bookmark_import_policy,
            globalrevs_store,
            synced_commit_mapping,
            lfs_helper,
            concurrent_changesets,
            concurrent_blobs,
            concurrent_lfs_imports,
            fixed_parent_order,
            has_globalrev,
            populate_git_mapping,
            small_repo_id,
            derived_data_types,
            origin_repo: origin_repo.map(|repo| BackupSourceRepo::from_blob_repo(&repo)),
        };

        let maybe_latest_imported_rev = if find_latest_imported_rev_only {
            blobimport.find_already_imported_revision().await?
        } else {
            blobimport.import().await?
        };

        match maybe_latest_imported_rev {
            Some((latest_imported_rev, latest_imported_cs_id)) => {
                info!(
                    ctx.logger(),
                    "latest imported revision {}",
                    latest_imported_rev.as_u32()
                );
                #[cfg(fbcode_build)]
                {
                    if let Some((manifold_key, bucket)) = manifold_key_bucket {
                        facebook::update_manifold_key(
                            fb,
                            latest_imported_rev,
                            &manifold_key,
                            &bucket,
                        )
                        .await?
                    }
                }
                #[cfg(not(fbcode_build))]
                {
                    assert!(
                        manifold_key_bucket.is_none(),
                        "Using Manifold is not supported in non fbcode builds"
                    );
                }

                maybe_update_highest_imported_generation_number(
                    ctx,
                    &blobrepo,
                    latest_imported_cs_id,
                )
                .await?;
            }
            None => info!(ctx.logger(), "didn't import any commits"),
        };

        Ok(())
    }
    .map_err({
        move |err| {
            // NOTE: We log the error immediatley, then provide another one for main's
            // Result (which will set our exit code).
            error!(ctx.logger(), "error while blobimporting"; SlogKVError(err));
            Error::msg("blobimport exited with a failure")
        }
    })
    .await
}

// Updating mutable_counters table to store the highest generation number that was imported.
// This in turn can be used to track which commits exist on both mercurial and Mononoke.
// For example, WarmBookmarkCache might consider a bookmark "warm" only if a commit is in both
// mercurial and Mononoke.
//
// Note that if a commit with a lower generation number was added (e.g. if this commit forked off from
// the main branch) then this hint will be misleading - i.e. the hint would store a higher generation
// number then the new commit which might not be processed by blobimport yet. In that case there are
// two options:
// 1) Use this hint only in single-branch repos
// 2) Accept that the hint might be incorrect sometimes.
async fn maybe_update_highest_imported_generation_number(
    ctx: &CoreContext,
    blobrepo: &BlobRepo,
    latest_imported_cs_id: ChangesetId,
) -> Result<(), Error> {
    let maybe_highest_imported_gen_num = blobrepo
        .mutable_counters()
        .get_counter(ctx, blobimport_lib::HIGHEST_IMPORTED_GEN_NUM);
    let new_gen_num = blobrepo
        .changeset_fetcher()
        .get_generation_number(ctx, latest_imported_cs_id);
    let (maybe_highest_imported_gen_num, new_gen_num) =
        try_join(maybe_highest_imported_gen_num, new_gen_num).await?;

    let new_gen_num = match maybe_highest_imported_gen_num {
        Some(highest_imported_gen_num) => {
            if new_gen_num.value() as i64 > highest_imported_gen_num {
                Some(new_gen_num)
            } else {
                None
            }
        }
        None => Some(new_gen_num),
    };

    if let Some(new_gen_num) = new_gen_num {
        blobrepo
            .mutable_counters()
            .set_counter(
                ctx,
                blobimport_lib::HIGHEST_IMPORTED_GEN_NUM,
                new_gen_num.value() as i64,
                maybe_highest_imported_gen_num,
            )
            .await?;
    }
    Ok(())
}

#[fbinit::main]
fn main(fb: FacebookInit) -> Result<()> {
    let (matches, _runtime) = setup_app().get_matches(fb)?;

    let logger = matches.logger();
    let scuba = matches.scuba_sample_builder();

    let ctx = &SessionContainer::new_with_defaults(fb).new_context(logger.clone(), scuba);

    block_execute(
        run_blobimport(fb, ctx, logger, &matches),
        fb,
        "blobimport",
        logger,
        &matches,
        cmdlib::monitoring::AliveService,
    )
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::path::PathBuf;
use std::str::FromStr;

use clap_old::App;
use clap_old::SubCommand;
use mercurial_revlog::revlog::Revlog;
use mercurial_revlog::RevlogChangeset;
use mercurial_types::HgNodeHash;

fn setup_app<'a, 'b>() -> App<'a, 'b> {
    App::new("read revlog repo content")
        .about("read revlogs")
        .args_from_usage(
            r#"
             <PATH>   'path to .hg folder with revlogs'
        "#,
        )
        .subcommand(
            SubCommand::with_name("changeset")
                .about("reads changesets")
                .args_from_usage(
                    r#"
                     <HASH>                 'sha1 hash to read'
                     -p, --parsed           'parse blob as RevlogChangeset'
                "#,
                ),
        )
}

fn main() {
    let matches = setup_app().get_matches();

    let base: PathBuf = matches.value_of("PATH").unwrap().into();
    let store = base.as_path().join("store");

    match matches.subcommand() {
        ("changeset", Some(sub_m)) => {
            let changelog =
                Revlog::from_idx_with_data(store.join("00changelog.i"), None as Option<String>)
                    .expect("Failed to load changelog");

            let hash = sub_m.value_of("HASH").unwrap();
            let hash = HgNodeHash::from_str(hash).expect("Incorrect Sha1 hash provided");

            let raw = changelog
                .get_idx_by_nodeid(hash)
                .and_then(|idx| changelog.get_rev(idx))
                .expect("Changeset not found");

            println!("RAW: {:?}", raw);

            let changeset = RevlogChangeset::new(raw.clone()).expect("Failed to deserialize CS");

            if sub_m.is_present("parsed") {
                println!();
                println!("PAR: {:?}", changeset);
            }

            let encoded = changeset.get_node().expect("Failed to serialize CS");

            if raw != encoded {
                println!();
                println!("POTENTIAL PROBLEM: {:?}", encoded);
            }
        }
        _ => {
            println!("{}", matches.usage());
            ::std::process::exit(1);
        }
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::time::Duration;

use anyhow::Context;
use anyhow::Error;
use blobrepo::BlobRepo;
use bytes::Bytes;
use changesets::deserialize_cs_entries;
use clap::Parser;
use cmdlib::helpers;
use context::SessionContainer;
use fbinit::FacebookInit;
use futures::future::join_all;
use futures::stream;
use mononoke_app::args::MultiRepoArgs;
use mononoke_app::args::RepoArg;
use mononoke_app::fb303::AliveService;
use mononoke_app::fb303::Fb303AppExtension;
use mononoke_app::MononokeApp;
use mononoke_app::MononokeAppBuilder;
use repo_identity::RepoIdentityRef;
use segmented_changelog::seedheads_from_config;
use segmented_changelog::OperationMode;
use segmented_changelog::SegmentedChangelogTailer;
use slog::info;
use slog::o;

/// Updates segmented changelog assets
#[derive(Parser)]
// `MultiRepoArgs` fields conflict with `repo_names` (`--repo`).
// See https://github.com/clap-rs/clap/issues/3269 for why this is the only implementation.
#[clap(mut_arg("repo_id", |a| a.conflicts_with("repo_names")))]
#[clap(mut_arg("repo_name", |a| a.conflicts_with("repo_names")))]
struct SegmentedChangelogTailerArgs {
    /// Repository to warm-up
    #[clap(flatten)]
    repos: MultiRepoArgs,
    /// Repository name to warm-up. Deprecated, use --repo-name/--repo-id instead
    // Deprecated, use repos instead
    #[clap(long = "repo", conflicts_with_all = ["repo_id", "repo_name"])]
    repo_names: Vec<String>,
    /// When set, the tailer will perform a single incremental build run. If no previous version exists it will perform full reseed instead
    #[clap(long)]
    once: bool,
    /// A file with a serialized list of ChangesetEntry, which can be used to speed up rebuilding of segmented changelog
    #[clap(long)]
    prefetched_commits_path: Option<String>,
    /// What heads to use for Segmented Changelog. If not provided, tailer will use the config to obtain heads
    #[clap(long)]
    head: Vec<String>,
    /// Force use of the configured heads, as well as any specified on the command line
    #[clap(long)]
    include_config_heads: bool,
    /// When set, the tailer will perform a single full reseed run
    #[clap(long, conflicts_with = "once")]
    force_reseed: bool,
}

#[fbinit::main]
fn main(fb: FacebookInit) -> Result<(), Error> {
    let app = MononokeAppBuilder::new(fb)
        .with_app_extension(Fb303AppExtension {})
        .build::<SegmentedChangelogTailerArgs>()?;

    app.run_with_monitoring_and_logging(async_main, "segmented_changelog_tailer", AliveService)
}

async fn async_main(app: MononokeApp) -> Result<(), Error> {
    let args: SegmentedChangelogTailerArgs = app.args()?;

    let repos = MultiRepoArgs {
        repo_id: args.repos.repo_id,
        repo_name: args
            .repos
            .repo_name
            .into_iter()
            .chain(args.repo_names.into_iter())
            .collect(),
    };

    // This is a bit weird from the dependency point of view but I think that it is best. The
    // BlobRepo may have a SegmentedChangelog attached to it but that doesn't hurt us in any
    // way.  On the other hand reconstructing the dependencies for SegmentedChangelog without
    // BlobRepo is probably prone to more problems from the maintenance perspective.
    let blobrepos: Vec<BlobRepo> = app.open_repos(&repos).await?;

    let prefetched_commits = match args.prefetched_commits_path {
        Some(path) => {
            info!(app.logger(), "reading prefetched commits from {}", path);
            let data = tokio::fs::read(&path).await?;
            deserialize_cs_entries(&Bytes::from(data))
                .with_context(|| format!("failed to parse serialized cs entries from {}", path))?
        }
        None => vec![],
    };

    let (env, logger) = (app.environment(), app.logger());
    let session = SessionContainer::new_with_defaults(env.fb);
    let ctx = session.new_context(logger.clone(), env.scuba_sample_builder.clone());

    let mut tasks = Vec::new();
    for (index, blobrepo) in blobrepos.into_iter().enumerate() {
        let repo_id = blobrepo.repo_identity().id();
        let (repo_name, config) = app.repo_config(&RepoArg::Id(repo_id))?;
        info!(
            ctx.logger(),
            "repo name '{}' translates to id {}", repo_name, repo_id
        );

        let prefetched_commits = stream::iter(prefetched_commits.iter().filter_map(|entry| {
            if entry.repo_id == repo_id {
                Some(Ok(entry.clone()))
            } else {
                None
            }
        }));

        let ctx = ctx.clone_with_logger(ctx.logger().new(o!("repo_id" => repo_id.to_string())));

        let seed_heads = {
            let mut heads = if args.head.is_empty() || args.include_config_heads {
                let mut heads = seedheads_from_config(
                    &ctx,
                    &config.segmented_changelog_config,
                    segmented_changelog::JobType::Background,
                )?;
                heads.reserve(args.head.len());
                heads
            } else {
                Vec::with_capacity(args.head.len())
            };

            for head_arg in &args.head {
                let head = helpers::csid_resolve(&ctx, blobrepo.clone(), head_arg)
                    .await
                    .with_context(|| {
                        format!("resolving head csid '{}' for repo {}", head_arg, repo_id)
                    })?;
                info!(ctx.logger(), "using '{}' for head", head);
                heads.push(head.into());
            }
            heads
        };

        let segmented_changelog_tailer = SegmentedChangelogTailer::build_from(
            &ctx,
            &blobrepo,
            &config.storage_config.metadata,
            app.mysql_options(),
            seed_heads,
            prefetched_commits,
            None,
        )
        .await?;

        info!(ctx.logger(), "SegmentedChangelogTailer initialized",);

        if args.once {
            segmented_changelog_tailer
                .run(&ctx, OperationMode::SingleIncrementalUpdate)
                .await;
            info!(ctx.logger(), "SegmentedChangelogTailer is done",);
        } else if args.force_reseed {
            segmented_changelog_tailer
                .run(&ctx, OperationMode::ForceReseed)
                .await;
            info!(ctx.logger(), "SegmentedChangelogTailer is done",);
        } else if let Some(period) = config.segmented_changelog_config.tailer_update_period {
            // spread out update operations, start updates on another repo after 7 seconds
            let wait_to_start = Duration::from_secs(7 * index as u64);
            let ctx = ctx.clone();
            tasks.push(async move {
                tokio::time::sleep(wait_to_start).await;
                segmented_changelog_tailer
                    .run(&ctx, OperationMode::ContinousIncrementalUpdate(period))
                    .await;
            });
        }
    }

    join_all(tasks).await;

    Ok(())
}