/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;
use std::collections::HashSet;
use std::sync::Arc;

use anyhow::anyhow;
use anyhow::Error;
use anyhow::Result;
use blobrepo::BlobRepo;
use blobrepo_hg::BlobRepoHg;
use blobstore::Loadable;
use bonsai_hg_mapping::BonsaiHgMappingRef;
use bytes::Bytes;
use bytes_old::Bytes as BytesOld;
use changeset_fetcher::ChangesetFetcherRef;
use changesets::ChangesetsRef;
use cloned::cloned;
use commit_graph::ArcCommitGraph;
use commit_graph::CommitGraphArc;
use context::CoreContext;
use context::PerfCounterType;
use derived_data::BonsaiDerived;
use filenodes_derivation::FilenodesOnlyPublic;
use filestore::FetchKey;
use futures::future;
use futures::future::FutureExt;
use futures::future::TryFutureExt;
use futures::stream;
use futures::stream::Stream;
use futures::stream::StreamExt;
use futures::stream::TryStreamExt;
use futures_01_ext::BoxFuture as OldBoxFuture;
use futures_01_ext::BoxStream as OldBoxStream;
use futures_01_ext::BufferedParams;
use futures_01_ext::FutureExt as _;
use futures_01_ext::StreamExt as OldStreamExt;
use futures_ext::stream::FbStreamExt;
use futures_old::future as old_future;
use futures_old::stream as old_stream;
use futures_old::Future as OldFuture;
use futures_old::Stream as OldStream;
use futures_stats::TimedTryFutureExt;
use futures_util::try_join;
use manifest::find_intersection_of_diffs_and_parents;
use manifest::Entry;
use mercurial_bundles::changegroup::CgVersion;
use mercurial_bundles::part_encode::PartEncodeBuilder;
use mercurial_bundles::parts;
use mercurial_bundles::parts::FilenodeEntry;
use mercurial_mutation::HgMutationStoreArc;
use mercurial_revlog::RevlogChangeset;
use mercurial_types::blobs::fetch_manifest_envelope;
use mercurial_types::blobs::File;
use mercurial_types::FileBytes;
use mercurial_types::HgBlobNode;
use mercurial_types::HgChangesetId;
use mercurial_types::HgFileNodeId;
use mercurial_types::HgManifestId;
use mercurial_types::HgParents;
use mercurial_types::MPath;
use mercurial_types::RevFlags;
use mercurial_types::NULL_CSID;
use mercurial_types::NULL_HASH;
use mononoke_types::hash::Sha256;
use mononoke_types::ChangesetId;
use mononoke_types::ContentId;
use mononoke_types::Generation;
use phases::Phase;
use phases::Phases;
use phases::PhasesRef;
use rate_limiting::Metric;
use repo_blobstore::RepoBlobstore;
use repo_blobstore::RepoBlobstoreRef;
use repo_derived_data::RepoDerivedDataRef;
use sha1::Digest;
use sha1::Sha1;
use slog::debug;
use slog::info;
use slog::o;
use tunables::tunables;

use crate::errors::ErrorKind;

mod errors;

pub const MAX_FILENODE_BYTES_IN_MEMORY: u64 = 100_000_000;
pub const GETBUNDLE_COMMIT_NUM_WARN: u64 = 1_000_000;
const UNEXPECTED_NONE_ERR_MSG: &str = "unexpected None while calling ancestors_difference_stream";

#[derive(PartialEq, Eq)]
pub enum PhasesPart {
    Yes,
    No,
}

#[derive(Clone)]
pub struct SessionLfsParams {
    pub threshold: Option<u64>,
}

pub async fn create_getbundle_response(
    ctx: &CoreContext,
    blobrepo: &BlobRepo,
    common: Vec<HgChangesetId>,
    heads: &[HgChangesetId],
    return_phases: PhasesPart,
    lfs_params: &SessionLfsParams,
) -> Result<Vec<PartEncodeBuilder>, Error> {
    let return_phases = return_phases == PhasesPart::Yes;
    debug!(ctx.logger(), "Return phases is: {:?}", return_phases);

    let heads_len = heads.len();
    let common: HashSet<_> = common.into_iter().collect();

    let phases = blobrepo.phases();
    let (draft_commits, commits_to_send) = try_join!(
        find_new_draft_commits_and_derive_filenodes_for_public_roots(
            ctx, blobrepo, &common, heads, phases
        ),
        find_commits_to_send(ctx, blobrepo, &common, heads),
    )?;

    report_draft_commits(ctx, &draft_commits);

    let mut parts = vec![];
    if heads_len != 0 {
        // no heads means bookmark-only pushrebase, and the client
        // does not expect a changegroup part in this case
        let cg_part =
            create_hg_changeset_part(ctx, blobrepo, commits_to_send.clone(), lfs_params).await?;
        parts.push(cg_part);

        if !draft_commits.is_empty() && tunables().mutation_generate_for_draft().unwrap_or_default()
        {
            let mutations_fut = {
                cloned!(ctx);
                let hg_mutation_store = blobrepo.hg_mutation_store_arc();
                async move {
                    hg_mutation_store
                        .all_predecessors(&ctx, draft_commits)
                        .await
                }
                .boxed()
                .compat()
            };
            let mut_part = parts::infinitepush_mutation_part(mutations_fut)?;
            parts.push(mut_part);
        }
    }

    // Phases part has to be after the changegroup part.
    if return_phases {
        let phase_heads = find_phase_heads(ctx, blobrepo, heads, phases).await?;
        parts.push(parts::phases_part(
            ctx.clone(),
            old_stream::iter_ok(phase_heads),
        )?);
    }

    Ok(parts)
}

fn report_draft_commits(ctx: &CoreContext, draft_commits: &HashSet<HgChangesetId>) {
    debug!(
        ctx.logger(),
        "Getbundle returning {} draft commits",
        draft_commits.len()
    );
    ctx.perf_counters().add_to_counter(
        PerfCounterType::GetbundleNumDrafts,
        draft_commits.len() as i64,
    );
}

/// return ancestors of heads with hint to exclude ancestors of common
pub async fn find_commits_to_send(
    ctx: &CoreContext,
    blobrepo: &BlobRepo,
    common: &HashSet<HgChangesetId>,
    heads: &[HgChangesetId],
) -> Result<Vec<ChangesetId>, Error> {
    let heads = hg_to_bonsai_stream(
        ctx,
        blobrepo,
        heads
            .iter()
            .filter(|head| !common.contains(head))
            .cloned()
            .collect(),
    );

    let excludes = hg_to_bonsai_stream(
        ctx,
        blobrepo,
        common
            .iter()
            .copied()
            .filter(|node| node.into_nodehash() != NULL_CSID.into_nodehash())
            .collect(),
    );

    let (heads, excludes) = try_join!(heads, excludes)?;

    let params = Params { heads, excludes };

    let nodes_to_send: Vec<_> = call_difference_of_union_of_ancestors_revset(
        ctx,
        &blobrepo.commit_graph_arc(),
        params,
        None,
    )
    .await?
    .ok_or_else(|| anyhow!(UNEXPECTED_NONE_ERR_MSG))?
    .into_iter()
    .rev()
    .collect();

    ctx.session()
        .bump_load(Metric::Commits, nodes_to_send.len() as f64);
    ctx.perf_counters().add_to_counter(
        PerfCounterType::GetbundleNumCommits,
        nodes_to_send.len() as i64,
    );

    ctx.scuba()
        .clone()
        .log_with_msg("Found commits to send to the client", None);
    Ok(nodes_to_send)
}

#[derive(Default, Clone)]
pub(crate) struct Params {
    heads: Vec<(ChangesetId, Generation)>,
    excludes: Vec<(ChangesetId, Generation)>,
}

impl Params {
    pub fn heads_signature(&self) -> Result<String, Error> {
        Self::signature(&self.heads)
    }

    pub fn excludes_signature(&self) -> Result<String, Error> {
        Self::signature(&self.excludes)
    }

    fn signature(v: &[(ChangesetId, Generation)]) -> Result<String, Error> {
        let mut csids = v.iter().map(|h| h.0).collect::<Vec<_>>();
        csids.sort();
        let mut hasher = Sha1::new();
        for csid in csids {
            hasher.update(csid.blake2().as_ref());
        }
        let res = faster_hex::hex_string(&hasher.finalize());
        Ok(res)
    }
}

async fn call_difference_of_union_of_ancestors_revset(
    ctx: &CoreContext,
    commit_graph: &ArcCommitGraph,
    params: Params,
    limit: Option<u64>,
) -> Result<Option<Vec<ChangesetId>>, Error> {
    let mut scuba = ctx.scuba().clone();
    scuba.add_opt("heads_signature", params.heads_signature().ok());
    scuba.add_opt("excludes_signature", params.excludes_signature().ok());
    scuba.add("heads_count", params.heads.len());
    scuba.add("excludes_count", params.excludes.len());

    let Params { heads, excludes } = params;

    let mut notified_expensive_getbundle = false;
    let min_heads_gen_num = heads.iter().map(|(_, gen)| gen).min();
    let max_excludes_gen_num = excludes.iter().map(|(_, gen)| gen).max();
    match (min_heads_gen_num, max_excludes_gen_num) {
        (Some(min_heads), Some(max_excludes)) => {
            if min_heads.difference_from(*max_excludes).unwrap_or(0) > GETBUNDLE_COMMIT_NUM_WARN {
                warn_expensive_getbundle(ctx);
                notified_expensive_getbundle = true;
            }
        }
        _ => {}
    };

    let nodes_to_send = commit_graph
        .ancestors_difference_stream(
            ctx,
            heads.into_iter().map(|(cs_id, _gen)| cs_id).collect(),
            excludes.into_iter().map(|(cs_id, _gen)| cs_id).collect(),
        )
        .await?
        .yield_periodically()
        .inspect({
            let mut i = 0;
            move |_| {
                i += 1;
                if i > GETBUNDLE_COMMIT_NUM_WARN && !notified_expensive_getbundle {
                    notified_expensive_getbundle = true;
                    warn_expensive_getbundle(ctx);
                }
            }
        });

    let (stats, res) = async move {
        if let Some(limit) = limit {
            let res: Vec<_> = nodes_to_send
                .take(limit.try_into().unwrap())
                .try_collect()
                .await?;
            if res.len() as u64 == limit {
                Ok(None)
            } else {
                Ok(Some(res))
            }
        } else {
            nodes_to_send.try_collect().map_ok(Some).await
        }
    }
    .try_timed()
    .await?;

    scuba.add_future_stats(&stats);
    scuba.log_with_msg("call_difference_of_union_of_ancestors_revset", None);

    Ok(res)
}

fn warn_expensive_getbundle(ctx: &CoreContext) {
    info!(
        ctx.logger(),
        "your repository is out of date and pulling new commits might take a long time. \
        Please consider recloning your repository since it might be much faster."
        ; o!("remote" => "true")
    );
}

async fn create_hg_changeset_part(
    ctx: &CoreContext,
    blobrepo: &BlobRepo,
    nodes_to_send: Vec<ChangesetId>,
    lfs_params: &SessionLfsParams,
) -> Result<PartEncodeBuilder> {
    let map_chunk_size = 100;
    let load_buffer_size = 1000;

    let changelogentries = stream::iter(nodes_to_send)
        .chunks(map_chunk_size)
        .then({
            cloned!(ctx, blobrepo);
            move |bonsais| {
                cloned!(ctx, blobrepo);
                async move {
                    let mapping = blobrepo
                        .get_hg_bonsai_mapping(ctx.clone(), bonsais.clone())
                        .await?
                        .into_iter()
                        .map(|(hg_cs_id, bonsai_cs_id)| (bonsai_cs_id, hg_cs_id))
                        .collect::<HashMap<_, _>>();

                    // We need to preserve ordering of the Bonsais for Mercurial on the client-side.

                    let ordered_mapping = bonsais
                        .into_iter()
                        .map(|bcs_id| {
                            let hg_cs_id = mapping.get(&bcs_id).ok_or_else(|| {
                                anyhow::format_err!("cs_id was missing from mapping: {:?}", bcs_id)
                            })?;
                            Ok((*hg_cs_id, bcs_id))
                        })
                        .collect::<Vec<_>>();

                    Result::<_, Error>::Ok(ordered_mapping)
                }
            }
        })
        .map_ok(stream::iter)
        .try_flatten()
        .map({
            cloned!(ctx, blobrepo);
            move |res| {
                cloned!(ctx, blobrepo);
                async move {
                    match res {
                        Ok((hg_cs_id, _bcs_id)) => {
                            let cs = hg_cs_id.load(&ctx, blobrepo.repo_blobstore()).await?;
                            Ok((hg_cs_id, cs))
                        }
                        Err(e) => Err(e),
                    }
                }
            }
        })
        .buffered(load_buffer_size)
        .and_then(|(hg_cs_id, cs)| async move {
            let node = hg_cs_id.into_nodehash();

            let revlogcs = RevlogChangeset::new_from_parts(
                cs.parents(),
                cs.manifestid(),
                cs.user().into(),
                cs.time().clone(),
                cs.extra().clone(),
                cs.files().into(),
                cs.message().into(),
            );

            let mut v = Vec::new();
            mercurial_revlog::changeset::serialize_cs(&revlogcs, &mut v)?;

            Ok((
                node,
                HgBlobNode::new(Bytes::from(v), revlogcs.p1(), revlogcs.p2()),
            ))
        })
        .boxed()
        .compat();

    let cg_version = if lfs_params.threshold.is_some() {
        CgVersion::Cg3Version
    } else {
        CgVersion::Cg2Version
    };

    parts::changegroup_part(changelogentries, None, cg_version)
}

async fn hg_to_bonsai_stream(
    ctx: &CoreContext,
    repo: &BlobRepo,
    nodes: Vec<HgChangesetId>,
) -> Result<Vec<(ChangesetId, Generation)>, Error> {
    stream::iter(nodes)
        .map({
            move |node| async move {
                let bcs_id = repo
                    .bonsai_hg_mapping()
                    .get_bonsai_from_hg(ctx, node)
                    .await?
                    .ok_or(ErrorKind::BonsaiNotFoundForHgChangeset(node))?;

                let gen_num = repo
                    .changeset_fetcher()
                    .get_generation_number(ctx, bcs_id)
                    .await?;
                Ok((bcs_id, gen_num))
            }
        })
        .buffered(100)
        .try_collect()
        .await
}

// Given a set of heads and a set of common heads, find the new draft commits,
// and ensure all the public heads and first public ancestors of the draft commits
// have had their filenodes derived.
pub async fn find_new_draft_commits_and_derive_filenodes_for_public_roots(
    ctx: &CoreContext,
    repo: &BlobRepo,
    common: &HashSet<HgChangesetId>,
    heads: &[HgChangesetId],
    phases: &dyn Phases,
) -> Result<HashSet<HgChangesetId>, Error> {
    let (draft, public_heads) =
        find_new_draft_commits_and_public_roots(ctx, repo, common, heads, phases).await?;

    ctx.scuba().clone().log_with_msg("Deriving filenodes", None);
    // Ensure filenodes are derived for all of the public heads.
    stream::iter(public_heads)
        .map(|bcs_id| FilenodesOnlyPublic::derive(ctx, repo, bcs_id))
        .buffered(100)
        .try_for_each(|_derive| async { Ok(()) })
        .await?;
    ctx.scuba()
        .clone()
        .log_with_msg("Derived all filenodes", None);

    Ok(draft)
}

/// Given a set of heads and set of common heads, find the new draft commits,
/// that is, draft commits that are ancestors of the heads but not ancestors of
/// the common heads, as well as the new public heads and the first public
/// ancestors of the new draft commits.
///
/// The draft commits are returned as `HgChangesetId`; the public heads are
/// returned as `ChangesetId`.
///
/// For example in the graph:
/// ```ignore
///   o F [public]
///   |
///   | o E [draft]
///   | |
///   | | o D [draft]
///   | | |
///   | | o C [draft]
///   | |/
///   | o B [public]
///   |/
///   o A [public]
/// ```
///
/// If `heads = [D, E, F]` and `common = [C]` then `new_draft_commits = [D, E]`,
/// and new_public_heads = `[A, B]`.
///
async fn find_new_draft_commits_and_public_roots(
    ctx: &CoreContext,
    repo: &BlobRepo,
    common: &HashSet<HgChangesetId>,
    heads: &[HgChangesetId],
    phases: &dyn Phases,
) -> Result<(HashSet<HgChangesetId>, HashSet<ChangesetId>), Error> {
    // Remove the common heads.
    let new_heads: Vec<_> = heads
        .iter()
        .filter(|hg_cs_id| !common.contains(hg_cs_id))
        .cloned()
        .collect();

    // Traverse the draft commits, accumulating all of the draft commits and the
    // public heads encountered.
    let mut new_hg_draft_commits = HashSet::new();
    let mut new_public_heads = HashSet::new();
    traverse_draft_commits(
        ctx,
        repo,
        phases,
        &new_heads,
        |public_bcs_id, _public_hg_cs_id| {
            new_public_heads.insert(public_bcs_id);
        },
        |_draft_head_bcs_id, _draft_head_hg_cs_id| {},
        |_draft_bcs_id, draft_hg_cs_id| {
            // If we encounter a common head, stop traversing there.  Ideally
            // we would stop if we encountered any ancestor of a common head,
            // however the common set may be large, and comparing ancestors
            // against all of these would be too expensive.
            let traverse = !common.contains(&draft_hg_cs_id);
            if traverse {
                new_hg_draft_commits.insert(draft_hg_cs_id);
            }
            traverse
        },
    )
    .await?;

    Ok((new_hg_draft_commits, new_public_heads))
}

/// Return phase heads for all public and draft heads, and the public roots of
/// the draft heads, that is, the first public ancestor of the draft heads.
///
/// For example in the graph:
/// ```ignore
///   o F [public]
///   |
///   | o E [draft]
///   | |
///   | | o D [draft]
///   | | |
///   | | o C [draft]
///   | |/
///   | o B [public]
///   |/
///   o A [public]
/// ```
///
/// If `heads = [D, E, F]` then this will return
/// `[(F, public), (E, draft), (D, draft), (B, public)]`
///
async fn find_phase_heads(
    ctx: &CoreContext,
    repo: &BlobRepo,
    heads: &[HgChangesetId],
    phases: &dyn Phases,
) -> Result<Vec<(HgChangesetId, Phase)>, Error> {
    // Traverse the draft commits, collecting phase heads for the draft heads
    // and public commits that we encounter.
    let mut phase_heads = Vec::new();
    let mut draft_heads = Vec::new();
    traverse_draft_commits(
        ctx,
        repo,
        phases,
        heads,
        |_public_bcs_id, public_hg_cs_id| {
            phase_heads.push((public_hg_cs_id, Phase::Public));
        },
        |_draft_head_bcs_id, draft_head_hg_cs_id| {
            draft_heads.push((draft_head_hg_cs_id, Phase::Draft));
        },
        |_draft_bcs_id, _draft_hg_cs_id| true,
    )
    .await?;
    phase_heads.append(&mut draft_heads);
    Ok(phase_heads)
}

/// Traverses all draft commits, calling `draft_head_callback` on each draft
/// head encountered, `draft_callback` on each draft commit encountered
/// (including heads) and `public_callback` on the first public commit
/// encountered. The parents of draft commits are only traversed if
/// `draft_callback` returns true.
async fn traverse_draft_commits(
    ctx: &CoreContext,
    repo: &(impl ChangesetsRef + RepoDerivedDataRef + BonsaiHgMappingRef + Send + Sync),
    phases: &dyn Phases,
    heads: &[HgChangesetId],
    mut public_callback: impl FnMut(ChangesetId, HgChangesetId),
    mut draft_head_callback: impl FnMut(ChangesetId, HgChangesetId),
    mut draft_callback: impl FnMut(ChangesetId, HgChangesetId) -> bool,
) -> Result<(), Error> {
    // Find the bonsai changeset id for all of the heads.
    let hg_bonsai_heads = repo
        .get_hg_bonsai_mapping(ctx.clone(), heads.to_vec())
        .await?;

    // Find the initial set of public changesets.
    let bonsai_heads = hg_bonsai_heads.iter().map(|(_, bcs_id)| *bcs_id).collect();
    let mut public_changesets = phases.get_cached_public(ctx, bonsai_heads).await?;

    // Call the draft head callback for each of the draft heads.
    let mut seen = HashSet::new();
    let mut next_changesets = Vec::new();
    for (hg_cs_id, bcs_id) in hg_bonsai_heads.into_iter() {
        if !public_changesets.contains(&bcs_id) {
            draft_head_callback(bcs_id, hg_cs_id);
        }
        next_changesets.push((hg_cs_id, bcs_id));
        seen.insert(bcs_id);
    }

    while !next_changesets.is_empty() {
        let mut traverse = Vec::new();
        for (hg_cs_id, bcs_id) in next_changesets {
            if public_changesets.contains(&bcs_id) {
                public_callback(bcs_id, hg_cs_id);
            } else if draft_callback(bcs_id, hg_cs_id) {
                traverse.push(bcs_id);
            }
        }

        if traverse.is_empty() {
            break;
        }

        // Get the parents of the changesets we are traversing.
        // TODO(mbthomas): After blobrepo refactoring, change to use a method that calls `Changesets::get_many`.
        let parents: Vec<_> = stream::iter(traverse)
            .map(move |csid| async move {
                let parents = repo
                    .changesets()
                    .get(ctx, csid)
                    .await?
                    .ok_or_else(|| {
                        anyhow::format_err!("Commit {} does not exist in the repo", csid)
                    })?
                    .parents;

                Result::<_, Error>::Ok(parents)
            })
            .buffered(100)
            .try_collect::<Vec<_>>()
            .await?
            .into_iter()
            .flatten()
            .filter(|csid| seen.insert(*csid))
            .collect();

        let (new_next_changesets, new_public_changesets) = try_join!(
            repo.get_hg_bonsai_mapping(ctx.clone(), parents.clone()),
            phases.get_cached_public(ctx, parents)
        )?;
        next_changesets = new_next_changesets;
        public_changesets = new_public_changesets;
    }

    Ok(())
}

pub enum FilenodeEntryContent {
    InlineV2(ContentId),
    InlineV3(ContentId),
    LfsV3(Sha256, u64),
}

pub struct PreparedFilenodeEntry {
    pub filenode: HgFileNodeId,
    pub linknode: HgChangesetId,
    pub parents: HgParents,
    pub metadata: Bytes,
    pub content: FilenodeEntryContent,
    /// This field represents the memory footprint of a single
    /// entry when streaming. For inline-stored entries, this is
    /// just the size of the contents, while for LFS this is a size
    /// of an LFS pointer. Of course, this does not have to be
    /// precise, as it just provides an estimate on when to stop
    /// buffering.
    pub entry_weight_hint: u64,
}

impl PreparedFilenodeEntry {
    async fn into_filenode<'a>(
        self,
        ctx: &'a CoreContext,
        blobstore: &'a RepoBlobstore,
    ) -> Result<(HgFileNodeId, HgChangesetId, HgBlobNode, Option<RevFlags>), Error> {
        let Self {
            filenode,
            linknode,
            parents,
            metadata,
            content,
            ..
        } = self;

        async fn fetch_and_wrap(
            ctx: &CoreContext,
            blobstore: &RepoBlobstore,
            content_id: ContentId,
        ) -> Result<FileBytes, Error> {
            let content = filestore::fetch_concat(blobstore, ctx, content_id).await?;

            Ok(FileBytes(content))
        }

        let (blob, flags) = match content {
            FilenodeEntryContent::InlineV2(content_id) => {
                let bytes = fetch_and_wrap(ctx, blobstore, content_id).await?;
                (generate_inline_file(&bytes, parents, &metadata), None)
            }
            FilenodeEntryContent::InlineV3(content_id) => {
                let bytes = fetch_and_wrap(ctx, blobstore, content_id).await?;
                (
                    generate_inline_file(&bytes, parents, &metadata),
                    Some(RevFlags::REVIDX_DEFAULT_FLAGS),
                )
            }
            FilenodeEntryContent::LfsV3(oid, size) => (
                generate_lfs_file(oid, parents, size, &metadata)?,
                Some(RevFlags::REVIDX_EXTSTORED),
            ),
        };

        Ok((filenode, linknode, blob, flags))
    }

    pub fn maybe_get_lfs_pointer(&self) -> Option<(Sha256, u64)> {
        match self.content {
            FilenodeEntryContent::LfsV3(sha256, size) => Some((sha256.clone(), size)),
            _ => None,
        }
    }
}

fn calculate_content_weight_hint(content_size: u64, content: &FilenodeEntryContent) -> u64 {
    match content {
        // Approximate calculation for LFS:
        // - 34 bytes for GitVersion
        // - 32 bytes for Sha256
        // - 8 bytes for size
        // - 40 bytes for parents
        FilenodeEntryContent::LfsV3(_, _) => 34 + 32 + 8 + 40,
        // Approximate calculation for inline:
        // - content_size
        // - parents
        FilenodeEntryContent::InlineV2(_) | FilenodeEntryContent::InlineV3(_) => content_size + 40,
    }
}

fn prepare_filenode_entries_stream<'a>(
    ctx: &'a CoreContext,
    blobstore: &'a RepoBlobstore,
    filenodes: Vec<(MPath, HgFileNodeId, HgChangesetId)>,
    lfs_session: &'a SessionLfsParams,
) -> impl Stream<Item = Result<(MPath, Vec<PreparedFilenodeEntry>), Error>> + 'a {
    stream::iter(filenodes.into_iter())
        .map({
            move |(path, filenode, linknode)| async move {
                let envelope = filenode.load(ctx, blobstore).await?;

                let file_size = envelope.content_size();

                let content = match lfs_session.threshold {
                    None => FilenodeEntryContent::InlineV2(envelope.content_id()),
                    Some(lfs_threshold) if file_size <= lfs_threshold => {
                        FilenodeEntryContent::InlineV3(envelope.content_id())
                    }
                    _ => {
                        let key = FetchKey::from(envelope.content_id());
                        let meta = filestore::get_metadata(blobstore, ctx, &key).await?;
                        let meta =
                            meta.ok_or_else(|| Error::from(ErrorKind::MissingContent(key)))?;
                        let oid = meta.sha256;
                        FilenodeEntryContent::LfsV3(oid, file_size)
                    }
                };

                let parents = envelope.hg_parents();
                let entry_weight_hint = calculate_content_weight_hint(file_size, &content);
                let prepared_filenode_entry = PreparedFilenodeEntry {
                    filenode,
                    linknode,
                    parents,
                    metadata: envelope.metadata().clone(),
                    content,
                    entry_weight_hint,
                };

                Ok((path, vec![prepared_filenode_entry]))
            }
        })
        .buffered(100)
}

fn generate_inline_file(content: &FileBytes, parents: HgParents, metadata: &Bytes) -> HgBlobNode {
    let mut parents = parents.into_iter();
    let p1 = parents.next();
    let p2 = parents.next();

    // Metadata is only used to store copy/rename information
    let no_rename_metadata = metadata.is_empty();
    let mut res = vec![];
    res.extend(metadata);
    res.extend(content.as_bytes());
    if no_rename_metadata {
        HgBlobNode::new(Bytes::from(res), p1, p2)
    } else {
        // Mercurial has a complicated logic regarding storing renames
        // If copy/rename metadata is stored then p1 is always "null"
        // (i.e. hash like "00000000....") - that's why we set it to None below.
        // p2 is null for a non-merge commit, but not-null for merges.
        // (See D6922881 for more details about merge logic)
        //
        // It boils down to the fact that we can't have both p1 and p2 to be
        // non-null if we have rename metadata.
        // `HgFileEnvelope::hg_parents()` returns HgParents structure, which
        // always makes p2 a null commit if at least one parent commit is null.
        // And that's why we set the second parent to p1 below.
        debug_assert!(p2.is_none());
        HgBlobNode::new(Bytes::from(res), None, p1)
    }
}

fn generate_lfs_file(
    oid: Sha256,
    parents: HgParents,
    file_size: u64,
    metadata: &Bytes,
) -> Result<HgBlobNode, Error> {
    let copy_from = File::extract_copied_from(metadata)?;
    let bytes = File::generate_lfs_file(oid, file_size, copy_from)?;

    let mut parents = parents.into_iter();
    let p1 = parents.next();
    let p2 = parents.next();
    Ok(HgBlobNode::new(bytes, p1, p2))
}

pub fn create_manifest_entries_stream(
    ctx: CoreContext,
    blobstore: RepoBlobstore,
    manifests: Vec<(Option<MPath>, HgManifestId, HgChangesetId)>,
) -> OldBoxStream<OldBoxFuture<parts::TreepackPartInput, Error>, Error> {
    old_stream::iter_ok(manifests.into_iter())
        .filter(|(fullpath, mf_id, _linknode)| {
            !(fullpath.is_none() && mf_id.clone().into_nodehash() == NULL_HASH)
        })
        .map({
            move |(fullpath, mf_id, linknode)| {
                cloned!(ctx, blobstore);
                async move {
                    fetch_manifest_envelope(&ctx, &blobstore.boxed(), mf_id)
                        .map_ok(move |mf_envelope| {
                            let (p1, p2) = mf_envelope.parents();
                            parts::TreepackPartInput {
                                node: mf_id.into_nodehash(),
                                p1,
                                p2,
                                content: BytesOld::from(mf_envelope.contents().as_ref()),
                                fullpath,
                                linknode: linknode.into_nodehash(),
                            }
                        })
                        .await
                }
                .boxed()
                .compat()
                .boxify()
            }
        })
        .boxify()
}

async fn diff_with_parents(
    ctx: &CoreContext,
    repo: &(
         impl ChangesetsRef + RepoDerivedDataRef + BonsaiHgMappingRef + RepoBlobstoreRef + Send + Sync
     ),
    hg_cs_id: HgChangesetId,
) -> Result<
    (
        Vec<(Option<MPath>, HgManifestId, HgChangesetId)>,
        Vec<(MPath, HgFileNodeId, HgChangesetId)>,
    ),
    Error,
> {
    let (mf_id, parent_mf_ids) = try_join!(
        fetch_manifest(ctx, repo.repo_blobstore(), &hg_cs_id),
        async {
            let parents = repo.get_hg_changeset_parents(ctx.clone(), hg_cs_id).await?;

            future::try_join_all(
                parents
                    .iter()
                    .map(|p| fetch_manifest(ctx, repo.repo_blobstore(), p)),
            )
            .await
        }
    )?;

    let blobstore = Arc::new(repo.repo_blobstore().clone());
    let new_entries: Vec<(Option<MPath>, Entry<_, _>, _)> =
        find_intersection_of_diffs_and_parents(ctx.clone(), blobstore, mf_id, parent_mf_ids)
            .try_collect()
            .await?;

    let mut mfs = vec![];
    let mut files = vec![];
    for (path, entry, parent_entries) in new_entries {
        match entry {
            Entry::Tree(mf) => {
                mfs.push((path, mf, hg_cs_id.clone()));
            }
            Entry::Leaf((_, file)) => {
                let mut found_same_in_parents = false;
                for p in parent_entries {
                    if let Entry::Leaf((_, parent_file)) = p {
                        if parent_file == file {
                            found_same_in_parents = true;
                            break;
                        }
                    }
                }
                if found_same_in_parents {
                    continue;
                }
                let path = path.expect("empty file paths?");
                files.push((path, file, hg_cs_id.clone()));
            }
        }
    }

    Ok((mfs, files))
}

fn create_filenodes_weighted(
    ctx: CoreContext,
    repo: impl RepoBlobstoreRef + Clone + Sync + Send + 'static,
    entries: HashMap<MPath, Vec<PreparedFilenodeEntry>>,
) -> impl OldStream<
    Item = (
        impl OldFuture<Item = (MPath, Vec<FilenodeEntry>), Error = Error>,
        u64,
    ),
    Error = Error,
> {
    let items = entries.into_iter().map({
        cloned!(ctx, repo);
        move |(path, prepared_entries)| {
            let total_weight: u64 = prepared_entries.iter().fold(0, |acc, prepared_entry| {
                acc + prepared_entry.entry_weight_hint
            });

            let entry_futs: Vec<_> = prepared_entries
                .into_iter()
                .map({
                    |entry| {
                        {
                            cloned!(ctx, repo);
                            async move { entry.into_filenode(&ctx, repo.repo_blobstore()).await }
                        }
                        .boxed()
                        .compat()
                    }
                })
                .collect();

            let fut = old_future::join_all(entry_futs).map(|entries| (path, entries));

            (fut, total_weight)
        }
    });
    old_stream::iter_ok(items)
}

pub fn create_filenodes(
    ctx: CoreContext,
    repo: impl RepoBlobstoreRef + Clone + Sync + Send + 'static,
    entries: HashMap<MPath, Vec<PreparedFilenodeEntry>>,
) -> impl OldStream<Item = (MPath, Vec<FilenodeEntry>), Error = Error> {
    let params = BufferedParams {
        weight_limit: MAX_FILENODE_BYTES_IN_MEMORY,
        buffer_size: 100,
    };
    create_filenodes_weighted(ctx, repo, entries).buffered_weight_limited(params)
}

// This function preserves the topological order of entries i.e. filenods or manifest
// created in an earlier commit will be earlier in the output.
pub async fn get_manifests_and_filenodes(
    ctx: &CoreContext,
    repo: &(
         impl ChangesetsRef + RepoDerivedDataRef + BonsaiHgMappingRef + RepoBlobstoreRef + Send + Sync
     ),
    commits: impl IntoIterator<Item = HgChangesetId>,
    lfs_params: &SessionLfsParams,
) -> Result<
    (
        Vec<(Option<MPath>, HgManifestId, HgChangesetId)>,
        HashMap<MPath, Vec<PreparedFilenodeEntry>>,
    ),
    Error,
> {
    let entries: Vec<_> = stream::iter(commits)
        .then({
            |hg_cs_id| async move {
                let (manifests, filenodes) = diff_with_parents(ctx, repo, hg_cs_id).await?;

                let filenodes: Vec<(MPath, Vec<PreparedFilenodeEntry>)> =
                    prepare_filenode_entries_stream(
                        ctx,
                        repo.repo_blobstore(),
                        filenodes,
                        lfs_params,
                    )
                    .try_collect()
                    .await?;
                Result::<_, Error>::Ok((manifests, filenodes))
            }
        })
        .try_collect()
        .await?;

    // We avoid duplicate manifests and filenodes, but we preserve the order of the entries
    // with respect to the commit i.e. entries from earlier commit are before entries
    // from the later commit.
    let mut all_mf_entries = vec![];
    let mut used_mfs = HashSet::new();

    let mut ordered_filenode_entries: HashMap<_, Vec<_>> = HashMap::new();
    let mut used_filenodes: HashMap<MPath, HashSet<HgFileNodeId>> = HashMap::new();
    for (mf_entries, file_entries) in entries {
        for (path, mf_id, linknode) in mf_entries {
            if used_mfs.insert((path.clone(), mf_id)) {
                all_mf_entries.push((path, mf_id, linknode));
            }
        }

        for (file_path, filenodes) in file_entries {
            let used_filenodes = used_filenodes.entry(file_path.clone()).or_default();
            let ordered_entries = ordered_filenode_entries.entry(file_path).or_default();

            for filenode in filenodes {
                if used_filenodes.insert(filenode.filenode) {
                    ordered_entries.push(filenode);
                }
            }
        }
    }

    Ok((all_mf_entries, ordered_filenode_entries))
}

async fn fetch_manifest(
    ctx: &CoreContext,
    blobstore: &RepoBlobstore,
    hg_cs_id: &HgChangesetId,
) -> Result<HgManifestId, Error> {
    let blob_cs = hg_cs_id.load(ctx, blobstore).await?;
    Ok(blob_cs.manifestid())
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::sync::Arc;

use anyhow::Result;
use cloned::cloned;
use context::CoreContext;
use futures::future;
use futures::stream::FuturesUnordered;
use futures::Stream;
use futures::StreamExt;
use futures::TryStreamExt;
use mercurial_bundles::obsmarkers::MetadataEntry;
use mercurial_bundles::part_encode::PartEncodeBuilder;
use mercurial_bundles::parts;
use mercurial_derivation::derive_hg_changeset;
use mercurial_types::HgChangesetId;
use mononoke_types::DateTime;
use repo_derived_data::RepoDerivedData;
use repo_derived_data::RepoDerivedDataArc;

pub fn pushrebased_changesets_to_obsmarkers_part(
    ctx: &CoreContext,
    repo: &impl RepoDerivedDataArc,
    pushrebased_changesets: Vec<pushrebase::PushrebaseChangesetPair>,
) -> Option<Result<PartEncodeBuilder>> {
    let filtered_changesets: Vec<_> = pushrebased_changesets
        .into_iter()
        .filter(|c| c.id_old != c.id_new)
        .collect();

    if filtered_changesets.is_empty() {
        return None;
    }

    let hg_pushrebased_changesets =
        pushrebased_changesets_to_hg_stream(ctx, repo.repo_derived_data_arc(), filtered_changesets);

    let time = DateTime::now();
    let mut metadata = vec![MetadataEntry::new("operation", "push")];

    if let Some(user) = ctx.metadata().unix_name() {
        metadata.push(MetadataEntry::new("user", user));
    }

    let part = parts::obsmarkers_part(hg_pushrebased_changesets.boxed().compat(), time, metadata);

    Some(part)
}

fn pushrebased_changesets_to_hg_stream(
    ctx: &CoreContext,
    repo_derived_data: Arc<RepoDerivedData>,
    pushrebased_changesets: Vec<pushrebase::PushrebaseChangesetPair>,
) -> impl Stream<Item = Result<(HgChangesetId, Vec<HgChangesetId>)>> {
    pushrebased_changesets
        .into_iter()
        .map(move |p| {
            cloned!(ctx, repo_derived_data);
            async move {
                let (old, new) = future::try_join(
                    derive_hg_changeset(&ctx, &repo_derived_data, p.id_old),
                    derive_hg_changeset(&ctx, &repo_derived_data, p.id_new),
                )
                .await?;
                Ok((old, vec![new]))
            }
        })
        .collect::<FuturesUnordered<_>>()
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashSet;
use std::fmt;

use anyhow::Result;
use blobrepo::AsBlobRepo;
use blobrepo_hg::file_history::get_file_history_maybe_incomplete;
use blobstore::Loadable;
use bytes::Bytes;
use bytes::BytesMut;
use cloned::cloned;
use context::CoreContext;
use filestore::FetchKey;
use futures::future;
use futures::future::BoxFuture;
use futures::stream::select_all::select_all;
use futures::Future;
use futures::FutureExt;
use futures::Stream;
use futures::StreamExt;
use futures::TryFutureExt;
use futures::TryStreamExt;
use getbundle_response::SessionLfsParams;
use mercurial_types::blobs::File;
use mercurial_types::calculate_hg_node_id;
use mercurial_types::FileBytes;
use mercurial_types::HgFileEnvelope;
use mercurial_types::HgFileEnvelopeMut;
use mercurial_types::HgFileHistoryEntry;
use mercurial_types::HgFileNodeId;
use mercurial_types::HgParents;
use mercurial_types::MPath;
use mercurial_types::RevFlags;
use redactedblobstore::has_redaction_root_cause;
use repo_blobstore::RepoBlobstore;
use repo_blobstore::RepoBlobstoreArc;
use repo_blobstore::RepoBlobstoreRef;
use revisionstore_types::Metadata;
use thiserror::Error;

#[facet::container]
pub struct Repo(RepoBlobstore);

#[derive(Debug, Error)]
pub enum ErrorKind {
    #[error("Corrupt hg filenode returned: {expected} != {actual}")]
    CorruptHgFileNode {
        expected: HgFileNodeId,
        actual: HgFileNodeId,
    },

    #[error("Invalid blob kind returned: {kind:?}")]
    InvalidKind { kind: RemotefilelogBlobKind },
    #[error("Missing content: {0:?}")]
    MissingContent(FetchKey),
}

#[derive(Debug)]
pub enum RemotefilelogBlobKind {
    /// An inline filenode. This represents its size.
    Inline(u64),
    /// An LFS filenode, together with the actual file size.
    Lfs(u64),
}

struct RemotefilelogBlob {
    kind: RemotefilelogBlobKind,
    /// data is a future of the metadata bytes and file bytes. For LFS blobs, the metadata bytes
    /// will be empty and the file bytes will contain a serialized LFS pointer.
    data: BoxFuture<'static, Result<(Bytes, FileBytes)>>,
}

impl fmt::Debug for RemotefilelogBlob {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "RemotefilelogBlob {{ kind: {:?} }}", self.kind)
    }
}

pub struct GetpackBlobInfo {
    pub filesize: u64,
    // weight is equal to file size if it's a non-lfs blobs
    // or it's zero for lfs blobs
    pub weight: u64,
}

fn rescue_redacted(res: Result<(Bytes, FileBytes)>) -> Result<(Bytes, FileBytes)> {
    /// Tombstone string to replace the content of redacted files with
    const REDACTED_CONTENT: &str = "PoUOK1GkdH6Xtx5j9WKYew3dZXspyfkahcNkhV6MJ4rhyNICTvX0nxmbCImFoT0oHAF9ivWGaC6ByswQZUgf1nlyxcDcahHknJS15Vl9Lvc4NokYhMg0mV1rapq1a4bhNoUI9EWTBiAkYmkadkO3YQXV0TAjyhUQWxxLVskjOwiiFPdL1l1pdYYCLTE3CpgOoxQV3EPVxGUPh1FGfk7F9Myv22qN1sUPSNN4h3IFfm2NNPRFgWPDsqAcaQ7BUSKa\n";
    match res {
        Ok(b) => Ok(b),
        Err(e) => {
            if has_redaction_root_cause(&e) {
                let ret = (Bytes::new(), FileBytes(REDACTED_CONTENT.as_bytes().into()));
                Ok(ret)
            } else {
                Err(e)
            }
        }
    }
}

/// Create a blob for getpack v1. This returns a future that resolves with an estimated weight for
/// this blob (this is NOT trying to be correct, it's just a rough estimate!), and the blob's
/// bytes.
pub async fn create_getpack_v1_blob(
    ctx: &CoreContext,
    repo: &impl RepoLike,
    node: HgFileNodeId,
    validate_hash: bool,
) -> Result<(
    GetpackBlobInfo,
    impl Future<Output = Result<(HgFileNodeId, Bytes)>>,
)> {
    let RemotefilelogBlob { kind, data } = prepare_blob(
        ctx,
        repo,
        node,
        SessionLfsParams { threshold: None },
        validate_hash,
    )
    .await?;
    use RemotefilelogBlobKind::*;

    let getpack_blob_data = match kind {
        Inline(size) => GetpackBlobInfo {
            filesize: size,
            weight: size,
        },
        Lfs(_) => unreachable!(), // lfs_threshold = None implies no LFS blobs.
    };

    let fut = data
        .map(rescue_redacted)
        .map_ok(move |(meta_bytes, file_bytes)| {
            // TODO (T30456231): Avoid this copy
            let mut buff = BytesMut::with_capacity(meta_bytes.len() + file_bytes.as_bytes().len());
            buff.extend_from_slice(&meta_bytes);
            buff.extend_from_slice(file_bytes.as_bytes());
            (node, buff.freeze())
        });

    Ok((getpack_blob_data, fut))
}

/// Create a blob for getpack v2. See v1 above for general details. This also returns Metadata,
/// which is present in the v2 version of the protocol.
pub async fn create_getpack_v2_blob(
    ctx: &CoreContext,
    repo: &impl RepoLike,
    node: HgFileNodeId,
    lfs_params: SessionLfsParams,
    validate_hash: bool,
) -> Result<(
    GetpackBlobInfo,
    impl Future<Output = Result<(HgFileNodeId, Bytes, Metadata)>>,
)> {
    let RemotefilelogBlob { kind, data } =
        prepare_blob(ctx, repo, node, lfs_params, validate_hash).await?;
    use RemotefilelogBlobKind::*;

    let (weight, metadata) = match kind {
        Inline(size) => {
            let getpack_blob_data = GetpackBlobInfo {
                filesize: size,
                weight: size,
            };
            (
                getpack_blob_data,
                Metadata {
                    size: None,
                    flags: None,
                },
            )
        }
        Lfs(size) => {
            let getpack_blob_data = GetpackBlobInfo {
                filesize: size,
                weight: 0,
            };
            let flags = Some(RevFlags::REVIDX_EXTSTORED.into());
            (getpack_blob_data, Metadata { size: None, flags })
        }
    };

    let fut = data
        .map(rescue_redacted)
        .map_ok(move |(meta_bytes, file_bytes)| {
            // TODO (T30456231): Avoid this copy
            let mut buff = BytesMut::with_capacity(meta_bytes.len() + file_bytes.as_bytes().len());
            buff.extend_from_slice(&meta_bytes);
            buff.extend_from_slice(file_bytes.as_bytes());
            (node, buff.freeze(), metadata)
        });

    Ok((weight, fut))
}

/// Retrieve the raw contents of a filenode. This does not substitute redacted content
/// (it'll just let the redacted error fall through).
pub async fn create_raw_filenode_blob(
    ctx: &CoreContext,
    repo: &impl RepoLike,
    node: HgFileNodeId,
    validate_hash: bool,
) -> Result<Bytes> {
    let RemotefilelogBlob { kind, data } = prepare_blob(
        ctx,
        repo,
        node,
        SessionLfsParams { threshold: None },
        validate_hash,
    )
    .await?;

    let (meta_bytes, file_bytes) = match kind {
        RemotefilelogBlobKind::Inline(_) => data.await?,
        kind => return Err(ErrorKind::InvalidKind { kind }.into()),
    };

    // TODO (T30456231): Avoid this copy
    let mut buff = BytesMut::with_capacity(meta_bytes.len() + file_bytes.as_bytes().len());
    buff.extend_from_slice(&meta_bytes);
    buff.extend_from_slice(file_bytes.as_bytes());
    Ok(buff.freeze())
}

/// Get ancestors of all filenodes
/// Current implementation might be inefficient because it might re-fetch the same filenode a few
/// times
pub fn get_unordered_file_history_for_multiple_nodes(
    ctx: &CoreContext,
    repo: &(impl RepoLike + AsBlobRepo),
    filenodes: HashSet<HgFileNodeId>,
    path: &MPath,
    allow_short_getpack_history: bool,
) -> impl Stream<Item = Result<HgFileHistoryEntry>> {
    let limit = if allow_short_getpack_history {
        let limit = tunables::tunables()
            .remotefilelog_file_history_limit()
            .unwrap_or_default();
        if limit <= 0 { None } else { Some(limit as u64) }
    } else {
        None
    };
    select_all(filenodes.into_iter().map(|filenode| {
        get_file_history_maybe_incomplete(
            ctx.clone(),
            repo.as_blob_repo().clone(),
            filenode,
            path.clone(),
            limit,
        )
        .boxed()
    }))
    .try_filter({
        let mut used_filenodes = HashSet::new();
        move |entry| future::ready(used_filenodes.insert(entry.filenode().clone()))
    })
}

async fn prepare_blob(
    ctx: &CoreContext,
    repo: &impl RepoLike,
    node: HgFileNodeId,
    lfs_params: SessionLfsParams,
    validate_hash: bool,
) -> Result<RemotefilelogBlob> {
    let envelope = node.load(ctx, repo.repo_blobstore()).await?;

    let inline_file = match lfs_params.threshold {
        Some(lfs_threshold) => envelope.content_size() <= lfs_threshold,
        None => true,
    };

    // NOTE: It'd be nice if we could hoist up redaction checks to this point. Doing so
    // would let us return a different kind based on whether the content is redacted or
    // not, and therefore would make it more obvious which methods do redaction or not
    // (based on their signature).
    if inline_file {
        Ok(prepare_blob_inline_file(
            envelope,
            ctx,
            repo,
            node,
            validate_hash,
        ))
    } else {
        Ok(prepare_blob_lfs_file(envelope, ctx, repo))
    }
}

fn prepare_blob_inline_file(
    envelope: HgFileEnvelope,
    ctx: &CoreContext,
    repo: &impl RepoLike,
    node: HgFileNodeId,
    validate_hash: bool,
) -> RemotefilelogBlob {
    cloned!(ctx, repo);
    let kind = RemotefilelogBlobKind::Inline(envelope.content_size());
    let blobstore = repo.repo_blobstore_arc();
    let data = async move {
        let file_bytes =
            FileBytes(filestore::fetch_concat(&blobstore, &ctx, envelope.content_id()).await?);

        let HgFileEnvelopeMut {
            p1, p2, metadata, ..
        } = envelope.into_mut();

        if validate_hash {
            let mut validation_bytes =
                BytesMut::with_capacity(metadata.len() + file_bytes.as_bytes().len());
            validation_bytes.extend_from_slice(&metadata);
            validation_bytes.extend_from_slice(file_bytes.as_bytes());

            let p1 = p1.map(|p| p.into_nodehash());
            let p2 = p2.map(|p| p.into_nodehash());
            let actual = HgFileNodeId::new(calculate_hg_node_id(
                &validation_bytes.freeze(),
                &HgParents::new(p1, p2),
            ));

            if actual != node {
                return Err(ErrorKind::CorruptHgFileNode {
                    expected: node,
                    actual,
                }
                .into());
            }
        }

        Ok((metadata, file_bytes))
    }
    .boxed();

    RemotefilelogBlob { kind, data }
}

/// For LFS blobs, we'll create the LFS pointer. Note that there is no hg-style
/// metadata encoded for LFS blobs (it's in the LFS pointer instead).
fn prepare_blob_lfs_file(
    envelope: HgFileEnvelope,
    ctx: &CoreContext,
    repo: &impl RepoLike,
) -> RemotefilelogBlob {
    cloned!(ctx);
    let file_size = envelope.content_size();
    let kind = RemotefilelogBlobKind::Lfs(file_size);
    let blobstore = repo.repo_blobstore_arc();
    let data = async move {
        let key = FetchKey::from(envelope.content_id());
        let oid = filestore::get_metadata(&blobstore, &ctx, &key)
            .await?
            .ok_or(ErrorKind::MissingContent(key))?
            .sha256;
        let copy_from = File::extract_copied_from(envelope.metadata())?;
        let bytes = File::generate_lfs_file(oid, file_size, copy_from)?;
        Ok((Bytes::new(), FileBytes(bytes)))
    }
    .boxed();

    RemotefilelogBlob { kind, data }
}

#[cfg(test)]
mod test {
    use anyhow::Error;
    use assert_matches::assert_matches;
    use borrowed::borrowed;
    use fbinit::FacebookInit;
    use manifest::Entry;
    use manifest::Manifest;
    use mercurial_derivation::DeriveHgChangeset;
    use metaconfig_types::FilestoreParams;
    use mononoke_types::MPathElement;
    use test_repo_factory::TestRepoFactory;
    use tests_utils::BasicTestRepo;
    use tests_utils::CreateCommitContext;

    use super::*;

    async fn roundtrip_blob(
        fb: FacebookInit,
        repo: &BasicTestRepo,
        content: &str,
        threshold: Option<u64>,
    ) -> Result<RemotefilelogBlobKind> {
        let filename = "f1";

        let ctx = CoreContext::test_mock(fb);
        borrowed!(ctx);

        let bcs = CreateCommitContext::new_root(ctx, repo)
            .add_file(filename, content)
            .commit()
            .await?;

        let hg_manifest = repo
            .derive_hg_changeset(ctx, bcs)
            .await?
            .load(ctx, repo.repo_blobstore())
            .await?
            .manifestid()
            .load(ctx, repo.repo_blobstore())
            .await?;

        let entry = hg_manifest
            .lookup(&MPathElement::new(filename.as_bytes().to_vec())?)
            .ok_or_else(|| Error::msg("file is missing"))?;

        let filenode = match entry {
            Entry::Leaf((_, filenode)) => filenode,
            _ => {
                return Err(Error::msg("file is not a leaf"));
            }
        };

        let blob = prepare_blob(ctx, repo, filenode, SessionLfsParams { threshold }, true).await?;

        let RemotefilelogBlob { kind, data } = blob;
        data.await?; // Await the blob data to make sure hash validation passes.

        Ok(kind)
    }

    #[fbinit::test]
    async fn test_prepare_blob(fb: FacebookInit) -> Result<()> {
        let repo: BasicTestRepo = test_repo_factory::build_empty(fb).await?;
        let blob = roundtrip_blob(fb, &repo, "foo", Some(3)).await?;
        assert_matches!(blob, RemotefilelogBlobKind::Inline(3));
        Ok(())
    }

    #[fbinit::test]
    async fn test_prepare_blob_chunked(fb: FacebookInit) -> Result<()> {
        let repo: BasicTestRepo = TestRepoFactory::new(fb)?
            .with_config_override(|config| {
                config.filestore = Some(FilestoreParams {
                    chunk_size: 1,
                    concurrency: 1,
                })
            })
            .build()
            .await?;

        let blob = roundtrip_blob(fb, &repo, "foo", None).await?;
        assert_matches!(blob, RemotefilelogBlobKind::Inline(3));
        Ok(())
    }

    #[fbinit::test]
    async fn test_prepare_blob_lfs(fb: FacebookInit) -> Result<()> {
        let repo: BasicTestRepo = test_repo_factory::build_empty(fb).await?;
        let blob = roundtrip_blob(fb, &repo, "foo", Some(2)).await?;
        assert_matches!(blob, RemotefilelogBlobKind::Lfs(3));
        Ok(())
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use anyhow::Error;
use metaconfig_types::HgsqlName;
use metaconfig_types::RepoReadOnly;
use sql::mysql;
use sql::mysql_async::prelude::ConvIr;
use sql::mysql_async::prelude::FromValue;
use sql::mysql_async::FromValueError;
use sql::mysql_async::Value;
use sql::Connection;
use sql_construct::SqlConstruct;
use sql_construct::SqlConstructFromMetadataDatabaseConfig;
use sql_ext::mononoke_queries;
use sql_ext::SqlConnections;

static DEFAULT_MSG: &str = "Defaulting to locked as the lock state isn't initialised for this repo";
static NOT_CONNECTED_MSG: &str = "Defaulting to locked as no database connection passed";
static DB_MSG: &str = "Repo is locked in DB";

#[derive(Clone, mysql::OptTryFromRowField)]
enum HgMononokeReadWrite {
    NoWrite,
    HgWrite,
    MononokeWrite,
}

impl From<HgMononokeReadWrite> for Value {
    fn from(read_write: HgMononokeReadWrite) -> Self {
        match read_write {
            HgMononokeReadWrite::NoWrite => Value::Int(0),
            HgMononokeReadWrite::HgWrite => Value::Int(1),
            HgMononokeReadWrite::MononokeWrite => Value::Int(2),
        }
    }
}

impl ConvIr<HgMononokeReadWrite> for HgMononokeReadWrite {
    fn new(val: Value) -> Result<Self, FromValueError> {
        match val {
            Value::Bytes(ref b) if b == &b"0" => Ok(HgMononokeReadWrite::NoWrite),
            Value::Int(0) => Ok(HgMononokeReadWrite::NoWrite),
            Value::Bytes(ref b) if b == &b"1" => Ok(HgMononokeReadWrite::HgWrite),
            Value::Int(1) => Ok(HgMononokeReadWrite::HgWrite),
            Value::Bytes(ref b) if b == &b"2" => Ok(HgMononokeReadWrite::MononokeWrite),
            Value::Int(2) => Ok(HgMononokeReadWrite::MononokeWrite),
            v => Err(FromValueError(v)),
        }
    }

    fn commit(self) -> Self {
        self
    }

    fn rollback(self) -> Value {
        self.into()
    }
}

impl FromValue for HgMononokeReadWrite {
    type Intermediate = HgMononokeReadWrite;
}

mononoke_queries! {
    read GetReadWriteStatus(repo_name: String) -> (HgMononokeReadWrite, Option<String>) {
        "SELECT state, reason FROM repo_lock
        WHERE repo = {repo_name}"
    }
    write SetReadWriteStatus(values: (repo_name: String, state: HgMononokeReadWrite, reason: str)) {
        none,
        mysql("INSERT INTO repo_lock (repo, state, reason) VALUES {values} ON DUPLICATE KEY UPDATE state = VALUES(state), reason = VALUES(reason)")
        sqlite("INSERT OR REPLACE INTO repo_lock (repo, state, reason) VALUES {values}")
    }
}

#[derive(Clone, Debug)]
pub struct SqlRepoReadWriteStatus {
    write_connection: Connection,
    read_connection: Connection,
}

impl SqlConstruct for SqlRepoReadWriteStatus {
    const LABEL: &'static str = "repo-lock";

    const CREATION_QUERY: &'static str = include_str!("../../schemas/sqlite-hg-repo-lock.sql");

    fn from_sql_connections(connections: SqlConnections) -> Self {
        Self {
            write_connection: connections.write_connection,
            read_connection: connections.read_connection,
        }
    }
}

impl SqlConstructFromMetadataDatabaseConfig for SqlRepoReadWriteStatus {}

impl SqlRepoReadWriteStatus {
    async fn query_read_write_state(
        &self,
        hgsql_name: &HgsqlName,
    ) -> Result<Option<(HgMononokeReadWrite, Option<String>)>, Error> {
        GetReadWriteStatus::query(&self.read_connection, hgsql_name.as_ref())
            .await
            .map(|rows| rows.into_iter().next())
    }

    async fn set_state(
        &self,
        hgsql_name: &HgsqlName,
        state: &HgMononokeReadWrite,
        reason: &String,
    ) -> Result<bool, Error> {
        SetReadWriteStatus::query(
            &self.write_connection,
            &[(hgsql_name.as_ref(), state, reason)],
        )
        .await
        .map(|res| res.affected_rows() > 0)
    }
}

#[derive(Clone, Debug)]
pub struct RepoReadWriteFetcher {
    sql_repo_read_write_status: Option<SqlRepoReadWriteStatus>,
    readonly_config: RepoReadOnly,
    hgsql_name: HgsqlName,
}

impl RepoReadWriteFetcher {
    pub fn new(
        sql_repo_read_write_status: Option<SqlRepoReadWriteStatus>,
        readonly_config: RepoReadOnly,
        hgsql_name: HgsqlName,
    ) -> Self {
        Self {
            sql_repo_read_write_status,
            readonly_config,
            hgsql_name,
        }
    }

    async fn query_read_write_state(&self) -> Result<RepoReadOnly, Error> {
        match &self.sql_repo_read_write_status {
            Some(status) => status
                .query_read_write_state(&self.hgsql_name)
                .await
                .map(|item| match item {
                    Some((HgMononokeReadWrite::MononokeWrite, _)) => RepoReadOnly::ReadWrite,
                    Some((_, reason)) => {
                        RepoReadOnly::ReadOnly(reason.unwrap_or_else(|| DB_MSG.to_string()))
                    }
                    None => RepoReadOnly::ReadOnly(DEFAULT_MSG.to_string()),
                }),
            None => Ok(RepoReadOnly::ReadOnly(NOT_CONNECTED_MSG.to_string())),
        }
    }

    pub async fn readonly(&self) -> Result<RepoReadOnly, Error> {
        if self.sql_repo_read_write_status.is_some() {
            match self.readonly_config {
                RepoReadOnly::ReadOnly(ref reason) => Ok(RepoReadOnly::ReadOnly(reason.clone())),
                RepoReadOnly::ReadWrite => self.query_read_write_state().await,
            }
        } else {
            Ok(self.readonly_config.clone())
        }
    }

    async fn set_state(&self, state: &HgMononokeReadWrite, reason: &String) -> Result<bool, Error> {
        match &self.sql_repo_read_write_status {
            Some(status) => status.set_state(&self.hgsql_name, state, reason).await,
            None => Err(Error::msg("db name is not specified")),
        }
    }

    pub async fn set_mononoke_read_write(&self, reason: &String) -> Result<bool, Error> {
        self.set_state(&HgMononokeReadWrite::MononokeWrite, reason)
            .await
    }

    pub async fn set_read_only(&self, reason: &String) -> Result<bool, Error> {
        self.set_state(&HgMononokeReadWrite::NoWrite, reason).await
    }
}

#[cfg(test)]
mod test {
    use metaconfig_types::RepoReadOnly::*;

    use super::*;

    static CONFIG_MSG: &str = "Set by config option";

    mononoke_queries! {
        write InsertState(values: (repo: str, state: HgMononokeReadWrite)) {
            none,
            "REPLACE INTO repo_lock(repo, state)
            VALUES {values}"
        }

        write InsertStateWithReason(values: (repo: str, state: HgMononokeReadWrite, reason: str)) {
            none,
            "REPLACE INTO repo_lock(repo, state, reason)
            VALUES {values}"
        }
    }

    #[tokio::test]
    async fn test_readonly_config_no_sqlite() {
        let fetcher = RepoReadWriteFetcher::new(
            None,
            ReadOnly(CONFIG_MSG.to_string()),
            HgsqlName("repo".to_string()),
        );

        assert_eq!(
            fetcher.readonly().await.unwrap(),
            ReadOnly(CONFIG_MSG.to_string())
        );
    }

    #[tokio::test]
    async fn test_readwrite_config_no_sqlite() {
        let fetcher = RepoReadWriteFetcher::new(None, ReadWrite, HgsqlName("repo".to_string()));
        assert_eq!(fetcher.readonly().await.unwrap(), ReadWrite);
    }

    #[tokio::test]
    async fn test_readonly_config_with_sqlite() {
        let sql_repo_read_write_status = SqlRepoReadWriteStatus::with_sqlite_in_memory().unwrap();
        let fetcher = RepoReadWriteFetcher::new(
            Some(sql_repo_read_write_status),
            ReadOnly(CONFIG_MSG.to_string()),
            HgsqlName("repo".to_string()),
        );
        assert_eq!(
            fetcher.readonly().await.unwrap(),
            ReadOnly(CONFIG_MSG.to_string())
        );
    }

    #[tokio::test]
    async fn test_readwrite_with_sqlite() {
        let sql_repo_read_write_status = SqlRepoReadWriteStatus::with_sqlite_in_memory().unwrap();
        let fetcher = RepoReadWriteFetcher::new(
            Some(sql_repo_read_write_status),
            ReadWrite,
            HgsqlName("repo".to_string()),
        );
        // As the DB hasn't been populated for this row, ensure that we mark the repo as locked.
        assert_eq!(
            fetcher.readonly().await.unwrap(),
            ReadOnly(DEFAULT_MSG.to_string())
        );

        InsertState::query(
            &fetcher
                .sql_repo_read_write_status
                .clone()
                .unwrap()
                .write_connection,
            &[("repo", &HgMononokeReadWrite::MononokeWrite)],
        )
        .await
        .unwrap();

        assert_eq!(fetcher.readonly().await.unwrap(), ReadWrite);

        InsertState::query(
            &fetcher
                .sql_repo_read_write_status
                .clone()
                .unwrap()
                .write_connection,
            &[("repo", &HgMononokeReadWrite::HgWrite)],
        )
        .await
        .unwrap();

        assert_eq!(
            fetcher.readonly().await.unwrap(),
            ReadOnly(DB_MSG.to_string())
        );
    }

    #[tokio::test]
    async fn test_readwrite_with_sqlite_and_reason() {
        let sql_repo_read_write_status = SqlRepoReadWriteStatus::with_sqlite_in_memory().unwrap();
        let fetcher = RepoReadWriteFetcher::new(
            Some(sql_repo_read_write_status),
            ReadWrite,
            HgsqlName("repo".to_string()),
        );

        InsertStateWithReason::query(
            &fetcher
                .sql_repo_read_write_status
                .clone()
                .unwrap()
                .write_connection,
            &[("repo", &HgMononokeReadWrite::HgWrite, "reason123")],
        )
        .await
        .unwrap();

        assert_eq!(
            fetcher.readonly().await.unwrap(),
            ReadOnly("reason123".to_string())
        );
    }

    #[tokio::test]
    async fn test_readwrite_with_sqlite_other_repo() {
        let sql_repo_read_write_status = SqlRepoReadWriteStatus::with_sqlite_in_memory().unwrap();
        let fetcher = RepoReadWriteFetcher::new(
            Some(sql_repo_read_write_status),
            ReadWrite,
            HgsqlName("repo".to_string()),
        );
        // As the DB hasn't been populated for this row, ensure that we mark the repo as locked.
        assert_eq!(
            fetcher.readonly().await.unwrap(),
            ReadOnly(DEFAULT_MSG.to_string())
        );

        InsertState::query(
            &fetcher
                .sql_repo_read_write_status
                .clone()
                .unwrap()
                .write_connection,
            &[("other_repo", &HgMononokeReadWrite::MononokeWrite)],
        )
        .await
        .unwrap();

        assert_eq!(
            fetcher.readonly().await.unwrap(),
            ReadOnly(DEFAULT_MSG.to_string())
        );

        InsertState::query(
            &fetcher
                .sql_repo_read_write_status
                .clone()
                .unwrap()
                .write_connection,
            &[("repo", &HgMononokeReadWrite::MononokeWrite)],
        )
        .await
        .unwrap();

        assert_eq!(fetcher.readonly().await.unwrap(), ReadWrite);
    }

    #[tokio::test]
    async fn test_write() {
        let sql_repo_read_write_status = SqlRepoReadWriteStatus::with_sqlite_in_memory().unwrap();
        let fetcher = RepoReadWriteFetcher::new(
            Some(sql_repo_read_write_status),
            ReadWrite,
            HgsqlName("repo".to_string()),
        );
        // As the DB hasn't been populated for this row, ensure that we mark the repo as locked.
        assert_eq!(
            fetcher.readonly().await.unwrap(),
            ReadOnly(DEFAULT_MSG.to_string())
        );

        fetcher
            .set_mononoke_read_write(&"repo is locked".to_string())
            .await
            .unwrap();
        assert_eq!(fetcher.readonly().await.unwrap(), ReadWrite);
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use mercurial_types::HgNodeHash;
use mercurial_types::RepoPath;
use rate_limiting::RateLimitReason;
use thiserror::Error;

#[derive(Debug, Error)]
pub enum ErrorKind {
    #[error("Data corruption for {path}: expected {expected}, actual {actual}!")]
    DataCorruption {
        path: RepoPath,
        expected: HgNodeHash,
        actual: HgNodeHash,
    },
    #[error("Request {request_name} was throttled")]
    RequestThrottled {
        request_name: String,
        #[source]
        reason: RateLimitReason,
    },
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

#![feature(async_closure)]

//! State for a single source control Repo

mod client;
mod errors;

pub use client::fetch_treepack_part_input;
pub use client::gettreepack_entries;
pub use client::RepoClient;
pub use getbundle_response::find_commits_to_send;
pub use getbundle_response::find_new_draft_commits_and_derive_filenodes_for_public_roots;
pub use unbundle::PushRedirector;
pub use unbundle::PushRedirectorArgs;


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;
use std::sync::Arc;

use context::CoreContext;
use context::PerfCounters;
use futures_stats::FutureStats;
use futures_stats::StreamStats;
use hgproto::GettreepackArgs;
use iterhelpers::chunk_by_accumulation;
use mercurial_types::HgManifestId;
use mononoke_types::MPath;
use scuba_ext::MononokeScubaSampleBuilder;
use scuba_ext::ScubaValue;
use scuba_ext::ScubaVerbosityLevel;

const COLUMN_SIZE_LIMIT: usize = 500_1000;
const FULL_ARGS_LOG_TAG: &str = "Full Command Args";

#[derive(Copy, Clone)]
pub enum CommandStats<'a> {
    Future(&'a FutureStats),
    Stream(&'a StreamStats),
}

impl<'a> CommandStats<'a> {
    fn insert_stats<'b>(
        &self,
        scuba: &'b mut MononokeScubaSampleBuilder,
    ) -> &'b mut MononokeScubaSampleBuilder {
        match self {
            Self::Future(stats) => scuba.add_future_stats(stats),
            Self::Stream(stats) => scuba.add_stream_stats(stats),
        }
    }
}

impl<'a> From<&'a FutureStats> for CommandStats<'a> {
    fn from(stats: &'a FutureStats) -> Self {
        Self::Future(stats)
    }
}

impl<'a> From<&'a StreamStats> for CommandStats<'a> {
    fn from(stats: &'a StreamStats) -> Self {
        Self::Stream(stats)
    }
}

/// Logs wireproto requests both to scuba.
/// Scuba logs are used for analysis of performance.
#[must_use = "A CommandLogger does not do anything if you don't use it"]
pub struct CommandLogger {
    inner: ScubaOnlyCommandLogger,
}

impl CommandLogger {
    pub fn new(ctx: CoreContext, request_perf_counters: Arc<PerfCounters>) -> Self {
        let inner = ScubaOnlyCommandLogger::new(ctx, request_perf_counters);

        Self { inner }
    }

    /// Opts-out of replaying the wireproto request on the shadow tier.
    /// Returns a simplified logger that does Scuba-only logging.
    pub fn without_wireproto(self) -> ScubaOnlyCommandLogger {
        self.inner
    }

    pub fn finalize_command<'a>(self, stats: impl Into<CommandStats<'a>>) {
        self.inner.log_command_processed(stats.into());
    }

    pub fn add_scuba_extra(&mut self, k: impl Into<String>, v: impl Into<ScubaValue>) {
        self.inner.add_scuba_extra(k, v);
    }

    pub fn add_trimmed_scuba_extra(&mut self, k: impl Into<String>, args: &serde_json::Value) {
        if let Ok(args) = serde_json::to_string(args) {
            let limit = ::std::cmp::min(args.len(), 1000);
            self.add_scuba_extra(k, &args[..limit]);
        }
    }
}

#[must_use = "A CommandLogger does not do anything if you don't use it"]
pub struct ScubaOnlyCommandLogger {
    ctx: CoreContext,
    request_perf_counters: Arc<PerfCounters>,
    extra: HashMap<String, ScubaValue>,
}

impl ScubaOnlyCommandLogger {
    fn new(ctx: CoreContext, request_perf_counters: Arc<PerfCounters>) -> Self {
        Self {
            ctx,
            request_perf_counters,
            extra: HashMap::new(),
        }
    }

    pub fn finalize_command<'a>(self, stats: impl Into<CommandStats<'a>>) {
        self.log_command_processed(stats.into());
    }

    pub fn add_scuba_extra(&mut self, k: impl Into<String>, v: impl Into<ScubaValue>) {
        self.extra.insert(k.into(), v.into());
    }

    fn log_command_processed(self, stats: CommandStats) {
        self.request_perf_counters
            .update_with_counters(self.ctx.perf_counters().top());
        let mut scuba = self.ctx.scuba().clone();
        stats.insert_stats(&mut scuba);
        self.ctx.perf_counters().insert_perf_counters(&mut scuba);

        for (k, v) in self.extra.into_iter() {
            scuba.add(k, v);
        }

        scuba.log_with_msg("Command processed", None);
    }
}

fn debug_format_directory<T: AsRef<[u8]>>(directory: &T) -> String {
    String::from_utf8_lossy(&hgproto::batch::escape(directory)).to_string()
}

pub fn debug_format_manifest(node: &HgManifestId) -> String {
    format!("{}", node)
}

pub fn debug_format_path(path: &Option<MPath>) -> String {
    match path {
        Some(p) => format!("{}", p),
        None => String::new(),
    }
}

fn greater_than_column_size(a: usize) -> bool {
    a > COLUMN_SIZE_LIMIT
}

pub fn log_gettreepack_params_verbose(ctx: &CoreContext, args: &GettreepackArgs) {
    if !ctx
        .scuba()
        .should_log_with_level(ScubaVerbosityLevel::Verbose)
    {
        return;
    }

    let mut sample = ctx.scuba().clone();
    sample.add("gettreepack_rootdir", debug_format_path(&args.rootdir));

    if let Some(depth) = args.depth {
        sample.add("gettreepack_depth", depth);
    }
    let msg = "gettreepack rootdir and depth".to_string();
    sample.log_with_msg_verbose(FULL_ARGS_LOG_TAG, msg);

    let mfnode_chunks = chunk_by_accumulation(
        args.mfnodes.iter().map(debug_format_manifest),
        0,
        |acc, s| acc + s.len(),
        greater_than_column_size,
    );

    let msg = "gettreepack mfnodes".to_string();
    for (i, mfnode_chunk) in mfnode_chunks.into_iter().enumerate() {
        ctx.scuba()
            .clone()
            .add("gettreepack_mfnode_chunk_idx", i)
            .add("gettreepack_mfnode_chunk", mfnode_chunk)
            .log_with_msg_verbose(FULL_ARGS_LOG_TAG, msg.clone());
    }

    let basemfnode_chunks = chunk_by_accumulation(
        args.basemfnodes.iter().map(debug_format_manifest),
        0,
        |acc, s| acc + s.len(),
        greater_than_column_size,
    );

    let msg = "gettreepack basemfnodes".to_string();
    for (i, basemfnode_chunk) in basemfnode_chunks.into_iter().enumerate() {
        ctx.scuba()
            .clone()
            .add("gettreepack_basemfnode_chunk_idx", i)
            .add("gettreepack_basemfnode_chunk", basemfnode_chunk)
            .log_with_msg_verbose(FULL_ARGS_LOG_TAG, msg.clone());
    }

    let directory_chunks = chunk_by_accumulation(
        args.directories.iter().map(debug_format_directory),
        0,
        |acc, s| acc + s.len(),
        greater_than_column_size,
    );
    let msg = "gettreepack directories".to_string();
    for (i, directory_chunk) in directory_chunks.into_iter().enumerate() {
        ctx.scuba()
            .clone()
            .add("gettreepack_directory_chunk_idx", i)
            .add("gettreepack_directory_chunk", directory_chunk)
            .log_with_msg_verbose(FULL_ARGS_LOG_TAG, msg.clone());
    }
}

pub fn log_getpack_params_verbose(ctx: &CoreContext, encoded_params: &[(String, Vec<String>)]) {
    if !ctx
        .scuba()
        .should_log_with_level(ScubaVerbosityLevel::Verbose)
    {
        return;
    }

    for (mpath, filenodes) in encoded_params {
        let filenode_chunks = chunk_by_accumulation(
            filenodes.iter().cloned(),
            0,
            |acc, s| acc + s.len(),
            greater_than_column_size,
        );
        for (i, filenode_chunk) in filenode_chunks.into_iter().enumerate() {
            ctx.scuba()
                .clone()
                .add("getpack_mpath", mpath.clone())
                .add("getpack_filenode_chunk_idx", i)
                .add("getpack_filenode_chunk", filenode_chunk)
                .log_with_msg_verbose(FULL_ARGS_LOG_TAG, None);
        }
    }
}