/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::any::Any;
use std::collections::HashMap;
use std::sync::Arc;

use anyhow::format_err;
use anyhow::Error;
use async_trait::async_trait;
use changesets::Changesets;
use context::CoreContext;
use mononoke_types::ChangesetId;
use mononoke_types::Generation;
use mononoke_types::RepositoryId;

mod prefetched;
pub use prefetched::PrefetchedChangesetsFetcher;

#[cfg(test)]
mod tests;

/// Trait that knows how to fetch DAG info about commits. Primary user is revsets
/// Concrete implementation may add more efficient caching logic to make request faster
#[facet::facet]
#[async_trait]
pub trait ChangesetFetcher: Send + Sync + 'static {
    async fn get_generation_number(
        &self,
        ctx: &CoreContext,
        cs_id: ChangesetId,
    ) -> Result<Generation, Error>;

    async fn get_parents(
        &self,
        ctx: &CoreContext,
        cs_id: ChangesetId,
    ) -> Result<Vec<ChangesetId>, Error>;

    fn get_stats(&self) -> HashMap<String, Box<dyn Any>> {
        HashMap::new()
    }
}

/// Simplest ChangesetFetcher implementation which is just a wrapper around `Changesets` object
pub struct SimpleChangesetFetcher {
    changesets: Arc<dyn Changesets>,
}

impl SimpleChangesetFetcher {
    pub fn new(changesets: Arc<dyn Changesets>, repo_id: RepositoryId) -> Self {
        assert_eq!(changesets.repo_id(), repo_id);
        Self { changesets }
    }
}

#[async_trait]
impl ChangesetFetcher for SimpleChangesetFetcher {
    async fn get_generation_number(
        &self,
        ctx: &CoreContext,
        cs_id: ChangesetId,
    ) -> Result<Generation, Error> {
        let maybe_cs = self.changesets.get(ctx, cs_id).await?;
        let cs = maybe_cs.ok_or_else(|| format_err!("{} not found", cs_id))?;
        Ok(Generation::new(cs.gen))
    }

    async fn get_parents(
        &self,
        ctx: &CoreContext,
        cs_id: ChangesetId,
    ) -> Result<Vec<ChangesetId>, Error> {
        let maybe_cs = self.changesets.get(ctx, cs_id).await?;
        let cs = maybe_cs.ok_or_else(|| format_err!("{} not found", cs_id))?;
        Ok(cs.parents)
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;
use std::sync::Arc;

use anyhow::bail;
use anyhow::format_err;
use anyhow::Error;
use anyhow::Result;
use changesets::ChangesetEntry;
use changesets::Changesets;
use context::CoreContext;
use futures::stream::Stream;
use futures::stream::TryStreamExt;
use mononoke_types::ChangesetId;
use mononoke_types::Generation;
use mononoke_types::RepositoryId;

use crate::ChangesetFetcher;

/// A [`ChangesetFetcher`] that uses prefetched data as an optimization to
/// speed up fetching by storing many entries in memory, rather than querying
/// changeset by changeset and relying on caching
pub struct PrefetchedChangesetsFetcher {
    changesets: Arc<dyn Changesets>,
    prefetched: HashMap<ChangesetId, ChangesetEntry>,
}

impl PrefetchedChangesetsFetcher {
    /// Construct with a fetcher to get from the backing store, and a prefetched set
    /// This can come directly from bulkops::PublicChangesetBulkFetch::fetch
    /// or from a deserialised file via `futures::stream::iter`
    pub async fn new(
        repo_id: RepositoryId,
        changesets: Arc<dyn Changesets>,
        prefetched: impl Stream<Item = Result<ChangesetEntry, Error>>,
    ) -> Result<Self> {
        if changesets.repo_id() != repo_id {
            bail!("Changesets object and supplied repo ID do not match");
        }
        let prefetched = prefetched
            .and_then(|entry| async move {
                if entry.repo_id != repo_id {
                    bail!("Prefetched changesets and supplied repo ID do not match");
                }
                Ok((entry.cs_id, entry))
            })
            .try_collect()
            .await?;
        Ok(Self {
            changesets,
            prefetched,
        })
    }

    pub async fn clone_with_extension(
        &self,
        extra: impl Stream<Item = Result<ChangesetEntry, Error>>,
    ) -> Result<Self> {
        let repo_id = self.changesets.repo_id();
        let mut prefetched = self.prefetched.clone();
        let extra: HashMap<_, _> = extra
            .and_then(|entry| async move {
                if entry.repo_id != repo_id {
                    bail!("Prefetched changesets and supplied repo ID do not match");
                }
                Ok((entry.cs_id, entry))
            })
            .try_collect()
            .await?;
        prefetched.extend(extra);
        Ok(Self {
            changesets: self.changesets.clone(),
            prefetched,
        })
    }

    async fn get_cs_entry(&self, ctx: &CoreContext, cs_id: ChangesetId) -> Result<ChangesetEntry> {
        let prefetched_entry = self.prefetched.get(&cs_id);
        if let Some(prefetched_entry) = prefetched_entry {
            Ok(prefetched_entry.clone())
        } else {
            let maybe_cs = self.changesets.get(ctx, cs_id).await?;
            maybe_cs.ok_or_else(|| format_err!("{} not found", cs_id))
        }
    }
}

#[async_trait::async_trait]
impl ChangesetFetcher for PrefetchedChangesetsFetcher {
    async fn get_generation_number(
        &self,
        ctx: &CoreContext,
        cs_id: ChangesetId,
    ) -> Result<Generation, Error> {
        let cs = self.get_cs_entry(ctx, cs_id).await?;
        Ok(Generation::new(cs.gen))
    }

    async fn get_parents(
        &self,
        ctx: &CoreContext,
        cs_id: ChangesetId,
    ) -> Result<Vec<ChangesetId>, Error> {
        let cs = self.get_cs_entry(ctx, cs_id).await?;
        Ok(cs.parents)
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::sync::Arc;

use anyhow::Result;
use changesets::ChangesetEntry;
use changesets::ChangesetInsert;
use changesets::Changesets;
use changesets_impl::SqlChangesetsBuilder;
use context::CoreContext;
use fbinit::FacebookInit;
use futures::stream;
use mononoke_types::Generation;
use mononoke_types_mocks::changesetid::*;
use mononoke_types_mocks::repo::*;
use rendezvous::RendezVousOptions;
use sql_construct::SqlConstruct;

use super::ChangesetFetcher;
use super::PrefetchedChangesetsFetcher;

#[fbinit::test]
async fn test_prefetched_fetcher_no_prefetching(fb: FacebookInit) -> Result<()> {
    let ctx = CoreContext::test_mock(fb);
    let ctx = &ctx;

    // Create our mock changesets with just ONES_CSID
    let changesets = Arc::new(
        SqlChangesetsBuilder::with_sqlite_in_memory()?
            .build(RendezVousOptions::for_test(), REPO_ZERO),
    );
    let row = ChangesetInsert {
        cs_id: ONES_CSID,
        parents: vec![],
    };
    changesets.add(ctx, row).await?;

    let prefetched_fetcher =
        PrefetchedChangesetsFetcher::new(REPO_ZERO, changesets, stream::empty()).await?;

    // Confirm the generation number and parents for ONES_CSID
    assert_eq!(
        prefetched_fetcher
            .get_generation_number(ctx, ONES_CSID)
            .await?,
        Generation::new(1)
    );
    assert_eq!(prefetched_fetcher.get_parents(ctx, ONES_CSID).await?, []);

    // Check that TWOS_CSID is an error
    assert!(
        prefetched_fetcher
            .get_parents(ctx, TWOS_CSID)
            .await
            .is_err()
    );

    Ok(())
}

#[fbinit::test]
async fn test_prefetched_fetcher_no_overlap(fb: FacebookInit) -> Result<()> {
    let ctx = CoreContext::test_mock(fb);
    let ctx = &ctx;

    // Create our mock changesets with just ONES_CSID
    let changesets = Arc::new(
        SqlChangesetsBuilder::with_sqlite_in_memory()?
            .build(RendezVousOptions::for_test(), REPO_ZERO),
    );
    let row = ChangesetInsert {
        cs_id: ONES_CSID,
        parents: vec![],
    };
    changesets.add(ctx, row).await?;

    let cs2 = Ok(ChangesetEntry {
        repo_id: REPO_ZERO,
        cs_id: TWOS_CSID,
        parents: vec![ONES_CSID],
        gen: 5,
    });

    let prefetched_fetcher =
        PrefetchedChangesetsFetcher::new(REPO_ZERO, changesets, stream::iter(vec![cs2])).await?;

    // Confirm the generation number and parents for ONES_CSID
    assert_eq!(
        prefetched_fetcher
            .get_generation_number(ctx, ONES_CSID)
            .await?,
        Generation::new(1)
    );
    assert_eq!(prefetched_fetcher.get_parents(ctx, ONES_CSID).await?, []);

    // Confirm the generation number and parents for TWOS_CSID
    assert_eq!(
        prefetched_fetcher
            .get_generation_number(ctx, TWOS_CSID)
            .await?,
        Generation::new(5)
    );
    assert_eq!(
        prefetched_fetcher.get_parents(ctx, TWOS_CSID).await?,
        [ONES_CSID]
    );

    // Check that THREES_CSID is an error
    assert!(
        prefetched_fetcher
            .get_parents(ctx, THREES_CSID)
            .await
            .is_err()
    );

    Ok(())
}

#[fbinit::test]
async fn test_prefetched_fetcher_overlap(fb: FacebookInit) -> Result<()> {
    let ctx = CoreContext::test_mock(fb);
    let ctx = &ctx;

    // Create our mock changesets with ONES_CSID and TWOS_CSID
    let changesets = Arc::new(
        SqlChangesetsBuilder::with_sqlite_in_memory()?
            .build(RendezVousOptions::for_test(), REPO_ZERO),
    );
    let row = ChangesetInsert {
        cs_id: ONES_CSID,
        parents: vec![],
    };
    changesets.add(ctx, row).await?;
    let row = ChangesetInsert {
        cs_id: TWOS_CSID,
        parents: vec![ONES_CSID],
    };
    changesets.add(ctx, row).await?;

    let cs2 = Ok(ChangesetEntry {
        repo_id: REPO_ZERO,
        cs_id: TWOS_CSID,
        parents: vec![THREES_CSID],
        gen: 5,
    });

    let prefetched_fetcher =
        PrefetchedChangesetsFetcher::new(REPO_ZERO, changesets, stream::iter(vec![cs2])).await?;

    // Confirm the generation number and parents for TWOS_CSID comes from the prefetch list
    assert_eq!(
        prefetched_fetcher
            .get_generation_number(ctx, TWOS_CSID)
            .await?,
        Generation::new(5)
    );
    assert_eq!(
        prefetched_fetcher.get_parents(ctx, TWOS_CSID).await?,
        [THREES_CSID]
    );

    // Check that THREES_CSID is an error
    assert!(
        prefetched_fetcher
            .get_parents(ctx, THREES_CSID)
            .await
            .is_err()
    );

    Ok(())
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashSet;
use std::sync::Arc;

use anyhow::Error;
use blobstore::Blobstore;
use context::CoreContext;
use futures::future;
use futures::StreamExt;
use futures::TryFutureExt;
use futures::TryStreamExt;
use manifest::Diff;
use manifest::Entry;
use manifest::ManifestOps;
use mercurial_types::HgFileNodeId;
use mercurial_types::HgManifestId;
use mononoke_types::FileType;
use mononoke_types::MPath;

/// NOTE: To be used only for generating list of files for old, Mercurial format of Changesets.
///
/// This function is used to extract any new files that the given root manifest has provided
/// compared to the provided p1 and p2 parents.
/// A files is considered new when it was not present in neither of parent manifests or it was
/// present, but with a different content.
/// It sorts the returned Vec<MPath> in the order expected by Mercurial.
pub async fn compute_changed_files(
    ctx: CoreContext,
    blobstore: Arc<dyn Blobstore>,
    root: HgManifestId,
    p1: Option<HgManifestId>,
    p2: Option<HgManifestId>,
) -> Result<Vec<MPath>, Error> {
    let files = match (p1, p2) {
        (None, None) => {
            root.list_leaf_entries(ctx, blobstore)
                .map_ok(|(path, _)| path)
                .try_collect()
                .await?
        }
        (Some(manifest), None) | (None, Some(manifest)) => {
            compute_changed_files_pair(ctx, blobstore.clone(), root, manifest).await?
        }
        (Some(p1), Some(p2)) => {
            let changed = future::try_join(
                compute_changed_files_pair(ctx.clone(), blobstore.clone(), root, p1),
                compute_changed_files_pair(ctx.clone(), blobstore.clone(), root, p2),
            )
            .map_ok(|(left, right)| left.intersection(&right).cloned().collect::<Vec<_>>());

            // Mercurial always includes removed files, we need to match this behaviour
            let (ch1, ch2, ch3) = future::try_join3(
                changed,
                compute_removed_files(&ctx, blobstore.clone(), root, Some(p1)),
                compute_removed_files(&ctx, blobstore.clone(), root, Some(p2)),
            )
            .await?;
            ch1.into_iter()
                .chain(ch2.into_iter())
                .chain(ch3.into_iter())
                .collect::<HashSet<_>>()
        }
    };

    let mut files: Vec<MPath> = files.into_iter().collect();
    files.sort_unstable_by(mercurial_mpath_comparator);
    Ok(files)
}

async fn compute_changed_files_pair(
    ctx: CoreContext,
    blobstore: Arc<dyn Blobstore>,
    to: HgManifestId,
    from: HgManifestId,
) -> Result<HashSet<MPath>, Error> {
    from.diff(ctx, blobstore, to)
        .try_filter_map(|diff| async move {
            let (path, entry) = match diff {
                Diff::Added(path, entry) | Diff::Removed(path, entry) => (path, entry),
                Diff::Changed(path, .., entry) => (path, entry),
            };

            match entry {
                Entry::Tree(_) => Ok(None),
                Entry::Leaf(_) => Ok(path),
            }
        })
        .try_collect()
        .await
}

async fn compute_removed_files(
    ctx: &CoreContext,
    blobstore: Arc<dyn Blobstore>,
    child: HgManifestId,
    parent: Option<HgManifestId>,
) -> Result<Vec<MPath>, Error> {
    compute_files_with_status(ctx, blobstore, child, parent, move |diff| match diff {
        Diff::Removed(path, entry) => match entry {
            Entry::Leaf(_) => path,
            Entry::Tree(_) => None,
        },
        _ => None,
    })
    .await
}

async fn compute_files_with_status(
    ctx: &CoreContext,
    blobstore: Arc<dyn Blobstore>,
    child: HgManifestId,
    parent: Option<HgManifestId>,
    filter_map: impl Fn(Diff<Entry<HgManifestId, (FileType, HgFileNodeId)>>) -> Option<MPath>,
) -> Result<Vec<MPath>, Error> {
    let s = match parent {
        Some(parent) => parent.diff(ctx.clone(), blobstore, child).left_stream(),
        None => child
            .list_all_entries(ctx.clone(), blobstore)
            .map_ok(|(path, entry)| Diff::Added(path, entry))
            .right_stream(),
    };

    s.try_filter_map(|e| async { Ok(filter_map(e)) })
        .try_collect()
        .await
}

fn mercurial_mpath_comparator(a: &MPath, b: &MPath) -> ::std::cmp::Ordering {
    a.to_vec().cmp(&b.to_vec())
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_mercurial_mpath_comparator() {
        let mut paths = vec![
            "foo/bar/baz/a.test",
            "foo/bar/baz-boo/a.test",
            "foo-faz/bar/baz/a.test",
        ];

        let mut mpaths: Vec<_> = paths
            .iter()
            .map(|path| MPath::new(path).expect("invalid path"))
            .collect();

        {
            mpaths.sort_unstable();
            let result: Vec<_> = mpaths
                .iter()
                .map(|mpath| String::from_utf8(mpath.to_vec()).unwrap())
                .collect();
            assert!(paths == result);
        }

        {
            paths.sort_unstable();
            mpaths.sort_unstable_by(mercurial_mpath_comparator);
            let result: Vec<_> = mpaths
                .iter()
                .map(|mpath| String::from_utf8(mpath.to_vec()).unwrap())
                .collect();
            assert!(paths == result);
        }
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;
use std::collections::VecDeque;

use anyhow::anyhow;
use anyhow::Context;
use anyhow::Error;
use blobrepo::BlobRepo;
use blobstore::Blobstore;
use blobstore::Loadable;
use bonsai_hg_mapping::BonsaiHgMappingRef;
use changesets::ChangesetsRef;
use cloned::cloned;
use context::CoreContext;
use context::PerfCounterType;
use filenodes::FilenodeInfo;
use filenodes::FilenodeRange;
use filenodes::FilenodeResult;
use filenodes::FilenodesRef;
use futures::future;
use futures::future::try_join;
use futures::stream;
use futures::FutureExt;
use futures::Stream;
use futures::StreamExt;
use futures::TryFutureExt;
use futures::TryStreamExt;
use maplit::hashset;
use mercurial_types::HgBlobEnvelope;
use mercurial_types::HgChangesetId;
use mercurial_types::HgFileHistoryEntry;
use mercurial_types::HgFileNodeId;
use mercurial_types::HgParents;
use mercurial_types::MPath;
use mercurial_types::RepoPath;
use mercurial_types::NULL_CSID;
use mercurial_types::NULL_HASH;
use mononoke_types::ChangesetId;
use repo_blobstore::RepoBlobstoreRef;
use repo_identity::RepoIdentityRef;
use slog::debug;
use stats::prelude::*;
use thiserror::Error;

#[derive(Debug, Error)]
pub enum ErrorKind {
    #[error("internal error: file {0} copied from directory {1}")]
    InconsistentCopyInfo(RepoPath, RepoPath),
    #[error("Filenode is missing: {0} {1}")]
    MissingFilenode(RepoPath, HgFileNodeId),
    #[error("Bonsai cs {0} not found")]
    BonsaiNotFound(ChangesetId),
    #[error("Bonsai changeset not found for hg changeset {0}")]
    BonsaiMappingNotFound(HgChangesetId),
}

pub enum FilenodesRelatedResult {
    Unrelated,
    FirstAncestorOfSecond,
    SecondAncestorOfFirst,
}

define_stats! {
    prefix = "mononoke.file_history";
    too_big: dynamic_timeseries("{}.too_big", (repo: String); Rate, Sum),
}

async fn get_filenode_generation(
    ctx: &CoreContext,
    repo: &BlobRepo,
    repo_path: &RepoPath,
    filenode: HgFileNodeId,
) -> Result<Option<u64>, Error> {
    let filenode_info = match repo
        .filenodes()
        .get_filenode(ctx, repo_path, filenode)
        .await?
        .do_not_handle_disabled_filenodes()?
    {
        Some(a) => a,
        None => return Ok(None),
    };
    let linknode = filenode_info.linknode;
    let bcs_id = match repo
        .bonsai_hg_mapping()
        .get_bonsai_from_hg(ctx, linknode)
        .await?
    {
        Some(a) => a,
        None => return Ok(None),
    };
    let bonsai = repo.changesets().get(ctx, bcs_id).await?;
    Ok(bonsai.map(|b| b.gen))
}

/// Checks if one filenode is ancestor of another
pub async fn check_if_related(
    ctx: CoreContext,
    repo: BlobRepo,
    filenode_a: HgFileNodeId,
    filenode_b: HgFileNodeId,
    path: MPath,
) -> Result<FilenodesRelatedResult, Error> {
    // Use linknodes to identify the older filenode
    let repo_path = RepoPath::file(path.clone())?;
    match try_join(
        get_filenode_generation(&ctx, &repo, &repo_path, filenode_a),
        get_filenode_generation(&ctx, &repo, &repo_path, filenode_b),
    )
    .await?
    {
        // We have linknodes, so know which is the "old" filenode.
        // Halve our data fetches by only fetching full history for the
        // "new" filenode
        (Some(gen_a), Some(gen_b)) => {
            let (old_node, history, relationship) = if gen_a < gen_b {
                (
                    filenode_a,
                    get_file_history(ctx, repo, filenode_b, path, None).await?,
                    FilenodesRelatedResult::FirstAncestorOfSecond,
                )
            } else {
                (
                    filenode_b,
                    get_file_history(ctx, repo, filenode_a, path, None).await?,
                    FilenodesRelatedResult::SecondAncestorOfFirst,
                )
            };

            if let FilenodeResult::Present(history) = history {
                if history.iter().any(|entry| entry.filenode() == &old_node) {
                    Ok(relationship)
                } else {
                    Ok(FilenodesRelatedResult::Unrelated)
                }
            } else {
                Err(anyhow!("filenodes are disabled"))
            }
        }
        // We don't have linknodes, so go down the slow path
        _ => {
            debug!(
                ctx.logger(),
                "No filenodes for parents. Using slow path for ancestry check"
            );
            match try_join(
                get_file_history(ctx.clone(), repo.clone(), filenode_a, path.clone(), None),
                get_file_history(ctx, repo, filenode_b, path, None),
            )
            .await?
            {
                (FilenodeResult::Present(history_a), FilenodeResult::Present(history_b)) => {
                    if history_a
                        .iter()
                        .any(|entry| entry.filenode() == &filenode_b)
                    {
                        Ok(FilenodesRelatedResult::SecondAncestorOfFirst)
                    } else if history_b
                        .iter()
                        .any(|entry| entry.filenode() == &filenode_a)
                    {
                        Ok(FilenodesRelatedResult::FirstAncestorOfSecond)
                    } else {
                        Ok(FilenodesRelatedResult::Unrelated)
                    }
                }
                _ => Err(anyhow!("filenodes are disabled")),
            }
        }
    }
}

/// Same as get_file_history(), but returns incomplete history if filenodes
/// are disabled
pub fn get_file_history_maybe_incomplete(
    ctx: CoreContext,
    repo: BlobRepo,
    filenode: HgFileNodeId,
    path: MPath,
    max_length: Option<u64>,
) -> impl Stream<Item = Result<HgFileHistoryEntry, Error>> {
    get_file_history(
        ctx.clone(),
        repo.clone(),
        filenode,
        path.clone(),
        max_length,
    )
    .and_then({
        cloned!(ctx, path, repo);
        move |file_history_res| {
            match file_history_res {
                FilenodeResult::Present(file_history) => future::ok(file_history).left_future(),
                FilenodeResult::Disabled => async move {
                    // Filenodes are disabled - fetch a single filenode
                    // from a blobstore
                    let path = RepoPath::FilePath(path);
                    let filenode_info = get_filenode_from_envelope(
                        repo.repo_blobstore().clone(),
                        &ctx,
                        &path,
                        filenode,
                        NULL_CSID,
                    )
                    .await?;
                    let filenode = filenode_to_history_entry(filenode, filenode_info, &path)?;
                    Ok(vec![filenode])
                }
                .right_future(),
            }
        }
    })
    .map_ok(|hist| stream::iter(hist.into_iter().map(Ok)))
    .try_flatten_stream()
}

/// Get the history of the file corresponding to the given filenode and path.
pub async fn get_file_history(
    ctx: CoreContext,
    repo: BlobRepo,
    filenode: HgFileNodeId,
    path: MPath,
    max_length: Option<u64>,
) -> Result<FilenodeResult<Vec<HgFileHistoryEntry>>, Error> {
    // Prefetch and cache filenode information. Performing these fetches in bulk upfront
    // prevents an excessive number of DB roundtrips when constructing file history.
    let prefetched_res = repo
        .filenodes()
        .get_all_filenodes_maybe_stale(&ctx, &RepoPath::FilePath(path.clone()), max_length)
        .await?;

    match prefetched_res {
        FilenodeResult::Present(FilenodeRange::Filenodes(prefetched)) => {
            let prefetched: HashMap<_, _> = prefetched
                .into_iter()
                .map(|filenode| (filenode.filenode, filenode))
                .collect();
            let history = get_file_history_using_prefetched(
                ctx, repo, filenode, path, max_length, prefetched,
            )
            .try_collect()
            .await?;
            Ok(FilenodeResult::Present(history))
        }
        FilenodeResult::Present(FilenodeRange::TooBig) => {
            ctx.perf_counters()
                .increment_counter(PerfCounterType::FilenodesTooBigHistory);
            STATS::too_big.add_value(1, (repo.repo_identity().name().to_string(),));
            let history = get_file_history_using_prefetched(
                ctx,
                repo,
                filenode,
                path,
                max_length,
                HashMap::new(),
            )
            .try_collect()
            .await?;
            Ok(FilenodeResult::Present(history))
        }
        FilenodeResult::Disabled => Ok(FilenodeResult::Disabled),
    }
}

/// Get the history of the file at the specified path, using the given
/// prefetched history map as a cache to speed up the operation.
///
/// FIXME: max_legth parameter is not necessary. We can use .take() method on the stream
/// i.e. get_file_history_using_prefetched().take(max_length)
fn get_file_history_using_prefetched(
    ctx: CoreContext,
    repo: BlobRepo,
    startnode: HgFileNodeId,
    path: MPath,
    max_length: Option<u64>,
    prefetched_history: HashMap<HgFileNodeId, FilenodeInfo>,
) -> impl Stream<Item = Result<HgFileHistoryEntry, Error>> {
    if startnode == HgFileNodeId::new(NULL_HASH) {
        return stream::empty().left_stream();
    }

    let mut startstate = VecDeque::new();
    startstate.push_back(startnode);
    let seen_nodes = hashset! {startnode};
    let path = RepoPath::FilePath(path);

    struct BfsContext {
        ctx: CoreContext,
        repo: BlobRepo,
        path: RepoPath,
        prefetched_history: HashMap<HgFileNodeId, FilenodeInfo>,
    }

    let bfs_context = BfsContext {
        ctx,
        repo,
        path,
        prefetched_history,
    };

    // TODO: There is probably another thundering herd problem here. If we change a file twice,
    // then the original cached value will be reused, and we'll keep going back to getting the
    // filenode individualy (perhaps not the end of the world?).
    stream::try_unfold(
        (bfs_context, startstate, seen_nodes, 0),
        move |(bfs_context, mut nodes, mut seen_nodes, length)| async move {
            match max_length {
                Some(max_length) if length >= max_length => return Ok(None),
                _ => {}
            }

            let node = match nodes.pop_front() {
                Some(node) => node,
                None => {
                    return Ok(None);
                }
            };

            let filenode = if let Some(filenode) = bfs_context.prefetched_history.get(&node) {
                filenode.clone()
            } else {
                get_maybe_missing_filenode(
                    &bfs_context.ctx,
                    &bfs_context.repo,
                    &bfs_context.path,
                    node,
                )
                .await?
            };

            let p1 = filenode.p1.map(|p| p.into_nodehash());
            let p2 = filenode.p2.map(|p| p.into_nodehash());
            let parents = HgParents::new(p1, p2);
            let entry = filenode_to_history_entry(node, filenode, &bfs_context.path)?;

            nodes.extend(
                parents
                    .into_iter()
                    .map(HgFileNodeId::new)
                    .filter(|p| seen_nodes.insert(*p)),
            );

            Ok(Some((entry, (bfs_context, nodes, seen_nodes, length + 1))))
        },
    )
    .right_stream()
}

pub fn filenode_to_history_entry(
    node: HgFileNodeId,
    filenode: FilenodeInfo,
    path: &RepoPath,
) -> Result<HgFileHistoryEntry, Error> {
    let p1 = filenode.p1.map(|p| p.into_nodehash());
    let p2 = filenode.p2.map(|p| p.into_nodehash());
    let parents = HgParents::new(p1, p2);

    let linknode = filenode.linknode;

    let copyfrom = match filenode.copyfrom {
        Some((RepoPath::FilePath(frompath), node)) => Some((frompath, node)),
        Some((frompath, _)) => {
            return Err(ErrorKind::InconsistentCopyInfo(path.clone(), frompath).into());
        }
        None => None,
    };

    Ok(HgFileHistoryEntry::new(node, parents, linknode, copyfrom))
}

async fn get_maybe_missing_filenode(
    ctx: &CoreContext,
    repo: &BlobRepo,
    path: &RepoPath,
    node: HgFileNodeId,
) -> Result<FilenodeInfo, Error> {
    let filenode_res = repo.filenodes().get_filenode(ctx, path, node).await?;
    match filenode_res {
        FilenodeResult::Present(Some(filenode)) => Ok(filenode),
        FilenodeResult::Present(None) | FilenodeResult::Disabled => {
            // The filenode couldn't be found.  This may be because it is a
            // draft node, which doesn't get stored in the database or because
            // filenodes were intentionally disabled.  Attempt
            // to reconstruct the filenode from the envelope.  Use `NULL_CSID`
            // to indicate a draft or missing linknode.
            get_filenode_from_envelope(repo.repo_blobstore().clone(), ctx, path, node, NULL_CSID)
                .await
        }
    }
}

async fn get_filenode_from_envelope(
    blobstore: impl Blobstore + 'static,
    ctx: &CoreContext,
    path: &RepoPath,
    node: HgFileNodeId,
    linknode: HgChangesetId,
) -> Result<FilenodeInfo, Error> {
    let envelope = node.load(ctx, &blobstore).await.with_context({
        cloned!(path);
        move || format!("While fetching filenode for {} {}", path, node)
    })?;
    let (p1, p2) = envelope.parents();
    let copyfrom = envelope
        .get_copy_info()
        .with_context({
            cloned!(path);
            move || format!("While parsing copy information for {} {}", path, node)
        })?
        .map(|(path, node)| (RepoPath::FilePath(path), node));
    Ok(FilenodeInfo {
        filenode: node,
        p1,
        p2,
        copyfrom,
        linknode,
    })
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

pub mod changed_files;
pub mod file_history;


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::fmt;

use ascii::AsciiString;
use mercurial_types::blobs::HgBlobChangeset;
use mercurial_types::HgBlob;
use mercurial_types::HgChangesetId;
use mercurial_types::HgFileNodeId;
use mercurial_types::HgManifestId;
use mercurial_types::HgNodeHash;
use mercurial_types::HgParents;
use mercurial_types::MPath;
use mercurial_types::RepoPath;
use mercurial_types::Type;
use mononoke_types::hash::Sha256;
use mononoke_types::ChangesetId;
use mononoke_types::FileType;
use thiserror::Error;

#[derive(Debug)]
pub enum StateOpenError {
    Heads,
    Bookmarks,
    Changesets,
    Filenodes,
    BonsaiGitMapping,
    BonsaiGlobalrevMapping,
    BonsaiSvnrevMapping,
    BonsaiHgMapping,
    Phases,
    HgMutationStore,
    SegmentedChangelog,
}

impl fmt::Display for StateOpenError {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            StateOpenError::Heads => write!(f, "heads"),
            StateOpenError::Bookmarks => write!(f, "bookmarks"),
            StateOpenError::Changesets => write!(f, "changesets"),
            StateOpenError::Filenodes => write!(f, "filenodes"),
            StateOpenError::BonsaiGitMapping => write!(f, "bonsai_git_mapping"),
            StateOpenError::BonsaiGlobalrevMapping => write!(f, "bonsai_globalrev_mapping"),
            StateOpenError::BonsaiSvnrevMapping => write!(f, "bonsai_svnrev_mapping"),
            StateOpenError::BonsaiHgMapping => write!(f, "bonsai_hg_mapping"),
            StateOpenError::Phases => write!(f, "phases"),
            StateOpenError::HgMutationStore => write!(f, "hg_mutation_store"),
            StateOpenError::SegmentedChangelog => write!(f, "segmented_changelog"),
        }
    }
}

#[derive(Debug, Error)]
pub enum ErrorKind {
    #[error("Error while opening state for {0}")]
    StateOpen(StateOpenError),
    #[error("Node id {0} is missing")]
    NodeMissing(HgNodeHash),
    #[error("Content missing nodeid {0}")]
    ContentMissing(HgNodeHash),
    #[error("Error while deserializing file contents retrieved from key '{0}'")]
    FileContentsDeserializeFailed(String),
    #[error("Content blob missing for id: {0}")]
    ContentBlobByAliasMissing(Sha256),
    #[error("Uploaded blob is incomplete {0:?}")]
    BadUploadBlob(HgBlob),
    #[error("HgParents are not in blob store {0:?}")]
    ParentsUnknown(HgParents),
    #[error("Serialization of node failed {0} ({1})")]
    SerializationFailed(HgNodeHash, bincode::Error),
    #[error("Root manifest is not a manifest (type {0})")]
    BadRootManifest(HgManifestId),
    #[error("Manifest type {0} does not match uploaded type {1}")]
    ManifestTypeMismatch(Type, Type),
    #[error("Node generation failed for unknown reason")]
    NodeGenerationFailed,
    #[error("Path {0} appears multiple times in manifests")]
    DuplicateEntry(RepoPath),
    #[error("Duplicate manifest hash {0}")]
    DuplicateManifest(HgNodeHash),
    #[error("Missing entries in new changeset {0}")]
    MissingEntries(HgNodeHash),
    #[error("Filenode is missing: {0} {1}")]
    MissingFilenode(RepoPath, HgFileNodeId),
    #[error("Some manifests do not exist")]
    MissingManifests,
    #[error("Expected {0} to be a manifest, found a {1} instead")]
    NotAManifest(HgNodeHash, Type),
    #[error(
        "Inconsistent node hash for changeset: provided: {0}, \
         computed: {1} for blob: {2:#?}"
    )]
    InconsistentChangesetHash(HgNodeHash, HgNodeHash, HgBlobChangeset),
    #[error("Bookmark {0} does not exist")]
    BookmarkNotFound(AsciiString),
    #[error("Unresolved conflict at {0} with parents: {1:?}")]
    UnresolvedConflicts(MPath, Vec<(FileType, HgFileNodeId)>),
    #[error("Manifest without parents did not get changed by a BonsaiChangeset")]
    UnchangedManifest,
    #[error("Saving empty manifest which is not a root: {0}")]
    SavingHgEmptyManifest(RepoPath),
    #[error("Trying to merge a manifest with two existing parents p1 {0} and p2 {1}")]
    ManifestAlreadyAMerge(HgNodeHash, HgNodeHash),
    #[error("Path not found: {0}")]
    PathNotFound(MPath),
    #[error("Remove called on non-directory")]
    NotADirectory,
    #[error("Empty file path")]
    EmptyFilePath,
    #[error("Memory manifest conflict can not contain single entry")]
    SingleEntryConflict,
    #[error("Cannot find cache pool {0}")]
    MissingCachePool(String),
    #[error("Bonsai cs {0} not found")]
    BonsaiNotFound(ChangesetId),
    #[error("Bonsai changeset not found for hg changeset {0}")]
    BonsaiMappingNotFound(HgChangesetId),
    #[error("Root path wasn't expected at this context")]
    UnexpectedRootPath,
    #[error(
        "Incorrect copy info: not found a file version {from_path} {from_node} the file {to_path} {to_node} was copied from"
    )]
    IncorrectCopyInfo {
        from_path: MPath,
        from_node: HgFileNodeId,
        to_path: MPath,
        to_node: HgFileNodeId,
    },
    #[error(
        "CaseConflict: the changes introduced by this commit have conflicting case. The first offending path is '{0}', and conflicted with '{1}'. Resolve the conflict."
    )]
    InternalCaseConflict(MPath, MPath),
    #[error(
        "CaseConflict: the changes introduced by this commit conflict with existing files in the repository. The first conflicting path in this commit was '{0}', and conflicted with '{1}' in the repository. Resolve the conflict."
    )]
    ExternalCaseConflict(MPath, MPath),
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::sync::Arc;

use blobrepo::BlobRepo;
use blobrepo::BlobRepoInner;
use blobstore::Blobstore;
use bonsai_hg_mapping::ArcBonsaiHgMapping;
use cacheblob::LeaseOps;
use changeset_fetcher::SimpleChangesetFetcher;
use changesets::ArcChangesets;
use filenodes::ArcFilenodes;
use repo_blobstore::RepoBlobstore;

/// Create new instance of implementing object with overridden field of specified type.
///
/// This override can be very dangerous, it should only be used in unittest, or if you
/// really know what you are doing.
pub trait DangerousOverride<T> {
    fn dangerous_override<F>(&self, modify: F) -> Self
    where
        F: FnOnce(T) -> T;
}

impl<T> DangerousOverride<T> for BlobRepo
where
    BlobRepoInner: DangerousOverride<T>,
{
    fn dangerous_override<F>(&self, modify: F) -> Self
    where
        F: FnOnce(T) -> T,
    {
        let inner = (*self.inner()).clone().dangerous_override(modify);
        BlobRepo::from_inner_dangerous(inner)
    }
}

impl DangerousOverride<Arc<dyn LeaseOps>> for BlobRepoInner {
    fn dangerous_override<F>(&self, modify: F) -> Self
    where
        F: FnOnce(Arc<dyn LeaseOps>) -> Arc<dyn LeaseOps>,
    {
        let derived_data_lease = modify(self.repo_derived_data.lease().clone());
        let repo_derived_data = Arc::new(
            self.repo_derived_data
                .with_replaced_lease(derived_data_lease),
        );
        Self {
            repo_derived_data,
            ..self.clone()
        }
    }
}

impl DangerousOverride<Arc<dyn Blobstore>> for BlobRepoInner {
    fn dangerous_override<F>(&self, modify: F) -> Self
    where
        F: FnOnce(Arc<dyn Blobstore>) -> Arc<dyn Blobstore>,
    {
        let blobstore = RepoBlobstore::new_with_wrapped_inner_blobstore(
            self.repo_blobstore.as_ref().clone(),
            modify,
        );
        let repo_derived_data = Arc::new(
            self.repo_derived_data
                .with_replaced_blobstore(blobstore.clone()),
        );
        let repo_blobstore = Arc::new(blobstore);
        Self {
            repo_blobstore,
            repo_derived_data,
            ..self.clone()
        }
    }
}

impl DangerousOverride<ArcChangesets> for BlobRepoInner {
    fn dangerous_override<F>(&self, modify: F) -> Self
    where
        F: FnOnce(ArcChangesets) -> ArcChangesets,
    {
        let changesets = modify(self.changesets.clone());
        let changeset_fetcher = Arc::new(SimpleChangesetFetcher::new(
            changesets.clone(),
            self.repo_identity.id(),
        ));
        let repo_derived_data = Arc::new(
            self.repo_derived_data
                .with_replaced_changesets(changesets.clone()),
        );

        Self {
            changesets,
            changeset_fetcher,
            repo_derived_data,
            ..self.clone()
        }
    }
}

impl DangerousOverride<ArcFilenodes> for BlobRepoInner {
    fn dangerous_override<F>(&self, modify: F) -> Self
    where
        F: FnOnce(ArcFilenodes) -> ArcFilenodes,
    {
        let filenodes = modify(self.filenodes.clone());
        let repo_derived_data = Arc::new(
            self.repo_derived_data
                .with_replaced_filenodes(filenodes.clone()),
        );
        Self {
            filenodes,
            repo_derived_data,
            ..self.clone()
        }
    }
}

impl DangerousOverride<ArcBonsaiHgMapping> for BlobRepoInner {
    fn dangerous_override<F>(&self, modify: F) -> Self
    where
        F: FnOnce(ArcBonsaiHgMapping) -> ArcBonsaiHgMapping,
    {
        let bonsai_hg_mapping = modify(self.bonsai_hg_mapping.clone());
        let repo_derived_data = Arc::new(
            self.repo_derived_data
                .with_replaced_bonsai_hg_mapping(bonsai_hg_mapping.clone()),
        );
        Self {
            bonsai_hg_mapping,
            repo_derived_data,
            ..self.clone()
        }
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::sync::Arc;

use anyhow::Result;
use async_trait::async_trait;
use blobstore::BlobCopier;
use blobstore::Blobstore;
use blobstore::BlobstoreGetData;
use blobstore::BlobstoreIsPresent;
use blobstore::GenericBlobstoreCopier;
use context::CoreContext;
use mononoke_types::BlobstoreBytes;
use mononoke_types::RepositoryId;
use prefixblob::PrefixBlobstore;
use redactedblobstore::RedactedBlobs;
use redactedblobstore::RedactedBlobstore;
use redactedblobstore::RedactedBlobstoreConfig;
use scuba_ext::MononokeScubaSampleBuilder;

/// RedactedBlobstore should be part of every blobstore since it is a layer
/// which adds security by preventing users to access sensitive content.

/// Making PrefixBlobstore part of every blobstore does two things:
/// 1. It ensures that the prefix applies first, which is important for shared caches like
///    memcache.
/// 2. It ensures that all possible blobrepos use a prefix.
type RepoBlobstoreStack<T> = RedactedBlobstore<PrefixBlobstore<T>>;

// NOTE: We parametize AbstractRepoBlobstore over T instead of explicitly using Arc<dyn Blobstore>
// so that even if we were to add a blobstore to the RepoBlobstoreStack that actually is a Arc<dyn
// Blobstore>, then we cannot accidentally forget to unwrap it below (since we wouldn't get a T
// back).
#[derive(Clone, Debug)]
pub struct AbstractRepoBlobstore<T>(RepoBlobstoreStack<T>);

impl<T: Blobstore + Clone> AbstractRepoBlobstore<T> {
    pub fn as_parts(&self) -> (T, RedactedBlobstoreConfig, String) {
        let (blobstore, redacted_blobstore_config) = self.0.as_parts();
        let prefix = blobstore.prefix();

        (blobstore.into_inner(), redacted_blobstore_config, prefix)
    }
}

#[facet::facet]
#[derive(Clone, Debug)]
pub struct RepoBlobstore(AbstractRepoBlobstore<Arc<dyn Blobstore>>);

impl RepoBlobstore {
    pub fn boxed(&self) -> Arc<dyn Blobstore> {
        self.0.0.boxed()
    }

    pub fn new(
        blobstore: Arc<dyn Blobstore>,
        redacted_blobs: Option<Arc<RedactedBlobs>>,
        repoid: RepositoryId,
        scuba_builder: MononokeScubaSampleBuilder,
    ) -> Self {
        let redacted_blobstore_config = RedactedBlobstoreConfig::new(redacted_blobs, scuba_builder);
        Self::build(blobstore, repoid.prefix(), redacted_blobstore_config)
    }

    pub fn new_with_wrapped_inner_blobstore<F>(blobstore: RepoBlobstore, wrapper: F) -> Self
    where
        F: FnOnce(Arc<dyn Blobstore>) -> Arc<dyn Blobstore>,
    {
        let (blobstore, redacted_blobstore_config, prefix) = blobstore.0.as_parts();
        let new_inner_blobstore = wrapper(blobstore);
        Self::build(new_inner_blobstore, prefix, redacted_blobstore_config)
    }

    #[allow(clippy::let_and_return)]
    fn build(
        blobstore: Arc<dyn Blobstore>,
        prefix: String,
        redacted_blobstore_config: RedactedBlobstoreConfig,
    ) -> Self {
        let blobstore = PrefixBlobstore::new(blobstore, prefix);
        let blobstore = RedactedBlobstore::new(blobstore, redacted_blobstore_config);
        let blobstore = RepoBlobstore(AbstractRepoBlobstore(blobstore));

        blobstore
    }

    pub fn copier_to<'a>(&'a self, other: &'a RepoBlobstore) -> RepoBlobstoreCopier<'a> {
        RepoBlobstoreCopier::new(self, other)
    }
}

impl std::fmt::Display for RepoBlobstore {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(f, "RepoBlobstore<{}>", self.0.0)
    }
}

#[async_trait]
impl Blobstore for RepoBlobstore {
    async fn get<'a>(
        &'a self,
        ctx: &'a CoreContext,
        key: &'a str,
    ) -> Result<Option<BlobstoreGetData>> {
        self.0.0.get(ctx, key).await
    }
    async fn put<'a>(
        &'a self,
        ctx: &'a CoreContext,
        key: String,
        value: BlobstoreBytes,
    ) -> Result<()> {
        self.0.0.put(ctx, key, value).await
    }
    async fn is_present<'a>(
        &'a self,
        ctx: &'a CoreContext,
        key: &'a str,
    ) -> Result<BlobstoreIsPresent> {
        self.0.0.is_present(ctx, key).await
    }
    async fn copy<'a>(
        &'a self,
        ctx: &'a CoreContext,
        old_key: &'a str,
        new_key: String,
    ) -> Result<()> {
        self.0.0.copy(ctx, old_key, new_key).await
    }
}

pub enum RepoBlobstoreCopier<'a> {
    Unoptimized(GenericBlobstoreCopier<'a, RepoBlobstore, RepoBlobstore>),
    /// We checked both repo blobstores have the same inner storage, but differ
    /// in prefixes
    Optimized {
        source: &'a PrefixBlobstore<Arc<dyn Blobstore>>,
        target: &'a PrefixBlobstore<Arc<dyn Blobstore>>,
    },
}

impl<'a> RepoBlobstoreCopier<'a> {
    fn new(source: &'a RepoBlobstore, target: &'a RepoBlobstore) -> Self {
        let inner_source = source.0.0.as_inner_unredacted();
        let inner_target = target.0.0.as_inner_unredacted();
        #[allow(clippy::vtable_address_comparisons)]
        if Arc::ptr_eq(inner_source.as_inner(), inner_target.as_inner()) {
            Self::Optimized {
                source: inner_source,
                target: inner_target,
            }
        } else {
            Self::Unoptimized(GenericBlobstoreCopier { source, target })
        }
    }

    pub fn is_optimized(&self) -> bool {
        matches!(self, RepoBlobstoreCopier::Optimized { .. })
    }
}

#[async_trait]
impl<'a> BlobCopier for RepoBlobstoreCopier<'a> {
    async fn copy(&self, ctx: &CoreContext, key: String) -> Result<()> {
        match self {
            Self::Unoptimized(generic) => generic.copy(ctx, key).await,
            Self::Optimized { source, target } => {
                // same as target.as_inner()
                let inner: &Arc<dyn Blobstore> = source.as_inner();
                let source_key = source.prepend(&key);
                let target_key = target.prepend(&key);
                inner.copy(ctx, &source_key, target_key).await
            }
        }
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

mod repo;

pub use changeset_fetcher::ChangesetFetcher;
pub use filestore::StoreRequest;

pub use crate::repo::save_bonsai_changesets;
pub use crate::repo::AsBlobRepo;
pub use crate::repo::BlobRepo;
pub use crate::repo::BlobRepoInner;


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::sync::Arc;

use anyhow::Error;
use bonsai_git_mapping::BonsaiGitMapping;
use bonsai_globalrev_mapping::BonsaiGlobalrevMapping;
use bonsai_hg_mapping::BonsaiHgMapping;
use bonsai_svnrev_mapping::BonsaiSvnrevMapping;
use bonsai_tag_mapping::BonsaiTagMapping;
use bookmarks::BookmarkUpdateLog;
use bookmarks::Bookmarks;
use changeset_fetcher::ChangesetFetcher;
use changeset_fetcher::SimpleChangesetFetcher;
use changesets::Changesets;
use changesets::ChangesetsRef;
use commit_graph::CommitGraph;
use context::CoreContext;
use ephemeral_blobstore::Bubble;
use filenodes::Filenodes;
use filestore::FilestoreConfig;
use mercurial_mutation::HgMutationStore;
use mononoke_types::BonsaiChangeset;
use mutable_counters::MutableCounters;
use phases::Phases;
use pushrebase_mutation_mapping::PushrebaseMutationMapping;
use repo_blobstore::RepoBlobstore;
use repo_blobstore::RepoBlobstoreRef;
use repo_bookmark_attrs::RepoBookmarkAttrs;
use repo_derived_data::RepoDerivedData;
use repo_identity::RepoIdentity;
use repo_identity::RepoIdentityRef;
use repo_lock::RepoLock;
use repo_permission_checker::RepoPermissionChecker;

// NOTE: this structure and its fields are public to enable `DangerousOverride` functionality
#[facet::container]
#[derive(Clone)]
pub struct BlobRepoInner {
    #[facet]
    pub repo_identity: RepoIdentity,

    #[init(repo_identity.name().to_string())]
    pub reponame: String,

    #[facet]
    pub repo_blobstore: RepoBlobstore,

    #[facet]
    pub changesets: dyn Changesets,

    #[facet]
    pub changeset_fetcher: dyn ChangesetFetcher,

    #[facet]
    pub commit_graph: CommitGraph,

    #[facet]
    pub bonsai_hg_mapping: dyn BonsaiHgMapping,

    #[facet]
    pub bonsai_git_mapping: dyn BonsaiGitMapping,

    #[facet]
    pub bonsai_tag_mapping: dyn BonsaiTagMapping,

    #[facet]
    pub bonsai_globalrev_mapping: dyn BonsaiGlobalrevMapping,

    #[facet]
    pub bonsai_svnrev_mapping: dyn BonsaiSvnrevMapping,

    #[facet]
    pub pushrebase_mutation_mapping: dyn PushrebaseMutationMapping,

    #[facet]
    pub bookmarks: dyn Bookmarks,

    #[facet]
    pub bookmark_update_log: dyn BookmarkUpdateLog,

    #[facet]
    pub phases: dyn Phases,

    #[facet]
    pub filestore_config: FilestoreConfig,

    #[facet]
    pub filenodes: dyn Filenodes,

    #[facet]
    pub hg_mutation_store: dyn HgMutationStore,

    #[facet]
    pub repo_derived_data: RepoDerivedData,

    #[facet]
    pub mutable_counters: dyn MutableCounters,

    #[facet]
    pub permission_checker: dyn RepoPermissionChecker,

    #[facet]
    pub repo_lock: dyn RepoLock,

    #[facet]
    pub repo_bookmark_attrs: RepoBookmarkAttrs,
}

#[facet::container]
#[derive(Clone)]
pub struct BlobRepo {
    #[delegate(
        RepoIdentity,
        RepoBlobstore,
        dyn Changesets,
        dyn ChangesetFetcher,
        CommitGraph,
        dyn BonsaiHgMapping,
        dyn BonsaiGitMapping,
        dyn BonsaiTagMapping,
        dyn BonsaiGlobalrevMapping,
        dyn BonsaiSvnrevMapping,
        dyn PushrebaseMutationMapping,
        dyn Bookmarks,
        dyn BookmarkUpdateLog,
        dyn Phases,
        FilestoreConfig,
        dyn Filenodes,
        dyn HgMutationStore,
        RepoDerivedData,
        dyn MutableCounters,
        dyn RepoPermissionChecker,
        dyn RepoLock,
        RepoBookmarkAttrs,
    )]
    inner: Arc<BlobRepoInner>,
}

impl BlobRepo {
    /// To be used by `DangerousOverride` only
    pub fn inner(&self) -> &Arc<BlobRepoInner> {
        &self.inner
    }

    /// To be used by `DagerouseOverride` only
    pub fn from_inner_dangerous(inner: BlobRepoInner) -> Self {
        Self {
            inner: Arc::new(inner),
        }
    }

    pub fn with_bubble(&self, bubble: Bubble) -> Self {
        let blobstore = bubble.wrap_repo_blobstore(self.repo_blobstore().clone());
        let changesets = Arc::new(bubble.changesets(self));
        let changeset_fetcher =
            SimpleChangesetFetcher::new(changesets.clone(), self.repo_identity().id());
        let new_manager = self
            .inner
            .repo_derived_data
            .manager()
            .clone()
            .for_bubble(bubble, self);
        let repo_derived_data = self.inner.repo_derived_data.with_manager(new_manager);
        let mut inner = (*self.inner).clone();
        inner.repo_derived_data = Arc::new(repo_derived_data);
        inner.changesets = changesets;
        inner.changeset_fetcher = Arc::new(changeset_fetcher);
        inner.repo_blobstore = Arc::new(blobstore);
        Self {
            inner: Arc::new(inner),
        }
    }
}

/// Compatibility trait for conversion between a facet-style repo (that
/// happens to contain a BlobRepo) and the blob repo (for calling things that
/// still require blobrepo).
pub trait AsBlobRepo {
    fn as_blob_repo(&self) -> &BlobRepo;
}

impl AsBlobRepo for BlobRepo {
    fn as_blob_repo(&self) -> &BlobRepo {
        self
    }
}

/// This function uploads bonsai changests object to blobstore in parallel, and then does
/// sequential writes to changesets table. Parents of the changesets should already by saved
/// in the repository.
pub async fn save_bonsai_changesets(
    bonsai_changesets: Vec<BonsaiChangeset>,
    ctx: CoreContext,
    container: &(impl ChangesetsRef + RepoBlobstoreRef),
) -> Result<(), Error> {
    changesets_creation::save_changesets(&ctx, container, bonsai_changesets).await
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::sync::Arc;

use anyhow::Error;
use anyhow::Result;
use blobrepo::BlobRepo;
use blobrepo_hg::BlobRepoHg;
use blobstore::Loadable;
use cloned::cloned;
use context::CoreContext;
use dashmap::DashMap;
use futures::compat::Future01CompatExt;
use futures::FutureExt;
use futures::TryFutureExt;
use futures_ext::send_discard;
use futures_ext::BoxFuture;
use futures_old::sync::mpsc;
use futures_old::sync::mpsc::Sender;
use futures_old::Future;
use futures_old::Stream;
use mercurial_types::blobs::HgBlobChangeset;
use mercurial_types::HgChangesetId;
use repo_blobstore::RepoBlobstoreRef;
use slog::o;
use slog::Logger;

/// This trait enables parallelized walks over changesets.
pub trait ChangesetVisitor: Clone + Send + Sync + 'static {
    type Item: Send;

    /// Visit a specific changeset.
    ///
    /// `logger` is already customized to be specific to this changeset.
    ///
    /// Each visit instance will get a fresh copy of the changeset visitor -- this is unfortunately
    /// unavoidable due to the way tokio works. To share state between instances, use an Arc.
    fn visit(
        self,
        ctx: CoreContext,
        logger: Logger,
        repo: BlobRepo,
        changeset: HgBlobChangeset,
        follow_remaining: usize,
    ) -> BoxFuture<Self::Item, Error>;
}

/// Information about the specific changeset whose result is provided.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ChangesetVisitMeta {
    pub changeset_id: HgChangesetId,
    pub follow_remaining: usize,
}

/// Walk over changesets in parallel, calling the visitor for each changeset.
///
/// Behind this scenes, this uses the default tokio executor (which is typically a thread pool, so
/// this is typically highly parallel). Dropping the returned stream will cause further visiting to
/// be canceled.
pub fn visit_changesets<V, I>(
    ctx: CoreContext,
    logger: Logger,
    repo: BlobRepo,
    visitor: V,
    start_points: I,
    follow_limit: usize,
) -> impl Stream<Item = (V::Item, ChangesetVisitMeta), Error = Error> + Send
where
    V: ChangesetVisitor,
    I: IntoIterator<Item = HgChangesetId>,
{
    // Some notes about this:
    //
    // * mpsc::channels will automatically end the stream when all senders are dropped. Each
    // individual visit call will have a sender associated with it, and when all verifications
    // are complete there will be no more senders, so it all works out.
    //
    // * The buffer is set to a relatively small number because the actual size of the buffer
    // is the number passed in here plus the number of senders that exist, and VisitOne below
    // creates at least one sender per changeset it traverses. If this proves to be a problem,
    // it might be worth wrapping the sender up in an Arc or similar.
    let (mut sender, receiver) = mpsc::channel(16);

    let inner = Arc::new(VisitOneShared {
        logger,
        repo,
        visitor,
        visit_started: DashMap::new(),
    });

    for changeset_id in start_points {
        // Start off with follow_limit + 1 because that's logically the previous follow_remaining.
        let visit_one = VisitOne::new(ctx.clone(), &inner, changeset_id, follow_limit, &mut sender);
        if let Some(visit_one) = visit_one {
            tokio::spawn(visit_one.visit().compat());
        }
    }

    receiver
        .map_err(|()| unreachable!("Receiver can never produce errors"))
        .and_then(|res| res)
}

struct VisitOneShared<V> {
    logger: Logger,
    repo: BlobRepo,
    visitor: V,
    visit_started: DashMap<HgChangesetId, ()>,
}

impl<V> VisitOneShared<V> {
    #[inline]
    fn visit_started(&self, changeset_id: HgChangesetId) -> bool {
        self.visit_started.contains_key(&changeset_id)
    }

    #[inline]
    fn mark_visit_started(&self, changeset_id: HgChangesetId) {
        self.visit_started.insert(changeset_id, ());
    }
}

struct VisitOne<V>
where
    V: ChangesetVisitor,
{
    ctx: CoreContext,
    shared: Arc<VisitOneShared<V>>,
    logger: Logger,
    changeset_id: HgChangesetId,
    follow_remaining: usize,
    sender: Sender<Result<(V::Item, ChangesetVisitMeta)>>,
}

impl<V> VisitOne<V>
where
    V: ChangesetVisitor,
{
    fn new(
        ctx: CoreContext,
        shared: &Arc<VisitOneShared<V>>,
        changeset_id: HgChangesetId,
        prev_follow_remaining: usize,
        sender: &mut Sender<Result<(V::Item, ChangesetVisitMeta)>>,
    ) -> Option<Self> {
        // Checks to figure out whether to terminate the visit.
        if prev_follow_remaining == 0 {
            return None;
        }
        if shared.visit_started(changeset_id) {
            return None;
        }
        if sender.poll_ready().is_err() {
            // The receiver is closed, so there's no point doing anything.
            return None;
        }

        let logger = shared
            .logger
            .new(o!["changeset_id" => format!("{}", changeset_id)]);

        Some(Self {
            ctx,
            shared: shared.clone(),
            logger,
            changeset_id,
            follow_remaining: prev_follow_remaining - 1,
            sender: sender.clone(),
        })
    }

    fn visit(self) -> impl Future<Item = (), Error = ()> + Send {
        let Self {
            ctx,
            shared,
            logger,
            changeset_id,
            follow_remaining,
            sender,
        } = self;

        shared.mark_visit_started(changeset_id);

        let parents_fut = {
            let repo = shared.repo.clone();
            cloned!(ctx);
            async move { repo.get_hg_changeset_parents(ctx, changeset_id).await }
        }
        .boxed()
        .compat()
        .map({
            cloned!(ctx, shared, mut sender);
            move |parent_hashes| {
                for parent_id in parent_hashes {
                    let visit_one = VisitOne::new(
                        ctx.clone(),
                        &shared,
                        parent_id,
                        follow_remaining,
                        &mut sender,
                    );
                    if let Some(visit_one) = visit_one {
                        // Avoid unbounded recursion by spawning separate futures for each parent
                        // directly on the executor.
                        tokio::spawn(visit_one.visit().compat());
                    }
                }
            }
        });

        let visit_fut = {
            cloned!(ctx, changeset_id, shared);
            async move { changeset_id.load(&ctx, shared.repo.repo_blobstore()).await }
        }
        .boxed()
        .compat()
        .from_err()
        .and_then({
            cloned!(ctx, shared.visitor, shared.repo);
            move |changeset| visitor.visit(ctx, logger, repo, changeset, follow_remaining)
        })
        .and_then({
            let sender = sender.clone();
            move |item| {
                send_discard(
                    sender,
                    Ok((
                        item,
                        ChangesetVisitMeta {
                            changeset_id,
                            follow_remaining,
                        },
                    )),
                )
            }
        });

        visit_fut
            .join(parents_fut)
            .map(|((), ())| ())
            .or_else(move |err| send_discard(sender, Err(err)))
    }
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn send_sync() {
        fn assert_send<T: Send>() {}
        fn assert_sync<T: Sync>() {}

        assert_send::<VisitOneShared<()>>();
        assert_sync::<VisitOneShared<()>>();
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use mercurial_types::HgChangesetId;
use thiserror::Error;

#[derive(Debug, Error)]
pub enum ErrorKind {
    #[error("While visiting changeset {0}")]
    VisitError(HgChangesetId),
    #[error("While verifying changeset {0}")]
    VerificationError(HgChangesetId),
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

//! Code for operations on blobrepos that are useful but not essential.

mod bonsai;
mod changeset;
mod errors;

use anyhow::Result;
use blobstore::Loadable;
use context::CoreContext;
use manifest::BonsaiDiffFileChange;
use mercurial_types::HgFileNodeId;
use mononoke_types::FileChange;
use mononoke_types::MPath;
use repo_blobstore::RepoBlobstoreRef;

pub use crate::bonsai::BonsaiMFVerify;
pub use crate::bonsai::BonsaiMFVerifyDifference;
pub use crate::bonsai::BonsaiMFVerifyResult;
pub use crate::changeset::visit_changesets;
pub use crate::changeset::ChangesetVisitor;
pub use crate::errors::ErrorKind;

/// This is a function that's used to generate additional file changes for rebased diamond merges.
/// It's used in a very specific use case - rebasing of a diamond merge and it should be used with
/// care. Primary consumer of this function is pushrebase, and pushrebase code
/// contains detailed explanation of why this function is necessary.
pub async fn convert_diff_result_into_file_change_for_diamond_merge(
    ctx: &CoreContext,
    repo: &impl RepoBlobstoreRef,
    diff_result: BonsaiDiffFileChange<HgFileNodeId>,
) -> Result<(MPath, FileChange)> {
    match diff_result {
        BonsaiDiffFileChange::Changed(path, ty, node_id)
        | BonsaiDiffFileChange::ChangedReusedId(path, ty, node_id) => {
            let envelope = node_id.load(ctx, repo.repo_blobstore()).await?;
            // Note that we intentionally do not set copy-from info,
            // even if a file has been copied from somewhere.
            // BonsaiChangeset requires that copy_from cs id points to one of
            // the parents of the commit, so if we just fetch the latest copy
            // info for a file change, then it might point to one of the
            // ancestors of `onto`, but not necessarily to the onto.
            let file_change =
                FileChange::tracked(envelope.content_id(), ty, envelope.content_size(), None);
            Ok((path, file_change))
        }
        BonsaiDiffFileChange::Deleted(path) => Ok((path, FileChange::Deletion)),
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashSet;
use std::fmt;
use std::sync::Arc;

use anyhow::bail;
use anyhow::Error;
use blobrepo::BlobRepo;
use blobrepo_override::DangerousOverride;
use blobstore::Blobstore;
use blobstore::Loadable;
use cacheblob::MemWritesBlobstore;
use cloned::cloned;
use context::CoreContext;
use futures::future::try_join;
use futures::FutureExt;
use futures::TryFutureExt;
use futures::TryStreamExt;
use futures_ext::try_boxfuture;
use futures_ext::BoxFuture;
use futures_ext::FutureExt as _;
use futures_ext::StreamExt as _;
use futures_old::future;
use futures_old::future::Either;
use futures_old::Future;
use futures_old::Stream;
use manifest::bonsai_diff;
use manifest::BonsaiDiffFileChange;
use manifest::Diff;
use manifest::Entry;
use manifest::ManifestOps;
use mercurial_derivation::derive_hg_manifest;
use mercurial_types::blobs::HgBlobChangeset;
use mercurial_types::blobs::HgBlobManifest;
use mercurial_types::HgChangesetId;
use mercurial_types::HgFileNodeId;
use mercurial_types::HgManifestId;
use mercurial_types::HgNodeHash;
use mononoke_types::DateTime;
use mononoke_types::FileType;
use repo_blobstore::RepoBlobstoreArc;
use repo_blobstore::RepoBlobstoreRef;
use slog::debug;
use slog::Logger;

use crate::changeset::visit_changesets;
use crate::changeset::ChangesetVisitMeta;
use crate::changeset::ChangesetVisitor;

#[derive(Clone, Debug)]
pub enum BonsaiMFVerifyResult {
    Valid {
        lookup_mf_id: HgNodeHash,
        computed_mf_id: HgNodeHash,
    },
    // ValidDifferentHash means that the root manifest ID didn't match up, but that that was
    // because of an expected difference in hash that isn't substantive.
    ValidDifferentId(BonsaiMFVerifyDifference),
    Invalid(BonsaiMFVerifyDifference),
    Ignored(HgChangesetId),
}

impl BonsaiMFVerifyResult {
    pub fn is_valid(&self) -> bool {
        match self {
            BonsaiMFVerifyResult::Valid { .. } | BonsaiMFVerifyResult::ValidDifferentId(..) => true,
            _ => false,
        }
    }

    pub fn is_ignored(&self) -> bool {
        match self {
            BonsaiMFVerifyResult::Ignored(..) => true,
            _ => false,
        }
    }
}

#[derive(Clone)]
pub struct BonsaiMFVerifyDifference {
    // Root manifests in treemanifest hybrid mode use a different ID than what's computed.
    // See the documentation in mercurial_types/if/mercurial_thrift.thrift's HgManifestEnvelope
    // for more.
    pub lookup_mf_id: HgNodeHash,
    // The difference/inconsistency is that expected_mf_id is not the same as roundtrip_mf_id.
    pub expected_mf_id: HgNodeHash,
    pub roundtrip_mf_id: HgNodeHash,
    repo: BlobRepo,
}

impl BonsaiMFVerifyDifference {
    /// What entries changed from the original manifest to the roundtripped one.
    pub fn changes(
        &self,
        ctx: CoreContext,
    ) -> impl Stream<Item = Diff<Entry<HgManifestId, (FileType, HgFileNodeId)>>, Error = Error> + Send
    {
        let lookup_mf_id = HgManifestId::new(self.lookup_mf_id);
        let roundtrip_mf_id = HgManifestId::new(self.roundtrip_mf_id);
        lookup_mf_id
            .diff(ctx, self.repo.repo_blobstore().clone(), roundtrip_mf_id)
            .compat()
    }

    /// Whether there are any changes beyond the root manifest ID being different.
    #[inline]
    pub fn has_changes(&self, ctx: CoreContext) -> impl Future<Item = bool, Error = Error> + Send {
        self.changes(ctx).not_empty()
    }

    /// Whether there are any files that changed.
    #[inline]
    pub fn has_file_changes(
        &self,
        ctx: CoreContext,
    ) -> impl Future<Item = bool, Error = Error> + Send {
        self.changes(ctx)
            .filter(|diff| {
                let entry = match diff {
                    Diff::Added(_, entry) | Diff::Removed(_, entry) => entry,
                    Diff::Changed(_, _, entry) => entry,
                };
                match entry {
                    Entry::Leaf(_) => true,
                    Entry::Tree(_) => false,
                }
            })
            .not_empty()
    }

    // XXX might need to return repo here if callers want to do direct queries
}

impl fmt::Debug for BonsaiMFVerifyDifference {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("BonsaiMFVerifyDifference")
            .field("lookup_mf_id", &format!("{}", self.lookup_mf_id))
            .field("expected_mf_id", &format!("{}", self.expected_mf_id))
            .field("roundtrip_mf_id", &format!("{}", self.roundtrip_mf_id))
            .finish()
    }
}

pub struct BonsaiMFVerify {
    pub ctx: CoreContext,
    pub logger: Logger,
    pub repo: BlobRepo,
    pub follow_limit: usize,
    pub ignores: HashSet<HgChangesetId>,
    pub broken_merges_before: Option<DateTime>,
    pub debug_bonsai_diff: bool,
}

impl BonsaiMFVerify {
    /// Verify that a list of changesets roundtrips through bonsai. Returns a stream of
    /// inconsistencies and errors encountered, which completes once verification is complete.
    pub fn verify(
        self,
        start_points: impl IntoIterator<Item = HgChangesetId>,
    ) -> impl Stream<Item = (BonsaiMFVerifyResult, ChangesetVisitMeta), Error = Error> + Send {
        let repo = self
            .repo
            .dangerous_override(|blobstore| -> Arc<dyn Blobstore> {
                Arc::new(MemWritesBlobstore::new(blobstore))
            });

        visit_changesets(
            self.ctx,
            self.logger,
            repo,
            BonsaiMFVerifyVisitor {
                ignores: Arc::new(self.ignores),
                broken_merges_before: self.broken_merges_before,
                debug_bonsai_diff: self.debug_bonsai_diff,
            },
            start_points,
            self.follow_limit,
        )
    }
}

#[derive(Clone, Debug)]
struct BonsaiMFVerifyVisitor {
    ignores: Arc<HashSet<HgChangesetId>>,
    broken_merges_before: Option<DateTime>,
    debug_bonsai_diff: bool,
}

impl ChangesetVisitor for BonsaiMFVerifyVisitor {
    type Item = BonsaiMFVerifyResult;

    fn visit(
        self,
        ctx: CoreContext,
        logger: Logger,
        repo: BlobRepo,
        changeset: HgBlobChangeset,
        _follow_remaining: usize,
    ) -> BoxFuture<Self::Item, Error> {
        let changeset_id = changeset.get_changeset_id();
        if self.ignores.contains(&changeset_id) {
            debug!(logger, "Changeset ignored");
            return future::ok(BonsaiMFVerifyResult::Ignored(changeset_id)).boxify();
        }

        let broken_merge = match &self.broken_merges_before {
            Some(before) => {
                changeset.p1().is_some() && changeset.p2().is_some() && changeset.time() <= before
            }
            None => false,
        };

        if broken_merge {
            debug!(
                logger,
                "Potentially broken merge -- will check for file changes, not just manifest hash"
            );
        }

        debug!(logger, "Starting bonsai diff computation");

        let mut parents = vec![];
        parents.extend(changeset.p1());
        parents.extend(changeset.p2());
        parents.extend(try_boxfuture!(changeset.step_parents()));

        let parents = parents
            .into_iter()
            .map(|p| {
                let id = HgChangesetId::new(p);
                cloned!(ctx, repo);
                async move { id.load(&ctx, repo.repo_blobstore()).await }
                    .boxed()
                    .compat()
                    .from_err()
                    .map(|cs| cs.manifestid())
            })
            .collect::<Vec<_>>();

        // TODO: Update this to support stepparents
        // Convert to bonsai first.
        let bonsai_diff_fut = future::join_all(parents).and_then({
            cloned!(ctx, repo);
            move |parents| {
                let root_mf_fut = {
                    cloned!(ctx, repo);
                    let mf_id = changeset.manifestid();
                    async move { HgBlobManifest::load(&ctx, repo.repo_blobstore(), mf_id).await }
                };

                try_join(
                    bonsai_diff(
                        ctx.clone(),
                        repo.repo_blobstore().clone(),
                        changeset.manifestid(),
                        parents.iter().cloned().collect(),
                    )
                    .try_collect::<Vec<_>>(),
                    root_mf_fut,
                )
                .and_then(move |(diff, root_mf)| async move {
                    match root_mf {
                        Some(root_mf) => Ok((diff, root_mf, parents)),
                        None => bail!(
                            "internal error: didn't find root manifest id {}",
                            changeset.manifestid()
                        ),
                    }
                })
                .boxed()
                .compat()
            }
        });

        bonsai_diff_fut
            .and_then({
                let logger = logger.clone();
                move |(diff_result, root_mf, manifestids)| {
                    let diff_count = diff_result.len();
                    debug!(
                        logger,
                        "Computed diff ({} entries), now applying it", diff_count,
                    );
                    if self.debug_bonsai_diff {
                        for diff in &diff_result {
                            debug!(logger, "diff result: {:?}", diff);
                        }
                    }

                    apply_diff(
                        ctx.clone(),
                        logger.clone(),
                        repo.clone(),
                        diff_result,
                        manifestids,
                    )
                    .and_then(move |roundtrip_mf_id| {
                        let lookup_mf_id = root_mf.node_id();
                        let computed_mf_id = root_mf.computed_node_id();
                        debug!(
                            logger,
                            "Saving complete: initial computed manifest ID: {} (original {}), \
                             roundtrip: {}",
                            computed_mf_id,
                            lookup_mf_id,
                            roundtrip_mf_id,
                        );

                        // If there's no diff, memory_manifest will return the same ID as the
                        // parent, which will be the lookup ID, not the computed one.
                        let expected_mf_id = if diff_count == 0 {
                            lookup_mf_id
                        } else {
                            computed_mf_id
                        };
                        if roundtrip_mf_id == expected_mf_id {
                            Either::A(future::ok(BonsaiMFVerifyResult::Valid {
                                lookup_mf_id,
                                computed_mf_id: roundtrip_mf_id,
                            }))
                        } else {
                            let difference = BonsaiMFVerifyDifference {
                                lookup_mf_id,
                                expected_mf_id,
                                roundtrip_mf_id,
                                repo,
                            };

                            if broken_merge {
                                // This is a (potentially) broken merge. Ignore tree changes and
                                // only check for file changes.
                                Either::B(Either::A(difference.has_file_changes(ctx).map(
                                    move |has_file_changes| {
                                        if has_file_changes {
                                            BonsaiMFVerifyResult::Invalid(difference)
                                        } else {
                                            BonsaiMFVerifyResult::ValidDifferentId(difference)
                                        }
                                    },
                                )))
                            } else if diff_count == 0 {
                                // This is an empty changeset. Mercurial is relatively inconsistent
                                // about creating new manifest nodes for such changesets, so it can
                                // happen.
                                Either::B(Either::B(difference.has_changes(ctx).map(
                                    move |has_changes| {
                                        if has_changes {
                                            BonsaiMFVerifyResult::Invalid(difference)
                                        } else {
                                            BonsaiMFVerifyResult::ValidDifferentId(difference)
                                        }
                                    },
                                )))
                            } else {
                                Either::A(future::ok(BonsaiMFVerifyResult::Invalid(difference)))
                            }
                        }
                    })
                }
            })
            .boxify()
    }
}

fn apply_diff(
    ctx: CoreContext,
    _logger: Logger,
    repo: BlobRepo,
    diff_result: Vec<BonsaiDiffFileChange<HgFileNodeId>>,
    manifestids: Vec<HgManifestId>,
) -> impl Future<Item = HgNodeHash, Error = Error> + Send {
    let changes: Vec<_> = diff_result
        .into_iter()
        .map(|result| (result.path().clone(), make_entry(&result)))
        .collect();
    derive_hg_manifest(ctx, repo.repo_blobstore_arc(), manifestids, changes)
        .boxed()
        .compat()
        .map(|manifest_id| manifest_id.into_nodehash())
}

// XXX should this be in a more central place?
fn make_entry(
    diff_result: &BonsaiDiffFileChange<HgFileNodeId>,
) -> Option<(FileType, HgFileNodeId)> {
    use self::BonsaiDiffFileChange::*;

    match diff_result {
        Changed(_, ft, entry_id) | ChangedReusedId(_, ft, entry_id) => Some((*ft, *entry_id)),
        Deleted(_path) => None,
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

mod manifest;

pub use self::manifest::BonsaiMFVerify;
pub use self::manifest::BonsaiMFVerifyDifference;
pub use self::manifest::BonsaiMFVerifyResult;


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use fixtures::*;

// An extra level of nesting is required to avoid clashes between crate and module names.
mod test {
    macro_rules! test_verify {
        ($test_name:ident, $repo:ident) => {
            mod $test_name {
                use std::collections::HashSet;

                use blobrepo_hg::BlobRepoHg;
                use blobrepo_utils::BonsaiMFVerify;
                use blobrepo_utils::BonsaiMFVerifyResult;
                use context::CoreContext;
                use fbinit::FacebookInit;
                use fixtures::TestRepoFixture;
                use futures::compat::Future01CompatExt;
                use futures::TryStreamExt;
                use futures_old::Future;
                use futures_old::Stream;

                use crate::$repo;

                #[fbinit::test]
                async fn test(fb: FacebookInit) {
                    let ctx = CoreContext::test_mock(fb);

                    let repo = $repo::getrepo(fb).await;
                    let heads = repo
                        .get_hg_heads_maybe_stale(ctx.clone())
                        .compat()
                        .collect();

                    let verify = BonsaiMFVerify {
                        ctx: ctx.clone(),
                        logger: ctx.logger().clone(),
                        repo: repo.clone(),
                        follow_limit: 1024,
                        ignores: HashSet::new(),
                        broken_merges_before: None,
                        debug_bonsai_diff: false,
                    };

                    let results = heads
                        .map_err(|err| panic!("cannot get the heads {}", err))
                        .and_then(|heads| verify.verify(heads).collect());
                    results
                        .and_then(move |results| {
                            let diffs =
                                results
                                    .into_iter()
                                    .filter_map(move |(res, meta)| match res {
                                        BonsaiMFVerifyResult::Invalid(difference) => {
                                            let cs_id = meta.changeset_id;
                                            Some(
                                                difference
                                                    .changes(ctx.clone())
                                                    .collect()
                                                    .map(move |changes| (cs_id, changes)),
                                            )
                                        }
                                        _ => None,
                                    });

                            futures_old::future::join_all(diffs)
                        })
                        .map(|diffs| {
                            let mut failed = false;
                            let mut desc = Vec::new();
                            for (changeset_id, changes) in diffs {
                                failed = true;
                                desc.push(format!(
                                    "*** Inconsistent roundtrip for {}",
                                    changeset_id,
                                ));
                                for changed_entry in changes {
                                    desc.push(format!("  - Changed entry: {:?}", changed_entry));
                                }
                                desc.push("".to_string());
                            }
                            let desc = desc.join("\n");
                            if failed {
                                panic!(
                                    "Inconsistencies detected, roundtrip test failed\n\n{}",
                                    desc
                                );
                            }
                        })
                        .map_err(|err| {
                            panic!("verify error {}", err);
                        })
                        .compat()
                        .await
                        .unwrap()
                }
            }
        };
    }

    test_verify!(branch_even, BranchEven);
    test_verify!(branch_uneven, BranchUneven);
    test_verify!(branch_wide, BranchWide);
    test_verify!(linear, Linear);
    test_verify!(merge_even, MergeEven);
    test_verify!(merge_uneven, MergeUneven);
    test_verify!(unshared_merge_even, UnsharedMergeEven);
    test_verify!(unshared_merge_uneven, UnsharedMergeUneven);
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::time::Duration;

use anyhow::Error;
use blobstore::BlobstoreGetData;
use blobstore::BlobstoreIsPresent;
use blobstore::OverwriteStatus;
use clap::ValueEnum;
use context::PerfCounters;
use futures_stats::FutureStats;
use metaconfig_types::BlobstoreId;
use scuba_ext::MononokeScubaSampleBuilder;
use scuba_ext::ScubaValue;
use strum::AsRefStr;
use strum::Display;
use strum::EnumString;
use strum::EnumVariantNames;
use time_ext::DurationExt;
use tunables::tunables;

const SLOW_REQUEST_THRESHOLD: Duration = Duration::from_secs(5);

pub const BLOBSTORE_ID: &str = "blobstore_id";
pub const BLOBSTORE_TYPE: &str = "blobstore_type";
pub const COMPLETION_TIME: &str = "completion_time";
pub const ERROR: &str = "error";
pub const KEY: &str = "key";
pub const OPERATION: &str = "operation";
pub const QUEUE: &str = "queue";
pub const SESSION: &str = "session";
pub const SIZE: &str = "size";
pub const WRITE_ORDER: &str = "write_order";
/// Was the blob found during the get/is_present operations?
pub const BLOB_PRESENT: &str = "blob_present";

const OVERWRITE_STATUS: &str = "overwrite_status";

#[derive(
    Clone,
    Copy,
    Debug,
    Eq,
    PartialEq,
    Hash,
    Display,
    AsRefStr,
    EnumString,
    EnumVariantNames,
    ValueEnum
)]
#[strum(serialize_all = "kebab_case")]
pub enum OperationType {
    Get,
    Put,
    ScrubGet,
    IsPresent,
    Link,
    Unlink,
    Enumerate,
}

impl From<OperationType> for ScubaValue {
    fn from(value: OperationType) -> ScubaValue {
        ScubaValue::from(value.as_ref())
    }
}

pub fn add_completion_time(
    scuba: &mut MononokeScubaSampleBuilder,
    session: &str,
    stats: FutureStats,
) {
    scuba.add(COMPLETION_TIME, stats.completion_time.as_micros_unchecked());
    if stats.completion_time >= SLOW_REQUEST_THRESHOLD {
        scuba.add(SESSION, session);
    }
}

fn add_common_values(
    scuba: &mut MononokeScubaSampleBuilder,
    pc: &PerfCounters,
    key: &str,
    session: &str,
    stats: FutureStats,
    operation: OperationType,
    blobstore_id: Option<BlobstoreId>,
    blobstore_type: impl ToString,
) {
    scuba
        .add(KEY, key)
        .add(OPERATION, operation)
        .add(BLOBSTORE_TYPE, blobstore_type.to_string());

    pc.insert_nonzero_perf_counters(scuba);

    if let Some(blobstore_id) = blobstore_id {
        scuba.add(BLOBSTORE_ID, blobstore_id);
    }

    add_completion_time(scuba, session, stats);
}

pub fn record_get_stats(
    scuba: &mut MononokeScubaSampleBuilder,
    pc: &PerfCounters,
    stats: FutureStats,
    result: Result<&Option<BlobstoreGetData>, &Error>,
    key: &str,
    session: &str,
    operation: OperationType,
    blobstore_id: Option<BlobstoreId>,
    blobstore_type: impl ToString,
) {
    add_common_values(
        scuba,
        pc,
        key,
        session,
        stats,
        operation,
        blobstore_id,
        blobstore_type,
    );

    match result {
        Ok(Some(data)) => {
            let size = data.as_bytes().len();
            let size_logging_threshold = tunables()
                .blobstore_read_size_logging_threshold()
                .unwrap_or_default();
            if size_logging_threshold > 0 && size > size_logging_threshold as usize {
                scuba.unsampled();
            }
            scuba.add(SIZE, size);
            scuba.add(BLOB_PRESENT, true);
        }
        Err(error) => {
            // Always log errors
            scuba.unsampled();
            scuba.add(ERROR, format!("{:#}", error));
        }
        Ok(None) => {
            scuba.add(BLOB_PRESENT, false);
        }
    }

    scuba.log();
}

pub fn record_is_present_stats(
    scuba: &mut MononokeScubaSampleBuilder,
    pc: &PerfCounters,
    stats: FutureStats,
    result: Result<&BlobstoreIsPresent, &Error>,
    key: &str,
    session: &str,
    blobstore_id: Option<BlobstoreId>,
    blobstore_type: impl ToString,
) {
    add_common_values(
        scuba,
        pc,
        key,
        session,
        stats,
        OperationType::IsPresent,
        blobstore_id,
        blobstore_type,
    );

    match result {
        Ok(BlobstoreIsPresent::Present) => {
            scuba.add(BLOB_PRESENT, true);
        }
        Ok(BlobstoreIsPresent::Absent) => {
            scuba.add(BLOB_PRESENT, false);
        }
        Ok(BlobstoreIsPresent::ProbablyNotPresent(error)) => {
            // Always log errors
            scuba.unsampled();
            scuba.add(BLOB_PRESENT, false);
            scuba.add(ERROR, format!("{:#}", error));
        }
        Err(error) => {
            scuba.unsampled();
            scuba.add(ERROR, format!("{:#}", error));
        }
    }

    scuba.log();
}

pub fn record_put_stats(
    scuba: &mut MononokeScubaSampleBuilder,
    pc: &PerfCounters,
    stats: FutureStats,
    result: Result<&OverwriteStatus, &Error>,
    key: &str,
    session: &str,
    size: usize,
    blobstore_id: Option<BlobstoreId>,
    blobstore_type: impl ToString,
    write_order: Option<usize>,
) {
    add_common_values(
        scuba,
        pc,
        key,
        session,
        stats,
        OperationType::Put,
        blobstore_id,
        blobstore_type,
    );
    scuba.add(SIZE, size);

    match result {
        Ok(overwrite_status) => {
            scuba.add(OVERWRITE_STATUS, overwrite_status.as_ref());
            if let Some(write_order) = write_order {
                scuba.add(WRITE_ORDER, write_order);
            }
        }
        Err(error) => {
            scuba.add(ERROR, format!("{:#}", error));
        }
    };

    scuba.log();
}

pub fn record_queue_stats(
    scuba: &mut MononokeScubaSampleBuilder,
    pc: &PerfCounters,
    stats: FutureStats,
    result: Result<&(), &Error>,
    key: &str,
    session: &str,
    operation: OperationType,
    blobstore_id: Option<BlobstoreId>,
    blobstore_type: impl ToString,
    queue: &str,
) {
    add_common_values(
        scuba,
        pc,
        key,
        session,
        stats,
        operation,
        blobstore_id,
        blobstore_type,
    );

    scuba.add(QUEUE, queue);

    if let Err(error) = result {
        scuba.add(ERROR, format!("{:#}", error));
    }

    scuba.log();
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::fmt;
use std::sync::Arc;

use async_trait::async_trait;
use blobstore::Blobstore;
use blobstore::BlobstoreBytes;
use blobstore::BlobstoreGetData;
use blobstore::CountedBlobstore;
use bytes::Bytes;
use cachelib::LruCachePool;
use context::PerfCounterType;

use crate::dummy::DummyLease;
use crate::in_process_lease::InProcessLease;
use crate::locking_cache::CacheBlobstore;
use crate::locking_cache::CacheOps;

const MAX_CACHELIB_VALUE_SIZE: u64 = 4 * 1024 * 1024;

#[derive(Clone, Copy, Debug)]
pub struct CachelibBlobstoreOptions {
    // Whether to attempt zstd compressing data so it will fit inside cachelibs threshold
    pub attempt_zstd: bool,
    // Whether to wait for cache write before returning. Usually false apart from tests.
    pub lazy_cache_put: bool,
}

impl CachelibBlobstoreOptions {
    pub fn new_lazy(attempt_zstd: Option<bool>) -> Self {
        Self {
            attempt_zstd: attempt_zstd.unwrap_or(true),
            lazy_cache_put: true,
        }
    }
    pub fn new_eager(attempt_zstd: Option<bool>) -> Self {
        Self {
            attempt_zstd: attempt_zstd.unwrap_or(true),
            lazy_cache_put: false,
        }
    }
}

impl Default for CachelibBlobstoreOptions {
    fn default() -> Self {
        Self::new_lazy(None)
    }
}

/// A caching layer over an existing blobstore, backed by cachelib
#[derive(Clone)]
pub struct CachelibOps {
    blob_pool: Arc<LruCachePool>,
    presence_pool: Arc<LruCachePool>,
    options: CachelibBlobstoreOptions,
}

impl std::fmt::Display for CachelibOps {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(f, "CachelibOps")
    }
}

impl CachelibOps {
    pub fn new(
        blob_pool: Arc<LruCachePool>,
        presence_pool: Arc<LruCachePool>,
        options: CachelibBlobstoreOptions,
    ) -> Self {
        Self {
            blob_pool,
            presence_pool,
            options,
        }
    }
}

pub fn new_cachelib_blobstore_no_lease<T>(
    blobstore: T,
    blob_pool: Arc<LruCachePool>,
    presence_pool: Arc<LruCachePool>,
    options: CachelibBlobstoreOptions,
) -> CountedBlobstore<CacheBlobstore<CachelibOps, DummyLease, T>>
where
    T: Blobstore,
{
    let cache_ops = CachelibOps::new(blob_pool, presence_pool, options);
    CountedBlobstore::new(
        "cachelib".to_string(),
        CacheBlobstore::new(cache_ops, DummyLease {}, blobstore, options.lazy_cache_put),
    )
}

pub fn new_cachelib_blobstore<T>(
    blobstore: T,
    blob_pool: Arc<LruCachePool>,
    presence_pool: Arc<LruCachePool>,
    options: CachelibBlobstoreOptions,
) -> CountedBlobstore<CacheBlobstore<CachelibOps, InProcessLease, T>>
where
    T: Blobstore + Clone,
{
    let cache_ops = CachelibOps::new(blob_pool, presence_pool, options);
    CountedBlobstore::new(
        "cachelib".to_string(),
        CacheBlobstore::new(
            cache_ops,
            InProcessLease::new(),
            blobstore,
            options.lazy_cache_put,
        ),
    )
}

#[async_trait]
impl CacheOps for CachelibOps {
    const HIT_COUNTER: Option<PerfCounterType> = Some(PerfCounterType::CachelibHits);
    const MISS_COUNTER: Option<PerfCounterType> = Some(PerfCounterType::CachelibMisses);
    const CACHE_NAME: &'static str = "cachelib";

    async fn get(&self, key: &str) -> Option<BlobstoreGetData> {
        let blob = self.blob_pool.get(key);
        let blob = blob.ok()??;
        let blob = BlobstoreBytes::decode(blob)?;
        Some(blob.into())
    }

    async fn put(&self, key: &str, value: BlobstoreGetData) {
        // A failure to set presence is considered fine, here.
        let _ = self.presence_pool.set(key, Bytes::from(b"P".as_ref()));

        let encode_limit = if self.options.attempt_zstd {
            Some(MAX_CACHELIB_VALUE_SIZE)
        } else {
            None
        };
        if let Some(bytes) = value.into_bytes().encode(encode_limit) {
            let _ = self.blob_pool.set(key, bytes);
        }
    }

    /// Ask the cache if it knows whether the backing store has a value for this key. Returns
    /// `true` if there is definitely a value (i.e. cache entry in Present or Known state), `false`
    /// otherwise (Empty or Leased states).
    async fn check_present(&self, key: &str) -> bool {
        let presence_pool = self
            .presence_pool
            .get(key)
            .map(|opt| opt.is_some())
            .unwrap_or(false);
        let blob_pool = self
            .blob_pool
            .get(key)
            .map(|opt| opt.is_some())
            .unwrap_or(false);

        presence_pool || blob_pool
    }
}

impl fmt::Debug for CachelibOps {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        // XXX possibly add more debug info here
        write!(f, "CachelibOps")
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use anyhow::Result;
use async_trait::async_trait;
use blobstore::BlobstoreGetData;
use context::CoreContext;
use futures::future::BoxFuture;

use crate::CacheOps;
use crate::LeaseOps;

/// A dummy implementation of LeaseOps that meets the letter of the spec, but uselessly
#[derive(Clone, Debug)]
pub struct DummyLease {}

impl std::fmt::Display for DummyLease {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(f, "DummyLease")
    }
}

#[async_trait]
impl LeaseOps for DummyLease {
    async fn try_add_put_lease(&self, _key: &str) -> Result<bool> {
        Ok(true)
    }

    fn renew_lease_until(&self, _ctx: CoreContext, _key: &str, _done: BoxFuture<'static, ()>) {}

    async fn wait_for_other_leases(&self, _key: &str) {}

    async fn release_lease(&self, _key: &str) {}
}

/// A dummy implementation of CacheOps that meets the letter of the spec, but uselessly
#[derive(Clone, Debug)]
pub struct DummyCache {}

impl std::fmt::Display for DummyCache {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(f, "DummyCache")
    }
}

#[async_trait]
impl CacheOps for DummyCache {
    async fn get(&self, _key: &str) -> Option<BlobstoreGetData> {
        None
    }

    async fn put(&self, _key: &str, _value: BlobstoreGetData) {}

    async fn check_present(&self, _key: &str) -> bool {
        false
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::hash_map::Entry;
use std::collections::HashMap;
use std::sync::Arc;

use anyhow::Result;
use async_trait::async_trait;
use context::CoreContext;
use futures::channel::oneshot::channel;
use futures::channel::oneshot::Receiver;
use futures::channel::oneshot::Sender;
use futures::future::BoxFuture;
use futures::future::FutureExt;
use futures::future::Shared;
use tokio::sync::Mutex;

use crate::LeaseOps;

/// LeaseOps that use in-memory data structures to avoid two separate tasks writing to the same key
#[derive(Clone, Debug)]
pub struct InProcessLease {
    leases: Arc<Mutex<HashMap<String, (Sender<()>, Shared<Receiver<()>>)>>>,
}

impl std::fmt::Display for InProcessLease {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(f, "InProcessLease")
    }
}

impl InProcessLease {
    pub fn new() -> Self {
        Self {
            leases: Arc::new(Mutex::new(HashMap::new())),
        }
    }
}

#[async_trait]
impl LeaseOps for InProcessLease {
    async fn try_add_put_lease(&self, key: &str) -> Result<bool> {
        let key = key.to_string();
        let mut in_flight_leases = self.leases.lock().await;

        let entry = in_flight_leases.entry(key);
        if let Entry::Occupied(_) = entry {
            Ok(false)
        } else {
            let (send, recv) = channel();
            entry.or_insert((send, recv.shared()));
            Ok(true)
        }
    }

    fn renew_lease_until(&self, _ctx: CoreContext, key: &str, done: BoxFuture<'static, ()>) {
        let this = self.clone();
        let key = key.to_string();
        tokio::spawn(async move {
            done.await;
            this.release_lease(&key).await;
        });
    }

    async fn wait_for_other_leases(&self, key: &str) {
        let key = key.to_string();
        let in_flight_leases = self.leases.lock().await;

        if let Some((_, fut)) = in_flight_leases.get(&key) {
            let _ = fut.clone().await;
        }
    }

    async fn release_lease(&self, key: &str) {
        let key = key.to_string();
        let mut in_flight_leases = self.leases.lock().await;

        if let Some((sender, _)) = in_flight_leases.remove(&key) {
            // Don't care if there's no-one listening - just want to wake them up if possible.
            let _ = sender.send(());
        }
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

mod cachelib_cache;
pub use crate::cachelib_cache::new_cachelib_blobstore;
pub use crate::cachelib_cache::new_cachelib_blobstore_no_lease;
pub use crate::cachelib_cache::CachelibBlobstoreOptions;

pub mod dummy;

mod in_process_lease;
pub use in_process_lease::InProcessLease;

mod locking_cache;
pub use crate::locking_cache::CacheBlobstore;
pub use crate::locking_cache::CacheBlobstoreExt;
pub use crate::locking_cache::CacheOps;
pub use crate::locking_cache::LeaseOps;

mod memcache_cache_lease;
pub use crate::memcache_cache_lease::new_memcache_blobstore;
pub use crate::memcache_cache_lease::MemcacheOps;

mod mem_writes;
pub use crate::mem_writes::MemWritesBlobstore;