/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

//! This library provides access to Mercurial's file and tree data. It manages
//! the lifetime of these and takes care of fetching them when not present
//! locally.
//!
//! # High-level overview
//!
//! After cloning a repository, no file and tree data are present locally,
//! these are either created during a `hg commit` operation, or on-demand fetched
//! when accessed.
//!
//! This functionality is provided by the `ContentStore` (for content), and
//! `MetadataStore` (for history) types and are the main entry-points of this
//! library. Both of these have the exact same behavior, they only differ by
//! what they store.
//!
//! The `ContentStore` (and `Metadatastore`) keeps track of data in 2 locations:
//!   - A shared store location,
//!   - A local store location.
//!
//! The shared store is where network fetched data is written into, and to
//! prevent excessive disk usage, is automatically size constrained with the
//! Mercurial config `remotefilelog.cachelimit` (for files) and
//! `remotefilelog.manifestlimit` (for trees). The assumption is that data
//! available on the server will always be available and can be fetched at any
//! time. Writing to this store is done automatically and no APIs are exposed
//! to write to it.
//!
//! The local store is where `hg commit` data goes into. As opposed to the
//! shared store, it is not automatically reclaimed and will grow unbounded.
//! The `ContentStore::add` (from `HgIdMutableDeltaStore`) allows adding data
//! to this store. Care must be taken to call `ContentStore::flush` (from
//! `HgIdMutableDeltaStore`) for the written data to be persisted on disk.
//!
//! # Types
//!
//! ## `Key`
//!
//! A `Key` is comprised of both a filenode hash, and a path. Old style of
//! addressing content.
//!
//! ## `StoreKey`
//!
//! A `StoreKey` allows mixing both `Key` based addressing, and content-only
//! hashed addressing. Used predominantly in the `LfsStore`.
//!
//! ## `UnionStore`
//!
//! Compose multiple stores into one and re-implement the main traits by
//! iterating over these stores.
//!
//! ## `MultiplexDeltaStore`, `MultiplexHgIdHistoryStore`
//!
//! Similarly to the `UnionStore`, this allows composing stores together but
//! for the purposes of duplicating all the writes to all the stores. Mainly
//! used to send data to both a fast caching server (ex: Memcache), and to a
//! shared store when receiving network data. It can also be used for data format
//! migration
//!
//! ## `DataPack`, `HistoryPack`
//!
//! Immutable file storage comprised of an index file that tracks the location
//! of the actual data in its associated pack file. Must be repacked frequently
//! to avoid linear searches in them during read operations.
//!
//! On repack, the pack files are squashed together by writing all their data into a
//! `ContentStore` (or `MetadataStore`), which is then committed to disk before the
//! squashed pack files are then deleted from disk. This ensures that a new Mercurial
//! process spawned while repack is running will still be able to read all the data.
//!
//! ## `IndexedLogHgIdDataStore`, `IndexedLogHgIdHistoryStore`
//!
//! Basic `IndexedLog` backed stores. As opposed to the packfiles described above,
//! these allow update in place (append-only).
//!
//! ## `LfsStore`
//!
//! Alternative store for large blobs. Data stored in it is bipartite: one pointer
//! that describe a blob, it's size and other metadata, and the pure blob itself.
//! The blob is addressed by its sha256 only, while the pointer can be retrieved
//! via either the blob hash, or a plain `Key`.
//!
//! # Traits
//!
//! ## `LocalStore`
//!
//! Badly named trait, initially intended to be implemented only by on-disk
//! stores, it provides a `get_missing` API that test whether some data is
//! present in the store. Addressed via a `StoreKey`.
//!
//! ## `HgIdDataStore`, `HgIdHistoryStore`
//!
//! Main interface to read data out of a store. For copied file data, the returned
//! data will contain a copy-from header which may need to be stripped with
//! `strip_metadata` to obtain the plain blob. Must implement the `LocalStore`
//! trait. Metadata can be also separated with separate_metadata that returns raw metadata blob.
//!
//! ## `ContentDataStore`
//!
//! Implemented by content-only stores. The hash of the returned blob will
//! match exactly the value of the passed in StoreKey hash. Used by the
//! `LfsStore`.
//!
//! ## `HgIdMutableDeltaStore`, `HgIdMutableHistoryStore`
//!
//! Main interface to write to a store. Unflushed data may be lost. Must implement
//! the `LocalStore` and `HgIdDataStore`/`HgIdHistoryStore` traits.
//!
//! ## `HgIdRemoteStore`, `RemoteDataStore`, `RemoteHistoryStore`
//!
//! The `HgIdRemoteStore` is implemented by raw remote stores, it is intended
//! to produce individual `RemoteDataStore` or `RemoteHistoryStore`. These stores
//! will automatically write fetched data to the passed mutable stores. This is
//! implemented by both the ssh and the edenapi remote store.
//!
//! The produced stores must implement the `HgIdDataStore` trait.
//!

mod contentstore;
mod dataindex;
#[cfg(all(fbcode_build, target_os = "linux"))]
mod facebook;
mod fanouttable;
mod fetch_logger;
mod historyindex;
mod indexedloghistorystore;
mod indexedlogutil;
mod lfs;
mod memcache;
mod metadatastore;
mod missing;
mod redacted;
mod remotestore;
mod repack;
mod sliceext;
mod types;
mod unionstore;

pub mod datapack;
pub mod datastore;
pub mod edenapi;
pub mod error;
pub mod historypack;
pub mod historystore;
pub mod indexedlogauxstore;
pub mod indexedlogdatastore;
pub mod localstore;
pub mod multiplexstore;
pub mod mutabledatapack;
pub mod mutablehistorypack;
pub mod mutablepack;
pub mod packstore;
pub mod packwriter;
pub mod scmstore;
pub mod trait_impls;
pub mod uniondatastore;
pub mod unionhistorystore;
pub mod util;

pub use revisionstore_types::*;

pub use crate::contentstore::ContentStore;
pub use crate::contentstore::ContentStoreBuilder;
pub use crate::datapack::DataEntry;
pub use crate::datapack::DataPack;
pub use crate::datapack::DataPackVersion;
pub use crate::datastore::ContentDataStore;
pub use crate::datastore::ContentMetadata;
pub use crate::datastore::Delta;
pub use crate::datastore::HgIdDataStore;
pub use crate::datastore::HgIdMutableDeltaStore;
pub use crate::datastore::LegacyStore;
pub use crate::datastore::RemoteDataStore;
pub use crate::datastore::StoreResult;
pub use crate::edenapi::EdenApiFileStore;
pub use crate::edenapi::EdenApiRemoteStore;
pub use crate::edenapi::EdenApiTreeStore;
pub use crate::historypack::HistoryEntry;
pub use crate::historypack::HistoryPack;
pub use crate::historypack::HistoryPackVersion;
pub use crate::historystore::HgIdHistoryStore;
pub use crate::historystore::HgIdMutableHistoryStore;
pub use crate::historystore::RemoteHistoryStore;
pub use crate::indexedlogauxstore::AuxStore;
pub use crate::indexedlogdatastore::IndexedLogHgIdDataStore;
pub use crate::indexedlogdatastore::IndexedLogHgIdDataStoreConfig;
pub use crate::indexedloghistorystore::IndexedLogHgIdHistoryStore;
pub use crate::indexedlogutil::StoreType;
pub use crate::localstore::ExtStoredPolicy;
pub use crate::localstore::LocalStore;
pub use crate::memcache::MemcacheStore;
pub use crate::metadatastore::MetadataStore;
pub use crate::metadatastore::MetadataStoreBuilder;
pub use crate::multiplexstore::MultiplexDeltaStore;
pub use crate::multiplexstore::MultiplexHgIdHistoryStore;
pub use crate::mutabledatapack::MutableDataPack;
pub use crate::mutablehistorypack::MutableHistoryPack;
pub use crate::packstore::CorruptionPolicy;
pub use crate::packstore::DataPackStore;
pub use crate::packstore::HistoryPackStore;
pub use crate::packstore::MutableDataPackStore;
pub use crate::packstore::MutableHistoryPackStore;
pub use crate::redacted::redact_if_needed;
pub use crate::remotestore::HgIdRemoteStore;
pub use crate::repack::repack;
pub use crate::repack::RepackKind;
pub use crate::repack::RepackLocation;
pub use crate::repack::Repackable;
pub use crate::repack::ToKeys;
pub use crate::types::ContentHash;
pub use crate::types::StoreKey;
pub use crate::uniondatastore::UnionHgIdDataStore;

#[cfg(any(test, feature = "for-tests"))]
pub mod testutil;

#[cfg(test)]
mod env_lock {
    use parking_lot::Mutex;

    pub static ENV_LOCK: Mutex<()> = Mutex::new(());

    fn env_reset() {
        for name in ["https_proxy", "http_proxy", "NO_PROXY"] {
            if std::env::var_os(name).is_some() {
                std::env::remove_var(name)
            }
        }
    }

    pub(crate) fn env_lock() -> impl Drop {
        let lock = ENV_LOCK.lock();
        env_reset();
        lock
    }
}

#[cfg(test)]
pub(crate) use env_lock::env_lock;


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

//! Common method shared between all mercurial stores

use std::ops::Deref;
use std::path::Path;

use anyhow::Result;

use crate::types::StoreKey;

/// Defines the behavior of the datapack code when encountering blobs that are externally stored.
/// This is typically found for LFS pointers stored in packfiles.
#[derive(Debug, PartialEq, Copy, Clone)]
pub enum ExtStoredPolicy {
    /// The datapack code will return it
    Use,
    /// The datapack code will pretend the blob isn't present
    Ignore,
}

pub trait StoreFromPath {
    /// Builds a Store from a filepath. The default implementation panics.
    fn from_path(_path: &Path, _extstored: ExtStoredPolicy) -> Result<Self>
    where
        Self: Sized;
}

pub trait LocalStore: Send + Sync {
    /// Returns all the keys that aren't present in this `Store`.
    fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>>;

    /// Test whether this `Store` contains a specific key.
    fn contains(&self, key: &StoreKey) -> Result<bool> {
        Ok(self.get_missing(&[key.clone()])?.is_empty())
    }
}

/// All the types that can `Deref` into a `Store` implements `Store`.
impl<T: LocalStore + ?Sized, U: Deref<Target = T> + Send + Sync> LocalStore for U {
    fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        T::get_missing(self, keys)
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

//! Adapters around Memcache to be transparently used as HgIdDataStore or HgIdHistoryStore.

use std::mem::size_of;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use std::time::Instant;

use anyhow::Result;
use minibytes::Bytes;
use serde_derive::Deserialize;
use serde_derive::Serialize;
use tracing::info_span;
use types::Key;
use types::NodeInfo;

use crate::datastore::Delta;
use crate::datastore::HgIdDataStore;
use crate::datastore::HgIdMutableDeltaStore;
use crate::datastore::Metadata;
use crate::datastore::RemoteDataStore;
use crate::datastore::StoreResult;
use crate::historystore::HgIdHistoryStore;
use crate::historystore::HgIdMutableHistoryStore;
use crate::historystore::RemoteHistoryStore;
use crate::localstore::LocalStore;
use crate::types::StoreKey;

/// Type of blobs stored in Memcache.
///
/// Whenever this type is changed, the `CODE_VERSION` value must be incremented to avoid
/// incompatibilities.
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
pub(crate) struct McData {
    #[serde(with = "types::serde_with::key::tuple")]
    pub key: Key,
    pub data: Bytes,
    pub metadata: Metadata,
}

/// Type of history info stored in Memcache.
///
/// Whenever this type is changed, the `CODE_VERSION` value must be incremented to avoid
/// incompatibilities.
#[derive(Serialize, Deserialize, Debug, PartialEq)]
pub(crate) struct McHist {
    #[serde(with = "types::serde_with::key::tuple")]
    pub key: Key,
    #[serde(with = "types::serde_with::nodeinfo::tuple")]
    pub nodeinfo: NodeInfo,
}

#[cfg(not(all(fbcode_build, target_os = "linux")))]
mod dummy {
    use std::iter::empty;

    use configmodel::Config;

    use super::*;

    /// Dummy memcache client for when Mercurial is compiled outside of fbcode.
    pub struct MemcacheStore;

    impl MemcacheStore {
        pub fn new(_config: &dyn Config) -> Result<Self> {
            Ok(MemcacheStore {})
        }

        pub(crate) fn get_data_iter(
            &self,
            _key: &[Key],
        ) -> Result<impl Iterator<Item = Result<McData>>> {
            Ok(empty())
        }

        pub(super) fn add_data(&self, _delta: &Delta, _metadata: &Metadata) {}
        pub(crate) fn add_mcdata(&self, _mcdata: McData) {}

        pub(super) fn get_hist_iter(
            &self,
            _key: &[Key],
        ) -> Result<impl Iterator<Item = Result<McHist>>> {
            Ok(empty())
        }

        pub(super) fn add_hist(&self, _key: &Key, _info: &NodeInfo) {}
    }
}

#[cfg(not(all(fbcode_build, target_os = "linux")))]
pub use dummy::MemcacheStore;

#[cfg(all(fbcode_build, target_os = "linux"))]
pub use crate::facebook::MemcacheStore;

impl HgIdDataStore for MemcacheStore {
    fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
        Ok(StoreResult::NotFound(key))
    }

    fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
        Ok(StoreResult::NotFound(key))
    }

    fn refresh(&self) -> Result<()> {
        Ok(())
    }
}

impl HgIdMutableDeltaStore for MemcacheStore {
    fn add(&self, delta: &Delta, metadata: &Metadata) -> Result<()> {
        self.add_data(delta, metadata);
        Ok(())
    }

    fn flush(&self) -> Result<Option<Vec<PathBuf>>> {
        Ok(None)
    }
}

impl HgIdHistoryStore for MemcacheStore {
    fn get_node_info(&self, _key: &Key) -> Result<Option<NodeInfo>> {
        Ok(None)
    }

    fn refresh(&self) -> Result<()> {
        Ok(())
    }
}

impl HgIdMutableHistoryStore for MemcacheStore {
    fn add(&self, key: &Key, info: &NodeInfo) -> Result<()> {
        self.add_hist(key, info);
        Ok(())
    }

    fn flush(&self) -> Result<Option<Vec<PathBuf>>> {
        Ok(None)
    }
}

impl LocalStore for MemcacheStore {
    fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        Ok(keys.to_vec())
    }
}

impl MemcacheStore {
    pub fn datastore(
        self: Arc<Self>,
        store: Arc<dyn HgIdMutableDeltaStore>,
    ) -> Arc<dyn HgIdMutableDeltaStore> {
        Arc::new(MemcacheHgIdDataStore::new(self, store))
    }

    pub fn remote_datastore(
        self: Arc<Self>,
        store: Arc<dyn HgIdMutableDeltaStore>,
    ) -> Arc<dyn RemoteDataStore> {
        Arc::new(MemcacheHgIdDataStore::new(self, store))
    }

    pub fn historystore(
        self: Arc<Self>,
        store: Arc<dyn HgIdMutableHistoryStore>,
    ) -> Arc<dyn HgIdMutableHistoryStore> {
        Arc::new(MemcacheHgIdHistoryStore::new(self, store))
    }

    pub fn remote_historystore(
        self: Arc<Self>,
        store: Arc<dyn HgIdMutableHistoryStore>,
    ) -> Arc<dyn RemoteHistoryStore> {
        Arc::new(MemcacheHgIdHistoryStore::new(self, store))
    }
}

struct MemcacheHgIdDataStore {
    store: Arc<dyn HgIdMutableDeltaStore>,
    memcache: Arc<MemcacheStore>,
    creation_time: Instant,
}

impl MemcacheHgIdDataStore {
    pub fn new(memcache: Arc<MemcacheStore>, store: Arc<dyn HgIdMutableDeltaStore>) -> Self {
        Self {
            memcache,
            store,
            creation_time: Instant::now(),
        }
    }

    fn use_memcache(&self) -> bool {
        self.creation_time.elapsed() > MEMCACHE_DELAY
    }
}

#[cfg(test)]
pub const MEMCACHE_DELAY: Duration = Duration::from_secs(0);
#[cfg(not(test))]
pub const MEMCACHE_DELAY: Duration = Duration::from_secs(10);

impl HgIdDataStore for MemcacheHgIdDataStore {
    fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
        match self.prefetch(&[key.clone()]) {
            Ok(_) => self.store.get(key),
            Err(_) => Ok(StoreResult::NotFound(key)),
        }
    }

    fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
        match self.prefetch(&[key.clone()]) {
            Ok(_) => self.store.get_meta(key),
            Err(_) => Ok(StoreResult::NotFound(key)),
        }
    }

    fn refresh(&self) -> Result<()> {
        Ok(())
    }
}

impl HgIdMutableDeltaStore for MemcacheHgIdDataStore {
    fn add(&self, delta: &Delta, metadata: &Metadata) -> Result<()> {
        if self.use_memcache() {
            self.memcache.add_data(delta, metadata);
        }
        Ok(())
    }

    fn flush(&self) -> Result<Option<Vec<PathBuf>>> {
        Ok(None)
    }
}

impl LocalStore for MemcacheHgIdDataStore {
    fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        Ok(keys.to_vec())
    }
}

impl RemoteDataStore for MemcacheHgIdDataStore {
    fn prefetch(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        if !self.use_memcache() {
            return self.store.get_missing(keys);
        }

        let span = info_span!(
            "MemcacheHgIdDataStore::prefetch",
            key_count = keys.len(),
            hit_count = &0,
            size = &0
        );
        let _guard = span.enter();

        let mut hits = 0;
        let mut size = 0;

        let hgidkeys = keys
            .iter()
            .filter_map(|k| match k {
                StoreKey::HgId(k) => Some(k.clone()),
                StoreKey::Content(_, _) => None,
            })
            .collect::<Vec<_>>();

        for mcdata in self.memcache.get_data_iter(&hgidkeys)? {
            if let Ok(mcdata) = mcdata {
                let metadata = mcdata.metadata;
                let delta = Delta {
                    data: mcdata.data,
                    base: None,
                    key: mcdata.key,
                };

                hits += 1;
                size += delta.data.len() + size_of::<Key>();

                self.store.add(&delta, &metadata)?;
            }
        }

        span.record("hit_count", &hits);
        span.record("size", &size);

        self.store.get_missing(keys)
    }

    fn upload(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        Ok(keys.to_vec())
    }
}

struct MemcacheHgIdHistoryStore {
    store: Arc<dyn HgIdMutableHistoryStore>,
    memcache: Arc<MemcacheStore>,
    creation_time: Instant,
}

impl MemcacheHgIdHistoryStore {
    pub fn new(memcache: Arc<MemcacheStore>, store: Arc<dyn HgIdMutableHistoryStore>) -> Self {
        Self {
            memcache,
            store,
            creation_time: Instant::now(),
        }
    }

    fn use_memcache(&self) -> bool {
        self.creation_time.elapsed() > MEMCACHE_DELAY
    }
}

impl HgIdHistoryStore for MemcacheHgIdHistoryStore {
    fn get_node_info(&self, key: &Key) -> Result<Option<NodeInfo>> {
        match self.prefetch(&[StoreKey::hgid(key.clone())]) {
            Ok(()) => self.store.get_node_info(key),
            Err(_) => Ok(None),
        }
    }

    fn refresh(&self) -> Result<()> {
        Ok(())
    }
}

impl HgIdMutableHistoryStore for MemcacheHgIdHistoryStore {
    fn add(&self, key: &Key, info: &NodeInfo) -> Result<()> {
        if self.use_memcache() {
            self.memcache.add_hist(key, info);
        }
        Ok(())
    }

    fn flush(&self) -> Result<Option<Vec<PathBuf>>> {
        Ok(None)
    }
}

impl LocalStore for MemcacheHgIdHistoryStore {
    fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        Ok(keys.to_vec())
    }
}

impl RemoteHistoryStore for MemcacheHgIdHistoryStore {
    fn prefetch(&self, keys: &[StoreKey]) -> Result<()> {
        if !self.use_memcache() {
            return Ok(());
        }

        let span = info_span!(
            "MemcacheHgIdHistoryStore::prefetch",
            key_count = keys.len(),
            hit_count = &0,
            size = &0
        );
        let _guard = span.enter();

        let keys = keys
            .iter()
            .filter_map(|k| match k {
                StoreKey::HgId(k) => Some(k.clone()),
                StoreKey::Content(_, _) => None,
            })
            .collect::<Vec<_>>();

        let mut hits = 0;
        let mut size = 0;

        for mchist in self.memcache.get_hist_iter(&keys)? {
            if let Ok(mchist) = mchist {
                self.store.add(&mchist.key, &mchist.nodeinfo)?;

                hits += 1;
                size += size_of::<McHist>();
            }
        }

        span.record("hit_count", &hits);
        span.record("size", &size);

        Ok(())
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;

use anyhow::format_err;
use anyhow::Result;
use configmodel::convert::ByteCount;
use configmodel::Config;
use configmodel::ConfigExt;
use types::Key;
use types::NodeInfo;

use crate::historystore::HgIdHistoryStore;
use crate::historystore::HgIdMutableHistoryStore;
use crate::historystore::RemoteHistoryStore;
use crate::indexedloghistorystore::IndexedLogHgIdHistoryStore;
use crate::indexedlogutil::StoreType;
use crate::localstore::LocalStore;
use crate::memcache::MemcacheStore;
use crate::multiplexstore::MultiplexHgIdHistoryStore;
use crate::packstore::CorruptionPolicy;
use crate::packstore::MutableHistoryPackStore;
use crate::remotestore::HgIdRemoteStore;
use crate::repack::RepackLocation;
use crate::types::StoreKey;
use crate::unionhistorystore::UnionHgIdHistoryStore;
use crate::util::get_cache_packs_path;
use crate::util::get_cache_path;
use crate::util::get_indexedloghistorystore_path;
use crate::util::get_local_path;
use crate::util::get_packs_path;

/// A `MetadataStore` aggregate all the local and remote stores and expose them as one. Both local and
/// remote stores can be queried and accessed via the `HgIdHistoryStore` trait. The local store can also
/// be written to via the `HgIdMutableHistoryStore` trait, this is intended to be used to store local
/// commit data.
pub struct MetadataStore {
    historystore: UnionHgIdHistoryStore<Arc<dyn HgIdHistoryStore>>,
    local_mutablehistorystore: Option<Arc<dyn HgIdMutableHistoryStore>>,
    shared_mutablehistorystore: Arc<dyn HgIdMutableHistoryStore>,
    remote_store: Option<Arc<dyn RemoteHistoryStore>>,
}

impl MetadataStore {
    pub fn new(local_path: impl AsRef<Path>, config: &dyn Config) -> Result<Self> {
        MetadataStoreBuilder::new(config)
            .local_path(&local_path)
            .build()
    }

    /// Attempt to repair the underlying stores that the `MetadataStore` is comprised of.
    ///
    /// As this may violate some of the stores asumptions, care must be taken to call this only
    /// when no other `MetadataStore` have been created for the `shared_path`.
    pub fn repair(
        shared_path: impl AsRef<Path>,
        local_path: Option<impl AsRef<Path>>,
        suffix: Option<impl AsRef<Path>>,
        config: &dyn Config,
    ) -> Result<String> {
        let mut repair_str = String::new();
        let mut shared_path = shared_path.as_ref().to_path_buf();
        if let Some(suffix) = suffix.as_ref() {
            shared_path.push(suffix);
        }
        let local_path = local_path
            .map(|p| get_local_path(p.as_ref().to_path_buf(), &suffix))
            .transpose()?;

        repair_str += &IndexedLogHgIdHistoryStore::repair(
            get_indexedloghistorystore_path(&shared_path)?,
            config,
            StoreType::Shared,
        )?;
        if let Some(local_path) = local_path {
            repair_str += &IndexedLogHgIdHistoryStore::repair(
                get_indexedloghistorystore_path(local_path)?,
                config,
                StoreType::Local,
            )?;
        }
        Ok(repair_str)
    }
}

// Repack specific methods, not to be used directly but by the repack code.
impl MetadataStore {
    pub(crate) fn add_pending(
        &self,
        key: &Key,
        info: NodeInfo,
        location: RepackLocation,
    ) -> Result<()> {
        match location {
            RepackLocation::Local => self.add(&key, &info),
            RepackLocation::Shared => self.shared_mutablehistorystore.add(&key, &info),
        }
    }

    pub(crate) fn commit_pending(&self, location: RepackLocation) -> Result<Option<Vec<PathBuf>>> {
        match location {
            RepackLocation::Local => self.flush(),
            RepackLocation::Shared => self.shared_mutablehistorystore.flush(),
        }
    }

    pub fn get_shared_mutable(&self) -> Arc<dyn HgIdMutableHistoryStore> {
        self.shared_mutablehistorystore.clone()
    }
}

impl HgIdHistoryStore for MetadataStore {
    fn get_node_info(&self, key: &Key) -> Result<Option<NodeInfo>> {
        self.historystore.get_node_info(key)
    }

    fn refresh(&self) -> Result<()> {
        self.historystore.refresh()
    }
}

impl RemoteHistoryStore for MetadataStore {
    fn prefetch(&self, keys: &[StoreKey]) -> Result<()> {
        if let Some(remote_store) = self.remote_store.as_ref() {
            let missing = self.get_missing(&keys)?;
            if missing == vec![] {
                Ok(())
            } else {
                remote_store.prefetch(&missing)
            }
        } else {
            // There is no remote store, let's pretend everything is fine.
            Ok(())
        }
    }
}

impl LocalStore for MetadataStore {
    fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        self.historystore.get_missing(keys)
    }
}

impl Drop for MetadataStore {
    /// The shared store is a cache, so let's flush all pending data when the `MetadataStore` goes
    /// out of scope.
    fn drop(&mut self) {
        let _ = self.shared_mutablehistorystore.flush();
    }
}

impl HgIdMutableHistoryStore for MetadataStore {
    fn add(&self, key: &Key, info: &NodeInfo) -> Result<()> {
        self.local_mutablehistorystore
            .as_ref()
            .ok_or_else(|| format_err!("writing to a non-local MetadataStore is not allowed"))?
            .add(key, info)
    }

    fn flush(&self) -> Result<Option<Vec<PathBuf>>> {
        self.shared_mutablehistorystore.as_ref().flush()?;
        self.local_mutablehistorystore
            .as_ref()
            .ok_or_else(|| format_err!("flushing a non-local MetadataStore is not allowed"))?
            .flush()
    }
}

/// Builder for `MetadataStore`. An `impl AsRef<Path>` represents the path to the store and a
/// `dyn Config` of the Mercurial configuration are required to build a `MetadataStore`.
pub struct MetadataStoreBuilder<'a> {
    local_path: Option<PathBuf>,
    no_local_store: bool,
    config: &'a dyn Config,
    remotestore: Option<Arc<dyn HgIdRemoteStore>>,
    suffix: Option<PathBuf>,
    memcachestore: Option<Arc<MemcacheStore>>,
}

impl<'a> MetadataStoreBuilder<'a> {
    pub fn new(config: &'a dyn Config) -> Self {
        Self {
            local_path: None,
            no_local_store: false,
            config,
            remotestore: None,
            suffix: None,
            memcachestore: None,
        }
    }

    /// Path to the local store.
    pub fn local_path(mut self, local_path: impl AsRef<Path>) -> Self {
        self.local_path = Some(local_path.as_ref().to_path_buf());
        self
    }

    /// Allows a MetadataStore to be created without a local store.
    ///
    /// This should be used in very specific cases that do not want a local store. Unless you know
    /// exactly that this is what you want, do not use.
    pub fn no_local_store(mut self) -> Self {
        self.no_local_store = true;
        self
    }

    pub fn remotestore(mut self, remotestore: Arc<dyn HgIdRemoteStore>) -> Self {
        self.remotestore = Some(remotestore);
        self
    }

    pub fn memcachestore(mut self, memcachestore: Arc<MemcacheStore>) -> Self {
        self.memcachestore = Some(memcachestore);
        self
    }

    pub fn suffix(mut self, suffix: impl AsRef<Path>) -> Self {
        self.suffix = Some(suffix.as_ref().to_path_buf());
        self
    }

    pub fn build(self) -> Result<MetadataStore> {
        let local_path = self
            .local_path
            .as_ref()
            .map(|p| get_local_path(p.clone(), &self.suffix))
            .transpose()?;
        let cache_path = get_cache_path(self.config, &self.suffix)?;
        let max_pending: u64 = self
            .config
            .get_or("packs", "maxhistorypending", || 10000000)?;
        let max_bytes = self
            .config
            .get_opt::<ByteCount>("packs", "maxhistorybytes")?
            .map(|v| v.value());

        let cache_packs_path = get_cache_packs_path(self.config, &self.suffix)?;
        let shared_pack_store = Arc::new(MutableHistoryPackStore::new(
            &cache_packs_path,
            CorruptionPolicy::REMOVE,
            max_pending,
            max_bytes,
        )?);
        let mut historystore: UnionHgIdHistoryStore<Arc<dyn HgIdHistoryStore>> =
            UnionHgIdHistoryStore::new();

        let shared_indexedloghistorystore = Arc::new(IndexedLogHgIdHistoryStore::new(
            get_indexedloghistorystore_path(&cache_path)?,
            &self.config,
            StoreType::Shared,
        )?);

        // The shared store should precede the local one for 2 reasons:
        //  - It is expected that the number of blobs and the number of requests satisfied by the
        //    shared cache to be significantly higher than ones in the local store
        //  - When pushing changes on a pushrebase server, the local linknode will become
        //    incorrect, future fetches will put that change in the shared cache where the linknode
        //    will be correct.
        let primary: Arc<dyn HgIdMutableHistoryStore> =
            if self
                .config
                .get_or("remotefilelog", "write-hgcache-to-indexedlog", || true)?
            {
                // Put the indexedlog first, since recent data will have gone there.
                historystore.add(shared_indexedloghistorystore.clone());
                historystore.add(shared_pack_store);
                shared_indexedloghistorystore
            } else {
                historystore.add(shared_pack_store.clone());
                historystore.add(shared_indexedloghistorystore);
                shared_pack_store
            };

        let local_mutablehistorystore: Option<Arc<dyn HgIdMutableHistoryStore>> =
            if let Some(unsuffixed_local_path) = self.local_path {
                let local_pack_store = Arc::new(MutableHistoryPackStore::new(
                    get_packs_path(&unsuffixed_local_path, &self.suffix)?,
                    CorruptionPolicy::IGNORE,
                    max_pending,
                    None,
                )?);
                let local_indexedloghistorystore = Arc::new(IndexedLogHgIdHistoryStore::new(
                    get_indexedloghistorystore_path(&local_path.unwrap())?,
                    &self.config,
                    StoreType::Local,
                )?);
                let primary: Arc<dyn HgIdMutableHistoryStore> =
                    if self
                        .config
                        .get_or("remotefilelog", "write-local-to-indexedlog", || true)?
                    {
                        // Put the indexedlog first, since recent data will have gone there.
                        historystore.add(local_indexedloghistorystore.clone());
                        historystore.add(local_pack_store);
                        local_indexedloghistorystore
                    } else {
                        historystore.add(local_pack_store.clone());
                        historystore.add(local_indexedloghistorystore);
                        local_pack_store
                    };

                Some(primary)
            } else {
                if !self.no_local_store {
                    return Err(format_err!(
                        "a MetadataStore cannot be built without a local store"
                    ));
                }
                None
            };

        let remote_store: Option<Arc<dyn RemoteHistoryStore>> =
            if let Some(remotestore) = self.remotestore {
                let (cache, shared_store) = if let Some(memcachestore) = self.memcachestore {
                    // Combine the memcache store with the other stores. The intent is that all remote
                    // requests will first go to the memcache store, and only reach the slower remote
                    // store after that.
                    //
                    // If data isn't found in the memcache store, once fetched from the remote store it
                    // will be written to the local cache, and will populate the memcache store, so
                    // other clients and future requests won't need to go to a network store.
                    let mut multiplexstore: MultiplexHgIdHistoryStore<
                        Arc<dyn HgIdMutableHistoryStore>,
                    > = MultiplexHgIdHistoryStore::new();
                    multiplexstore.add_store(memcachestore.clone().historystore(primary.clone()));
                    multiplexstore.add_store(primary.clone());

                    (
                        Some(memcachestore.clone().remote_historystore(primary.clone())),
                        Arc::new(multiplexstore) as Arc<dyn HgIdMutableHistoryStore>,
                    )
                } else {
                    (None, primary.clone() as Arc<dyn HgIdMutableHistoryStore>)
                };

                let store = remotestore.historystore(shared_store);

                let remotestores = if let Some(cache) = cache {
                    let mut remotestores = UnionHgIdHistoryStore::new();
                    remotestores.add(cache.clone());
                    remotestores.add(store.clone());
                    Arc::new(remotestores)
                } else {
                    store
                };

                historystore.add(Arc::new(remotestores.clone()));
                Some(remotestores)
            } else {
                None
            };

        let shared_mutablehistorystore: Arc<dyn HgIdMutableHistoryStore> = primary;

        Ok(MetadataStore {
            historystore,
            local_mutablehistorystore,
            shared_mutablehistorystore,
            remote_store,
        })
    }
}

#[cfg(test)]
mod tests {
    use std::collections::HashMap;

    use tempfile::TempDir;
    use types::testutil::*;

    use super::*;
    use crate::testutil::make_config;
    use crate::testutil::setconfig;
    use crate::testutil::FakeHgIdRemoteStore;

    #[test]
    fn test_new() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_config(&cachedir);

        MetadataStore::new(&localdir, &config)?;
        Ok(())
    }

    #[test]
    fn test_add_get() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_config(&cachedir);

        let store = MetadataStore::new(&localdir, &config)?;

        let k = key("a", "1");
        let nodeinfo = NodeInfo {
            parents: [key("a", "2"), null_key("a")],
            linknode: hgid("3"),
        };

        store.add(&k, &nodeinfo)?;
        assert_eq!(store.get_node_info(&k)?, Some(nodeinfo));
        Ok(())
    }

    #[test]
    fn test_add_dropped() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_config(&cachedir);

        let store = MetadataStore::new(&localdir, &config)?;

        let k = key("a", "1");
        let nodeinfo = NodeInfo {
            parents: [key("a", "2"), null_key("a")],
            linknode: hgid("3"),
        };

        store.add(&k, &nodeinfo)?;
        drop(store);

        let store = MetadataStore::new(&localdir, &config)?;
        assert!(store.get_node_info(&k)?.is_none());
        Ok(())
    }

    #[test]
    fn test_add_flush_get() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_config(&cachedir);

        let store = MetadataStore::new(&localdir, &config)?;

        let k = key("a", "1");
        let nodeinfo = NodeInfo {
            parents: [key("a", "2"), null_key("a")],
            linknode: hgid("3"),
        };

        store.add(&k, &nodeinfo)?;
        store.flush()?;
        assert_eq!(store.get_node_info(&k)?, Some(nodeinfo));
        Ok(())
    }

    #[test]
    fn test_add_flush_drop_get() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_config(&cachedir);

        let store = MetadataStore::new(&localdir, &config)?;

        let k = key("a", "1");
        let nodeinfo = NodeInfo {
            parents: [key("a", "2"), null_key("a")],
            linknode: hgid("3"),
        };

        store.add(&k, &nodeinfo)?;
        store.flush()?;
        drop(store);

        let store = MetadataStore::new(&localdir, &config)?;
        assert_eq!(store.get_node_info(&k)?, Some(nodeinfo));
        Ok(())
    }

    #[test]
    fn test_remote_store() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_config(&cachedir);

        let k = key("a", "1");
        let nodeinfo = NodeInfo {
            parents: [key("a", "2"), null_key("a")],
            linknode: hgid("3"),
        };

        let mut map = HashMap::new();
        map.insert(k.clone(), nodeinfo.clone());
        let mut remotestore = FakeHgIdRemoteStore::new();
        remotestore.hist(map);

        let store = MetadataStoreBuilder::new(&config)
            .local_path(&localdir)
            .remotestore(Arc::new(remotestore))
            .build()?;
        assert_eq!(store.get_node_info(&k)?, Some(nodeinfo));
        Ok(())
    }

    #[test]
    fn test_remote_store_cached() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_config(&cachedir);

        let k = key("a", "1");
        let nodeinfo = NodeInfo {
            parents: [key("a", "2"), null_key("a")],
            linknode: hgid("3"),
        };

        let mut map = HashMap::new();
        map.insert(k.clone(), nodeinfo.clone());
        let mut remotestore = FakeHgIdRemoteStore::new();
        remotestore.hist(map);

        let store = MetadataStoreBuilder::new(&config)
            .local_path(&localdir)
            .remotestore(Arc::new(remotestore))
            .build()?;
        store.get_node_info(&k)?;
        drop(store);

        let store = MetadataStore::new(&localdir, &config)?;
        assert_eq!(store.get_node_info(&k)?, Some(nodeinfo));
        Ok(())
    }

    #[test]
    fn test_not_in_remote_store() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_config(&cachedir);

        let map = HashMap::new();
        let mut remotestore = FakeHgIdRemoteStore::new();
        remotestore.hist(map);

        let store = MetadataStoreBuilder::new(&config)
            .local_path(&localdir)
            .remotestore(Arc::new(remotestore))
            .build()?;

        let k = key("a", "1");
        assert_eq!(store.get_node_info(&k)?, None);
        Ok(())
    }

    #[test]
    fn test_fetch_location() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_config(&cachedir);

        let k = key("a", "1");
        let nodeinfo = NodeInfo {
            parents: [key("a", "2"), null_key("a")],
            linknode: hgid("3"),
        };

        let mut map = HashMap::new();
        map.insert(k.clone(), nodeinfo.clone());
        let mut remotestore = FakeHgIdRemoteStore::new();
        remotestore.hist(map);

        let store = MetadataStoreBuilder::new(&config)
            .local_path(&localdir)
            .remotestore(Arc::new(remotestore))
            .build()?;
        store.get_node_info(&k)?;
        assert_eq!(
            store.shared_mutablehistorystore.get_node_info(&k)?,
            Some(nodeinfo)
        );
        assert!(
            store
                .local_mutablehistorystore
                .as_ref()
                .unwrap()
                .get_node_info(&k)?
                .is_none()
        );
        Ok(())
    }

    #[test]
    fn test_add_shared_only_store() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_config(&cachedir);

        let store = MetadataStore::new(&localdir, &config)?;

        let k = key("a", "1");
        let nodeinfo = NodeInfo {
            parents: [key("a", "2"), null_key("a")],
            linknode: hgid("3"),
        };

        store.add(&k, &nodeinfo)?;
        store.flush()?;

        let store = MetadataStoreBuilder::new(&config)
            .no_local_store()
            .build()?;
        assert_eq!(store.get_node_info(&k)?, None);
        Ok(())
    }

    #[test]
    fn test_no_local_store() -> Result<()> {
        let cachedir = TempDir::new()?;
        let config = make_config(&cachedir);
        assert!(MetadataStoreBuilder::new(&config).build().is_err());
        Ok(())
    }

    #[test]
    fn test_local_indexedlog_write() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let mut config = make_config(&cachedir);
        setconfig(
            &mut config,
            "remotefilelog",
            "write-local-to-indexedlog",
            "True",
        );

        let store = MetadataStoreBuilder::new(&config)
            .local_path(&localdir)
            .build()?;

        let k1 = key("a", "1");
        let nodeinfo = NodeInfo {
            parents: [key("a", "2"), null_key("a")],
            linknode: hgid("3"),
        };

        store.add(&k1, &nodeinfo)?;
        store.flush()?;
        drop(store);

        let store = IndexedLogHgIdHistoryStore::new(
            get_indexedloghistorystore_path(&localdir)?,
            &config,
            StoreType::Local,
        )?;
        assert_eq!(store.get_node_info(&k1)?, Some(nodeinfo));
        Ok(())
    }

    #[cfg(all(fbcode_build, target_os = "linux"))]
    mod fbcode_tests {
        use memcache::MockMemcache;
        use once_cell::sync::Lazy;

        use super::*;

        static MOCK: Lazy<MockMemcache> = Lazy::new(|| MockMemcache::new());

        #[fbinit::test]
        fn test_memcache_get() -> Result<()> {
            let _mock = Lazy::force(&MOCK);

            let cachedir = TempDir::new()?;
            let localdir = TempDir::new()?;
            let config = make_config(&cachedir);

            let k = key("a", "1");
            let nodeinfo = NodeInfo {
                parents: [key("a", "2"), null_key("a")],
                linknode: hgid("3"),
            };

            let mut map = HashMap::new();
            map.insert(k.clone(), nodeinfo.clone());
            let mut remotestore = FakeHgIdRemoteStore::new();
            remotestore.hist(map);

            let memcache = Arc::new(MemcacheStore::new(&config)?);
            let store = MetadataStoreBuilder::new(&config)
                .local_path(&localdir)
                .remotestore(Arc::new(remotestore))
                .memcachestore(memcache.clone())
                .build()?;
            let nodeinfo_get = store.get_node_info(&k)?;
            assert_eq!(nodeinfo_get, Some(nodeinfo.clone()));

            loop {
                let memcache_nodeinfo = memcache
                    .get_hist_iter(&[k.clone()])
                    .unwrap()
                    .collect::<Result<Vec<_>>>()?;
                if !memcache_nodeinfo.is_empty() {
                    assert_eq!(memcache_nodeinfo[0].nodeinfo, nodeinfo);
                    break;
                }
            }
            Ok(())
        }
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::env;

use types::RepoPath;

pub struct MissingInjection {
    inner: Option<MissingInjectionInner>,
}

struct MissingInjectionInner {
    missing: Vec<String>,
}

impl MissingInjection {
    pub fn new_from_env(name: &str) -> Self {
        match env::var(name) {
            Ok(v) => {
                let mut missing = vec![];
                for part in v.split(",") {
                    missing.push(part.to_string());
                }
                Self {
                    inner: Some(MissingInjectionInner { missing }),
                }
            }
            Err(_) => Self { inner: None },
        }
    }

    pub fn is_missing(&self, path: &RepoPath) -> bool {
        match self.inner {
            None => false,
            Some(ref inner) => {
                for matcher in &inner.missing {
                    if path.as_str().starts_with(matcher.as_str()) {
                        return true;
                    }
                }
                false
            }
        }
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::path::PathBuf;

use anyhow::Result;
use types::Key;
use types::NodeInfo;

use crate::datastore::Delta;
use crate::datastore::HgIdDataStore;
use crate::datastore::HgIdMutableDeltaStore;
use crate::datastore::Metadata;
use crate::datastore::StoreResult;
use crate::historystore::HgIdHistoryStore;
use crate::historystore::HgIdMutableHistoryStore;
use crate::localstore::LocalStore;
use crate::types::StoreKey;

/// A `MultiplexDeltaStore` is a store that will duplicate all the writes to all the
/// delta stores that it is made of.
pub struct MultiplexDeltaStore<T: HgIdMutableDeltaStore> {
    stores: Vec<T>,
}

/// A `MultiplexHgIdHistoryStore` is a store that will duplicate all the writes to all the
/// history stores that it is made of.
pub struct MultiplexHgIdHistoryStore<T: HgIdMutableHistoryStore> {
    stores: Vec<T>,
}

impl<T: HgIdMutableDeltaStore> MultiplexDeltaStore<T> {
    pub fn new() -> Self {
        Self { stores: Vec::new() }
    }

    pub fn add_store(&mut self, store: T) {
        self.stores.push(store)
    }
}

impl<T: HgIdMutableHistoryStore> MultiplexHgIdHistoryStore<T> {
    pub fn new() -> Self {
        Self { stores: Vec::new() }
    }

    pub fn add_store(&mut self, store: T) {
        self.stores.push(store)
    }
}

impl<T: HgIdMutableDeltaStore> HgIdMutableDeltaStore for MultiplexDeltaStore<T> {
    /// Write the `Delta` and `Metadata` to all the stores
    fn add(&self, delta: &Delta, metadata: &Metadata) -> Result<()> {
        for store in self.stores.iter() {
            store.add(delta, metadata)?;
        }

        Ok(())
    }

    fn flush(&self) -> Result<Option<Vec<PathBuf>>> {
        let mut ret = None;
        for store in self.stores.iter() {
            let opt = store.flush()?;
            // It's non sensical for the MultiplexStore to be built with multiple pack stores,
            // therefore let's assert that only one store can ever return a PathBuf.
            assert!(opt.is_none() || !ret.is_some());
            ret = ret.or(opt);
        }

        Ok(ret)
    }
}

impl<T: HgIdMutableDeltaStore> HgIdDataStore for MultiplexDeltaStore<T> {
    fn get(&self, mut key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
        for store in self.stores.iter() {
            match store.get(key)? {
                StoreResult::Found(data) => return Ok(StoreResult::Found(data)),
                StoreResult::NotFound(next) => key = next,
            }
        }

        Ok(StoreResult::NotFound(key))
    }

    fn get_meta(&self, mut key: StoreKey) -> Result<StoreResult<Metadata>> {
        for store in self.stores.iter() {
            match store.get_meta(key)? {
                StoreResult::Found(data) => return Ok(StoreResult::Found(data)),
                StoreResult::NotFound(next) => key = next,
            }
        }

        Ok(StoreResult::NotFound(key))
    }

    fn refresh(&self) -> Result<()> {
        for store in self.stores.iter() {
            store.refresh()?;
        }
        Ok(())
    }
}

impl<T: HgIdMutableDeltaStore> LocalStore for MultiplexDeltaStore<T> {
    fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        let initial_keys = Ok(keys.to_vec());
        self.stores
            .iter()
            .fold(initial_keys, |missing_keys, store| match missing_keys {
                Ok(missing_keys) => store.get_missing(&missing_keys),
                Err(e) => Err(e),
            })
    }
}

impl<T: HgIdMutableHistoryStore> HgIdMutableHistoryStore for MultiplexHgIdHistoryStore<T> {
    fn add(&self, key: &Key, info: &NodeInfo) -> Result<()> {
        for store in self.stores.iter() {
            store.add(key, info)?;
        }

        Ok(())
    }

    fn flush(&self) -> Result<Option<Vec<PathBuf>>> {
        let mut ret = None;
        for store in self.stores.iter() {
            let opt = store.flush()?;
            // It's non sensical for the MultiplexStore to be built with multiple pack stores,
            // therefore let's assert that only one store can ever return a PathBuf.
            assert!(opt.is_none() || !ret.is_some());
            ret = ret.or(opt);
        }

        Ok(ret)
    }
}

impl<T: HgIdMutableHistoryStore> HgIdHistoryStore for MultiplexHgIdHistoryStore<T> {
    fn get_node_info(&self, key: &Key) -> Result<Option<NodeInfo>> {
        for store in self.stores.iter() {
            if let Some(nodeinfo) = store.get_node_info(key)? {
                return Ok(Some(nodeinfo));
            }
        }

        Ok(None)
    }

    fn refresh(&self) -> Result<()> {
        for store in self.stores.iter() {
            store.refresh()?;
        }
        Ok(())
    }
}

impl<T: HgIdMutableHistoryStore> LocalStore for MultiplexHgIdHistoryStore<T> {
    fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        let initial_keys = Ok(keys.to_vec());
        self.stores
            .iter()
            .fold(initial_keys, |missing_keys, store| match missing_keys {
                Ok(missing_keys) => store.get_missing(&missing_keys),
                Err(e) => Err(e),
            })
    }
}

#[cfg(test)]
mod tests {
    use minibytes::Bytes;
    use tempfile::TempDir;
    use types::testutil::*;

    use super::*;
    use crate::datapack::DataPackVersion;
    use crate::datastore::HgIdDataStore;
    use crate::historypack::HistoryPackVersion;
    use crate::historystore::HgIdHistoryStore;
    use crate::indexedlogdatastore::IndexedLogHgIdDataStore;
    use crate::indexedlogdatastore::IndexedLogHgIdDataStoreConfig;
    use crate::indexedlogutil::StoreType;
    use crate::localstore::ExtStoredPolicy;
    use crate::mutabledatapack::MutableDataPack;
    use crate::mutablehistorypack::MutableHistoryPack;

    #[test]
    fn test_delta_add_static() -> Result<()> {
        let tempdir = TempDir::new()?;
        let config = IndexedLogHgIdDataStoreConfig {
            max_log_count: None,
            max_bytes_per_log: None,
            max_bytes: None,
        };
        let mut log = IndexedLogHgIdDataStore::new(
            &tempdir,
            ExtStoredPolicy::Ignore,
            &config,
            StoreType::Shared,
        )?;
        let mut multiplex = MultiplexDeltaStore::new();
        multiplex.add_store(Box::new(&mut log));

        let delta = Delta {
            data: Bytes::from(&[1, 2, 3, 4][..]),
            base: None,
            key: key("a", "1"),
        };
        let metadata = Default::default();

        multiplex.add(&delta, &metadata)?;
        drop(multiplex);
        let read_data = log.get(StoreKey::hgid(delta.key))?;
        assert_eq!(StoreResult::Found(delta.data.as_ref().to_vec()), read_data);
        log.flush()?;
        Ok(())
    }

    #[test]
    fn test_delta_add_dynamic() -> Result<()> {
        let tempdir = TempDir::new()?;
        let config = IndexedLogHgIdDataStoreConfig {
            max_log_count: None,
            max_bytes_per_log: None,
            max_bytes: None,
        };
        let mut log = IndexedLogHgIdDataStore::new(
            &tempdir,
            ExtStoredPolicy::Ignore,
            &config,
            StoreType::Shared,
        )?;
        let mut pack = MutableDataPack::new(&tempdir, DataPackVersion::One);
        let mut multiplex: MultiplexDeltaStore<Box<dyn HgIdMutableDeltaStore>> =
            MultiplexDeltaStore::new();
        multiplex.add_store(Box::new(&mut log));
        multiplex.add_store(Box::new(&mut pack));

        let delta = Delta {
            data: Bytes::from(&[1, 2, 3, 4][..]),
            base: None,
            key: key("a", "1"),
        };
        let metadata = Default::default();

        multiplex.add(&delta, &metadata)?;
        drop(multiplex);

        let k = StoreKey::hgid(delta.key);
        let read_data = log.get(k.clone())?;
        assert_eq!(StoreResult::Found(delta.data.as_ref().to_vec()), read_data);

        let read_data = pack.get(k)?;
        assert_eq!(StoreResult::Found(delta.data.as_ref().to_vec()), read_data);

        log.flush()?;
        pack.flush()?;
        Ok(())
    }

    #[test]
    fn test_history_add_static() -> Result<()> {
        let tempdir = TempDir::new()?;
        let mut pack = MutableHistoryPack::new(&tempdir, HistoryPackVersion::One);
        let mut multiplex = MultiplexHgIdHistoryStore::new();
        multiplex.add_store(Box::new(&mut pack));

        let k = key("a", "1");
        let nodeinfo = NodeInfo {
            parents: [key("a", "2"), key("a", "3")],
            linknode: hgid("4"),
        };

        multiplex.add(&k, &nodeinfo)?;
        drop(multiplex);

        let read_hgid = pack.get_node_info(&k)?;
        assert_eq!(Some(nodeinfo), read_hgid);

        pack.flush()?;
        Ok(())
    }

    #[test]
    fn test_history_add_dynamic() -> Result<()> {
        let tempdir = TempDir::new()?;
        let mut pack1 = MutableHistoryPack::new(&tempdir, HistoryPackVersion::One);
        let mut pack2 = MutableHistoryPack::new(&tempdir, HistoryPackVersion::One);
        let mut multiplex = MultiplexHgIdHistoryStore::new();
        multiplex.add_store(Box::new(&mut pack1));
        multiplex.add_store(Box::new(&mut pack2));

        let k = key("a", "1");
        let nodeinfo = NodeInfo {
            parents: [key("a", "2"), key("a", "3")],
            linknode: hgid("4"),
        };

        multiplex.add(&k, &nodeinfo)?;
        drop(multiplex);

        let read_hgid = pack1.get_node_info(&k)?;
        assert_eq!(Some(nodeinfo.clone()), read_hgid);

        let read_hgid = pack2.get_node_info(&k)?;
        assert_eq!(Some(nodeinfo), read_hgid);

        pack1.flush()?;
        pack2.flush()?;
        Ok(())
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;
use std::io::Read;
use std::io::Seek;
use std::io::SeekFrom;
use std::io::Write;
use std::mem::replace;
use std::path::Path;
use std::path::PathBuf;
use std::u16;

use anyhow::format_err;
use anyhow::Error;
use anyhow::Result;
use byteorder::BigEndian;
use byteorder::WriteBytesExt;
use lz4_pyframe::compress;
use mpatch::mpatch::get_full_text;
use parking_lot::Mutex;
use sha1::Digest;
use sha1::Sha1;
use tempfile::Builder;
use tempfile::NamedTempFile;
use thiserror::Error;
use types::HgId;
use types::Key;

use crate::dataindex::DataIndex;
use crate::dataindex::DeltaLocation;
use crate::datapack::DataEntry;
use crate::datapack::DataPackVersion;
use crate::datastore::Delta;
use crate::datastore::HgIdDataStore;
use crate::datastore::HgIdMutableDeltaStore;
use crate::datastore::Metadata;
use crate::datastore::StoreResult;
use crate::error::EmptyMutablePack;
use crate::localstore::LocalStore;
use crate::mutablepack::MutablePack;
use crate::packwriter::PackWriter;
use crate::types::StoreKey;

struct MutableDataPackInner {
    dir: PathBuf,
    data_file: PackWriter<NamedTempFile>,
    mem_index: HashMap<HgId, DeltaLocation>,
    hasher: Sha1,
}

pub struct MutableDataPack {
    dir: PathBuf,
    version: DataPackVersion,
    inner: Mutex<Option<MutableDataPackInner>>,
}

#[derive(Debug, Error)]
#[error("Mutable Data Pack Error: {0:?}")]
struct MutableDataPackError(String);

impl MutableDataPackInner {
    /// Creates a new MutableDataPack for producing datapack files.
    ///
    /// The data is written to a temporary file, and renamed to the final location
    /// when flush() is called, at which point the MutableDataPack is consumed. If
    /// flush() is not called, the temporary file is cleaned up when the object is
    /// release.
    pub fn new(dir: impl AsRef<Path>, version: DataPackVersion) -> Result<Self> {
        let dir = dir.as_ref();
        if !dir.is_dir() {
            return Err(format_err!(
                "cannot create mutable datapack in non-directory '{:?}'",
                dir
            ));
        }

        if version == DataPackVersion::Zero {
            return Err(format_err!("cannot create a v0 datapack"));
        }

        let tempfile = Builder::new().append(true).tempfile_in(&dir)?;
        let mut data_file = PackWriter::new(tempfile);
        let mut hasher = Sha1::new();
        let version_u8: u8 = version.into();
        data_file.write_u8(version_u8)?;
        hasher.update(&[version_u8]);

        Ok(Self {
            dir: dir.to_path_buf(),
            data_file,
            mem_index: HashMap::new(),
            hasher,
        })
    }

    fn read_entry(&self, key: &Key) -> Result<Option<(Delta, Metadata)>> {
        let location: &DeltaLocation = match self.mem_index.get(&key.hgid) {
            None => return Ok(None),
            Some(location) => location,
        };

        // Make sure the buffers are empty so the reads below are consistent with what is being
        // written.
        self.data_file.flush_inner()?;
        let mut file = self.data_file.get_mut();

        let mut data = Vec::with_capacity(location.size as usize);
        unsafe { data.set_len(location.size as usize) };

        file.seek(SeekFrom::Start(location.offset))?;
        file.read_exact(&mut data)?;

        let entry = DataEntry::new(&data, 0, DataPackVersion::One)?;
        Ok(Some((
            Delta {
                data: entry.delta()?,
                base: entry
                    .delta_base()
                    .map(|delta_base| Key::new(key.path.clone(), delta_base.clone())),
                key: Key::new(key.path.clone(), entry.hgid().clone()),
            },
            entry.metadata().clone(),
        )))
    }

    fn add(&mut self, delta: &Delta, metadata: &Metadata) -> Result<()> {
        let path_slice = delta.key.path.as_byte_slice();
        if path_slice.len() >= u16::MAX as usize {
            return Err(MutableDataPackError("delta path is longer than 2^16".into()).into());
        }

        let offset = self.data_file.bytes_written();

        let compressed = compress(&delta.data)?;

        // Preallocate with approximately the size we need:
        // (namelen(2) + name + hgid(20) + hgid(20) + datalen(8) + data + metadata(~22))
        let mut buf = Vec::with_capacity(path_slice.len() + compressed.len() + 72);
        buf.write_u16::<BigEndian>(path_slice.len() as u16)?;
        buf.write_all(path_slice)?;
        buf.write_all(delta.key.hgid.as_ref())?;

        buf.write_all(
            delta
                .base
                .as_ref()
                .map_or_else(|| HgId::null_id(), |k| &k.hgid)
                .as_ref(),
        )?;
        buf.write_u64::<BigEndian>(compressed.len() as u64)?;
        buf.write_all(&compressed)?;

        metadata.write(&mut buf)?;

        self.data_file.write_all(&buf)?;
        self.hasher.update(&buf);

        let delta_location = DeltaLocation {
            delta_base: delta.base.as_ref().map(|k| k.hgid.clone()),
            offset,
            size: buf.len() as u64,
        };
        self.mem_index
            .insert(delta.key.hgid.clone(), delta_location);
        Ok(())
    }
}

impl MutableDataPack {
    pub fn new(dir: impl AsRef<Path>, version: DataPackVersion) -> Self {
        Self {
            dir: dir.as_ref().to_path_buf(),
            version,
            inner: Mutex::new(None),
        }
    }

    fn get_pack<'a>(
        &self,
        inner: &'a mut Option<MutableDataPackInner>,
    ) -> Result<&'a mut MutableDataPackInner> {
        if inner.is_none() {
            inner.replace(MutableDataPackInner::new(&self.dir, self.version.clone())?);
        }
        Ok(inner.as_mut().unwrap())
    }

    fn get_delta_chain(&self, key: &Key) -> Result<Option<Vec<Delta>>> {
        let mut guard = self.inner.lock();
        if let Some(pack) = guard.as_mut() {
            let mut chain: Vec<Delta> = Default::default();
            let mut next_key = Some(key.clone());
            while let Some(key) = next_key {
                let (delta, _metadata) = match pack.read_entry(&key) {
                    Ok(Some(entry)) => entry,
                    Ok(None) => {
                        if chain.is_empty() {
                            return Ok(None);
                        } else {
                            return Ok(Some(chain));
                        }
                    }
                    Err(e) => {
                        if chain.is_empty() {
                            return Err(e);
                        } else {
                            return Ok(Some(chain));
                        }
                    }
                };
                next_key = delta.base.clone();
                chain.push(delta);
            }
            Ok(Some(chain))
        } else {
            Ok(None)
        }
    }
}

impl HgIdMutableDeltaStore for MutableDataPack {
    /// Adds the given entry to the mutable datapack.
    fn add(&self, delta: &Delta, metadata: &Metadata) -> Result<()> {
        let mut guard = self.inner.lock();
        let pack = self.get_pack(&mut guard)?;
        pack.add(delta, metadata)
    }

    fn flush(&self) -> Result<Option<Vec<PathBuf>>> {
        let mut guard = self.inner.lock();
        let old_inner = replace(&mut *guard, None);

        if let Some(old_inner) = old_inner {
            Ok(match old_inner.close_pack()? {
                Some(pack) => Some(vec![pack]),
                None => Some(vec![]),
            })
        } else {
            Ok(None)
        }
    }
}

impl MutablePack for MutableDataPackInner {
    fn build_files(self) -> Result<(NamedTempFile, NamedTempFile, PathBuf)> {
        if self.mem_index.is_empty() {
            return Err(EmptyMutablePack.into());
        }

        let mut index_file = PackWriter::new(NamedTempFile::new_in(&self.dir)?);
        DataIndex::write(&mut index_file, &self.mem_index)?;

        Ok((
            self.data_file.into_inner()?,
            index_file.into_inner()?,
            self.dir.join(&hex::encode(self.hasher.finalize())),
        ))
    }

    fn extension(&self) -> &'static str {
        "data"
    }
}

impl MutablePack for MutableDataPack {
    fn build_files(self) -> Result<(NamedTempFile, NamedTempFile, PathBuf)> {
        let old_inner = (*self.inner.lock()).take();
        if let Some(old_inner) = old_inner {
            old_inner.build_files()
        } else {
            Err(EmptyMutablePack.into())
        }
    }

    fn extension(&self) -> &'static str {
        "data"
    }
}

impl HgIdDataStore for MutableDataPack {
    fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
        let key = match key {
            StoreKey::HgId(key) => key,
            content => return Ok(StoreResult::NotFound(content)),
        };

        let delta_chain = self.get_delta_chain(&key)?;
        let delta_chain = match delta_chain {
            Some(chain) => chain,
            None => return Ok(StoreResult::NotFound(StoreKey::HgId(key))),
        };

        let (basetext, deltas) = match delta_chain.split_last() {
            Some((base, delta)) => (base, delta),
            None => return Ok(StoreResult::NotFound(StoreKey::HgId(key))),
        };

        let deltas: Vec<&[u8]> = deltas
            .iter()
            .rev()
            .map(|delta| delta.data.as_ref())
            .collect();

        Ok(StoreResult::Found(
            get_full_text(basetext.data.as_ref(), &deltas).map_err(Error::msg)?,
        ))
    }

    fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
        let mut guard = self.inner.lock();
        if let Some(pack) = guard.as_mut() {
            let key = match key {
                StoreKey::HgId(key) => key,
                content => return Ok(StoreResult::NotFound(content)),
            };

            match pack.read_entry(&key)? {
                None => Ok(StoreResult::NotFound(StoreKey::HgId(key))),
                Some((_, metadata)) => Ok(StoreResult::Found(metadata)),
            }
        } else {
            Ok(StoreResult::NotFound(key))
        }
    }

    fn refresh(&self) -> Result<()> {
        Ok(())
    }
}

impl LocalStore for MutableDataPack {
    fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        let mut guard = self.inner.lock();
        if let Some(pack) = guard.as_mut() {
            Ok(keys
                .iter()
                .filter(|k| match k {
                    StoreKey::HgId(k) => pack.mem_index.get(&k.hgid).is_none(),
                    StoreKey::Content(_, _) => true,
                })
                .cloned()
                .collect())
        } else {
            Ok(keys.to_vec())
        }
    }
}

#[cfg(test)]
mod tests {
    use std::fs;
    use std::fs::File;
    use std::io::Read;

    use minibytes::Bytes;
    use tempfile::tempdir;
    use types::testutil::*;
    use types::Key;
    use types::RepoPathBuf;

    use super::*;

    #[test]
    fn test_basic_creation() {
        let tempdir = tempdir().unwrap();
        let mutdatapack = MutableDataPack::new(tempdir.path(), DataPackVersion::One);
        let delta = Delta {
            data: Bytes::from(&[0, 1, 2][..]),
            base: None,
            key: Key::new(RepoPathBuf::new(), Default::default()),
        };
        mutdatapack.add(&delta, &Default::default()).expect("add");
        let datapackbase = mutdatapack.flush().expect("flush").unwrap()[0].clone();
        let datapackpath = datapackbase.with_extension("datapack");
        let dataindexpath = datapackbase.with_extension("dataidx");

        assert!(datapackpath.exists());
        assert!(dataindexpath.exists());

        // Verify the hash
        let mut temppath = datapackpath.clone();
        // The file's name is the hash of it's content, so drop the extension to get just the name
        temppath.set_extension("");

        let filename_hash = temppath.file_name().unwrap().to_str().unwrap();
        let mut hasher = Sha1::new();
        let mut file = File::open(datapackpath).expect("file");
        let mut buf = vec![];
        file.read_to_end(&mut buf).expect("read to end");
        hasher.update(&buf);
        let hash = hex::encode(hasher.finalize());
        assert!(hash == filename_hash);
    }

    #[test]
    fn test_basic_abort() {
        let tempdir = tempdir().unwrap();
        {
            let mutdatapack = MutableDataPack::new(tempdir.path(), DataPackVersion::One);
            let delta = Delta {
                data: Bytes::from(&[0, 1, 2][..]),
                base: None,
                key: Key::new(RepoPathBuf::new(), Default::default()),
            };
            mutdatapack.add(&delta, &Default::default()).expect("add");
        }

        assert_eq!(fs::read_dir(tempdir.path()).unwrap().count(), 0);
    }

    #[test]
    fn test_get_delta_chain() {
        let tempdir = tempdir().unwrap();
        let mutdatapack = MutableDataPack::new(tempdir.path(), DataPackVersion::One);
        let delta = Delta {
            data: Bytes::from(&[0, 1, 2][..]),
            base: None,
            key: Key::new(RepoPathBuf::new(), hgid("1")),
        };
        mutdatapack.add(&delta, &Default::default()).unwrap();
        let delta2 = Delta {
            data: Bytes::from(&[0, 1, 2][..]),
            base: Some(Key::new(RepoPathBuf::new(), delta.key.hgid.clone())),
            key: Key::new(RepoPathBuf::new(), hgid("2")),
        };
        mutdatapack.add(&delta2, &Default::default()).unwrap();

        let chain = mutdatapack.get_delta_chain(&delta.key).unwrap();
        assert_eq!(&vec![delta.clone()], &chain.unwrap());

        let chain = mutdatapack.get_delta_chain(&delta2.key).unwrap();
        assert_eq!(&vec![delta2.clone(), delta.clone()], &chain.unwrap());
    }

    #[test]
    fn test_get_partial_delta_chain() -> Result<()> {
        let tempdir = tempdir()?;
        let mutdatapack = MutableDataPack::new(tempdir.path(), DataPackVersion::One);

        let delta = Delta {
            data: Bytes::from(&[0, 1, 2][..]),
            base: Some(key("a", "1")),
            key: key("a", "2"),
        };

        mutdatapack.add(&delta, &Default::default())?;
        let chain = mutdatapack.get_delta_chain(&delta.key)?.unwrap();
        assert_eq!(chain.len(), 1);
        assert_eq!(chain.get(0), Some(&delta));

        Ok(())
    }

    #[test]
    fn test_get_meta() {
        let tempdir = tempdir().unwrap();

        let mutdatapack = MutableDataPack::new(tempdir.path(), DataPackVersion::One);
        let delta = Delta {
            data: Bytes::from(&[0, 1, 2][..]),
            base: None,
            key: Key::new(RepoPathBuf::new(), hgid("1")),
        };
        mutdatapack.add(&delta, &Default::default()).unwrap();
        let delta2 = Delta {
            data: Bytes::from(&[0, 1, 2][..]),
            base: None,
            key: Key::new(RepoPathBuf::new(), hgid("2")),
        };
        let meta2 = Metadata {
            flags: Some(2),
            size: Some(1000),
        };
        mutdatapack.add(&delta2, &meta2).unwrap();

        // Requesting a default metadata
        let found_meta = mutdatapack.get_meta(StoreKey::hgid(delta.key)).unwrap();
        assert_eq!(found_meta, StoreResult::Found(Metadata::default()));

        // Requesting a specified metadata
        let found_meta = mutdatapack.get_meta(StoreKey::hgid(delta2.key)).unwrap();
        assert_eq!(found_meta, StoreResult::Found(meta2));

        // Requesting a non-existent metadata
        let not = StoreKey::hgid(key("not", "10000"));
        assert_eq!(
            mutdatapack.get_meta(not.clone()).unwrap(),
            StoreResult::NotFound(not)
        );
    }

    #[test]
    fn test_get_missing() {
        let tempdir = tempdir().unwrap();

        let mutdatapack = MutableDataPack::new(tempdir.path(), DataPackVersion::One);
        let delta = Delta {
            data: Bytes::from(&[0, 1, 2][..]),
            base: None,
            key: Key::new(RepoPathBuf::new(), Default::default()),
        };
        mutdatapack.add(&delta, &Default::default()).unwrap();

        let not = key("not", "10000");
        let missing = mutdatapack
            .get_missing(&vec![StoreKey::from(delta.key), StoreKey::from(&not)])
            .unwrap();
        assert_eq!(missing, vec![StoreKey::from(not)]);
    }

    #[test]
    fn test_empty() {
        let tempdir = tempdir().unwrap();

        let mutdatapack = MutableDataPack::new(tempdir.path(), DataPackVersion::One);
        assert_eq!(mutdatapack.flush().unwrap(), None);
        drop(mutdatapack);
        assert_eq!(fs::read_dir(tempdir.path()).unwrap().count(), 0);
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;
use std::collections::HashSet;
use std::collections::VecDeque;
use std::io::Write;
use std::path::Path;
use std::path::PathBuf;

use anyhow::Result;
use byteorder::WriteBytesExt;
use parking_lot::Mutex;
use sha1::Digest;
use sha1::Sha1;
use tempfile::NamedTempFile;
use thiserror::Error;
use types::Key;
use types::NodeInfo;
use types::RepoPath;
use types::RepoPathBuf;

use crate::error::EmptyMutablePack;
use crate::historyindex::FileSectionLocation;
use crate::historyindex::HistoryIndex;
use crate::historyindex::NodeLocation;
use crate::historypack::FileSectionHeader;
use crate::historypack::HistoryEntry;
use crate::historypack::HistoryPackVersion;
use crate::historystore::HgIdHistoryStore;
use crate::historystore::HgIdMutableHistoryStore;
use crate::localstore::LocalStore;
use crate::mutablepack::MutablePack;
use crate::packwriter::PackWriter;
use crate::types::StoreKey;

#[derive(Debug, Error)]
#[error("Mutable History Pack Error: {0:?}")]
struct MutableHistoryPackError(String);

struct MutableHistoryPackInner {
    version: HistoryPackVersion,
    dir: PathBuf,
    mem_index: HashMap<RepoPathBuf, HashMap<Key, NodeInfo>>,
}

pub struct MutableHistoryPack {
    dir: PathBuf,
    version: HistoryPackVersion,
    inner: Mutex<Option<MutableHistoryPackInner>>,
}

impl MutableHistoryPackInner {
    pub fn new(dir: impl AsRef<Path>, version: HistoryPackVersion) -> Result<Self> {
        let dir = dir.as_ref();
        if !dir.is_dir() {
            return Err(MutableHistoryPackError(format!(
                "cannot create mutable historypack in non-directory '{:?}'",
                dir
            ))
            .into());
        }

        Ok(Self {
            version,
            dir: dir.to_path_buf(),
            mem_index: HashMap::new(),
        })
    }

    fn write_section<'a>(
        &self,
        writer: &mut Vec<u8>,
        file_name: &'a RepoPath,
        hgid_map: &HashMap<Key, NodeInfo>,
        section_offset: usize,
        nodes: &mut HashMap<&'a RepoPath, HashMap<Key, NodeLocation>>,
    ) -> Result<()> {
        let mut hgid_locations = HashMap::<Key, NodeLocation>::with_capacity(hgid_map.len());

        // Write section header
        FileSectionHeader {
            file_name: &file_name,
            count: hgid_map.len() as u32,
        }
        .write(writer)?;

        // Sort the nodes in topological order (ancestors first), as required by the histpack spec
        let hgid_map = topo_sort(hgid_map)?;

        // Write nodes
        for (key, node_info) in hgid_map.iter() {
            let p1 = &node_info.parents[0];
            let copyfrom = if !p1.hgid.is_null() && p1.path != key.path {
                Some(p1.path.as_ref())
            } else {
                None
            };

            let hgid_offset = section_offset + writer.len() as usize;
            HistoryEntry::write(
                writer,
                &key.hgid,
                &node_info.parents[0].hgid,
                &node_info.parents[1].hgid,
                &node_info.linknode,
                &copyfrom,
            )?;

            hgid_locations.insert(
                (*key).clone(),
                NodeLocation {
                    offset: hgid_offset as u64,
                },
            );
        }

        nodes.insert(file_name, hgid_locations);
        Ok(())
    }
}

impl MutableHistoryPack {
    pub fn new(dir: impl AsRef<Path>, version: HistoryPackVersion) -> Self {
        Self {
            dir: dir.as_ref().to_path_buf(),
            version,
            inner: Mutex::new(None),
        }
    }

    fn get_pack<'a>(
        &self,
        inner: &'a mut Option<MutableHistoryPackInner>,
    ) -> Result<&'a mut MutableHistoryPackInner> {
        if inner.is_none() {
            inner.replace(MutableHistoryPackInner::new(
                &self.dir,
                self.version.clone(),
            )?);
        }
        Ok(inner.as_mut().unwrap())
    }
}

impl HgIdMutableHistoryStore for MutableHistoryPack {
    fn add(&self, key: &Key, info: &NodeInfo) -> Result<()> {
        let mut guard = self.inner.lock();
        let pack = self.get_pack(&mut guard)?;
        // Loops in the graph aren't allowed. Since this is a logic error in the code, let's
        // assert.
        assert_ne!(key.hgid, info.parents[0].hgid);
        assert_ne!(key.hgid, info.parents[1].hgid);

        // Ideally we could use something like:
        //     self.mem_index.entry(key.name()).or_insert_with(|| HashMap::new())
        // To get the inner map, then insert our new NodeInfo. Unfortunately it requires
        // key.name().clone() though. So we have to do it the long way to avoid the allocation.
        let entries = pack
            .mem_index
            .entry(key.path.clone())
            .or_insert_with(HashMap::new);
        entries.insert(key.clone(), info.clone());
        Ok(())
    }

    fn flush(&self) -> Result<Option<Vec<PathBuf>>> {
        let mut guard = self.inner.lock();
        let old_inner = (*guard).take();

        if let Some(old_inner) = old_inner {
            Ok(match old_inner.close_pack()? {
                Some(pack) => Some(vec![pack]),
                None => Some(vec![]),
            })
        } else {
            Ok(None)
        }
    }
}

impl MutablePack for MutableHistoryPackInner {
    fn build_files(self) -> Result<(NamedTempFile, NamedTempFile, PathBuf)> {
        if self.mem_index.is_empty() {
            return Err(EmptyMutablePack.into());
        }

        let mut data_file = PackWriter::new(NamedTempFile::new_in(&self.dir)?);
        let mut hasher = Sha1::new();

        // Write the header
        let version_u8: u8 = self.version.clone().into();
        data_file.write_u8(version_u8)?;
        hasher.update(&[version_u8]);

        // Store data for the index
        let mut file_sections: Vec<(&RepoPath, FileSectionLocation)> = Default::default();
        let mut nodes: HashMap<&RepoPath, HashMap<Key, NodeLocation>> = Default::default();

        // Write the historypack
        let mut section_buf = Vec::new();
        let mut section_offset = data_file.bytes_written();
        // - In sorted order for deterministic hashes.
        let mut keys = self.mem_index.keys().collect::<Vec<_>>();
        keys.sort_unstable();
        for file_name in keys {
            let hgid_map = self.mem_index.get(file_name).unwrap();
            self.write_section(
                &mut section_buf,
                file_name,
                hgid_map,
                section_offset as usize,
                &mut nodes,
            )?;
            hasher.update(&section_buf);
            data_file.write_all(&section_buf)?;

            let section_location = FileSectionLocation {
                offset: section_offset,
                size: section_buf.len() as u64,
            };
            file_sections.push((file_name, section_location));

            section_offset += section_buf.len() as u64;
            section_buf.clear();
        }

        // Compute the index
        let mut index_file = PackWriter::new(NamedTempFile::new_in(&self.dir)?);
        HistoryIndex::write(&mut index_file, &file_sections, &nodes)?;

        Ok((
            data_file.into_inner()?,
            index_file.into_inner()?,
            self.dir.join(hex::encode(hasher.finalize())),
        ))
    }

    fn extension(&self) -> &'static str {
        "hist"
    }
}

impl MutablePack for MutableHistoryPack {
    fn build_files(self) -> Result<(NamedTempFile, NamedTempFile, PathBuf)> {
        let old_inner = (*self.inner.lock()).take();
        if let Some(old_inner) = old_inner {
            old_inner.build_files()
        } else {
            Err(EmptyMutablePack.into())
        }
    }

    fn extension(&self) -> &'static str {
        "hist"
    }
}

fn topo_sort(hgid_map: &HashMap<Key, NodeInfo>) -> Result<Vec<(&Key, &NodeInfo)>> {
    // Sorts the given keys into newest-first topological order
    let mut roots = Vec::<&Key>::new();

    // Child map will be used to perform an oldest-first walk later.
    let mut child_map = HashMap::<&Key, HashSet<&Key>>::with_capacity(hgid_map.len());
    // Parent count will be used to keep track of when all a commit's parents have been processed.
    let mut parent_counts = HashMap::with_capacity(hgid_map.len());

    for (key, info) in hgid_map.iter() {
        let mut parent_count = 0;
        for parent in &info.parents {
            // Only record the relationship if the parent is also in the provided hgid_map.
            // This also filters out null parents.
            if hgid_map.contains_key(parent) {
                let children = child_map.entry(parent).or_default();
                if !children.contains(key) {
                    children.insert(key);
                    parent_count += 1;
                }
            }
        }

        if parent_count == 0 {
            roots.push(key);
        } else {
            parent_counts.insert(key, parent_count);
        }
    }

    // Sort the roots so things are deterministic.
    roots.sort_unstable();

    // Process roots, adding children to the queue once all their parents are processed.
    let mut pending = VecDeque::<&Key>::from_iter(roots.iter().cloned());
    let mut results = Vec::new();
    while let Some(key) = pending.pop_front() {
        results.push((key, hgid_map.get(key).unwrap()));

        if let Some(children) = child_map.get(key) {
            for child in children.iter() {
                let mut parent_count = parent_counts
                    .get(child)
                    .ok_or_else(|| {
                        MutableHistoryPackError(format!("missing {:?} during topo sort", child))
                    })?
                    .clone();
                parent_count -= 1;
                parent_counts.insert(child, parent_count);
                if parent_count == 0 {
                    // If a child has no more parents, its a root and is ready for processing.
                    // Put it at the front so ancestor chains are processed contiguously.
                    pending.push_front(child);
                }
            }
        }
    }

    // We built the result in oldest first order, but we need it in newest first order.
    results.reverse();

    assert_eq!(results.len(), hgid_map.len());
    Ok(results)
}

impl HgIdHistoryStore for MutableHistoryPack {
    fn get_node_info(&self, key: &Key) -> Result<Option<NodeInfo>> {
        let mut guard = self.inner.lock();
        if let Some(pack) = guard.as_mut() {
            Ok(pack
                .mem_index
                .get(&key.path)
                .and_then(|nodes| nodes.get(key))
                .cloned())
        } else {
            Ok(None)
        }
    }

    fn refresh(&self) -> Result<()> {
        Ok(())
    }
}

impl LocalStore for MutableHistoryPack {
    fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        let mut guard = self.inner.lock();
        if let Some(pack) = guard.as_mut() {
            Ok(keys
                .iter()
                .filter(|k| match k {
                    StoreKey::HgId(k) => match pack.mem_index.get(&k.path) {
                        Some(e) => e.get(k).is_none(),
                        None => true,
                    },
                    StoreKey::Content(_, _) => true,
                })
                .cloned()
                .collect())
        } else {
            Ok(keys.to_vec())
        }
    }
}

#[cfg(test)]
mod tests {
    use std::fs;

    use quickcheck::quickcheck;
    use rand::seq::SliceRandom;
    use rand::SeedableRng;
    use rand_chacha::ChaChaRng;
    use tempfile::tempdir;
    use types::hgid::HgId;
    use types::testutil::key;

    use super::*;
    use crate::historypack::HistoryPack;
    use crate::repack::ToKeys;

    #[test]
    fn test_topo_order() {
        // Tests for exponential time complexity in a merge ancestory. This doesn't won't fail,
        // but may take a long time if there is bad time complexity.
        let mut rng = ChaChaRng::from_seed([0u8; 32]);
        let tempdir = tempdir().unwrap();
        let muthistorypack = MutableHistoryPack::new(tempdir.path(), HistoryPackVersion::One);
        let null_key = Key::new(RepoPathBuf::new(), HgId::null_id().clone());

        let chain_count = 2;
        let chain_len = 3;

        let mut chains = HashMap::<Key, Vec<(Key, NodeInfo)>>::new();
        let mut entries = Vec::<(Key, NodeInfo)>::new();
        for _ in 0..chain_count {
            let mut chain = Vec::<(Key, NodeInfo)>::new();
            for i in 0..chain_len {
                let p1 = if i > 0 {
                    chain[i - 1].0.clone()
                } else {
                    null_key.clone()
                };
                let p2 = if i > 1 {
                    chain[i - 2].0.clone()
                } else {
                    null_key.clone()
                };

                let key = Key::new(RepoPathBuf::new(), HgId::random(&mut rng));
                let info = NodeInfo {
                    parents: [p1, p2],
                    linknode: HgId::random(&mut rng),
                };
                entries.push((key.clone(), info.clone()));
                chain.push((key.clone(), info.clone()));
                if i == chain_len - 1 {
                    // Reverse it so the newest key is first.
                    chain.reverse();
                    chains.insert(key, chain.clone());
                }
            }
        }

        // Add them in random order, so we can verify they get sorted correctly
        entries.shuffle(&mut rng);
        for (key, info) in entries.iter() {
            muthistorypack.add(&key, &info).unwrap();
        }
        let path = &muthistorypack.flush().unwrap().unwrap()[0];
        let pack = HistoryPack::new(&path).unwrap();

        let actual_order = pack
            .to_keys()
            .into_iter()
            .collect::<Result<Vec<Key>>>()
            .unwrap();

        // Compute the expected order
        let mut chains = chains.iter().collect::<Vec<_>>();
        chains.sort_unstable();
        chains.reverse();
        let mut expected_order = vec![];
        for (_, chain) in chains.iter() {
            for (key, _) in chain.iter() {
                expected_order.push(key.clone());
            }
        }

        assert_eq!(actual_order, expected_order);
    }

    #[test]
    #[should_panic]
    fn test_loop() {
        let tempdir = tempdir().unwrap();
        let muthistorypack = MutableHistoryPack::new(tempdir.path(), HistoryPackVersion::One);

        let k = key("a", "1");
        let nodeinfo = NodeInfo {
            parents: [k.clone(), k.clone()],
            linknode: Default::default(),
        };

        muthistorypack.add(&k, &nodeinfo).unwrap();
    }

    #[test]
    fn test_empty() {
        let tempdir = tempdir().unwrap();
        let muthistorypack = MutableHistoryPack::new(tempdir.path(), HistoryPackVersion::One);
        assert!(muthistorypack.flush().unwrap().is_none());
        drop(muthistorypack);
        assert_eq!(fs::read_dir(tempdir.path()).unwrap().count(), 0);
    }

    quickcheck! {
        fn test_get_node_info(insert: HashMap<Key, NodeInfo>, notinsert: Vec<Key>) -> bool {
            let tempdir = tempdir().unwrap();
            let muthistorypack =
                MutableHistoryPack::new(tempdir.path(), HistoryPackVersion::One);

            for (key, info) in insert.iter() {
                muthistorypack.add(&key, &info).unwrap();
            }

            for (key, info) in insert.iter() {
                if *info != muthistorypack.get_node_info(key).unwrap().unwrap() {
                    return false;
                }
            }

            for key in notinsert.iter() {
                if muthistorypack.get_node_info(key).unwrap().is_some() {
                    return false;
                }
            }

            true
        }

        fn test_get_missing(insert: HashMap<Key, NodeInfo>, notinsert: Vec<StoreKey>) -> bool {
            let tempdir = tempdir().unwrap();
            let muthistorypack =
                MutableHistoryPack::new(tempdir.path(), HistoryPackVersion::One);

            for (key, info) in insert.iter() {
                muthistorypack.add(&key, &info).unwrap();
            }

            let mut lookup = notinsert.clone();
            lookup.extend(insert.keys().map(|k| StoreKey::from(k)));

            let missing = muthistorypack.get_missing(&lookup).unwrap();
            missing == notinsert
        }
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::fs::Permissions;
use std::io::ErrorKind;
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
use std::path::PathBuf;

use anyhow::Result;
use tempfile::NamedTempFile;

use crate::error::EmptyMutablePack;

/// Mark the permission as read-only for user-group-other.
#[cfg(not(unix))]
fn make_readonly(perms: &mut Permissions) {
    perms.set_readonly(true);
}

#[cfg(unix)]
fn make_readonly(perms: &mut Permissions) {
    perms.set_mode(0o444);
}

/// Persist the temporary file.
///
/// Since packfiles are named based on their content, a rename failure due to an already existing
/// file isn't an error, as both files have effectively the same content.
fn persist(file: NamedTempFile, path: PathBuf) -> Result<()> {
    match file.persist_noclobber(path) {
        Ok(_) => Ok(()),
        Err(e) => {
            if e.error.kind() != ErrorKind::AlreadyExists {
                Err(e.into())
            } else {
                Ok(())
            }
        }
    }
}

pub trait MutablePack {
    /// Make the data and index pack files with the data added to it. Also returns the fullpath of
    /// the files. After calling this function, the `MutablePack` is consumed and is no longer usable.
    fn build_files(self) -> Result<(NamedTempFile, NamedTempFile, PathBuf)>;

    /// Returns the extension for this kind of pack files.
    fn extension(&self) -> &'static str;

    /// Close the packfile, returning the path of the final immutable pack on disk. The
    /// `MutablePack` is no longer usable after being closed.
    fn close_pack(self) -> Result<Option<PathBuf>>
    where
        Self: Sized,
    {
        let extension = self.extension().to_string();
        let pack_extension = extension.clone() + "pack";
        let index_extension = extension + "idx";

        let (packfile, indexfile, base_filepath) = match self.build_files() {
            Err(err) => {
                if err.downcast_ref::<EmptyMutablePack>().is_some() {
                    return Ok(None);
                } else {
                    return Err(err);
                }
            }
            Ok(files) => files,
        };

        let mut perms = packfile.as_file().metadata()?.permissions();
        make_readonly(&mut perms);

        packfile.as_file().set_permissions(perms.clone())?;
        indexfile.as_file().set_permissions(perms)?;

        let packfile_path = base_filepath.with_extension(pack_extension);
        let indexfile_path = base_filepath.with_extension(index_extension);

        persist(packfile, packfile_path)?;
        persist(indexfile, indexfile_path)?;

        Ok(Some(base_filepath))
    }
}