/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashSet;
use std::fs;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;

use anyhow::format_err;
use anyhow::Result;
use configmodel::convert::ByteCount;
use configmodel::Config;
use configmodel::ConfigExt;
use hgstore::strip_metadata;
use hgtime::HgTime;
use minibytes::Bytes;
use regex::Regex;
use tracing::info_span;
use types::Key;
use types::RepoPathBuf;

use crate::datastore::ContentDataStore;
use crate::datastore::ContentMetadata;
use crate::datastore::Delta;
use crate::datastore::HgIdDataStore;
use crate::datastore::HgIdMutableDeltaStore;
use crate::datastore::LegacyStore;
use crate::datastore::Metadata;
use crate::datastore::RemoteDataStore;
use crate::datastore::ReportingRemoteDataStore;
use crate::datastore::StoreResult;
use crate::indexedlogdatastore::IndexedLogHgIdDataStore;
use crate::indexedlogdatastore::IndexedLogHgIdDataStoreConfig;
use crate::indexedlogutil::StoreType;
use crate::lfs::LfsFallbackRemoteStore;
use crate::lfs::LfsMultiplexer;
use crate::lfs::LfsRemote;
use crate::lfs::LfsStore;
use crate::localstore::ExtStoredPolicy;
use crate::localstore::LocalStore;
use crate::memcache::MemcacheStore;
use crate::multiplexstore::MultiplexDeltaStore;
use crate::packstore::CorruptionPolicy;
use crate::packstore::MutableDataPackStore;
use crate::remotestore::HgIdRemoteStore;
use crate::repack::RepackLocation;
use crate::types::StoreKey;
use crate::uniondatastore::UnionContentDataStore;
use crate::uniondatastore::UnionHgIdDataStore;
use crate::util::check_run_once;
use crate::util::get_cache_packs_path;
use crate::util::get_cache_path;
use crate::util::get_indexedlogdatastore_path;
use crate::util::get_local_path;
use crate::util::get_packs_path;
use crate::util::RUN_ONCE_FILENAME;

/// A `ContentStore` aggregate all the local and remote stores and expose them as one. Both local and
/// remote stores can be queried and accessed via the `HgIdDataStore` trait. The local store can also
/// be written to via the `HgIdMutableDeltaStore` trait, this is intended to be used to store local
/// commit data.
pub struct ContentStore {
    datastore: UnionHgIdDataStore<Arc<dyn HgIdDataStore>>,
    local_mutabledatastore: Option<Arc<dyn HgIdMutableDeltaStore>>,
    shared_mutabledatastore: Arc<dyn HgIdMutableDeltaStore>,
    remote_store: Option<Arc<ReportingRemoteDataStore>>,

    blob_stores: UnionContentDataStore<Arc<dyn ContentDataStore>>,
}

impl ContentStore {
    pub fn new(local_path: impl AsRef<Path>, config: &dyn Config) -> Result<Self> {
        ContentStoreBuilder::new(config)
            .local_path(&local_path)
            .build()
    }

    /// Attempt to repair the underlying stores that the `ContentStore` is comprised of.
    ///
    /// As this may violate some of the stores asumptions, care must be taken to call this only
    /// when no other `ContentStore` have been created for the `shared_path`.
    pub fn repair(
        shared_path: impl AsRef<Path>,
        local_path: Option<impl AsRef<Path>>,
        suffix: Option<impl AsRef<Path>>,
        config: &dyn Config,
    ) -> Result<String> {
        let mut repair_str = String::new();
        let mut shared_path = shared_path.as_ref().to_path_buf();
        if let Some(suffix) = suffix.as_ref() {
            shared_path.push(suffix);
        }
        let local_path = local_path
            .map(|p| get_local_path(p.as_ref().to_path_buf(), &suffix))
            .transpose()?;

        let max_log_count = config.get_opt::<u8>("indexedlog", "data.max-log-count")?;
        let max_bytes_per_log =
            config.get_opt::<ByteCount>("indexedlog", "data.max-bytes-per-log")?;
        let max_bytes = config.get_opt::<ByteCount>("remotefilelog", "cachelimit")?;
        let config = IndexedLogHgIdDataStoreConfig {
            max_log_count,
            max_bytes_per_log,
            max_bytes,
        };

        repair_str += &IndexedLogHgIdDataStore::repair(
            get_indexedlogdatastore_path(&shared_path)?,
            &config,
            StoreType::Shared,
        )?;
        if let Some(local_path) = local_path {
            repair_str += &IndexedLogHgIdDataStore::repair(
                get_indexedlogdatastore_path(local_path)?,
                &config,
                StoreType::Local,
            )?;
        }
        repair_str += &LfsStore::repair(shared_path)?;

        Ok(repair_str)
    }
}

impl LegacyStore for ContentStore {
    /// Some blobs may contain copy-from metadata, let's strip it. For more details about the
    /// copy-from metadata, see `strip_metadata`.
    ///
    /// XXX: This should only be used on `ContentStore` that are storing actual
    /// file content, tree stores should use the `get` method instead.
    fn get_file_content(&self, key: &Key) -> Result<Option<Bytes>> {
        if let StoreResult::Found(vec) = self.get(StoreKey::hgid(key.clone()))? {
            let bytes = vec.into();
            let (bytes, _) = strip_metadata(&bytes)?;
            Ok(Some(bytes))
        } else {
            Ok(None)
        }
    }

    fn get_logged_fetches(&self) -> HashSet<RepoPathBuf> {
        if let Some(remote_store) = &self.remote_store {
            remote_store.take_seen()
        } else {
            HashSet::new()
        }
    }

    fn get_shared_mutable(&self) -> Arc<dyn HgIdMutableDeltaStore> {
        self.shared_mutabledatastore.clone()
    }

    // Repack specific methods, not to be used directly but by the repack code.
    fn add_pending(
        &self,
        key: &Key,
        data: Bytes,
        meta: Metadata,
        location: RepackLocation,
    ) -> Result<()> {
        let delta = Delta {
            data,
            base: None,
            key: key.clone(),
        };

        match location {
            RepackLocation::Local => self.add(&delta, &meta),
            RepackLocation::Shared => self.shared_mutabledatastore.add(&delta, &meta),
        }
    }

    fn commit_pending(&self, location: RepackLocation) -> Result<Option<Vec<PathBuf>>> {
        match location {
            RepackLocation::Local => self.flush(),
            RepackLocation::Shared => self.shared_mutabledatastore.flush(),
        }
    }
}

impl HgIdDataStore for ContentStore {
    fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
        self.datastore.get(key)
    }

    fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
        self.datastore.get_meta(key)
    }

    fn refresh(&self) -> Result<()> {
        self.datastore.refresh()
    }
}

impl RemoteDataStore for ContentStore {
    fn prefetch(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        if let Some(remote_store) = self.remote_store.as_ref() {
            let missing = self.get_missing(keys)?;
            if missing == vec![] {
                Ok(vec![])
            } else {
                remote_store.prefetch(&missing)
            }
        } else {
            // There is no remote store, let's pretend everything is fine.
            Ok(vec![])
        }
    }

    fn upload(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        if let Some(remote_store) = self.remote_store.as_ref() {
            remote_store.upload(keys)
        } else {
            Ok(keys.to_vec())
        }
    }
}

impl LocalStore for ContentStore {
    fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        let span = info_span!("Get Missing", keys = keys.len(),);
        span.in_scope(|| self.datastore.get_missing(keys))
    }
}

impl Drop for ContentStore {
    /// The shared store is a cache, so let's flush all pending data when the `ContentStore` goes
    /// out of scope.
    fn drop(&mut self) {
        let _ = self.shared_mutabledatastore.flush();
    }
}

/// HgIdMutableDeltaStore is only implemented for the local store and not for the remote ones. The
/// remote stores will be automatically written to while calling the various `HgIdDataStore` methods.
///
/// These methods can only be used when the ContentStore was created with a local store.
impl HgIdMutableDeltaStore for ContentStore {
    /// Add the data to the local store.
    fn add(&self, delta: &Delta, metadata: &Metadata) -> Result<()> {
        self.local_mutabledatastore
            .as_ref()
            .ok_or_else(|| format_err!("writing to a non-local ContentStore is not allowed"))?
            .add(delta, metadata)
    }

    /// Commit the data written to the local store.
    fn flush(&self) -> Result<Option<Vec<PathBuf>>> {
        self.shared_mutabledatastore.as_ref().flush()?;
        self.local_mutabledatastore
            .as_ref()
            .ok_or_else(|| format_err!("flushing a non-local ContentStore is not allowed"))?
            .flush()
    }
}

impl ContentDataStore for ContentStore {
    /// Fetch a raw blob from the LFS stores.
    fn blob(&self, key: StoreKey) -> Result<StoreResult<Bytes>> {
        self.blob_stores.blob(key)
    }

    fn metadata(&self, key: StoreKey) -> Result<StoreResult<ContentMetadata>> {
        self.blob_stores.metadata(key)
    }
}

/// Builder for `ContentStore`. An `impl AsRef<Path>` represents the path to the store and a
/// `dyn Config` of the Mercurial configuration are required to build a `ContentStore`. Users can
/// use this builder to add optional `HgIdRemoteStore` to enable remote data fetchingï¼Œ and a `Path`
/// suffix to specify other type of stores.
pub struct ContentStoreBuilder<'a> {
    local_path: Option<PathBuf>,
    no_local_store: bool,
    config: &'a dyn Config,
    remotestore: Option<Arc<dyn HgIdRemoteStore>>,
    suffix: Option<PathBuf>,
    memcachestore: Option<Arc<MemcacheStore>>,
    correlator: Option<String>,
    shared_indexedlog_local: Option<Arc<IndexedLogHgIdDataStore>>,
    shared_indexedlog_shared: Option<Arc<IndexedLogHgIdDataStore>>,
    shared_lfs_local: Option<Arc<LfsStore>>,
    shared_lfs_shared: Option<Arc<LfsStore>>,
}

impl<'a> ContentStoreBuilder<'a> {
    pub fn new(config: &'a dyn Config) -> Self {
        Self {
            local_path: None,
            no_local_store: false,
            config,
            remotestore: None,
            memcachestore: None,
            suffix: None,
            correlator: None,
            shared_indexedlog_shared: None,
            shared_indexedlog_local: None,
            shared_lfs_shared: None,
            shared_lfs_local: None,
        }
    }

    /// Path to the local store.
    pub fn local_path(mut self, local_path: impl AsRef<Path>) -> Self {
        self.local_path = Some(local_path.as_ref().to_path_buf());
        self
    }

    /// Allows a ContentStore to be created without a local store.
    ///
    /// This should be used in very specific cases that do not want a local store. Unless you know
    /// exactly that this is what you want, do not use.
    pub fn no_local_store(mut self) -> Self {
        self.no_local_store = true;
        self
    }

    pub fn remotestore(mut self, remotestore: Arc<dyn HgIdRemoteStore>) -> Self {
        self.remotestore = Some(remotestore);
        self
    }

    pub fn memcachestore(mut self, memcachestore: Arc<MemcacheStore>) -> Self {
        self.memcachestore = Some(memcachestore);
        self
    }

    pub fn suffix(mut self, suffix: impl AsRef<Path>) -> Self {
        self.suffix = Some(suffix.as_ref().to_path_buf());
        self
    }

    pub fn correlator(mut self, correlator: Option<impl ToString>) -> Self {
        self.correlator = correlator.map(|s| s.to_string());
        self
    }

    pub fn shared_indexedlog_local(mut self, indexedlog: Arc<IndexedLogHgIdDataStore>) -> Self {
        self.shared_indexedlog_local = Some(indexedlog);
        self
    }

    pub fn shared_indexedlog_shared(mut self, indexedlog: Arc<IndexedLogHgIdDataStore>) -> Self {
        self.shared_indexedlog_shared = Some(indexedlog);
        self
    }

    pub fn shared_lfs_local(mut self, lfs: Arc<LfsStore>) -> Self {
        self.shared_lfs_local = Some(lfs);
        self
    }

    pub fn shared_lfs_shared(mut self, lfs: Arc<LfsStore>) -> Self {
        self.shared_lfs_shared = Some(lfs);
        self
    }

    pub fn build(self) -> Result<ContentStore> {
        let local_path = self
            .local_path
            .as_ref()
            .map(|p| get_local_path(p.clone(), &self.suffix))
            .transpose()?;
        let cache_path = get_cache_path(self.config, &self.suffix)?;
        check_cache_buster(&self.config, &cache_path);

        // Do this after the cache busting, since this will recreate the necessary directories.
        let cache_packs_path = get_cache_packs_path(self.config, &self.suffix)?;
        let max_pending_bytes = self
            .config
            .get_or("packs", "maxdatapendingbytes", || {
                // Default to 4GB
                ByteCount::from(4 * (1024 * 1024 * 1024))
            })?
            .value();
        let max_bytes = self
            .config
            .get_opt::<ByteCount>("packs", "maxdatabytes")?
            .map(|v| v.value());

        let mut datastore: UnionHgIdDataStore<Arc<dyn HgIdDataStore>> = UnionHgIdDataStore::new();
        let mut blob_stores: UnionContentDataStore<Arc<dyn ContentDataStore>> =
            UnionContentDataStore::new();

        let enable_lfs = self.config.get_or_default::<bool>("remotefilelog", "lfs")?;
        let extstored_policy = if enable_lfs {
            if self
                .config
                .get_or_default::<bool>("remotefilelog", "useextstored")?
            {
                ExtStoredPolicy::Use
            } else {
                ExtStoredPolicy::Ignore
            }
        } else {
            ExtStoredPolicy::Use
        };

        let shared_pack_store = Arc::new(MutableDataPackStore::new(
            &cache_packs_path,
            CorruptionPolicy::REMOVE,
            max_pending_bytes,
            max_bytes,
            extstored_policy,
        )?);
        let shared_indexedlogdatastore =
            if let Some(shared_indexedlog_shared) = self.shared_indexedlog_shared {
                shared_indexedlog_shared
            } else {
                let max_log_count = self
                    .config
                    .get_opt::<u8>("indexedlog", "data.max-log-count")?;
                let max_bytes_per_log = self
                    .config
                    .get_opt::<ByteCount>("indexedlog", "data.max-bytes-per-log")?;
                let max_bytes = self
                    .config
                    .get_opt::<ByteCount>("remotefilelog", "cachelimit")?;
                let config = IndexedLogHgIdDataStoreConfig {
                    max_log_count,
                    max_bytes_per_log,
                    max_bytes,
                };
                Arc::new(IndexedLogHgIdDataStore::new(
                    get_indexedlogdatastore_path(&cache_path)?,
                    extstored_policy,
                    &config,
                    StoreType::Shared,
                )?)
            };

        // The shared stores should precede the local one since we expect both the number of blobs,
        // and the number of requests satisfied by the shared cache to be significantly higher than
        // ones in the local store.

        let lfs_threshold = if enable_lfs {
            self.config.get_opt::<ByteCount>("lfs", "threshold")?
        } else {
            None
        };

        let shared_lfs_store = if let Some(shared_lfs_shared) = self.shared_lfs_shared {
            shared_lfs_shared
        } else {
            Arc::new(LfsStore::shared(&cache_path, self.config)?)
        };
        blob_stores.add(shared_lfs_store.clone());

        let primary: Arc<dyn HgIdMutableDeltaStore> =
            if self
                .config
                .get_or("remotefilelog", "write-hgcache-to-indexedlog", || true)?
            {
                // Put the indexedlog first, since recent data will have gone there.
                datastore.add(shared_indexedlogdatastore.clone());
                datastore.add(shared_pack_store);
                shared_indexedlogdatastore
            } else {
                datastore.add(shared_pack_store.clone());
                datastore.add(shared_indexedlogdatastore);
                shared_pack_store
            };
        datastore.add(shared_lfs_store.clone());

        let shared_mutabledatastore: Arc<dyn HgIdMutableDeltaStore> = {
            if let Some(lfs_threshold) = lfs_threshold {
                let lfs_store = Arc::new(LfsMultiplexer::new(
                    shared_lfs_store.clone(),
                    primary,
                    lfs_threshold.value() as usize,
                ));
                lfs_store
            } else {
                primary
            }
        };

        let (local_mutabledatastore, local_lfs_store): (Option<Arc<dyn HgIdMutableDeltaStore>>, _) =
            if let Some(unsuffixed_local_path) = self.local_path {
                let local_pack_store = Arc::new(MutableDataPackStore::new(
                    get_packs_path(&unsuffixed_local_path, &self.suffix)?,
                    CorruptionPolicy::IGNORE,
                    max_pending_bytes,
                    None,
                    extstored_policy,
                )?);
                let local_indexedlogdatastore =
                    if let Some(shared_indexedlog_local) = self.shared_indexedlog_local {
                        shared_indexedlog_local
                    } else {
                        let config = IndexedLogHgIdDataStoreConfig {
                            max_log_count: None,
                            max_bytes_per_log: None,
                            max_bytes: None,
                        };
                        Arc::new(IndexedLogHgIdDataStore::new(
                            get_indexedlogdatastore_path(local_path.as_ref().unwrap())?,
                            extstored_policy,
                            &config,
                            StoreType::Local,
                        )?)
                    };

                let primary: Arc<dyn HgIdMutableDeltaStore> =
                    if self
                        .config
                        .get_or("remotefilelog", "write-local-to-indexedlog", || true)?
                    {
                        // Put the indexedlog first, since recent data will have gone there.
                        datastore.add(local_indexedlogdatastore.clone());
                        datastore.add(local_pack_store);
                        local_indexedlogdatastore
                    } else {
                        datastore.add(local_pack_store.clone());
                        datastore.add(local_indexedlogdatastore);
                        local_pack_store
                    };

                let local_lfs_store = if let Some(shared_lfs_local) = self.shared_lfs_local {
                    shared_lfs_local
                } else {
                    Arc::new(LfsStore::local(&local_path.unwrap(), self.config)?)
                };
                blob_stores.add(local_lfs_store.clone());
                datastore.add(local_lfs_store.clone());

                let local_mutabledatastore: Arc<dyn HgIdMutableDeltaStore> = {
                    if let Some(lfs_threshold) = lfs_threshold {
                        Arc::new(LfsMultiplexer::new(
                            local_lfs_store.clone(),
                            primary,
                            lfs_threshold.value() as usize,
                        ))
                    } else {
                        primary
                    }
                };

                (Some(local_mutabledatastore), Some(local_lfs_store))
            } else {
                if !self.no_local_store {
                    return Err(format_err!(
                        "a ContentStore cannot be built without a local store"
                    ));
                }
                (None, None)
            };

        let remote_store: Option<Arc<ReportingRemoteDataStore>> = if let Some(remotestore) =
            self.remotestore
        {
            let (cache, shared_store) = if let Some(memcachestore) = self.memcachestore {
                // Combine the memcache store with the other stores. The intent is that all
                // remote requests will first go to the memcache store, and only reach the
                // slower remote store after that.
                //
                // If data isn't found in the memcache store, once fetched from the remote
                // store it will be written to the local cache, and will populate the memcache
                // store, so other clients and future requests won't need to go to a network
                // store.
                let mut multiplexstore: MultiplexDeltaStore<Arc<dyn HgIdMutableDeltaStore>> =
                    MultiplexDeltaStore::new();
                multiplexstore.add_store(
                    memcachestore
                        .clone()
                        .datastore(shared_mutabledatastore.clone()),
                );
                multiplexstore.add_store(shared_mutabledatastore.clone());

                (
                    Some(
                        memcachestore
                            .clone()
                            .remote_datastore(shared_mutabledatastore.clone()),
                    ),
                    Arc::new(multiplexstore) as Arc<dyn HgIdMutableDeltaStore>,
                )
            } else {
                (None, shared_mutabledatastore.clone())
            };

            let mut remotestores = UnionHgIdDataStore::new();

            // First, the fast memcache store
            if let Some(cache) = cache {
                remotestores.add(cache.clone());
            };

            // Second, the slower remotestore. For LFS blobs, the LFS pointers will be fetched
            // at this step and be written to the LFS store.
            let filenode_remotestore = remotestore.datastore(shared_store.clone());
            remotestores.add(filenode_remotestore.clone());

            // Third, the LFS remote store. The previously fetched LFS pointers will be used to
            // fetch the actual blobs in this store.
            if enable_lfs {
                let lfs_remote_store = Arc::new(LfsRemote::new(
                    shared_lfs_store,
                    local_lfs_store,
                    self.config,
                    self.correlator,
                )?);
                remotestores.add(lfs_remote_store.datastore(shared_store.clone()));

                // Fallback store if the LFS one is dead.
                let lfs_fallback = LfsFallbackRemoteStore::new(filenode_remotestore);
                remotestores.add(lfs_fallback);
            }

            let remotestores: Box<dyn RemoteDataStore> = Box::new(remotestores);
            let logging_regex = self
                .config
                .get_opt::<String>("remotefilelog", "undesiredfileregex")?
                .map(|s| Regex::new(&s))
                .transpose()?;
            let remotestores = Arc::new(ReportingRemoteDataStore::new(remotestores, logging_regex));
            datastore.add(remotestores.clone());
            Some(remotestores)
        } else {
            None
        };

        Ok(ContentStore {
            datastore,
            local_mutabledatastore,
            shared_mutabledatastore,
            remote_store,
            blob_stores,
        })
    }
}

/// Reads the configs and deletes the hgcache if a hgcache-purge.$KEY=$DATE value hasn't already
/// been processed.
pub fn check_cache_buster(config: &dyn Config, store_path: &Path) {
    for key in config.keys("hgcache-purge").into_iter() {
        if let Some(cutoff) = config
            .get("hgcache-purge", &key)
            .map(|c| HgTime::parse(&c))
            .flatten()
        {
            if check_run_once(store_path, &key, cutoff) {
                let _ = delete_hgcache(store_path);
                break;
            }
        }
    }
}

/// Recursively deletes the contents of the path, excluding the run-once marker file.
/// Ignores errors on individual files or directories.
fn delete_hgcache(store_path: &Path) -> Result<()> {
    for file in fs::read_dir(store_path)? {
        let _ = (|| -> Result<()> {
            let file = file?;
            if file.file_name() == RUN_ONCE_FILENAME {
                return Ok(());
            }

            let path = file.path();
            let file_type = file.file_type()?;
            if file_type.is_dir() {
                fs::remove_dir_all(path)?;
            } else if file_type.is_file() || file_type.is_symlink() {
                fs::remove_file(path)?;
            }
            Ok(())
        })();
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use std::collections::HashMap;
    use std::ffi::OsString;
    use std::ops::Add;
    use std::ops::Sub;

    use minibytes::Bytes;
    use mockito::Mock;
    use tempfile::TempDir;
    use types::testutil::*;
    use util::path::create_dir;

    use super::*;
    use crate::metadatastore::MetadataStore;
    use crate::repack::repack;
    use crate::repack::RepackKind;
    use crate::repack::RepackLocation;
    #[cfg(feature = "fb")]
    use crate::testutil::example_blob;
    use crate::testutil::get_lfs_batch_mock;
    use crate::testutil::get_lfs_download_mock;
    use crate::testutil::make_config;
    use crate::testutil::make_lfs_config;
    use crate::testutil::setconfig;
    use crate::testutil::FakeHgIdRemoteStore;
    use crate::testutil::TestBlob;
    use crate::types::ContentHash;

    #[cfg(feature = "fb")]
    fn prepare_lfs_mocks(blob: &TestBlob) -> Vec<Mock> {
        let m1 = get_lfs_batch_mock(200, &[blob]);
        let mut m2 = get_lfs_download_mock(200, blob);
        m2.push(m1);
        m2
    }

    #[test]
    fn test_new() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_config(&cachedir);

        let _store = ContentStore::new(&localdir, &config)?;
        Ok(())
    }

    #[test]
    fn test_add_get() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_config(&cachedir);

        let store = ContentStore::new(&localdir, &config)?;

        let k1 = key("a", "2");
        let delta = Delta {
            data: Bytes::from(&[1, 2, 3, 4][..]),
            base: None,
            key: k1.clone(),
        };
        store.add(&delta, &Default::default())?;
        let stored = store.get(StoreKey::hgid(k1))?;
        assert_eq!(stored, StoreResult::Found(delta.data.as_ref().to_vec()));
        Ok(())
    }

    #[test]
    fn test_add_dropped() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let mut config = make_config(&cachedir);
        setconfig(
            &mut config,
            "remotefilelog",
            "write-local-to-indexedlog",
            "False",
        );

        let store = ContentStore::new(&localdir, &config)?;

        let k1 = key("a", "2");
        let delta = Delta {
            data: Bytes::from(&[1, 2, 3, 4][..]),
            base: Some(key("a", "1")),
            key: k1.clone(),
        };
        store.add(&delta, &Default::default())?;
        drop(store);

        let store = ContentStore::new(&localdir, &config)?;
        let k1 = StoreKey::hgid(k1);
        assert_eq!(store.get(k1.clone())?, StoreResult::NotFound(k1));
        Ok(())
    }

    #[test]
    fn test_add_flush_get() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_config(&cachedir);

        let store = ContentStore::new(&localdir, &config)?;

        let k1 = key("a", "2");
        let delta = Delta {
            data: Bytes::from(&[1, 2, 3, 4][..]),
            base: None,
            key: k1.clone(),
        };
        store.add(&delta, &Default::default())?;
        store.flush()?;
        let stored = store.get(StoreKey::hgid(k1))?;
        assert_eq!(stored, StoreResult::Found(delta.data.as_ref().to_vec()));
        Ok(())
    }

    #[test]
    fn test_add_flush_drop_get() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_config(&cachedir);

        let store = ContentStore::new(&localdir, &config)?;

        let k1 = key("a", "2");
        let delta = Delta {
            data: Bytes::from(&[1, 2, 3, 4][..]),
            base: None,
            key: k1.clone(),
        };
        store.add(&delta, &Default::default())?;
        store.flush()?;
        drop(store);

        let store = ContentStore::new(&localdir, &config)?;
        let stored = store.get(StoreKey::hgid(k1))?;
        assert_eq!(stored, StoreResult::Found(delta.data.as_ref().to_vec()));
        Ok(())
    }

    #[test]
    fn test_remote_store() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_config(&cachedir);

        let k = key("a", "1");
        let data = Bytes::from(&[1, 2, 3, 4][..]);

        let mut map = HashMap::new();
        map.insert(k.clone(), (data.clone(), None));
        let mut remotestore = FakeHgIdRemoteStore::new();
        remotestore.data(map);

        let store = ContentStoreBuilder::new(&config)
            .local_path(&localdir)
            .remotestore(Arc::new(remotestore))
            .build()?;
        let data_get = store.get(StoreKey::hgid(k))?;

        assert_eq!(data_get, StoreResult::Found(data.as_ref().to_vec()));
        Ok(())
    }

    #[test]
    fn test_remote_store_cached() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_config(&cachedir);

        let k = key("a", "1");
        let data = Bytes::from(&[1, 2, 3, 4][..]);

        let mut map = HashMap::new();
        map.insert(k.clone(), (data.clone(), None));

        let mut remotestore = FakeHgIdRemoteStore::new();
        remotestore.data(map);

        let store = ContentStoreBuilder::new(&config)
            .local_path(&localdir)
            .remotestore(Arc::new(remotestore))
            .build()?;
        store.get(StoreKey::hgid(k.clone()))?;
        drop(store);

        let store = ContentStore::new(&localdir, &config)?;
        let data_get = store.get(StoreKey::hgid(k))?;

        assert_eq!(data_get, StoreResult::Found(data.as_ref().to_vec()));

        Ok(())
    }

    #[test]
    fn test_not_in_remote_store() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_config(&cachedir);

        let map = HashMap::new();
        let mut remotestore = FakeHgIdRemoteStore::new();
        remotestore.data(map);

        let store = ContentStoreBuilder::new(&config)
            .local_path(&localdir)
            .remotestore(Arc::new(remotestore))
            .build()?;

        let k = StoreKey::hgid(key("a", "1"));
        assert_eq!(store.get(k.clone())?, StoreResult::NotFound(k));
        Ok(())
    }

    #[test]
    fn test_local_indexedlog_write() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let mut config = make_config(&cachedir);
        setconfig(
            &mut config,
            "remotefilelog",
            "write-local-to-indexedlog",
            "True",
        );

        let store = ContentStoreBuilder::new(&config)
            .local_path(&localdir)
            .build()?;

        let k1 = key("a", "2");
        let delta = Delta {
            data: Bytes::from(&[1, 2, 3, 4][..]),
            base: None,
            key: k1.clone(),
        };
        store.add(&delta, &Default::default())?;
        store.flush()?;
        drop(store);

        let indexed_log_config = IndexedLogHgIdDataStoreConfig {
            max_log_count: None,
            max_bytes_per_log: None,
            max_bytes: None,
        };
        let store = IndexedLogHgIdDataStore::new(
            get_indexedlogdatastore_path(&localdir)?,
            ExtStoredPolicy::Use,
            &indexed_log_config,
            StoreType::Local,
        )?;
        assert_eq!(
            store.get(StoreKey::hgid(k1))?,
            StoreResult::Found(delta.data.as_ref().to_vec())
        );
        Ok(())
    }

    #[test]
    fn test_fetch_location() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_config(&cachedir);

        let k = key("a", "1");
        let data = Bytes::from(&[1, 2, 3, 4][..]);

        let mut map = HashMap::new();
        map.insert(k.clone(), (data.clone(), None));

        let mut remotestore = FakeHgIdRemoteStore::new();
        remotestore.data(map);

        let store = ContentStoreBuilder::new(&config)
            .local_path(&localdir)
            .remotestore(Arc::new(remotestore))
            .build()?;
        store.get(StoreKey::hgid(k.clone()))?;
        store
            .shared_mutabledatastore
            .get(StoreKey::hgid(k.clone()))?;
        let k = StoreKey::hgid(k);
        let res = store
            .local_mutabledatastore
            .as_ref()
            .unwrap()
            .get(k.clone())?;
        assert_eq!(res, StoreResult::NotFound(k));
        Ok(())
    }

    #[test]
    fn test_add_shared_only_store() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let mut config = make_config(&cachedir);
        setconfig(
            &mut config,
            "remotefilelog",
            "write-local-to-indexedlog",
            "False",
        );

        let store = ContentStore::new(&localdir, &config)?;

        let k1 = key("a", "2");
        let delta = Delta {
            data: Bytes::from(&[1, 2, 3, 4][..]),
            base: Some(key("a", "1")),
            key: k1.clone(),
        };
        store.add(&delta, &Default::default())?;
        store.flush()?;

        let store = ContentStoreBuilder::new(&config).no_local_store().build()?;
        let k = StoreKey::hgid(k1);
        assert_eq!(store.get(k.clone())?, StoreResult::NotFound(k));
        Ok(())
    }

    #[test]
    fn test_no_local_store() -> Result<()> {
        let cachedir = TempDir::new()?;
        let config = make_config(&cachedir);
        assert!(ContentStoreBuilder::new(&config).build().is_err());
        Ok(())
    }

    #[test]
    fn test_lfs_local() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_config(&cachedir);

        let lfs_store = LfsStore::local(&localdir, &config)?;
        let k1 = key("a", "2");
        let delta = Delta {
            data: Bytes::from(&[1, 2, 3, 4][..]),
            base: None,
            key: k1.clone(),
        };
        lfs_store.add(&delta, &Default::default())?;
        lfs_store.flush()?;

        let store = ContentStore::new(&localdir, &config)?;
        assert_eq!(
            store.get(StoreKey::hgid(k1))?,
            StoreResult::Found(delta.data.as_ref().to_vec())
        );
        Ok(())
    }

    #[test]
    fn test_lfs_shared() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_config(&cachedir);

        let mut lfs_cache_dir = cachedir.path().to_path_buf();
        lfs_cache_dir.push("test");
        create_dir(&lfs_cache_dir)?;
        let lfs_store = LfsStore::shared(&lfs_cache_dir, &config)?;
        let k1 = key("a", "2");
        let delta = Delta {
            data: Bytes::from(&[1, 2, 3, 4][..]),
            base: None,
            key: k1.clone(),
        };
        lfs_store.add(&delta, &Default::default())?;
        lfs_store.flush()?;

        let store = ContentStore::new(&localdir, &config)?;
        assert_eq!(
            store.get(StoreKey::hgid(k1))?,
            StoreResult::Found(delta.data.as_ref().to_vec())
        );
        Ok(())
    }

    #[test]
    fn test_lfs_blob() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_lfs_config(&cachedir, "test_lfs_blob");

        let k1 = key("a", "2");
        let delta = Delta {
            data: Bytes::from(&[1, 2, 3, 4, 5][..]),
            base: None,
            key: k1.clone(),
        };

        let store = ContentStore::new(&localdir, &config)?;
        store.add(&delta, &Default::default())?;

        let blob = store.blob(StoreKey::from(k1))?;
        assert_eq!(blob, StoreResult::Found(delta.data));

        Ok(())
    }

    #[test]
    fn test_lfs_metadata() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_lfs_config(&cachedir, "test_lfs_metadata");

        let k1 = key("a", "2");
        let data = Bytes::from(&[1, 2, 3, 4, 5][..]);
        let hash = ContentHash::sha256(&data);
        let delta = Delta {
            data,
            base: None,
            key: k1.clone(),
        };

        let store = ContentStore::new(&localdir, &config)?;
        store.add(&delta, &Default::default())?;

        let metadata = store.metadata(StoreKey::from(k1))?;
        assert_eq!(
            metadata,
            StoreResult::Found(ContentMetadata {
                size: 5,
                is_binary: false,
                hash,
            })
        );

        Ok(())
    }

    #[test]
    fn test_lfs_multiplexer() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let config = make_lfs_config(&cachedir, "test_lfs_multiplexer");

        let k1 = key("a", "2");
        let delta = Delta {
            data: Bytes::from(&[1, 2, 3, 4, 5][..]),
            base: None,
            key: k1.clone(),
        };

        let store = ContentStore::new(&localdir, &config)?;
        store.add(&delta, &Default::default())?;
        store.flush()?;

        let lfs_store = LfsStore::local(&localdir, &config)?;
        let stored = lfs_store.get(StoreKey::hgid(k1))?;
        assert_eq!(stored, StoreResult::Found(delta.data.as_ref().to_vec()));
        Ok(())
    }

    #[test]
    fn test_repack_one_datapack_lfs() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let mut config = make_lfs_config(&cachedir, "test_repack_one_datapack_lfs");
        setconfig(&mut config, "lfs", "threshold", "10M");

        let k1 = key("a", "2");
        let delta = Delta {
            data: Bytes::from(&[1, 2, 3, 4, 5][..]),
            base: None,
            key: k1.clone(),
        };

        let store = Arc::new(ContentStore::new(&localdir, &config)?);
        store.add(&delta, &Default::default())?;
        store.flush()?;

        let metadata = Arc::new(MetadataStore::new(&localdir, &config)?);

        repack(
            get_packs_path(&localdir, &None)?,
            Some((store, metadata)),
            RepackKind::Full,
            RepackLocation::Local,
            &config,
        )?;

        let store = Arc::new(ContentStore::new(&localdir, &config)?);
        let stored = store.get(StoreKey::hgid(k1))?;
        assert_eq!(stored, StoreResult::Found(delta.data.as_ref().to_vec()));
        Ok(())
    }

    #[test]
    fn test_purge_cache() -> Result<()> {
        let cachedir = TempDir::new()?;
        let localdir = TempDir::new()?;
        let mut config = make_config(&cachedir);
        setconfig(
            &mut config,
            "remotefilelog",
            "write-local-to-indexedlog",
            "False",
        );
        setconfig(
            &mut config,
            "remotefilelog",
            "write-hgcache-to-indexedlog",
            "False",
        );

        let k = key("a", "2");
        let store_key = StoreKey::hgid(k.clone());
        let data = Bytes::from(&[1, 2, 3, 4, 5][..]);

        let mut map = HashMap::new();
        map.insert(k.clone(), (data.clone(), None));
        let mut remotestore = FakeHgIdRemoteStore::new();
        remotestore.data(map);
        let remotestore = Arc::new(remotestore);

        let create_store = |config: &dyn Config| -> ContentStore {
            ContentStoreBuilder::new(&config)
                .local_path(&localdir)
                .remotestore(remotestore.clone())
                .build()
                .unwrap()
        };

        // Populate the cache
        let store = create_store(&mut config);
        let stored = store.get(store_key.clone())?;
        assert_eq!(stored, StoreResult::Found(data.as_ref().to_vec()));

        // Drop the store so any temp files (for mutable packs) are deleted.
        drop(store);

        let get_subdirs = || -> Vec<OsString> {
            fs::read_dir(cachedir.path().join("test/packs"))
                .unwrap()
                .map(|f| f.unwrap().file_name())
                .collect::<Vec<_>>()
        };

        // Ensure pack files exist
        assert!(!get_subdirs().is_empty());

        // Set a purge that ended yesterday.
        let yesterday = HgTime::now().unwrap().sub(86000).unwrap();
        setconfig(
            &mut config,
            "hgcache-purge",
            "marker",
            &yesterday.to_utc().to_string(),
        );

        // Recreate the store, which should not activate the purge.
        let store = create_store(&mut config);
        drop(store);

        assert!(!get_subdirs().is_empty());

        // Set a purge that lasts until tomorrow.
        let tomorrow = HgTime::now().unwrap().add(86000).unwrap();
        setconfig(
            &mut config,
            "hgcache-purge",
            "marker",
            &tomorrow.to_utc().to_string(),
        );

        // Recreate the store, which will activate the purge.
        let store = create_store(&mut config);
        drop(store);

        assert!(get_subdirs().is_empty());

        // Populate the store again
        let store = create_store(&mut config);
        let _ = store.get(store_key.clone())?;

        // Construct a store again and verify it doesn't purge the cache
        let store = create_store(&mut config);
        drop(store);

        assert!(!get_subdirs().is_empty());
        Ok(())
    }

    #[cfg(feature = "fb")]
    mod fb_tests {
        use std::str::FromStr;

        use types::Sha256;
        use url::Url;

        use super::*;

        #[test]
        fn test_lfs_remote() -> Result<()> {
            let _env_lock = crate::env_lock();

            let cachedir = TempDir::new()?;
            let localdir = TempDir::new()?;
            let config = make_lfs_config(&cachedir, "test_lfs_remote");
            let blob = example_blob();
            let _lfs_mocks = prepare_lfs_mocks(&blob);

            let k = key("a", "1");

            let pointer = format!(
                "version https://git-lfs.github.com/spec/v1\noid sha256:{}\nsize {}\nx-is-binary 0\n",
                blob.sha.to_hex(),
                blob.size,
            );

            let data = Bytes::from(pointer);

            let mut map = HashMap::new();
            map.insert(k.clone(), (data, Some(0x2000)));
            let mut remotestore = FakeHgIdRemoteStore::new();
            remotestore.data(map);

            let store = ContentStoreBuilder::new(&config)
                .local_path(&localdir)
                .remotestore(Arc::new(remotestore))
                .build()?;

            let data = store.get(StoreKey::hgid(k))?;

            assert_eq!(
                data,
                StoreResult::Found(Bytes::from(&b"master"[..]).as_ref().to_vec())
            );

            Ok(())
        }

        #[test]
        fn test_lfs_fallback_on_missing_blob() -> Result<()> {
            let cachedir = TempDir::new()?;
            let localdir = TempDir::new()?;
            let mut config = make_lfs_config(&cachedir, "test_lfs_fallback_on_missing_blob");

            let lfsdir = TempDir::new()?;
            setconfig(
                &mut config,
                "lfs",
                "url",
                &Url::from_file_path(&lfsdir).unwrap().to_string(),
            );

            let k = key("a", "1");
            // This should be a missing blob.
            let sha256 = Sha256::from_str(
                "0000000000000000000000000000000000000000000000000000000000000042",
            )?;
            let size = 4;

            let pointer = format!(
                "version https://git-lfs.github.com/spec/v1\noid sha256:{}\nsize {}\nx-is-binary 0\n",
                sha256.to_hex(),
                size
            );

            let data = Bytes::from("AAAA");

            let mut map = HashMap::new();
            map.insert(k.clone(), (data.clone(), None));
            let mut remotestore = FakeHgIdRemoteStore::new();
            remotestore.data(map);

            let store = ContentStoreBuilder::new(&config)
                .local_path(&localdir)
                .remotestore(Arc::new(remotestore))
                .build()?;

            let delta = Delta {
                data: Bytes::from(pointer),
                base: None,
                key: k.clone(),
            };

            // Add the pointer the the shared store, but not the blob
            store.shared_mutabledatastore.add(
                &delta,
                &Metadata {
                    size: None,
                    flags: Some(0x2000),
                },
            )?;

            assert_eq!(
                store.get_missing(&[StoreKey::from(k.clone())])?,
                vec![StoreKey::Content(
                    ContentHash::Sha256(sha256),
                    Some(k.clone())
                )]
            );
            store.prefetch(&[StoreKey::from(k.clone())])?;
            // Even though the blob was missing, we got it!
            assert_eq!(store.get_missing(&[StoreKey::from(k.clone())])?, vec![]);

            Ok(())
        }

        #[test]
        fn test_lfs_prefetch_once() -> Result<()> {
            let _env_lock = crate::env_lock();
            let blob = example_blob();
            let _lfs_mocks = prepare_lfs_mocks(&blob);

            let cachedir = TempDir::new()?;
            let localdir = TempDir::new()?;
            let config = make_lfs_config(&cachedir, "test_lfs_prefetch_once");

            let k1 = key("a", "1");
            let k2 = key("a", "2");
            let sha256 = Sha256::from_str(
                "fc613b4dfd6736a7bd268c8a0e74ed0d1c04a959f59dd74ef2874983fd443fc9",
            )?;
            let size = 6;

            let pointer = format!(
                "version https://git-lfs.github.com/spec/v1\noid sha256:{}\nsize {}\nx-is-binary 0\n",
                sha256.to_hex(),
                size
            );

            let data = Bytes::from(pointer);

            let mut map = HashMap::new();
            map.insert(k1.clone(), (data.clone(), Some(0x2000)));
            map.insert(k2.clone(), (data, Some(0x2000)));
            let mut remotestore = FakeHgIdRemoteStore::new();
            remotestore.data(map);

            let store = ContentStoreBuilder::new(&config)
                .local_path(&localdir)
                .remotestore(Arc::new(remotestore))
                .build()?;

            let k1 = StoreKey::from(k1.clone());
            let k2 = StoreKey::from(k2.clone());
            assert_eq!(store.prefetch(&[k1, k2])?, vec![]);

            Ok(())
        }
    }

    #[cfg(all(fbcode_build, target_os = "linux"))]
    mod fbcode_tests {
        use std::str::FromStr;

        use memcache::MockMemcache;
        use once_cell::sync::Lazy;
        use types::Sha256;

        use super::*;

        static MOCK: Lazy<MockMemcache> = Lazy::new(|| MockMemcache::new());

        #[fbinit::test]
        fn test_memcache_get() -> Result<()> {
            let _mock = Lazy::force(&MOCK);

            let cachedir = TempDir::new()?;
            let localdir = TempDir::new()?;
            let config = make_config(&cachedir);

            let k = key("a", "1234");
            let data = Bytes::from(&[1, 2, 3, 4][..]);

            let mut map = HashMap::new();
            map.insert(k.clone(), (data.clone(), None));
            let mut remotestore = FakeHgIdRemoteStore::new();
            remotestore.data(map);

            let memcache = Arc::new(MemcacheStore::new(&config)?);
            let store = ContentStoreBuilder::new(&config)
                .local_path(&localdir)
                .remotestore(Arc::new(remotestore))
                .memcachestore(memcache.clone())
                .build()?;
            let data_get = store.get(StoreKey::hgid(k.clone()))?;
            assert_eq!(data_get, StoreResult::Found(data.as_ref().to_vec()));

            loop {
                let memcache_data = memcache
                    .get_data_iter(&[k.clone()])
                    .unwrap()
                    .collect::<Result<Vec<_>>>()?;
                if !memcache_data.is_empty() {
                    assert_eq!(memcache_data[0].data, data);
                    break;
                }
            }
            Ok(())
        }

        #[fbinit::test]
        fn test_memcache_get_large() -> Result<()> {
            let _mock = Lazy::force(&MOCK);

            let cachedir = TempDir::new()?;
            let localdir = TempDir::new()?;
            let config = make_config(&cachedir);

            let k = key("a", "abcd");

            let data: Bytes = (0..10 * 1024 * 1024)
                .map(|_| rand::random::<u8>())
                .collect::<Vec<u8>>()
                .into();
            assert_eq!(data.len(), 10 * 1024 * 1024);

            let mut map = HashMap::new();
            map.insert(k.clone(), (data.clone(), None));
            let mut remotestore = FakeHgIdRemoteStore::new();
            remotestore.data(map);

            let memcache = Arc::new(MemcacheStore::new(&config)?);
            let store = ContentStoreBuilder::new(&config)
                .local_path(&localdir)
                .remotestore(Arc::new(remotestore))
                .memcachestore(memcache.clone())
                .build()?;
            let data_get = store.get(StoreKey::hgid(k.clone()))?;
            assert_eq!(data_get, StoreResult::Found(data.as_ref().to_vec()));

            let memcache_data = memcache
                .get_data_iter(&[k])
                .unwrap()
                .collect::<Result<Vec<_>>>()?;
            assert_eq!(memcache_data, vec![]);
            Ok(())
        }

        #[fbinit::test]
        fn test_memcache_no_wait() -> Result<()> {
            let _mock = Lazy::force(&MOCK);

            let cachedir = TempDir::new()?;
            let localdir = TempDir::new()?;
            let mut config = make_config(&cachedir);
            setconfig(&mut config, "remotefilelog", "waitformemcache", "false");

            let k = key("a", "1");
            let data = Bytes::from(&[1, 2, 3, 4][..]);

            let mut map = HashMap::new();
            map.insert(k.clone(), (data.clone(), None));
            let mut remotestore = FakeHgIdRemoteStore::new();
            remotestore.data(map);

            let memcache = Arc::new(MemcacheStore::new(&config)?);
            let store = ContentStoreBuilder::new(&config)
                .local_path(&localdir)
                .remotestore(Arc::new(remotestore))
                .memcachestore(memcache)
                .build()?;
            let data_get = store.get(StoreKey::hgid(k))?;
            assert_eq!(data_get, StoreResult::Found(data.as_ref().to_vec()));

            // Ideally, we should check that we didn't wait for memcache, but that's timing
            // related and thus a bit hard to test.
            Ok(())
        }

        #[fbinit::test]
        fn test_memcache_no_duplicate_fetch() -> Result<()> {
            let _mock = Lazy::force(&MOCK);

            let cachedir = TempDir::new()?;
            let localdir = TempDir::new()?;
            let config = make_config(&cachedir);

            let k = key("a", "1234");
            let data = Bytes::from(&[1, 2, 3, 4][..]);

            let mut map = HashMap::new();
            map.insert(k.clone(), (data.clone(), None));
            let mut remotestore = FakeHgIdRemoteStore::new();
            remotestore.data(map);

            let memcache = Arc::new(MemcacheStore::new(&config)?);
            let store = ContentStoreBuilder::new(&config)
                .local_path(&localdir)
                .remotestore(Arc::new(remotestore))
                .memcachestore(memcache.clone())
                .build()?;

            // This populates memcache
            let data_get = store.get(StoreKey::hgid(k.clone()))?;
            assert_eq!(data_get, StoreResult::Found(data.as_ref().to_vec()));

            // Wait to make sure it's there.
            loop {
                let memcache_data = memcache
                    .get_data_iter(&[k.clone()])
                    .unwrap()
                    .collect::<Result<Vec<_>>>()?;
                if !memcache_data.is_empty() {
                    assert_eq!(memcache_data[0].data, data);
                    break;
                }
            }

            let cachedir = TempDir::new()?;
            let localdir = TempDir::new()?;
            let config = make_config(&cachedir);

            let mut remotestore = FakeHgIdRemoteStore::new();
            remotestore.data(HashMap::new());
            let store = ContentStoreBuilder::new(&config)
                .local_path(&localdir)
                .remotestore(Arc::new(remotestore))
                .memcachestore(memcache.clone())
                .build()?;

            // The data is only in the memcache store, this should succeed.
            assert_eq!(store.prefetch(&vec![StoreKey::hgid(k.clone())])?, vec![]);

            Ok(())
        }

        #[fbinit::test]
        fn test_memcache_lfs() -> Result<()> {
            let _mock = Lazy::force(&MOCK);
            let blob = example_blob();
            let _lfs_mocks = prepare_lfs_mocks(&blob);

            let cachedir = TempDir::new()?;
            let localdir = TempDir::new()?;
            let config = make_lfs_config(&cachedir, "test_memcache_lfs");

            let k = key("a", "1f5");
            let sha256 = Sha256::from_str(
                "fc613b4dfd6736a7bd268c8a0e74ed0d1c04a959f59dd74ef2874983fd443fc9",
            )?;
            let size = 6;

            let pointer = format!(
                "version https://git-lfs.github.com/spec/v1\noid sha256:{}\nsize {}\nx-is-binary 0\n",
                sha256.to_hex(),
                size
            );

            let pointer_data = Bytes::from(pointer);

            let mut map = HashMap::new();
            map.insert(k.clone(), (pointer_data.clone(), Some(0x2000)));
            let mut remotestore = FakeHgIdRemoteStore::new();
            remotestore.data(map);
            let remotestore = Arc::new(remotestore);

            let memcache = Arc::new(MemcacheStore::new(&config)?);
            let store = ContentStoreBuilder::new(&config)
                .local_path(&localdir)
                .remotestore(remotestore.clone())
                .memcachestore(memcache.clone())
                .build()?;

            let data = store.get(StoreKey::hgid(k.clone()))?;
            assert_eq!(
                data,
                StoreResult::Found(Bytes::from(&b"master"[..]).as_ref().to_vec())
            );

            loop {
                let memcache_data = memcache
                    .get_data_iter(&[k.clone()])
                    .unwrap()
                    .collect::<Result<Vec<_>>>()?;
                if !memcache_data.is_empty() {
                    assert_eq!(memcache_data[0].data, pointer_data);
                    break;
                }
            }

            let cachedir = TempDir::new()?;
            let localdir = TempDir::new()?;
            let config = make_lfs_config(&cachedir, "test_memcache_lfs2");

            let store = ContentStoreBuilder::new(&config)
                .local_path(&localdir)
                .remotestore(remotestore)
                .memcachestore(memcache)
                .build()?;

            let data = store.get(StoreKey::hgid(k))?;
            assert_eq!(
                data,
                StoreResult::Found(Bytes::from(&b"master"[..]).as_ref().to_vec())
            );

            Ok(())
        }
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;
use std::fs::File;
use std::io::Cursor;
use std::io::Read;
use std::io::Write;
use std::path::Path;

use anyhow::Result;
use byteorder::BigEndian;
use byteorder::ReadBytesExt;
use byteorder::WriteBytesExt;
use memmap2::Mmap;
use memmap2::MmapOptions;
use thiserror::Error;
use types::HgId;

use crate::fanouttable::FanoutTable;
use crate::sliceext::SliceExt;

const ENTRY_LEN: usize = 40;
const SMALL_FANOUT_CUTOFF: usize = 8192; // 2^16 / 8

#[derive(Debug, Error)]
#[error("DataIndex Error: {0:?}")]
struct DataIndexError(String);

#[derive(Debug, PartialEq)]
struct DataIndexOptions {
    version: u8,
    // Indicates whether to use the large fanout (2 bytes) or the small (1 byte)
    large: bool,
}

#[derive(Debug)]
pub struct DeltaLocation {
    pub delta_base: Option<HgId>,
    pub offset: u64,
    pub size: u64,
}

#[derive(Debug)]
pub struct IndexEntry {
    hgid: HgId,
    delta_base_offset: u32,
    pack_entry_offset: u64,
    pack_entry_size: u64,
}

impl IndexEntry {
    pub fn new(
        hgid: HgId,
        delta_base_offset: DeltaBaseOffset,
        pack_entry_offset: u64,
        pack_entry_size: u64,
    ) -> Self {
        IndexEntry {
            hgid,
            delta_base_offset: match delta_base_offset {
                DeltaBaseOffset::FullText => 0xffffffff,
                DeltaBaseOffset::Missing => 0xfffffffe,
                DeltaBaseOffset::Offset(value) => value,
            },
            pack_entry_offset,
            pack_entry_size,
        }
    }

    pub fn hgid(&self) -> &HgId {
        &self.hgid
    }

    pub fn delta_base_offset(&self) -> DeltaBaseOffset {
        if self.delta_base_offset == 0xffffffff {
            DeltaBaseOffset::FullText
        } else if self.delta_base_offset == 0xfffffffe {
            DeltaBaseOffset::Missing
        } else {
            DeltaBaseOffset::Offset(self.delta_base_offset.clone())
        }
    }

    pub fn pack_entry_offset(&self) -> u64 {
        self.pack_entry_offset.clone()
    }

    pub fn pack_entry_size(&self) -> u64 {
        self.pack_entry_size.clone()
    }
}

#[derive(Debug, PartialEq)]
pub enum DeltaBaseOffset {
    Offset(u32),
    FullText,
    Missing,
}

impl DeltaBaseOffset {
    fn new(value: i32) -> Result<Self> {
        if value >= 0 {
            Ok(DeltaBaseOffset::Offset(value as u32))
        } else if value == -1 {
            Ok(DeltaBaseOffset::FullText)
        } else if value == -2 {
            Ok(DeltaBaseOffset::Missing)
        } else {
            Err(DataIndexError(format!("invalid delta base offset value '{:?}'", value)).into())
        }
    }

    fn to_i32(&self) -> i32 {
        match *self {
            DeltaBaseOffset::Offset(value) => value as i32,
            DeltaBaseOffset::FullText => -1,
            DeltaBaseOffset::Missing => -2,
        }
    }
}

impl IndexEntry {
    pub fn read(buf: &[u8]) -> Result<Self> {
        let mut cur = Cursor::new(buf);
        cur.set_position(20);
        let hgid_slice: &[u8] = buf.get_err(0..20)?;
        let hgid = HgId::from_slice(hgid_slice)?;
        let delta_base_offset = cur.read_i32::<BigEndian>()?;
        let delta_base_offset = DeltaBaseOffset::new(delta_base_offset)?;
        let pack_entry_offset = cur.read_u64::<BigEndian>()?;
        let pack_entry_size = cur.read_u64::<BigEndian>()?;
        Ok(IndexEntry::new(
            hgid,
            delta_base_offset,
            pack_entry_offset,
            pack_entry_size,
        ))
    }

    fn write<T: Write>(&self, writer: &mut T) -> Result<()> {
        writer.write_all(self.hgid().as_ref())?;
        writer.write_i32::<BigEndian>(self.delta_base_offset().to_i32())?;
        writer.write_u64::<BigEndian>(self.pack_entry_offset())?;
        writer.write_u64::<BigEndian>(self.pack_entry_size())?;
        Ok(())
    }
}

impl DataIndexOptions {
    pub fn read<T: Read>(reader: &mut T) -> Result<DataIndexOptions> {
        let version = reader.read_u8()?;
        if version > 1 {
            return Err(DataIndexError(format!("unsupported version '{:?}'", version)).into());
        };

        let raw_config = reader.read_u8()?;
        let large = match raw_config {
            0b10000000 => true,
            0 => false,
            _ => {
                return Err(
                    DataIndexError(format!("invalid data index '{:?}'", raw_config)).into(),
                );
            }
        };
        Ok(DataIndexOptions { version, large })
    }

    pub fn write<T: Write>(&self, writer: &mut T) -> Result<()> {
        writer.write_u8(self.version)?;
        writer.write_u8(if self.large { 0b10000000 } else { 0 })?;
        Ok(())
    }
}

pub struct DataIndex {
    mmap: Mmap,
    fanout_size: usize,
    index_start: usize,
}

impl DataIndex {
    pub fn new(path: &Path) -> Result<Self> {
        let file = File::open(path)?;
        let len = file.metadata()?.len();
        if len < 1 {
            return Err(DataIndexError(format!(
                "empty dataidx '{:?}' is invalid",
                path.to_str().unwrap_or("<unknown>")
            ))
            .into());
        }

        let mmap = unsafe { MmapOptions::new().len(len as usize).map(&file)? };
        let options = DataIndexOptions::read(&mut Cursor::new(&mmap))?;
        let fanout_size = FanoutTable::get_size(options.large);
        let mut index_start = 2 + fanout_size;

        // Version one records the number of entries in the index
        if options.version == 1 {
            index_start += 8;
        }

        Ok(DataIndex {
            mmap,
            fanout_size,
            index_start,
        })
    }

    pub fn write<T: Write>(writer: &mut T, values: &HashMap<HgId, DeltaLocation>) -> Result<()> {
        // Write header
        let options = DataIndexOptions {
            version: 1,
            large: values.len() > SMALL_FANOUT_CUTOFF,
        };
        options.write(writer)?;

        let mut values: Vec<(&HgId, &DeltaLocation)> = values.iter().collect();
        // They must be written in sorted order
        values.sort_by_key(|x| x.0);

        // Write fanout
        // `locations` will contain the eventual offset that each value will be written to.
        let mut locations: Vec<u32> = Vec::with_capacity(values.len());
        unsafe { locations.set_len(values.len()) };
        FanoutTable::write(
            writer,
            if options.large { 2 } else { 1 },
            &mut values.iter().map(|x| x.0),
            ENTRY_LEN,
            Some(&mut locations),
        )?;

        // Map from hgid to location
        let mut nodelocations: HashMap<HgId, u32> = HashMap::new();
        for (i, &(hgid, _value)) in values.iter().enumerate() {
            nodelocations.insert(hgid.clone(), locations[i]);
        }

        // Write index
        writer.write_u64::<BigEndian>(values.len() as u64)?;
        for &(hgid, value) in values.iter() {
            let delta_base_offset =
                value
                    .delta_base
                    .map_or(DeltaBaseOffset::FullText, |delta_base| {
                        nodelocations
                            .get(&delta_base)
                            .map_or(DeltaBaseOffset::Missing, |x| {
                                DeltaBaseOffset::Offset(*x as u32)
                            })
                    });

            let entry = IndexEntry::new(hgid.clone(), delta_base_offset, value.offset, value.size);

            entry.write(writer)?;
        }

        Ok(())
    }

    pub fn get_entry(&self, hgid: &HgId) -> Result<Option<IndexEntry>> {
        let (start, end) = FanoutTable::get_bounds(self.get_fanout_slice(), hgid)?;
        let start = start + self.index_start;
        let end = match end {
            Option::None => self.mmap.len(),
            Option::Some(pos) => pos + self.index_start,
        };

        let entry_offset = match self.binary_search(hgid, &self.mmap[start..end]) {
            None => return Ok(None),
            Some(offset) => offset,
        };
        self.read_entry((start + entry_offset) - self.index_start)
            .map(Some)
    }

    pub fn read_entry(&self, offset: usize) -> Result<IndexEntry> {
        let offset = offset + self.index_start;
        let raw_entry = self.mmap.get_err(offset..offset + ENTRY_LEN)?;
        IndexEntry::read(raw_entry)
    }

    fn binary_search(&self, key: &HgId, slice: &[u8]) -> Option<usize> {
        let size = slice.len() / ENTRY_LEN;
        // Cast the slice into an array of entry buffers so we can bisect across them
        let slice: &[[u8; ENTRY_LEN]] =
            unsafe { ::std::slice::from_raw_parts(slice.as_ptr() as *const [u8; ENTRY_LEN], size) };
        match slice.binary_search_by(|entry| entry[0..20].cmp(key.as_ref())) {
            Ok(offset) => Some(offset * ENTRY_LEN),
            Err(_offset) => None,
        }
    }

    fn get_fanout_slice(&self) -> &[u8] {
        self.mmap[2..2 + self.fanout_size].as_ref()
    }
}

#[cfg(test)]
mod tests {
    use quickcheck::quickcheck;
    use rand::SeedableRng;
    use rand_chacha::ChaChaRng;
    use tempfile::NamedTempFile;

    use super::*;

    fn make_index(values: &HashMap<HgId, DeltaLocation>) -> DataIndex {
        let mut file = NamedTempFile::new().expect("file");
        DataIndex::write(&mut file, &values).expect("write dataindex");
        let path = file.into_temp_path();

        DataIndex::new(&path).expect("dataindex")
    }

    #[test]
    fn test_header_invalid() {
        let buf: Vec<u8> = vec![2, 0];
        DataIndexOptions::read(&mut Cursor::new(buf)).expect_err("invalid read");

        let buf: Vec<u8> = vec![0, 1];
        DataIndexOptions::read(&mut Cursor::new(buf)).expect_err("invalid read");
    }

    #[test]
    fn test_missing_delta_base() {
        let mut rng = ChaChaRng::from_seed([0u8; 32]);
        let mut values: HashMap<HgId, DeltaLocation> = HashMap::new();
        let hgid = HgId::random(&mut rng);
        let base = HgId::random(&mut rng);
        values.insert(
            hgid.clone(),
            DeltaLocation {
                delta_base: Some(base),
                offset: 1,
                size: 2,
            },
        );
        let index = make_index(&values);

        let delta = index.get_entry(&hgid).unwrap().unwrap();
        assert_eq!(delta.delta_base_offset(), DeltaBaseOffset::Missing);
    }

    #[test]
    fn test_missing_key() {
        let mut rng = ChaChaRng::from_seed([0u8; 32]);
        let mut values: HashMap<HgId, DeltaLocation> = HashMap::new();
        let hgid = HgId::random(&mut rng);
        values.insert(
            hgid.clone(),
            DeltaLocation {
                delta_base: None,
                offset: 1,
                size: 2,
            },
        );
        let index = make_index(&values);

        let other = HgId::random(&mut rng);
        assert!(index.get_entry(&other).unwrap().is_none());
    }

    quickcheck! {
        fn test_header_serialization(version: u8, large: bool) -> bool {
            let version = version % 2;
            let options = DataIndexOptions { version, large };
            let mut buf: Vec<u8> = vec![];
            options.write(&mut buf).expect("write");
            let parsed_options = DataIndexOptions::read(&mut Cursor::new(buf)).expect("read");
            options == parsed_options
        }

        fn test_roundtrip_index(nodes: Vec<(HgId, u32)>) -> bool {
            let mut values: HashMap<HgId, DeltaLocation> = HashMap::new();

            let mut nodes = nodes;
            let last_hgid = nodes.pop();

            let mut offset = 0;
            for &(hgid, size) in nodes.iter() {
                let size = (size as u64) + 1;
                values.insert(
                    hgid.clone(),
                    DeltaLocation {
                        delta_base: Default::default(),
                        offset: offset,
                        size: size,
                    },
                );

                offset += size;
            }

            let index = make_index(&values);

            let mut offset = 0;
            for &(hgid, size) in nodes.iter() {
                let size = (size as u64) + 1;
                let entry = index.get_entry(&hgid).expect("get_entry").unwrap();
                assert_eq!(entry.hgid(), &hgid);
                assert_eq!(entry.delta_base_offset(), DeltaBaseOffset::FullText);
                assert_eq!(entry.pack_entry_offset(), offset);
                assert_eq!(entry.pack_entry_size(), size);
                offset += size;
            }

            let mut rng = ChaChaRng::from_seed([0u8; 32]);
            let last_hgid = last_hgid.unwrap_or((HgId::random(&mut rng), 0)).0;
            index.get_entry(&last_hgid).unwrap().is_none()
        }
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

//! Classes for constructing and serializing a datapack file and index.
//!
//! A datapack is a pair of files that contain the revision contents for various
//! file revisions in Mercurial. It contains only revision contents (like file
//! contents), not any history information.
//!
//! It consists of two files, with the following format. All bytes are in
//! network byte order (big endian).
//!
//! ```text
//!
//! .datapack
//!     The pack itself is a series of revision deltas with some basic header
//!     information on each. A revision delta may be a fulltext, represented by
//!     a deltabasenode equal to the nullid.
//!
//!     datapack = <version: 1 byte>
//!                [<revision>,...]
//!     revision = <filename len: 2 byte unsigned int>
//!                <filename>
//!                <hgid: 20 byte>
//!                <deltabasenode: 20 byte>
//!                <delta len: 8 byte unsigned int>
//!                <delta>
//!                <metadata-list len: 4 byte unsigned int> [1]
//!                <metadata-list>                          [1]
//!     metadata-list = [<metadata-item>, ...]
//!     metadata-item = <metadata-key: 1 byte>
//!                     <metadata-value len: 2 byte unsigned>
//!                     <metadata-value>
//!
//!     metadata-key could be METAKEYFLAG or METAKEYSIZE or other single byte
//!     value in the future.
//!
//! .dataidx
//!     The index file consists of two parts, the fanout and the index.
//!
//!     The index is a list of index entries, sorted by hgid (one per revision
//!     in the pack). Each entry has:
//!
//!     - hgid (The 20 byte hgid of the entry; i.e. the commit hash, file hgid
//!             hash, etc)
//!     - deltabase index offset (The location in the index of the deltabase for
//!                               this entry. The deltabase is the next delta in
//!                               the chain, with the chain eventually
//!                               terminating in a full-text, represented by a
//!                               deltabase offset of -1. This lets us compute
//!                               delta chains from the index, then do
//!                               sequential reads from the pack if the revision
//!                               are nearby on disk.)
//!     - pack entry offset (The location of this entry in the datapack)
//!     - pack content size (The on-disk length of this entry's pack data)
//!
//!     The fanout is a quick lookup table to reduce the number of steps for
//!     bisecting the index. It is a series of 4 byte pointers to positions
//!     within the index. It has 2^16 entries, which corresponds to hash
//!     prefixes [0000, 0001,..., FFFE, FFFF]. Example: the pointer in slot
//!     4F0A points to the index position of the first revision whose hgid
//!     starts with 4F0A. This saves log(2^16)=16 bisect steps.
//!
//!     dataidx = <version: 1 byte>
//!               <config: 1 byte>
//!               <fanouttable>
//!               <index>
//!     fanouttable = [<index offset: 4 byte unsigned int>,...] (2^8 or 2^16 entries)
//!     index = [<index entry>,...]
//!     indexentry = <hgid: 20 byte>
//!                  <deltabase location: 4 byte signed int>
//!                  <pack entry offset: 8 byte unsigned int>
//!                  <pack entry size: 8 byte unsigned int>
//!
//! ```
//! [1]: new in version 1.

use std::cell::RefCell;
use std::fmt;
use std::fs::File;
use std::io::Cursor;
use std::io::Read;
use std::mem::take;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;

use anyhow::format_err;
use anyhow::Error;
use anyhow::Result;
use byteorder::BigEndian;
use byteorder::ReadBytesExt;
use lz4_pyframe::decompress;
use memmap2::Mmap;
use memmap2::MmapOptions;
use minibytes::Bytes;
use mpatch::mpatch::get_full_text;
use thiserror::Error;
use types::HgId;
use types::Key;
use types::RepoPath;
use util::path::remove_file;

use crate::dataindex::DataIndex;
use crate::dataindex::DeltaBaseOffset;
use crate::datastore::Delta;
use crate::datastore::HgIdDataStore;
use crate::datastore::Metadata;
use crate::datastore::StoreResult;
use crate::localstore::ExtStoredPolicy;
use crate::localstore::LocalStore;
use crate::localstore::StoreFromPath;
use crate::repack::Repackable;
use crate::repack::ToKeys;
use crate::sliceext::SliceExt;
use crate::types::StoreKey;

#[derive(Debug, Error)]
#[error("Datapack Error: {0:?}")]
struct DataPackError(String);

#[derive(Clone, PartialEq)]
pub enum DataPackVersion {
    Zero,
    One,
}

pub struct DataPack {
    mmap: Mmap,
    version: DataPackVersion,
    index: DataIndex,
    base_path: Arc<PathBuf>,
    pack_path: PathBuf,
    index_path: PathBuf,
    extstored_policy: ExtStoredPolicy,
}

pub struct DataEntry<'a> {
    offset: u64,
    filename: &'a RepoPath,
    hgid: HgId,
    delta_base: Option<HgId>,
    compressed_data: &'a [u8],
    data: RefCell<Option<Bytes>>,
    metadata: Metadata,
    next_offset: u64,
}

impl DataPackVersion {
    fn new(value: u8) -> Result<Self> {
        match value {
            0 => Ok(DataPackVersion::Zero),
            1 => Ok(DataPackVersion::One),
            _ => {
                Err(DataPackError(format!("invalid datapack version number '{:?}'", value)).into())
            }
        }
    }
}

impl From<DataPackVersion> for u8 {
    fn from(version: DataPackVersion) -> u8 {
        match version {
            DataPackVersion::Zero => 0,
            DataPackVersion::One => 1,
        }
    }
}

impl<'a> DataEntry<'a> {
    pub fn new(buf: &'a [u8], offset: u64, version: DataPackVersion) -> Result<Self> {
        let mut cur = Cursor::new(buf);
        cur.set_position(offset);

        // Filename
        let filename_len = cur.read_u16::<BigEndian>()? as u64;
        let filename_slice =
            buf.get_err(cur.position() as usize..(cur.position() + filename_len) as usize)?;
        let filename = RepoPath::from_utf8(filename_slice)?;
        let cur_pos = cur.position();
        cur.set_position(cur_pos + filename_len);

        // HgId
        let mut hgid_buf: [u8; 20] = Default::default();
        cur.read_exact(&mut hgid_buf)?;
        let hgid = HgId::from(&hgid_buf);

        // Delta
        cur.read_exact(&mut hgid_buf)?;
        let delta_base = HgId::from(&hgid_buf);
        let delta_base = if delta_base.is_null() {
            None
        } else {
            Some(delta_base)
        };

        let delta_len = cur.read_u64::<BigEndian>()?;
        let compressed_data =
            buf.get_err(cur.position() as usize..(cur.position() + delta_len) as usize)?;
        let data = RefCell::new(None);
        let cur_pos = cur.position();
        cur.set_position(cur_pos + delta_len);

        // Metadata
        let metadata = if version == DataPackVersion::One {
            Metadata::read(&mut cur)?
        } else {
            Default::default()
        };

        let next_offset = cur.position();

        Ok(DataEntry {
            offset,
            filename,
            hgid,
            delta_base,
            compressed_data,
            data,
            metadata,
            next_offset,
        })
    }

    pub fn offset(&self) -> u64 {
        self.offset
    }

    pub fn filename(&self) -> &RepoPath {
        self.filename
    }

    pub fn hgid(&self) -> &HgId {
        &self.hgid
    }

    pub fn delta_base(&self) -> &Option<HgId> {
        &self.delta_base
    }

    pub fn delta(&self) -> Result<Bytes> {
        let mut cell = self.data.borrow_mut();
        if cell.is_none() {
            *cell = Some(decompress(&self.compressed_data)?.into());
        }

        Ok(cell.as_ref().unwrap().clone())
    }

    pub fn metadata(&self) -> &Metadata {
        &self.metadata
    }
}

impl<'a> fmt::Debug for DataEntry<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let delta = self
            .delta()
            .unwrap_or_else(|e| Bytes::copy_from_slice(format!("{:?}", e).as_bytes()));
        write!(
            f,
            "DataEntry {{\n  offset: {:?}\n  filename: {:?}\n  \
             hgid: {:?}\n  delta_base: {:?}\n  compressed_len: {:?}\n  \
             data_len: {:?}\n  data: {:?}\n  metadata: N/A\n}}",
            self.offset,
            self.filename,
            self.hgid,
            self.delta_base,
            self.compressed_data.len(),
            delta.len(),
            delta.iter().map(|b| *b as char).collect::<String>(),
        )
    }
}

impl DataPack {
    pub fn new(p: impl AsRef<Path>, extstored_policy: ExtStoredPolicy) -> Result<Self> {
        DataPack::with_path(p.as_ref(), extstored_policy)
    }

    fn with_path(path: &Path, extstored_policy: ExtStoredPolicy) -> Result<Self> {
        let base_path = PathBuf::from(path);
        let pack_path = path.with_extension("datapack");
        let file = File::open(&pack_path)?;
        let len = file.metadata()?.len();
        if len < 1 {
            return Err(format_err!(
                "empty datapack '{:?}' is invalid",
                path.to_str().unwrap_or("<unknown>")
            ));
        }

        let mmap = unsafe { MmapOptions::new().len(len as usize).map(&file)? };
        let version = DataPackVersion::new(mmap[0])?;
        let index_path = path.with_extension("dataidx");
        Ok(DataPack {
            mmap,
            version,
            index: DataIndex::new(&index_path)?,
            base_path: Arc::new(base_path),
            pack_path,
            index_path,
            extstored_policy,
        })
    }

    pub fn len(&self) -> usize {
        self.mmap.len()
    }

    pub fn read_entry(&self, offset: u64) -> Result<DataEntry> {
        DataEntry::new(self.mmap.as_ref(), offset, self.version.clone())
    }

    pub fn base_path(&self) -> &Path {
        &self.base_path
    }

    pub fn pack_path(&self) -> &Path {
        &self.pack_path
    }

    pub fn index_path(&self) -> &Path {
        &self.index_path
    }

    pub(crate) fn get_delta_chain(&self, key: &Key) -> Result<Option<Vec<Delta>>> {
        let mut chain: Vec<Delta> = Default::default();
        let mut next_entry = match self.index.get_entry(&key.hgid)? {
            None => return Ok(None),
            Some(entry) => entry,
        };
        loop {
            // Due to either storage corruption, or wrongly added data to the datapack, we could
            // end up in an unbounded loop due to a never ending delta chain. Let's avoid this and
            // thus error out if the delta chain is overly long.
            if chain.len() > 1000 {
                return Err(format_err!("Delta chain too long"));
            }

            let data_entry = self.read_entry(next_entry.pack_entry_offset())?;
            if self.extstored_policy == ExtStoredPolicy::Ignore && data_entry.metadata.is_lfs() {
                return Ok(None);
            }

            chain.push(Delta {
                data: data_entry.delta()?,
                base: data_entry
                    .delta_base()
                    .map(|delta_base| Key::new(data_entry.filename.to_owned(), delta_base.clone())),
                key: Key::new(data_entry.filename.to_owned(), data_entry.hgid().clone()),
            });

            if let DeltaBaseOffset::Offset(offset) = next_entry.delta_base_offset() {
                next_entry = self.index.read_entry(offset as usize)?;
            } else {
                break;
            }
        }

        Ok(Some(chain))
    }
}

impl HgIdDataStore for DataPack {
    fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
        let key = match key {
            StoreKey::HgId(key) => key,
            content => return Ok(StoreResult::NotFound(content)),
        };

        let delta_chain = self.get_delta_chain(&key)?;
        let delta_chain = match delta_chain {
            Some(chain) => chain,
            None => return Ok(StoreResult::NotFound(StoreKey::hgid(key))),
        };

        let (basetext, deltas) = match delta_chain.split_last() {
            Some((base, delta)) => (base, delta),
            None => return Ok(StoreResult::NotFound(StoreKey::hgid(key))),
        };

        let deltas: Vec<&[u8]> = deltas
            .iter()
            .rev()
            .map(|delta| delta.data.as_ref())
            .collect();

        Ok(StoreResult::Found(
            get_full_text(basetext.data.as_ref(), &deltas).map_err(Error::msg)?,
        ))
    }

    fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
        let key = match key {
            StoreKey::HgId(key) => key,
            content => return Ok(StoreResult::NotFound(content)),
        };

        let index_entry = match self.index.get_entry(&key.hgid)? {
            None => return Ok(StoreResult::NotFound(StoreKey::hgid(key))),
            Some(entry) => entry,
        };

        let entry = self.read_entry(index_entry.pack_entry_offset())?;
        if self.extstored_policy == ExtStoredPolicy::Ignore && entry.metadata.is_lfs() {
            Ok(StoreResult::NotFound(StoreKey::hgid(key)))
        } else {
            Ok(StoreResult::Found(entry.metadata))
        }
    }

    fn refresh(&self) -> Result<()> {
        Ok(())
    }
}

impl StoreFromPath for DataPack {
    fn from_path(path: &Path, extstored: ExtStoredPolicy) -> Result<Self> {
        DataPack::new(path, extstored)
    }
}

impl LocalStore for DataPack {
    fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        Ok(keys
            .iter()
            .filter(|k| match k {
                StoreKey::HgId(k) => match self.index.get_entry(&k.hgid) {
                    Ok(None) | Err(_) => true,
                    Ok(Some(_)) => false,
                },
                StoreKey::Content(_, _) => true,
            })
            .cloned()
            .collect())
    }
}

impl ToKeys for DataPack {
    fn to_keys(&self) -> Vec<Result<Key>> {
        DataPackIterator::new(self).collect()
    }
}

impl Repackable for DataPack {
    fn delete(mut self) -> Result<()> {
        // On some platforms, removing a file can fail if it's still opened or mapped, let's make
        // sure we close and unmap them before deletion.
        let pack_path = take(&mut self.pack_path);
        let index_path = take(&mut self.index_path);
        drop(self);

        let result1 = remove_file(&pack_path);
        let result2 = remove_file(&index_path);
        // Only check for errors after both have run. That way if pack_path doesn't exist,
        // index_path is still deleted.
        result1?;
        result2?;
        Ok(())
    }

    fn size(&self) -> u64 {
        self.mmap.len() as u64
    }
}

struct DataPackIterator<'a> {
    pack: &'a DataPack,
    offset: u64,
}

impl<'a> DataPackIterator<'a> {
    pub fn new(pack: &'a DataPack) -> Self {
        DataPackIterator {
            pack,
            offset: 1, // Start after the header byte
        }
    }
}

impl<'a> Iterator for DataPackIterator<'a> {
    type Item = Result<Key>;

    fn next(&mut self) -> Option<Self::Item> {
        if self.offset as usize >= self.pack.len() {
            return None;
        }
        let entry = self.pack.read_entry(self.offset);
        Some(match entry {
            Ok(ref e) => {
                self.offset = e.next_offset;
                Ok(Key::new(e.filename.to_owned(), e.hgid))
            }
            Err(e) => {
                // The entry is corrupted, and we have no way to know where the next one is
                // located, let's forcibly stop the iteration.
                self.offset = self.pack.len() as u64;
                Err(e)
            }
        })
    }
}

#[cfg(test)]
pub mod tests {
    use std::rc::Rc;

    use quickcheck::quickcheck;
    use tempfile::TempDir;
    use types::testutil::*;

    use super::*;
    use crate::datastore::Delta;
    use crate::datastore::HgIdMutableDeltaStore;
    use crate::datastore::Metadata;
    use crate::mutabledatapack::MutableDataPack;

    pub fn make_datapack(tempdir: &TempDir, deltas: &Vec<(Delta, Metadata)>) -> DataPack {
        let mutdatapack = MutableDataPack::new(tempdir.path(), DataPackVersion::One);
        for (delta, metadata) in deltas.iter() {
            mutdatapack.add(delta, metadata).unwrap();
        }

        let path = mutdatapack.flush().unwrap().unwrap()[0].clone();

        DataPack::new(&path, ExtStoredPolicy::Use).unwrap()
    }

    #[test]
    fn test_get_missing() {
        let tempdir = TempDir::new().unwrap();

        let revisions = vec![(
            Delta {
                data: Bytes::from(&[1, 2, 3, 4][..]),
                base: Some(key("a", "1")),
                key: key("a", "2"),
            },
            Default::default(),
        )];
        let pack = make_datapack(&tempdir, &revisions);
        for &(ref delta, ref _metadata) in revisions.iter() {
            let missing = pack.get_missing(&[StoreKey::from(&delta.key)]).unwrap();
            assert_eq!(missing.len(), 0);
        }

        let not = key("b", "3");
        let missing = pack.get_missing(&vec![StoreKey::from(&not)]).unwrap();
        assert_eq!(missing, vec![StoreKey::from(not)]);
    }

    #[test]
    fn test_get_meta() {
        let tempdir = TempDir::new().unwrap();

        let revisions = vec![
            (
                Delta {
                    data: Bytes::from(&[1, 2, 3, 4][..]),
                    base: Some(key("a", "1")),
                    key: key("a", "2"),
                },
                Default::default(),
            ),
            (
                Delta {
                    data: Bytes::from(&[1, 2, 3, 4][..]),
                    base: Some(key("a", "3")),
                    key: key("a", "4"),
                },
                Metadata {
                    size: Some(1000),
                    flags: Some(7),
                },
            ),
        ];

        let pack = make_datapack(&tempdir, &revisions);
        for (delta, metadata) in revisions {
            let meta = pack.get_meta(StoreKey::hgid(delta.key)).unwrap();
            assert_eq!(meta, StoreResult::Found(metadata));
        }
    }

    #[test]
    fn test_get_delta_chain_single() {
        let tempdir = TempDir::new().unwrap();

        let revisions = vec![
            (
                Delta {
                    data: Bytes::from(&[1, 2, 3, 4][..]),
                    base: Some(key("a", "1")),
                    key: key("a", "2"),
                },
                Default::default(),
            ),
            (
                Delta {
                    data: Bytes::from(&[1, 2, 3, 4][..]),
                    base: Some(key("a", "3")),
                    key: key("a", "4"),
                },
                Default::default(),
            ),
        ];

        let pack = make_datapack(&tempdir, &revisions);
        for &(ref delta, ref _metadata) in revisions.iter() {
            let chain = pack.get_delta_chain(&delta.key).unwrap().unwrap();
            assert_eq!(chain[0], *delta);
        }
    }

    #[test]
    fn test_get_delta_chain_multiple() {
        let tempdir = TempDir::new().unwrap();

        let mut revisions = vec![(
            Delta {
                data: Bytes::from(&[1, 2, 3, 4][..]),
                base: Some(key("a", "1")),
                key: key("a", "2"),
            },
            Default::default(),
        )];
        let base0 = revisions[0].0.key.clone();
        revisions.push((
            Delta {
                data: Bytes::from(&[1, 2, 3, 4][..]),
                base: Some(base0),
                key: key("a", "3"),
            },
            Default::default(),
        ));
        let base1 = revisions[1].0.key.clone();
        revisions.push((
            Delta {
                data: Bytes::from(&[1, 2, 3, 4][..]),
                base: Some(base1),
                key: key("a", "4"),
            },
            Default::default(),
        ));

        let pack = make_datapack(&tempdir, &revisions);

        let chains = [
            vec![revisions[0].0.clone()],
            vec![revisions[1].0.clone(), revisions[0].0.clone()],
            vec![
                revisions[2].0.clone(),
                revisions[1].0.clone(),
                revisions[0].0.clone(),
            ],
        ];

        for i in 0..2 {
            let chain = pack.get_delta_chain(&revisions[i].0.key).unwrap().unwrap();
            assert_eq!(&chains[i], &chain);
        }
    }

    #[test]
    fn test_iter() {
        let tempdir = TempDir::new().unwrap();

        let revisions = vec![
            (
                Delta {
                    data: Bytes::from(&[1, 2, 3, 4][..]),
                    base: Some(key("a", "1")),
                    key: key("a", "2"),
                },
                Default::default(),
            ),
            (
                Delta {
                    data: Bytes::from(&[1, 2, 3, 4][..]),
                    base: Some(key("a", "3")),
                    key: key("a", "4"),
                },
                Default::default(),
            ),
        ];

        let pack = make_datapack(&tempdir, &revisions);
        assert_eq!(
            pack.to_keys()
                .into_iter()
                .collect::<Result<Vec<Key>>>()
                .unwrap(),
            revisions
                .iter()
                .map(|d| d.0.key.clone())
                .collect::<Vec<Key>>()
        );
    }

    #[test]
    fn test_delete() {
        let tempdir = TempDir::new().unwrap();

        let revisions = vec![(
            Delta {
                data: Bytes::from(&[1, 2, 3, 4][..]),
                base: None,
                key: key("a", "1"),
            },
            Default::default(),
        )];

        let pack = make_datapack(&tempdir, &revisions);
        assert_eq!(
            tempdir.path().read_dir().unwrap().collect::<Vec<_>>().len(),
            2
        );
        pack.delete().unwrap();
        assert_eq!(
            tempdir.path().read_dir().unwrap().collect::<Vec<_>>().len(),
            0
        );
    }

    #[test]
    fn test_delete_while_open() {
        let tempdir = TempDir::new().unwrap();

        let revisions = vec![(
            Delta {
                data: Bytes::from(&[1, 2, 3, 4][..]),
                base: None,
                key: key("a", "1"),
            },
            Default::default(),
        )];

        let pack = make_datapack(&tempdir, &revisions);
        let pack2 = DataPack::new(pack.base_path(), ExtStoredPolicy::Use).unwrap();
        assert!(pack.delete().is_ok());
        assert!(!pack2.pack_path().exists());
        assert!(!pack2.index_path().exists());
    }

    #[test]
    fn test_rc() {
        let tempdir = TempDir::new().unwrap();

        let revisions = vec![(
            Delta {
                data: Bytes::from(&[1, 2, 3, 4][..]),
                base: None,
                key: key("a", "1"),
            },
            Default::default(),
        )];

        let pack = Rc::new(make_datapack(&tempdir, &revisions));
        let data = pack
            .get(StoreKey::hgid(revisions[0].0.key.clone()))
            .unwrap();
        assert_eq!(
            data,
            StoreResult::Found(revisions[0].0.data.as_ref().to_vec())
        );
    }

    #[test]
    fn test_extstored_ignore() -> Result<()> {
        let tempdir = TempDir::new()?;

        let revisions = vec![(
            Delta {
                data: Bytes::from(&[1, 2, 3, 4][..]),
                base: None,
                key: key("a", "1"),
            },
            Metadata {
                size: None,
                flags: Some(Metadata::LFS_FLAG),
            },
        )];
        let pack = make_datapack(&tempdir, &revisions);
        let pack2 = DataPack::new(pack.base_path(), ExtStoredPolicy::Ignore)?;

        let k = StoreKey::hgid(revisions[0].0.key.clone());
        let res = pack2.get(k.clone())?;
        assert_eq!(res, StoreResult::NotFound(k));

        Ok(())
    }

    quickcheck! {
        fn test_iter_quickcheck(keys: Vec<(Vec<u8>, Key)>) -> bool {
            if keys.is_empty() {
                return true;
            }

            let tempdir = TempDir::new().unwrap();

            let mut revisions = vec![];
            for (data, key) in keys {
                revisions.push((
                    Delta {
                        data: data.into(),
                        base: None,
                        key: key.clone(),
                    },
                    Default::default(),
                ));
            }

            let pack = make_datapack(&tempdir, &revisions);
            let same = pack.to_keys().into_iter().collect::<Result<Vec<Key>>>().unwrap()
                == revisions
                    .iter()
                    .map(|d| d.0.key.clone())
                    .collect::<Vec<Key>>();
            same
        }
    }
}