/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

// Union data store
use anyhow::Result;
use minibytes::Bytes;

use crate::datastore::ContentDataStore;
use crate::datastore::ContentMetadata;
use crate::datastore::HgIdDataStore;
use crate::datastore::Metadata;
use crate::datastore::RemoteDataStore;
use crate::datastore::StoreResult;
use crate::types::StoreKey;
use crate::unionstore::UnionStore;

pub type UnionHgIdDataStore<T> = UnionStore<T>;

impl<T: HgIdDataStore> HgIdDataStore for UnionHgIdDataStore<T> {
    fn get(&self, mut key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
        for store in self {
            match store.get(key)? {
                StoreResult::Found(data) => return Ok(StoreResult::Found(data)),
                StoreResult::NotFound(next) => key = next,
            }
        }

        Ok(StoreResult::NotFound(key))
    }

    fn get_meta(&self, mut key: StoreKey) -> Result<StoreResult<Metadata>> {
        for store in self {
            match store.get_meta(key)? {
                StoreResult::Found(meta) => return Ok(StoreResult::Found(meta)),
                StoreResult::NotFound(next) => key = next,
            }
        }

        Ok(StoreResult::NotFound(key))
    }

    fn refresh(&self) -> Result<()> {
        for store in self {
            store.refresh()?;
        }
        Ok(())
    }
}

impl<T: RemoteDataStore> RemoteDataStore for UnionHgIdDataStore<T> {
    fn prefetch(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        let initial_keys = Ok(keys.to_vec());
        self.into_iter()
            .fold(initial_keys, |missing_keys, store| match missing_keys {
                Ok(missing_keys) => {
                    if !missing_keys.is_empty() {
                        store.prefetch(&missing_keys)
                    } else {
                        Ok(vec![])
                    }
                }
                Err(e) => Err(e),
            })
    }

    fn upload(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        self.into_iter()
            .fold(Ok(keys.to_vec()), |not_sent, store| match not_sent {
                Ok(not_sent) => {
                    if !not_sent.is_empty() {
                        store.upload(&not_sent)
                    } else {
                        Ok(Vec::new())
                    }
                }
                Err(e) => Err(e),
            })
    }
}

pub type UnionContentDataStore<T> = UnionStore<T>;

impl<T: ContentDataStore> ContentDataStore for UnionContentDataStore<T> {
    fn blob(&self, mut key: StoreKey) -> Result<StoreResult<Bytes>> {
        for store in self {
            match store.blob(key)? {
                StoreResult::Found(blob) => return Ok(StoreResult::Found(blob)),
                StoreResult::NotFound(next) => key = next,
            }
        }

        Ok(StoreResult::NotFound(key))
    }

    fn metadata(&self, mut key: StoreKey) -> Result<StoreResult<ContentMetadata>> {
        for store in self {
            match store.metadata(key)? {
                StoreResult::Found(meta) => return Ok(StoreResult::Found(meta)),
                StoreResult::NotFound(next) => key = next,
            }
        }

        Ok(StoreResult::NotFound(key))
    }
}

#[cfg(test)]
mod tests {
    use quickcheck::quickcheck;
    use thiserror::Error;
    use types::Key;

    use super::*;
    use crate::localstore::LocalStore;
    use crate::types::StoreKey;

    struct BadHgIdDataStore;

    #[derive(Debug, Error)]
    #[error("Bad data store always has error which is not KeyError")]
    struct BadHgIdDataStoreError;

    struct EmptyHgIdDataStore;

    impl HgIdDataStore for EmptyHgIdDataStore {
        fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
            Ok(StoreResult::NotFound(key))
        }

        fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
            Ok(StoreResult::NotFound(key))
        }

        fn refresh(&self) -> Result<()> {
            Ok(())
        }
    }

    impl LocalStore for EmptyHgIdDataStore {
        fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
            Ok(keys.iter().cloned().collect())
        }
    }

    impl HgIdDataStore for BadHgIdDataStore {
        fn get(&self, _key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
            Err(BadHgIdDataStoreError.into())
        }

        fn get_meta(&self, _key: StoreKey) -> Result<StoreResult<Metadata>> {
            Err(BadHgIdDataStoreError.into())
        }

        fn refresh(&self) -> Result<()> {
            Ok(())
        }
    }

    impl LocalStore for BadHgIdDataStore {
        fn get_missing(&self, _keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
            Err(BadHgIdDataStoreError.into())
        }
    }

    quickcheck! {
        fn test_empty_unionstore_get(key: Key) -> bool {
            match UnionHgIdDataStore::<EmptyHgIdDataStore>::new().get(StoreKey::hgid(key)) {
                Ok(StoreResult::NotFound(_)) => true,
                _ => false,
            }
        }

        fn test_empty_datastore_get(key: Key) -> bool {
            let mut unionstore = UnionHgIdDataStore::new();
            unionstore.add(EmptyHgIdDataStore);
            match unionstore.get(StoreKey::hgid(key)) {
                Ok(StoreResult::NotFound(_)) => true,
                _ => false,
            }
        }

        fn test_bad_datastore_get(key: Key) -> bool {
            let mut unionstore = UnionHgIdDataStore::new();
            unionstore.add(BadHgIdDataStore);
            match unionstore.get(StoreKey::hgid(key)) {
                Err(_) => true,
                _ => false,
            }
        }

        fn test_empty_unionstore_get_meta(key: Key) -> bool {
            match UnionHgIdDataStore::<EmptyHgIdDataStore>::new().get_meta(StoreKey::hgid(key)) {
                Ok(StoreResult::NotFound(_)) => true,
                _ => false,
            }
        }

        fn test_empty_datastore_get_meta(key: Key) -> bool {
            let mut unionstore = UnionHgIdDataStore::new();
            unionstore.add(EmptyHgIdDataStore);
            match unionstore.get_meta(StoreKey::hgid(key)) {
                Ok(StoreResult::NotFound(_)) => true,
                _ => false,
            }
        }

        fn test_bad_datastore_get_meta(key: Key) -> bool {
            let mut unionstore = UnionHgIdDataStore::new();
            unionstore.add(BadHgIdDataStore);
            match unionstore.get_meta(StoreKey::hgid(key)) {
                Err(_) => true,
                _ => false,
            }
        }

        fn test_empty_unionstore_get_missing(keys: Vec<StoreKey>) -> bool {
            keys == UnionHgIdDataStore::<EmptyHgIdDataStore>::new().get_missing(&keys).unwrap()
        }

        fn test_empty_datastore_get_missing(keys: Vec<StoreKey>) -> bool {
            let mut unionstore = UnionHgIdDataStore::new();
            unionstore.add(EmptyHgIdDataStore);
            keys == unionstore.get_missing(&keys).unwrap()
        }

        fn test_bad_datastore_get_missing(keys: Vec<StoreKey>) -> bool {
            let mut unionstore = UnionHgIdDataStore::new();
            unionstore.add(BadHgIdDataStore);
            match unionstore.get_missing(&keys) {
                Ok(_) => false,
                Err(_) => true,
            }
        }
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

// Union history store
use anyhow::Result;
use types::Key;
use types::NodeInfo;

use crate::historystore::HgIdHistoryStore;
use crate::historystore::RemoteHistoryStore;
use crate::types::StoreKey;
use crate::unionstore::UnionStore;

pub type UnionHgIdHistoryStore<T> = UnionStore<T>;

impl<T: HgIdHistoryStore> HgIdHistoryStore for UnionHgIdHistoryStore<T> {
    fn get_node_info(&self, key: &Key) -> Result<Option<NodeInfo>> {
        for store in self {
            match store.get_node_info(key)? {
                None => continue,
                Some(res) => return Ok(Some(res)),
            }
        }

        Ok(None)
    }

    fn refresh(&self) -> Result<()> {
        for store in self {
            store.refresh()?;
        }
        Ok(())
    }
}

impl<T: RemoteHistoryStore> RemoteHistoryStore for UnionHgIdHistoryStore<T> {
    fn prefetch(&self, keys: &[StoreKey]) -> Result<()> {
        let initial_keys = Ok(keys.to_vec());
        self.into_iter()
            .fold(initial_keys, |missing_keys, store| match missing_keys {
                Ok(missing_keys) => {
                    if !missing_keys.is_empty() {
                        store.prefetch(&missing_keys)?;
                        store.get_missing(&missing_keys)
                    } else {
                        Ok(vec![])
                    }
                }
                Err(e) => Err(e),
            })?;

        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use quickcheck::quickcheck;
    use thiserror::Error;

    use super::*;
    use crate::localstore::LocalStore;
    use crate::types::StoreKey;

    struct BadHgIdHistoryStore;

    struct EmptyHgIdHistoryStore;

    #[derive(Debug, Error)]
    #[error("Bad history store always has error which is not KeyError")]
    struct BadHgIdHistoryStoreError;

    impl HgIdHistoryStore for EmptyHgIdHistoryStore {
        fn get_node_info(&self, _key: &Key) -> Result<Option<NodeInfo>> {
            Ok(None)
        }

        fn refresh(&self) -> Result<()> {
            Ok(())
        }
    }

    impl LocalStore for EmptyHgIdHistoryStore {
        fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
            Ok(keys.iter().cloned().collect())
        }
    }

    impl HgIdHistoryStore for BadHgIdHistoryStore {
        fn get_node_info(&self, _key: &Key) -> Result<Option<NodeInfo>> {
            Err(BadHgIdHistoryStoreError.into())
        }

        fn refresh(&self) -> Result<()> {
            Ok(())
        }
    }

    impl LocalStore for BadHgIdHistoryStore {
        fn get_missing(&self, _keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
            Err(BadHgIdHistoryStoreError.into())
        }
    }

    quickcheck! {
        fn test_empty_unionstore_get_node_info(key: Key) -> bool {
            match UnionHgIdHistoryStore::<EmptyHgIdHistoryStore>::new().get_node_info(&key) {
                Ok(None) => true,
                _ => false,
            }
        }

        fn test_empty_historystore_get_node_info(key: Key) -> bool {
            let mut unionstore = UnionHgIdHistoryStore::new();
            unionstore.add(EmptyHgIdHistoryStore);
            match unionstore.get_node_info(&key) {
                Ok(None) => true,
                _ => false,
            }
        }

        fn test_bad_historystore_get_node_info(key: Key) -> bool {
            let mut unionstore = UnionHgIdHistoryStore::new();
            unionstore.add(BadHgIdHistoryStore);
            match unionstore.get_node_info(&key) {
                Err(_) => true,
                _ => false,
            }
        }

        fn test_empty_unionstore_get_missing(keys: Vec<StoreKey>) -> bool {
            keys == UnionHgIdHistoryStore::<EmptyHgIdHistoryStore>::new().get_missing(&keys).unwrap()
        }

        fn test_empty_historystore_get_missing(keys: Vec<StoreKey>) -> bool {
            let mut unionstore = UnionHgIdHistoryStore::new();
            unionstore.add(EmptyHgIdHistoryStore);
            keys == unionstore.get_missing(&keys).unwrap()
        }

        fn test_bad_historystore_get_missing(keys: Vec<StoreKey>) -> bool {
            let mut unionstore = UnionHgIdHistoryStore::new();
            unionstore.add(BadHgIdHistoryStore);
            match unionstore.get_missing(&keys) {
                Err(_) => true,
                _ => false,
            }
        }
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

// Union store

use std::slice::Iter;
use std::vec::IntoIter;

use anyhow::Result;
use types::Key;

use crate::localstore::LocalStore;
use crate::repack::ToKeys;
use crate::types::StoreKey;

pub struct UnionStore<T> {
    stores: Vec<T>,
}

impl<T> UnionStore<T> {
    pub fn new() -> UnionStore<T> {
        UnionStore { stores: Vec::new() }
    }

    pub fn add(&mut self, item: T) {
        self.stores.push(item)
    }
}

impl<T: LocalStore> LocalStore for UnionStore<T> {
    fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        let initial_keys = Ok(keys.to_vec());
        self.into_iter()
            .fold(initial_keys, |missing_keys, store| match missing_keys {
                Ok(missing_keys) => store.get_missing(&missing_keys),
                Err(e) => Err(e),
            })
    }
}

impl<T> IntoIterator for UnionStore<T> {
    type Item = T;
    type IntoIter = IntoIter<T>;

    fn into_iter(self) -> Self::IntoIter {
        self.stores.into_iter()
    }
}

impl<'a, T> IntoIterator for &'a UnionStore<T> {
    type Item = &'a T;
    type IntoIter = Iter<'a, T>;

    fn into_iter(self) -> Self::IntoIter {
        self.stores.iter()
    }
}

impl<T: ToKeys> ToKeys for UnionStore<T> {
    fn to_keys(&self) -> Vec<Result<Key>> {
        self.into_iter()
            .map(|store| store.to_keys())
            .flatten()
            .collect()
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::fs::read_to_string;
use std::fs::OpenOptions;
use std::io::ErrorKind;
use std::io::Write;
use std::path::Path;
use std::path::PathBuf;

use anyhow::Result;
use configmodel::Config;
use configmodel::ConfigExt;
use edenapi::Stats;
use hgtime::HgTime;
use repo_name::encode_repo_name;
use tracing::Span;
use util::path::create_dir;
use util::path::create_shared_dir;

pub fn get_repo_name(config: &dyn Config) -> Result<String> {
    Ok(config.must_get("remotefilelog", "reponame")?)
}

fn get_config_cache_path(config: &dyn Config) -> Result<PathBuf> {
    let reponame = get_repo_name(config)?;
    let mut path: PathBuf = config.must_get("remotefilelog", "cachepath")?;
    create_shared_dir(&path)?;
    path.push(encode_repo_name(reponame));
    create_shared_dir(&path)?;

    Ok(path)
}

pub fn get_cache_path(config: &dyn Config, suffix: &Option<impl AsRef<Path>>) -> Result<PathBuf> {
    let mut path = get_config_cache_path(config)?;

    if let Some(ref suffix) = suffix {
        path.push(suffix);
        create_shared_dir(&path)?;
    }

    Ok(path)
}

pub fn get_local_path(local_path: PathBuf, suffix: &Option<impl AsRef<Path>>) -> Result<PathBuf> {
    let mut path = local_path;
    create_dir(&path)?;

    if let Some(ref suffix) = suffix {
        path.push(suffix);
        create_dir(&path)?;
    }

    Ok(path)
}

pub fn get_indexedlogdatastore_path(path: impl AsRef<Path>) -> Result<PathBuf> {
    let mut path = path.as_ref().to_owned();
    path.push("indexedlogdatastore");
    create_shared_dir(&path)?;
    Ok(path)
}

pub fn get_indexedlogdatastore_aux_path(path: impl AsRef<Path>) -> Result<PathBuf> {
    let mut path = path.as_ref().to_owned();
    path.push("indexedlogdatastore_aux");
    create_shared_dir(&path)?;
    Ok(path)
}

pub fn get_indexedloghistorystore_path(path: impl AsRef<Path>) -> Result<PathBuf> {
    let mut path = path.as_ref().to_owned();
    path.push("indexedloghistorystore");
    create_shared_dir(&path)?;
    Ok(path)
}

pub fn get_packs_path(path: impl AsRef<Path>, suffix: &Option<PathBuf>) -> Result<PathBuf> {
    let mut path = path.as_ref().to_owned();
    path.push("packs");
    create_shared_dir(&path)?;

    if let Some(suffix) = suffix {
        path.push(suffix);
        create_shared_dir(&path)?;
    }

    Ok(path)
}

pub fn get_cache_packs_path(config: &dyn Config, suffix: &Option<PathBuf>) -> Result<PathBuf> {
    get_packs_path(get_config_cache_path(config)?, suffix)
}

fn get_lfs_path(store_path: impl AsRef<Path>) -> Result<PathBuf> {
    let mut path = store_path.as_ref().to_owned();
    path.push("lfs");
    create_shared_dir(&path)?;

    Ok(path)
}

pub fn get_lfs_pointers_path(store_path: impl AsRef<Path>) -> Result<PathBuf> {
    let mut path = get_lfs_path(store_path)?;
    path.push("pointers");
    create_shared_dir(&path)?;

    Ok(path)
}

pub fn get_lfs_objects_path(store_path: impl AsRef<Path>) -> Result<PathBuf> {
    let mut path = get_lfs_path(store_path)?;
    path.push("objects");
    create_shared_dir(&path)?;

    Ok(path)
}

pub fn get_lfs_blobs_path(store_path: impl AsRef<Path>) -> Result<PathBuf> {
    let mut path = get_lfs_path(store_path)?;
    path.push("blobs");
    create_shared_dir(&path)?;

    Ok(path)
}

pub const RUN_ONCE_FILENAME: &str = "runoncemarker";
pub fn check_run_once(store_path: impl AsRef<Path>, key: &str, cutoff: HgTime) -> bool {
    if HgTime::now() > Some(cutoff) {
        return false;
    }

    let marker_path = store_path.as_ref().join(RUN_ONCE_FILENAME);
    let line = format!("\n{}\n", key);
    let marked = match read_to_string(&marker_path) {
        Ok(contents) => contents.contains(&line),
        // If the file doesn't exist, it hasn't run yet.
        Err(e) if e.kind() == ErrorKind::NotFound => false,
        // If it's some other IO error (permission denied, etc), just give up.
        _ => return false,
    };

    if !marked {
        let mut fp = OpenOptions::new()
            .create(true)
            .append(true)
            .open(marker_path)
            .unwrap();
        return write!(fp, "{}", line).is_ok();
    }

    return false;
}

pub fn record_edenapi_stats(span: &Span, stats: &Stats) {
    // Bytes
    span.record("downloaded", &stats.downloaded);
    // Bytes
    span.record("uploaded", &stats.uploaded);
    span.record("requests", &stats.requests);
    // Milliseconds
    span.record(
        "time",
        &u64::try_from(stats.time.as_millis()).unwrap_or(u64::MAX),
    );
    // Milliseconds
    span.record(
        "latency",
        &u64::try_from(stats.latency.as_millis()).unwrap_or(u64::MAX),
    );
    // Compute the speed in MB/s
    let time = stats.time.as_millis() as f64 / 1000.0;
    let size = stats.downloaded as f64 / 1024.0 / 1024.0;
    span.record("download_speed", &format!("{:.2}", size / time).as_str());
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::sync::Arc;

use anyhow::Result;
use async_runtime::block_on;
use async_runtime::spawn_blocking;
use futures::prelude::*;
use progress_model::ProgressBar;
use tracing::field;

use super::hgid_keys;
use super::EdenApiRemoteStore;
use super::EdenApiStoreKind;
use super::File;
use super::Tree;
use crate::datastore::HgIdDataStore;
use crate::datastore::HgIdMutableDeltaStore;
use crate::datastore::Metadata;
use crate::datastore::RemoteDataStore;
use crate::datastore::StoreResult;
use crate::localstore::LocalStore;
use crate::types::StoreKey;
use crate::util;

/// A data store backed by an `EdenApiRemoteStore` and a mutable store.
///
/// Data will be fetched over the network via the remote store and stored in the
/// mutable store before being returned to the caller. This type is not exported
/// because it is intended to be used as a trait object.
pub(super) struct EdenApiDataStore<T> {
    remote: Arc<EdenApiRemoteStore<T>>,
    store: Arc<dyn HgIdMutableDeltaStore>,
}

impl<T: EdenApiStoreKind> EdenApiDataStore<T> {
    pub(super) fn new(
        remote: Arc<EdenApiRemoteStore<T>>,
        store: Arc<dyn HgIdMutableDeltaStore>,
    ) -> Self {
        Self { remote, store }
    }
}

impl RemoteDataStore for EdenApiDataStore<File> {
    fn prefetch(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        let client = self.remote.client.clone();
        let hgidkeys = hgid_keys(keys);

        let response = async move {
            let prog = ProgressBar::register_new(
                "Downloading files over HTTP",
                hgidkeys.len() as u64,
                "files",
            );

            let response = File::prefetch_files(client, hgidkeys).await?;
            // store.add_file() may compress the data before writing it to the store. This can slow
            // things down enough that we don't pull responses off the queue fast enough and
            // edenapi starts queueing all the responses in memory. Let's write to the store in
            // parallel, so we have at least a few threads doing decompression for us.
            let mut entries = response
                .entries
                .map(|entry| {
                    let store = self.store.clone();
                    spawn_blocking(move || {
                        entry.map(|e| {
                            if let Ok(entry) = e.result {
                                store.add_file(&entry)
                            } else {
                                Ok(())
                            }
                        })
                    })
                })
                .buffer_unordered(4);

            while let Some(result) = entries.try_next().await? {
                let _ = result??;
                prog.increase_position(1);
            }
            // Explicitly force the result type here, since otherwise it can't infer the error
            // type.
            let result: Result<_> = Ok((self.store.get_missing(keys)?, response.stats.await?));
            result
        };

        let span = tracing::info_span!(
            "fetch_edenapi",
            downloaded = field::Empty,
            uploaded = field::Empty,
            requests = field::Empty,
            time = field::Empty,
            latency = field::Empty,
            download_speed = field::Empty,
            scmstore = false,
        );
        let _enter = span.enter();
        let (keys, stats) = block_on(response)?;
        util::record_edenapi_stats(&span, &stats);
        Ok(keys)
    }

    fn upload(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        // XXX: EdenAPI does not presently support uploads.
        Ok(keys.to_vec())
    }
}

impl RemoteDataStore for EdenApiDataStore<Tree> {
    fn prefetch(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        let client = self.remote.client.clone();
        let hgidkeys = hgid_keys(keys);

        let response = async move {
            let prog = ProgressBar::register_new(
                "Downloading trees over HTTP",
                hgidkeys.len() as u64,
                "trees",
            );

            let mut response = Tree::prefetch_trees(client, hgidkeys, None).await?;
            while let Some(Ok(entry)) = response.entries.try_next().await? {
                self.store.add_tree(&entry)?;
                prog.increase_position(1);
            }
            // Explicitly force the result type here, since otherwise it can't infer the error
            // type.
            let result: Result<_> = Ok((self.store.get_missing(keys)?, response.stats.await?));
            result
        };

        let span = tracing::info_span!(
            "fetch_edenapi",
            downloaded = field::Empty,
            uploaded = field::Empty,
            requests = field::Empty,
            time = field::Empty,
            latency = field::Empty,
            download_speed = field::Empty,
            scmstore = false,
        );
        let _enter = span.enter();
        let (keys, stats) = block_on(response)?;
        util::record_edenapi_stats(&span, &stats);
        Ok(keys)
    }

    fn upload(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        // XXX: EdenAPI does not presently support uploads.
        Ok(keys.to_vec())
    }
}

impl HgIdDataStore for EdenApiDataStore<File> {
    fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
        self.prefetch(&[key.clone()])?;
        self.store.get(key)
    }

    fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
        self.prefetch(&[key.clone()])?;
        self.store.get_meta(key)
    }

    fn refresh(&self) -> Result<()> {
        Ok(())
    }
}

impl HgIdDataStore for EdenApiDataStore<Tree> {
    fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
        self.prefetch(&[key.clone()])?;
        self.store.get(key)
    }

    fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
        self.prefetch(&[key.clone()])?;
        self.store.get_meta(key)
    }

    fn refresh(&self) -> Result<()> {
        Ok(())
    }
}

impl<T: EdenApiStoreKind> LocalStore for EdenApiDataStore<T> {
    fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        Ok(keys.to_vec())
    }
}

#[cfg(test)]
mod tests {
    use std::str::FromStr;

    use edenapi_types::Blake3;
    use edenapi_types::ContentId;
    use edenapi_types::Sha1;
    use maplit::hashmap;
    use tempfile::TempDir;
    use types::testutil::*;
    use types::Sha256;

    use super::*;
    use crate::edenapi::File;
    use crate::edenapi::Tree;
    use crate::indexedlogauxstore::AuxStore;
    use crate::indexedlogdatastore::IndexedLogHgIdDataStore;
    use crate::indexedlogdatastore::IndexedLogHgIdDataStoreConfig;
    use crate::indexedlogutil::StoreType;
    use crate::localstore::ExtStoredPolicy;
    use crate::scmstore::FetchMode;
    use crate::scmstore::FileAttributes;
    use crate::scmstore::FileAuxData;
    use crate::scmstore::FileStore;
    use crate::scmstore::TreeStore;
    use crate::testutil::*;

    #[test]
    fn test_get_file() -> Result<()> {
        // Set up mocked EdenAPI file and tree stores.
        let k = key("a", "def6f29d7b61f9cb70b2f14f79cd5c43c38e21b2");
        let d = delta("1234", None, k.clone());
        let files = hashmap! { k.clone() => d.data.clone() };

        let client = FakeEdenApi::new().files(files).into_arc();
        let remote_files = EdenApiRemoteStore::<File>::new(client);

        // Set up local cache store to write received data.
        let mut store = FileStore::empty();

        let tmp = TempDir::new()?;
        let config = IndexedLogHgIdDataStoreConfig {
            max_log_count: None,
            max_bytes_per_log: None,
            max_bytes: None,
        };
        let cache = Arc::new(IndexedLogHgIdDataStore::new(
            &tmp,
            ExtStoredPolicy::Ignore,
            &config,
            StoreType::Shared,
        )?);
        store.indexedlog_cache = Some(cache.clone());
        store.edenapi = Some(remote_files);

        // Attempt fetch.
        let mut fetched = store
            .fetch(
                std::iter::once(k.clone()),
                FileAttributes::CONTENT,
                FetchMode::AllowRemote,
            )
            .single()?
            .expect("key not found");
        assert_eq!(fetched.file_content()?.to_vec(), d.data.as_ref().to_vec());

        // Check that data was written to the local store.
        let mut fetched = cache.get_entry(k.clone())?.expect("key not found");
        assert_eq!(fetched.content()?.to_vec(), d.data.as_ref().to_vec());

        Ok(())
    }

    #[test]
    fn test_get_tree() -> Result<()> {
        // Set up mocked EdenAPI file and tree stores.
        let k = key("a", "def6f29d7b61f9cb70b2f14f79cd5c43c38e21b2");
        let d = delta("1234", None, k.clone());
        let trees = hashmap! { k.clone() => d.data.clone() };

        let client = FakeEdenApi::new().trees(trees).into_arc();
        let remote_trees = EdenApiRemoteStore::<Tree>::new(client);

        // Set up local cache store to write received data.
        let mut store = TreeStore::empty();

        let tmp = TempDir::new()?;
        let config = IndexedLogHgIdDataStoreConfig {
            max_log_count: None,
            max_bytes_per_log: None,
            max_bytes: None,
        };
        let cache = Arc::new(IndexedLogHgIdDataStore::new(
            &tmp,
            ExtStoredPolicy::Ignore,
            &config,
            StoreType::Shared,
        )?);
        store.indexedlog_cache = Some(cache.clone());
        store.edenapi = Some(remote_trees);

        // Attempt fetch.
        let mut fetched = store
            .fetch_batch(std::iter::once(k.clone()), FetchMode::AllowRemote)
            .single()?
            .expect("key not found");
        assert_eq!(
            fetched.manifest_tree_entry()?.0.to_vec(),
            d.data.as_ref().to_vec()
        );

        // Check that data was written to the local store.
        let mut fetched = cache.get_entry(k.clone())?.expect("key not found");
        assert_eq!(fetched.content()?.to_vec(), d.data.as_ref().to_vec());

        Ok(())
    }

    #[test]
    fn test_not_found() -> Result<()> {
        let client = FakeEdenApi::new().into_arc();
        let remote_trees = EdenApiRemoteStore::<Tree>::new(client);

        // Set up local cache store to write received data.
        let mut store = TreeStore::empty();
        store.edenapi = Some(remote_trees);

        let k = key("a", "def6f29d7b61f9cb70b2f14f79cd5c43c38e21b2");

        // Attempt fetch.
        let fetched = store.fetch_batch(std::iter::once(k.clone()), FetchMode::AllowRemote);
        let (found, missing, _errors) = fetched.consume();
        assert_eq!(found.len(), 0);
        assert_eq!(missing.into_keys().collect::<Vec<_>>(), vec![k]);

        Ok(())
    }

    #[test]
    fn test_get_aux_cache() -> Result<()> {
        // Set up mocked EdenAPI file and tree stores.
        let k = key("a", "def6f29d7b61f9cb70b2f14f79cd5c43c38e21b2");
        let d = delta("1234", None, k.clone());
        let files = hashmap! { k.clone() => d.data.clone() };

        let client = FakeEdenApi::new().files(files).into_arc();
        let remote_files = EdenApiRemoteStore::<File>::new(client);

        // Set up local cache store to write received data.
        let mut store = FileStore::empty();
        store.edenapi = Some(remote_files);

        // Empty aux cache
        let tmp = TempDir::new()?;
        let aux_cache = Arc::new(AuxStore::new(&tmp, &empty_config(), StoreType::Shared)?);
        store.aux_cache = Some(aux_cache.clone());

        // Empty content cache
        let tmp = TempDir::new()?;
        let config = IndexedLogHgIdDataStoreConfig {
            max_log_count: None,
            max_bytes_per_log: None,
            max_bytes: None,
        };
        let cache = Arc::new(IndexedLogHgIdDataStore::new(
            &tmp,
            ExtStoredPolicy::Ignore,
            &config,
            StoreType::Shared,
        )?);
        store.indexedlog_cache = Some(cache.clone());

        let expected = FileAuxData {
            total_size: 4,
            content_id: ContentId::from_str(
                "aa6ab85da77ca480b7624172fe44aa9906b6c3f00f06ff23c3e5f60bfd0c414e",
            )?,
            content_sha1: Sha1::from_str("7110eda4d09e062aa5e4a390b0a572ac0d2c0220")?,
            content_sha256: Sha256::from_str(
                "03ac674216f3e15c761ee1a5e255f067953623c8b388b4459e13f978d7c846f4",
            )?,
            content_seeded_blake3: Some(Blake3::from_str(
                "2078b4229b5353de0268efc7f64b68f3c99fb8829e9c052117b4e1e090b2603a",
            )?),
        };

        // Test that we can read aux data from EdenApi
        let fetched = store
            .fetch(
                std::iter::once(k.clone()),
                FileAttributes::AUX,
                FetchMode::AllowRemote,
            )
            .single()?
            .expect("key not found");
        assert_eq!(fetched.aux_data().expect("no aux data found"), expected);

        // Disable EdenApi and local cache, make sure we can read from aux cache.
        store.edenapi = None;
        store.indexedlog_cache = None;
        let fetched = store
            .fetch(
                std::iter::once(k.clone()),
                FileAttributes::AUX,
                FetchMode::AllowRemote,
            )
            .single()?
            .expect("key not found");
        assert_eq!(fetched.aux_data().expect("no aux data found"), expected);

        // Content shouldn't have been cached
        assert_eq!(cache.get_entry(k.clone())?, None);

        Ok(())
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::sync::Arc;

use anyhow::Result;
use async_runtime::block_on;
use futures::prelude::*;
use progress_model::ProgressBar;
use types::Key;
use types::NodeInfo;

use super::hgid_keys;
use super::EdenApiRemoteStore;
use super::File;
use crate::historystore::HgIdHistoryStore;
use crate::historystore::HgIdMutableHistoryStore;
use crate::historystore::RemoteHistoryStore;
use crate::localstore::LocalStore;
use crate::types::StoreKey;

/// A history store backed by an `EdenApiRemoteStore` and a mutable store.
///
/// This type can only be created from an `EdenApiRemoteStore<File>`; attempting
/// to create one from a remote store for trees will panic since EdenAPI does
/// not support fetching tree history.
///
/// Data will be fetched over the network via the remote store and stored in the
/// mutable store before being returned to the caller. This type is not exported
/// because it is intended to be used as a trait object.
pub(super) struct EdenApiHistoryStore {
    remote: Arc<EdenApiRemoteStore<File>>,
    store: Arc<dyn HgIdMutableHistoryStore>,
}

impl EdenApiHistoryStore {
    pub(super) fn new(
        remote: Arc<EdenApiRemoteStore<File>>,
        store: Arc<dyn HgIdMutableHistoryStore>,
    ) -> Self {
        Self { remote, store }
    }
}

impl RemoteHistoryStore for EdenApiHistoryStore {
    fn prefetch(&self, keys: &[StoreKey]) -> Result<()> {
        let client = self.remote.client.clone();
        let keys = hgid_keys(keys);

        let response = async move {
            let prog =
                ProgressBar::register_new("Downloading file history over HTTP", 0, "entries");

            let mut response = client.history(keys, None).await?;
            while let Some(entry) = response.entries.try_next().await? {
                self.store.add_entry(&entry)?;
                prog.increase_position(1);
            }

            Ok(())
        };

        block_on(response)
    }
}

impl HgIdHistoryStore for EdenApiHistoryStore {
    fn get_node_info(&self, key: &Key) -> Result<Option<NodeInfo>> {
        self.prefetch(&[StoreKey::hgid(key.clone())])?;
        self.store.get_node_info(key)
    }

    fn refresh(&self) -> Result<()> {
        Ok(())
    }
}

impl LocalStore for EdenApiHistoryStore {
    fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        Ok(keys.to_vec())
    }
}

#[cfg(test)]
mod tests {
    use maplit::hashmap;
    use tempfile::TempDir;
    use types::testutil::*;

    use super::*;
    use crate::edenapi::File;
    use crate::edenapi::Tree;
    use crate::indexedloghistorystore::IndexedLogHgIdHistoryStore;
    use crate::indexedlogutil::StoreType;
    use crate::remotestore::HgIdRemoteStore;
    use crate::testutil::*;

    #[test]
    fn test_file_history() -> Result<()> {
        // Set up mocked EdenAPI store.
        let k = key("a", "1");
        let n = NodeInfo {
            parents: [key("b", "2"), null_key("a")],
            linknode: hgid("3"),
        };
        let history = hashmap! { k.clone() => n.clone() };

        let client = FakeEdenApi::new().history(history).into_arc();
        let remote = EdenApiRemoteStore::<File>::new(client);

        // Set up local mutable store to write received data.
        let tmp = TempDir::new()?;
        let local = Arc::new(IndexedLogHgIdHistoryStore::new(
            &tmp,
            &empty_config(),
            StoreType::Shared,
        )?);

        // Set up `EdenApiHistoryStore`.
        let edenapi = remote.historystore(local.clone());

        // Attempt fetch.
        let nodeinfo = edenapi.get_node_info(&k)?.expect("history not found");
        assert_eq!(&nodeinfo, &n);

        // Check that data was written to the local store.
        let nodeinfo = local.get_node_info(&k)?.expect("history not found");
        assert_eq!(&nodeinfo, &n);

        Ok(())
    }

    #[test]
    #[should_panic]
    fn test_tree_history() {
        let client = FakeEdenApi::new().into_arc();
        let remote = EdenApiRemoteStore::<Tree>::new(client);

        // Set up local mutable store to write received data.
        let tmp = TempDir::new().unwrap();
        let local = Arc::new(
            IndexedLogHgIdHistoryStore::new(&tmp, &empty_config(), StoreType::Shared).unwrap(),
        );

        // EdenAPI does not support fetching tree history, so it should
        // not be possible to get a history store from a tree store.
        // The following line should panic.
        let _ = remote.historystore(local);
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::marker::PhantomData;
use std::sync::Arc;

use async_trait::async_trait;
use edenapi::BlockingResponse;
use edenapi::EdenApi;
use edenapi::EdenApiError;
use edenapi::Response;
use edenapi_types::EdenApiServerError;
use edenapi_types::FileResponse;
use edenapi_types::FileSpec;
use edenapi_types::TreeAttributes;
use edenapi_types::TreeEntry;
use types::Key;

use crate::datastore::HgIdMutableDeltaStore;
use crate::datastore::RemoteDataStore;
use crate::historystore::HgIdMutableHistoryStore;
use crate::historystore::RemoteHistoryStore;
use crate::remotestore::HgIdRemoteStore;
use crate::types::StoreKey;

mod data;
mod history;

use data::EdenApiDataStore;
use history::EdenApiHistoryStore;

/// Convenience aliases for file and tree stores.
pub type EdenApiFileStore = EdenApiRemoteStore<File>;
pub type EdenApiTreeStore = EdenApiRemoteStore<Tree>;

/// A shim around an EdenAPI client that implements the various traits of
/// Mercurial's storage layer, allowing a type that implements `EdenApi` to be
/// used alongside other Mercurial data and history stores.
///
/// Note that this struct does not allow for data fetching on its own, because
/// it does not contain a mutable store into which to write the fetched data.
/// Use the methods from the `HgIdRemoteStore` trait to provide an appropriate
/// mutable store.
#[derive(Clone)]
pub struct EdenApiRemoteStore<T> {
    client: Arc<dyn EdenApi>,
    _phantom: PhantomData<T>,
}

impl<T: EdenApiStoreKind> EdenApiRemoteStore<T> {
    /// Create a new EdenApiRemoteStore using the given EdenAPI client.
    ///
    /// The current design of the storage layer also requires a distinction
    /// between stores that provide file data and stores that provide tree data.
    /// (This is because both kinds of data are fetched via the `prefetch()`
    /// method from the `RemoteDataStore` trait.)
    ///
    /// The kind of data fetched by a store can be specified via a marker type;
    /// in particular, `File` or `Tree`. For example, a store that fetches file
    /// data would be created as follows:
    ///
    /// ```rust,ignore
    /// let store = EdenApiStore::<File>::new(edenapi);
    /// ```
    pub fn new(client: Arc<dyn EdenApi>) -> Arc<Self> {
        Arc::new(Self {
            client,
            _phantom: PhantomData,
        })
    }

    /// Obtain the URL from the EdenApi client.
    pub fn url(&self) -> Option<String> {
        self.client.url()
    }
}

impl HgIdRemoteStore for EdenApiRemoteStore<File> {
    fn datastore(
        self: Arc<Self>,
        store: Arc<dyn HgIdMutableDeltaStore>,
    ) -> Arc<dyn RemoteDataStore> {
        Arc::new(EdenApiDataStore::new(self, store))
    }

    fn historystore(
        self: Arc<Self>,
        store: Arc<dyn HgIdMutableHistoryStore>,
    ) -> Arc<dyn RemoteHistoryStore> {
        Arc::new(EdenApiHistoryStore::new(self, store))
    }
}

impl HgIdRemoteStore for EdenApiRemoteStore<Tree> {
    fn datastore(
        self: Arc<Self>,
        store: Arc<dyn HgIdMutableDeltaStore>,
    ) -> Arc<dyn RemoteDataStore> {
        Arc::new(EdenApiDataStore::new(self, store))
    }

    fn historystore(
        self: Arc<Self>,
        _store: Arc<dyn HgIdMutableHistoryStore>,
    ) -> Arc<dyn RemoteHistoryStore> {
        unimplemented!("EdenAPI does not support fetching tree history")
    }
}

/// Marker type indicating that the store fetches file data.
pub enum File {}

/// Marker type indicating that the store fetches tree data.
pub enum Tree {}

impl EdenApiFileStore {
    pub fn files_blocking(
        &self,
        keys: Vec<Key>,
    ) -> Result<BlockingResponse<FileResponse>, EdenApiError> {
        BlockingResponse::from_async(self.client.files(keys))
    }

    pub fn files_attrs_blocking(
        &self,
        reqs: Vec<FileSpec>,
    ) -> Result<BlockingResponse<FileResponse>, EdenApiError> {
        BlockingResponse::from_async(self.client.files_attrs(reqs))
    }

    pub async fn files_attrs(
        &self,
        reqs: Vec<FileSpec>,
    ) -> Result<Response<FileResponse>, EdenApiError> {
        self.client.files_attrs(reqs).await
    }
}

impl EdenApiTreeStore {
    pub fn trees_blocking(
        &self,
        keys: Vec<Key>,
        attributes: Option<TreeAttributes>,
    ) -> Result<BlockingResponse<Result<TreeEntry, EdenApiServerError>>, EdenApiError> {
        BlockingResponse::from_async(self.client.trees(keys, attributes))
    }
}

/// Trait that provides a common interface for calling the `files` and `trees`
/// methods on an EdenAPI client.
#[async_trait]
pub trait EdenApiStoreKind: Send + Sync + 'static {
    async fn prefetch_files(
        _client: Arc<dyn EdenApi>,
        _keys: Vec<Key>,
    ) -> Result<Response<FileResponse>, EdenApiError> {
        unimplemented!("fetching files not supported for this store")
    }

    async fn prefetch_trees(
        _client: Arc<dyn EdenApi>,
        _keys: Vec<Key>,
        _attributes: Option<TreeAttributes>,
    ) -> Result<Response<Result<TreeEntry, EdenApiServerError>>, EdenApiError> {
        unimplemented!("fetching trees not supported for this store")
    }
}

#[async_trait]
impl EdenApiStoreKind for File {
    async fn prefetch_files(
        client: Arc<dyn EdenApi>,
        keys: Vec<Key>,
    ) -> Result<Response<FileResponse>, EdenApiError> {
        client.files(keys).await
    }
}

#[async_trait]
impl EdenApiStoreKind for Tree {
    async fn prefetch_trees(
        client: Arc<dyn EdenApi>,
        keys: Vec<Key>,
        attributes: Option<TreeAttributes>,
    ) -> Result<Response<Result<TreeEntry, EdenApiServerError>>, EdenApiError> {
        client.trees(keys, attributes).await
    }
}

/// Return only the HgId keys from the given iterator.
/// EdenAPI cannot fetch content-addressed LFS blobs.
fn hgid_keys<'a>(keys: impl IntoIterator<Item = &'a StoreKey>) -> Vec<Key> {
    keys.into_iter()
        .filter_map(|k| match k {
            StoreKey::HgId(k) => Some(k.clone()),
            StoreKey::Content(..) => None,
        })
        .collect()
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::fs::File;
use std::io::BufRead;
use std::io::BufReader;
use std::io::Write;
use std::path::Path;
use std::time::Duration;
use std::time::SystemTime;

use anyhow::Result;
use serde::Deserialize;
use serde::Serialize;
use types::Key;

use super::FileAttributes;

pub(crate) struct ActivityLogger {
    f: File,
}

#[derive(Serialize, Deserialize, Debug)]
pub struct ActivityLog {
    pub op: ActivityType,
    pub keys: Vec<Key>,
    pub attrs: FileAttributes,
    pub start_millis: u128,
    pub duration_millis: u128,
}

#[derive(Serialize, Deserialize, Debug)]
pub enum ActivityType {
    FileFetch,
}

impl ActivityLogger {
    pub(crate) fn new(f: File) -> Self {
        ActivityLogger { f }
    }

    pub(crate) fn log_file_fetch(
        &mut self,
        keys: Vec<Key>,
        attrs: FileAttributes,
        dur: Duration,
    ) -> Result<()> {
        serde_json::to_writer(
            &mut self.f,
            &ActivityLog {
                op: ActivityType::FileFetch,
                keys,
                attrs,
                start_millis: (SystemTime::now() - dur)
                    .duration_since(SystemTime::UNIX_EPOCH)?
                    .as_millis(),
                duration_millis: dur.as_millis(),
            },
        )?;
        self.f.write_all(&[b'\n'])?;
        Ok(())
    }
}

pub fn log_iter<P: AsRef<Path>>(path: P) -> Result<impl Iterator<Item = Result<ActivityLog>>> {
    let file = File::open(path)?;
    Ok(BufReader::new(file).lines().map(|line| {
        let log: ActivityLog = serde_json::from_str(&line?)?;
        Ok(log)
    }))
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::ops::BitAnd;
use std::ops::BitOr;
use std::ops::Not;
use std::ops::Sub;

pub trait StoreAttrs:
    Copy
    + Clone
    + BitAnd<Output = Self>
    + BitOr<Output = Self>
    + Not<Output = Self>
    + Sub<Output = Self>
    + PartialEq
    + std::fmt::Debug
    + Sized
{
    const NONE: Self;
    fn with_computable(&self) -> Self;

    /// Returns true if no attributes are set, otherwise false.
    fn none(&self) -> bool {
        *self == Self::NONE
    }

    /// Returns true if at least one attribute is set, otherwise false.
    fn any(&self) -> bool {
        *self != Self::NONE
    }

    /// Returns true if all attributes are set, otherwise false.
    fn all(&self) -> bool {
        !*self == Self::NONE
    }

    /// Returns true if all the specified attributes are set, otherwise false.
    fn has(&self, attrs: Self) -> bool {
        (attrs - *self).none()
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Instant;

use anyhow::Result;
use configmodel::convert::ByteCount;
use configmodel::Config;
use configmodel::ConfigExt;
use edenapi::Builder;
use parking_lot::Mutex;
use progress_model::AggregatingProgressBar;
use regex::Regex;

use crate::contentstore::check_cache_buster;
use crate::fetch_logger::FetchLogger;
use crate::indexedlogauxstore::AuxStore;
use crate::indexedlogdatastore::IndexedLogHgIdDataStore;
use crate::indexedlogdatastore::IndexedLogHgIdDataStoreConfig;
use crate::indexedlogutil::StoreType;
use crate::lfs::LfsRemote;
use crate::lfs::LfsStore;
use crate::scmstore::activitylogger::ActivityLogger;
use crate::scmstore::file::FileStoreMetrics;
use crate::scmstore::FileStore;
use crate::scmstore::TreeStore;
use crate::util::get_indexedlogdatastore_aux_path;
use crate::util::get_indexedlogdatastore_path;
use crate::util::get_local_path;
use crate::ContentStore;
use crate::EdenApiFileStore;
use crate::EdenApiTreeStore;
use crate::ExtStoredPolicy;
use crate::MemcacheStore;

pub struct FileStoreBuilder<'a> {
    config: &'a dyn Config,
    local_path: Option<PathBuf>,
    suffix: Option<PathBuf>,
    correlator: Option<String>,
    store_aux_data: bool,
    override_edenapi: Option<bool>,

    indexedlog_local: Option<Arc<IndexedLogHgIdDataStore>>,
    indexedlog_cache: Option<Arc<IndexedLogHgIdDataStore>>,
    lfs_local: Option<Arc<LfsStore>>,
    lfs_cache: Option<Arc<LfsStore>>,

    edenapi: Option<Arc<EdenApiFileStore>>,
    memcache: Option<Arc<MemcacheStore>>,

    contentstore: Option<Arc<ContentStore>>,
}

impl<'a> FileStoreBuilder<'a> {
    pub fn new(config: &'a dyn Config) -> Self {
        Self {
            config,
            local_path: None,
            suffix: None,
            correlator: None,
            store_aux_data: false,
            override_edenapi: None,
            indexedlog_local: None,
            indexedlog_cache: None,
            lfs_local: None,
            lfs_cache: None,
            edenapi: None,
            memcache: None,
            contentstore: None,
        }
    }

    pub fn local_path(mut self, path: impl AsRef<Path>) -> Self {
        self.local_path = Some(path.as_ref().to_path_buf());
        self
    }

    pub fn suffix(mut self, suffix: impl AsRef<Path>) -> Self {
        self.suffix = Some(suffix.as_ref().to_path_buf());
        self
    }

    pub fn correlator(mut self, correlator: impl ToString) -> Self {
        self.correlator = Some(correlator.to_string());
        self
    }

    pub fn store_aux_data(mut self) -> Self {
        self.store_aux_data = true;
        self
    }

    pub fn override_edenapi(mut self, use_edenapi: bool) -> Self {
        self.override_edenapi = Some(use_edenapi);
        self
    }

    pub fn edenapi(mut self, edenapi: Arc<EdenApiFileStore>) -> Self {
        self.edenapi = Some(edenapi);
        self
    }

    pub fn memcache(mut self, memcache: Arc<MemcacheStore>) -> Self {
        self.memcache = Some(memcache);
        self
    }

    pub fn indexedlog_cache(mut self, indexedlog: Arc<IndexedLogHgIdDataStore>) -> Self {
        self.indexedlog_cache = Some(indexedlog);
        self
    }

    pub fn indexedlog_local(mut self, indexedlog: Arc<IndexedLogHgIdDataStore>) -> Self {
        self.indexedlog_local = Some(indexedlog);
        self
    }

    pub fn lfs_cache(mut self, lfs_cache: Arc<LfsStore>) -> Self {
        self.lfs_cache = Some(lfs_cache);
        self
    }

    pub fn lfs_local(mut self, lfs_local: Arc<LfsStore>) -> Self {
        self.lfs_local = Some(lfs_local);
        self
    }

    pub fn contentstore(mut self, contentstore: Arc<ContentStore>) -> Self {
        self.contentstore = Some(contentstore);
        self
    }

    fn get_extstored_policy(&self) -> Result<ExtStoredPolicy> {
        // This is to keep compatibility w/ the Python lfs extension.
        // Contentstore would "upgrade" Python LFS pointers from the pack store
        // to indexedlog store during repack, but scmstore doesn't have a repack
        // notion. It doesn't seem bad to just always allow LFS pointers stored
        // in the non-LFS pointer storage.
        Ok(ExtStoredPolicy::Use)
    }

    fn get_lfs_threshold(&self) -> Result<Option<ByteCount>> {
        let enable_lfs = self.config.get_or_default::<bool>("remotefilelog", "lfs")?;
        let lfs_threshold = if enable_lfs {
            self.config.get_opt::<ByteCount>("lfs", "threshold")?
        } else {
            None
        };

        Ok(lfs_threshold)
    }

    fn get_edenapi_retries(&self) -> i32 {
        self.config
            .get_or_default::<i32>("scmstore", "retries")
            .unwrap_or_default()
    }

    fn use_edenapi(&self) -> Result<bool> {
        Ok(if let Some(use_edenapi) = self.override_edenapi {
            use_edenapi
        } else {
            self.edenapi.is_some() || use_edenapi_via_config(self.config)?
        })
    }

    fn use_lfs(&self) -> Result<bool> {
        Ok(self.get_lfs_threshold()?.is_some())
    }

    fn build_edenapi(&self) -> Result<Arc<EdenApiFileStore>> {
        let client = Builder::from_config(self.config)?.build()?;

        Ok(EdenApiFileStore::new(client))
    }

    pub fn build_indexedlog_local(&self) -> Result<Option<Arc<IndexedLogHgIdDataStore>>> {
        Ok(if let Some(local_path) = self.local_path.clone() {
            let local_path = get_local_path(local_path, &self.suffix)?;
            let config = IndexedLogHgIdDataStoreConfig {
                max_log_count: None,
                max_bytes_per_log: None,
                max_bytes: None,
            };
            Some(Arc::new(IndexedLogHgIdDataStore::new(
                get_indexedlogdatastore_path(&local_path)?,
                self.get_extstored_policy()?,
                &config,
                StoreType::Local,
            )?))
        } else {
            None
        })
    }

    pub fn build_indexedlog_cache(&self) -> Result<Option<Arc<IndexedLogHgIdDataStore>>> {
        let cache_path = match cache_path(self.config, &self.suffix)? {
            Some(p) => p,
            None => return Ok(None),
        };

        let max_log_count = self
            .config
            .get_opt::<u8>("indexedlog", "data.max-log-count")?;
        let max_bytes_per_log = self
            .config
            .get_opt::<ByteCount>("indexedlog", "data.max-bytes-per-log")?;
        let max_bytes = self
            .config
            .get_opt::<ByteCount>("remotefilelog", "cachelimit")?;
        let config = IndexedLogHgIdDataStoreConfig {
            max_log_count,
            max_bytes_per_log,
            max_bytes,
        };
        Ok(Some(Arc::new(IndexedLogHgIdDataStore::new(
            get_indexedlogdatastore_path(&cache_path)?,
            self.get_extstored_policy()?,
            &config,
            StoreType::Shared,
        )?)))
    }

    pub fn build_aux_local(&self) -> Result<Option<Arc<AuxStore>>> {
        Ok(if let Some(local_path) = self.local_path.clone() {
            let local_path = get_local_path(local_path, &self.suffix)?;
            let local_path = get_indexedlogdatastore_aux_path(&local_path)?;
            Some(Arc::new(AuxStore::new(
                local_path,
                self.config,
                StoreType::Local,
            )?))
        } else {
            None
        })
    }

    pub fn build_aux_cache(&self) -> Result<Option<Arc<AuxStore>>> {
        let cache_path = match cache_path(self.config, &self.suffix)? {
            Some(p) => p,
            None => return Ok(None),
        };

        let cache_path = get_indexedlogdatastore_aux_path(&cache_path)?;
        Ok(Some(Arc::new(AuxStore::new(
            cache_path,
            self.config,
            StoreType::Shared,
        )?)))
    }

    pub fn build_lfs_local(&self) -> Result<Option<Arc<LfsStore>>> {
        if !self.use_lfs()? {
            return Ok(None);
        }

        Ok(if let Some(local_path) = self.local_path.clone() {
            let local_path = get_local_path(local_path, &self.suffix)?;
            Some(Arc::new(LfsStore::local(&local_path, self.config)?))
        } else {
            None
        })
    }

    pub fn build_lfs_cache(&self) -> Result<Option<Arc<LfsStore>>> {
        if !self.use_lfs()? {
            return Ok(None);
        }

        let cache_path = match cache_path(self.config, &self.suffix)? {
            Some(p) => p,
            None => return Ok(None),
        };

        Ok(Some(Arc::new(LfsStore::shared(&cache_path, self.config)?)))
    }

    pub fn build(mut self) -> Result<FileStore> {
        tracing::trace!(target: "revisionstore::filestore", "checking cache");
        if self.contentstore.is_none() {
            if let Some(cache_path) = cache_path(self.config, &self.suffix)? {
                check_cache_buster(&self.config, &cache_path);
            }
        }

        tracing::trace!(target: "revisionstore::filestore", "processing extstored policy");
        let extstored_policy = self.get_extstored_policy()?;

        tracing::trace!(target: "revisionstore::filestore", "processing lfs threshold");
        let lfs_threshold_bytes = self.get_lfs_threshold()?.map(|b| b.value());

        let edenapi_retries = self.get_edenapi_retries();

        tracing::trace!(target: "revisionstore::filestore", "processing local");
        let indexedlog_local = if let Some(indexedlog_local) = self.indexedlog_local.take() {
            Some(indexedlog_local)
        } else {
            self.build_indexedlog_local()?
        };

        tracing::trace!(target: "revisionstore::filestore", "processing cache");
        let indexedlog_cache = if let Some(indexedlog_cache) = self.indexedlog_cache.take() {
            Some(indexedlog_cache)
        } else {
            self.build_indexedlog_cache()?
        };

        tracing::trace!(target: "revisionstore::filestore", "processing lfs local");
        let lfs_local = if let Some(lfs_local) = self.lfs_local.take() {
            Some(lfs_local)
        } else {
            self.build_lfs_local()?
        };

        tracing::trace!(target: "revisionstore::filestore", "processing lfs cache");
        let lfs_cache = if let Some(lfs_cache) = self.lfs_cache.take() {
            Some(lfs_cache)
        } else {
            self.build_lfs_cache()?
        };

        tracing::trace!(target: "revisionstore::filestore", "processing aux data");
        let (aux_local, aux_cache) = if self.store_aux_data {
            let aux_local = self.build_aux_local()?;
            let aux_cache = self.build_aux_cache()?;
            (aux_local, aux_cache)
        } else {
            (None, None)
        };

        tracing::trace!(target: "revisionstore::filestore", "processing lfs remote");
        let lfs_remote = if self.use_lfs()? {
            if let Some(ref lfs_cache) = lfs_cache {
                // TODO(meyer): Refactor upload functionality so we don't need to use LfsRemote with it's own references to the
                // underlying stores.
                Some(Arc::new(LfsRemote::new(
                    lfs_cache.clone(),
                    lfs_local.clone(),
                    self.config,
                    self.correlator.take(),
                )?))
            } else {
                None
            }
        } else {
            None
        };

        let memcache = self.memcache.take();

        tracing::trace!(target: "revisionstore::filestore", "processing edenapi");
        let edenapi = if self.use_edenapi()? {
            if let Some(edenapi) = self.edenapi.take() {
                Some(edenapi)
            } else {
                Some(self.build_edenapi()?)
            }
        } else {
            None
        };

        tracing::trace!(target: "revisionstore::filestore", "processing contentstore");
        let contentstore = if self
            .config
            .get_or_default::<bool>("scmstore", "contentstorefallback")?
        {
            self.contentstore
        } else {
            None
        };

        tracing::trace!(target: "revisionstore::filestore", "processing fetch logger");
        let logging_regex = self
            .config
            .get_opt::<String>("remotefilelog", "undesiredfileregex")?
            .map(|s| Regex::new(&s))
            .transpose()?;
        let fetch_logger = Some(Arc::new(FetchLogger::new(logging_regex)));

        let allow_write_lfs_ptrs = self
            .config
            .get_or_default::<bool>("scmstore", "lfsptrwrites")?;

        let prefer_computing_aux_data = self
            .config
            .get_or_default::<bool>("scmstore", "prefercomputingauxdata")?;

        let activity_logger =
            if let Some(path) = self.config.get_opt::<String>("scmstore", "activitylog")? {
                let f = std::fs::OpenOptions::new()
                    .append(true)
                    .create(true)
                    .open(path)?;
                Some(Arc::new(Mutex::new(ActivityLogger::new(f))))
            } else {
                None
            };

        tracing::trace!(target: "revisionstore::filestore", "constructing FileStore");
        Ok(FileStore {
            extstored_policy,
            lfs_threshold_bytes,
            edenapi_retries,
            allow_write_lfs_ptrs,
            prefer_computing_aux_data,

            indexedlog_local,
            lfs_local,

            indexedlog_cache,
            lfs_cache,

            memcache,
            cache_to_memcache: true,

            edenapi,
            lfs_remote,

            activity_logger,
            contentstore,
            fetch_logger,
            metrics: FileStoreMetrics::new(),

            aux_local,
            aux_cache,

            creation_time: Instant::now(),
            lfs_progress: AggregatingProgressBar::new("fetching", "LFS"),
            flush_on_drop: true,
        })
    }
}

// Return remotefilelog cache path, or None if there is no cache path
// (e.g. because we have no repo name).
fn cache_path(config: &dyn Config, suffix: &Option<PathBuf>) -> Result<Option<PathBuf>> {
    match crate::util::get_cache_path(config, suffix) {
        Ok(p) => Ok(Some(p)),
        Err(err) => {
            if matches!(
                err.downcast_ref::<configmodel::Error>(),
                Some(configmodel::Error::NotSet(_, _))
            ) {
                Ok(None)
            } else {
                Err(err)
            }
        }
    }
}

pub struct TreeStoreBuilder<'a> {
    config: &'a dyn Config,
    local_path: Option<PathBuf>,
    suffix: Option<PathBuf>,
    override_edenapi: Option<bool>,

    indexedlog_local: Option<Arc<IndexedLogHgIdDataStore>>,
    indexedlog_cache: Option<Arc<IndexedLogHgIdDataStore>>,
    edenapi: Option<Arc<EdenApiTreeStore>>,
    memcache: Option<Arc<MemcacheStore>>,
    contentstore: Option<Arc<ContentStore>>,
    filestore: Option<Arc<FileStore>>,
}

impl<'a> TreeStoreBuilder<'a> {
    pub fn new(config: &'a dyn Config) -> Self {
        Self {
            config,
            local_path: None,
            suffix: None,
            override_edenapi: None,
            indexedlog_local: None,
            indexedlog_cache: None,
            edenapi: None,
            memcache: None,
            contentstore: None,
            filestore: None,
        }
    }

    pub fn local_path(mut self, path: impl AsRef<Path>) -> Self {
        self.local_path = Some(path.as_ref().to_path_buf());
        self
    }

    // TODO(meyer): Can we remove this since we have seprate builders for files and trees?
    // Is this configurable somewhere we can directly check from Config instead of having the
    // caller pass in, or is it just hardcoded elsewhere and we should hardcode it here?
    /// Cache path suffix for the associated indexedlog. For files, this will not be given.
    /// For trees, it will be "manifests".
    pub fn suffix(mut self, suffix: impl AsRef<Path>) -> Self {
        self.suffix = Some(suffix.as_ref().to_path_buf());
        self
    }

    pub fn edenapi(mut self, edenapi: Arc<EdenApiTreeStore>) -> Self {
        self.edenapi = Some(edenapi);
        self
    }

    pub fn memcache(mut self, memcache: Arc<MemcacheStore>) -> Self {
        self.memcache = Some(memcache);
        self
    }

    pub fn indexedlog_cache(mut self, indexedlog: Arc<IndexedLogHgIdDataStore>) -> Self {
        self.indexedlog_cache = Some(indexedlog);
        self
    }

    pub fn indexedlog_local(mut self, indexedlog: Arc<IndexedLogHgIdDataStore>) -> Self {
        self.indexedlog_local = Some(indexedlog);
        self
    }

    pub fn contentstore(mut self, contentstore: Arc<ContentStore>) -> Self {
        self.contentstore = Some(contentstore);
        self
    }

    pub fn override_edenapi(mut self, use_edenapi: bool) -> Self {
        self.override_edenapi = Some(use_edenapi);
        self
    }

    pub fn filestore(mut self, filestore: Arc<FileStore>) -> Self {
        self.filestore = Some(filestore);
        self
    }

    fn use_edenapi(&self) -> Result<bool> {
        Ok(if let Some(use_edenapi) = self.override_edenapi {
            use_edenapi
        } else {
            self.edenapi.is_some() || use_edenapi_via_config(self.config)?
        })
    }

    fn build_edenapi(&self) -> Result<Arc<EdenApiTreeStore>> {
        let client = Builder::from_config(self.config)?.build()?;

        Ok(EdenApiTreeStore::new(client))
    }

    pub fn build_indexedlog_local(&self) -> Result<Option<Arc<IndexedLogHgIdDataStore>>> {
        Ok(if let Some(local_path) = self.local_path.clone() {
            let local_path = get_local_path(local_path, &self.suffix)?;
            let config = IndexedLogHgIdDataStoreConfig {
                max_log_count: None,
                max_bytes_per_log: None,
                max_bytes: None,
            };
            Some(Arc::new(IndexedLogHgIdDataStore::new(
                get_indexedlogdatastore_path(&local_path)?,
                ExtStoredPolicy::Use,
                &config,
                StoreType::Local,
            )?))
        } else {
            None
        })
    }

    pub fn build_indexedlog_cache(&self) -> Result<Option<Arc<IndexedLogHgIdDataStore>>> {
        let cache_path = match cache_path(self.config, &self.suffix)? {
            Some(p) => p,
            None => return Ok(None),
        };

        let max_log_count = self
            .config
            .get_opt::<u8>("indexedlog", "manifest.max-log-count")?;
        let max_bytes_per_log = self
            .config
            .get_opt::<ByteCount>("indexedlog", "manifest.max-bytes-per-log")?;
        let max_bytes = self
            .config
            .get_opt::<ByteCount>("remotefilelog", "manifestlimit")?;
        let config = IndexedLogHgIdDataStoreConfig {
            max_log_count,
            max_bytes_per_log,
            max_bytes,
        };

        Ok(Some(Arc::new(IndexedLogHgIdDataStore::new(
            get_indexedlogdatastore_path(&cache_path)?,
            ExtStoredPolicy::Use,
            &config,
            StoreType::Shared,
        )?)))
    }

    pub fn build(mut self) -> Result<TreeStore> {
        // TODO(meyer): Clean this up, just copied and pasted from the other version & did some ugly hacks to get this
        // (the EdenApiAdapter stuff needs to be fixed in particular)
        tracing::trace!(target: "revisionstore::treestore", "checking cache");
        if self.contentstore.is_none() {
            if let Some(cache_path) = cache_path(self.config, &self.suffix)? {
                check_cache_buster(&self.config, &cache_path);
            }
        }

        tracing::trace!(target: "revisionstore::treestore", "processing local");
        let indexedlog_local = if let Some(indexedlog_local) = self.indexedlog_local.take() {
            Some(indexedlog_local)
        } else {
            self.build_indexedlog_local()?
        };

        tracing::trace!(target: "revisionstore::treestore", "processing cache");
        let indexedlog_cache = if let Some(indexedlog_cache) = self.indexedlog_cache.take() {
            Some(indexedlog_cache)
        } else {
            self.build_indexedlog_cache()?
        };

        let memcache = self.memcache.take();

        tracing::trace!(target: "revisionstore::treestore", "processing edenapi");
        let edenapi = if self.use_edenapi()? {
            if let Some(edenapi) = self.edenapi.take() {
                Some(edenapi)
            } else {
                Some(self.build_edenapi()?)
            }
        } else {
            None
        };

        tracing::trace!(target: "revisionstore::treestore", "processing contentstore");
        let contentstore = if self
            .config
            .get_or_default::<bool>("scmstore", "contentstorefallback")?
        {
            self.contentstore
        } else {
            None
        };

        tracing::trace!(target: "revisionstore::treestore", "constructing TreeStore");
        Ok(TreeStore {
            indexedlog_local,

            indexedlog_cache,
            cache_to_local_cache: true,

            memcache,
            cache_to_memcache: true,

            edenapi,

            contentstore,
            filestore: self.filestore,

            creation_time: Instant::now(),
            flush_on_drop: true,
        })
    }
}

fn use_edenapi_via_config(config: &dyn Config) -> Result<bool> {
    let mut use_edenapi: bool = config.get_or_default("remotefilelog", "http")?;
    if use_edenapi {
        // If paths.default is not set, there is no server repo. Therefore, do
        // not use edenapi.
        let path: Option<String> = config.get_opt("paths", "default")?;
        if path.is_none() {
            use_edenapi = false;
        }
    }
    Ok(use_edenapi)
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::hash_map;
use std::collections::HashMap;
use std::collections::HashSet;
use std::fmt;

use anyhow::anyhow;
use anyhow::Error;
use anyhow::Result;
use crossbeam::channel::Sender;
use types::Key;

use crate::scmstore::attrs::StoreAttrs;
use crate::scmstore::value::StoreValue;

#[derive(Debug)]
pub enum FetchMode {
    /// The fetch may hit memcache or other servers.
    AllowRemote,
    /// The fetch is limited to RAM and disk.
    LocalOnly,
}

pub(crate) struct CommonFetchState<T: StoreValue> {
    /// Requested keys for which at least some attributes haven't been found.
    pub pending: HashSet<Key>,

    /// Which attributes were requested
    pub request_attrs: T::Attrs,

    /// All attributes which have been found so far
    pub found: HashMap<Key, T>,

    pub found_tx: Sender<Result<(Key, T), KeyFetchError>>,
}

impl<T: StoreValue> CommonFetchState<T> {
    pub(crate) fn new(
        keys: impl Iterator<Item = Key>,
        attrs: T::Attrs,
        found_tx: Sender<Result<(Key, T), KeyFetchError>>,
    ) -> Self {
        Self {
            pending: keys.collect(),
            request_attrs: attrs,
            found: HashMap::new(),
            found_tx,
        }
    }

    pub(crate) fn pending_len(&self) -> usize {
        self.pending.len()
    }

    pub(crate) fn pending<'a>(
        &'a self,
        fetchable: T::Attrs,
        with_computable: bool,
    ) -> impl Iterator<Item = (&'a Key, T::Attrs)> + 'a {
        self.pending.iter().filter_map(move |key| {
            let actionable = self.actionable(key, fetchable, with_computable);
            if actionable.any() {
                Some((key, actionable))
            } else {
                None
            }
        })
    }

    pub(crate) fn found(&mut self, key: Key, value: T) -> bool {
        use hash_map::Entry::*;
        match self.found.entry(key.clone()) {
            Occupied(mut entry) => {
                tracing::debug!("merging into previously fetched attributes");
                // Combine the existing and newly-found attributes, overwriting existing attributes with the new ones
                // if applicable (so that we can re-use this function to replace in-memory files with mmap-ed files)
                let available = entry.get_mut();
                let new = value | std::mem::take(available);

                if new.attrs().has(self.request_attrs) {
                    self.found.remove(&key);
                    self.pending.remove(&key);
                    let new = new.mask(self.request_attrs);
                    let _ = self.found_tx.send(Ok((key, new)));
                    return true;
                } else {
                    *available = new;
                }
            }
            Vacant(entry) => {
                if value.attrs().has(self.request_attrs) {
                    self.pending.remove(&key);
                    let value = value.mask(self.request_attrs);
                    let _ = self.found_tx.send(Ok((key, value)));
                    return true;
                } else {
                    entry.insert(value);
                }
            }
        };

        return false;
    }

    pub(crate) fn results(mut self, errors: FetchErrors) {
        // Combine and collect errors
        let mut incomplete = errors.fetch_errors;
        for key in self.pending.into_iter() {
            self.found.remove(&key);
            incomplete.entry(key).or_insert_with(|| {
                // This should really never happen. If a key fails to fetch, it should've been
                // associated with a keyed error and put in incomplete already.
                vec![anyhow!("unknown error while fetching")]
            });
        }

        for (key, _) in self.found.iter_mut() {
            // Don't return errors for keys we eventually found.
            incomplete.remove(key);
        }

        for (key, errors) in incomplete {
            let _ = self
                .found_tx
                .send(Err(KeyFetchError::KeyedError { key, errors }.into()));
        }

        for err in errors.other_errors {
            let _ = self.found_tx.send(Err(KeyFetchError::Other(err)));
        }
    }

    pub(crate) fn actionable(
        &self,
        key: &Key,
        fetchable: T::Attrs,
        with_computable: bool,
    ) -> T::Attrs {
        if fetchable.none() {
            return T::Attrs::NONE;
        }

        let available = self.found.get(key).map_or(T::Attrs::NONE, |f| f.attrs());
        let (available, fetchable) = if with_computable {
            (available.with_computable(), fetchable.with_computable())
        } else {
            (available, fetchable)
        };
        let missing = self.request_attrs - available;
        let actionable = missing & fetchable;
        actionable
    }
}

#[derive(Debug)]
pub enum KeyFetchError {
    KeyedError { key: Key, errors: Vec<Error> },
    Other(Error),
}

// Manual std::error impl to pick a source() for KeyedError.
impl std::error::Error for KeyFetchError {
    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
        match self {
            Self::Other(err) => Some(err.as_ref()),
            Self::KeyedError { errors, .. } => errors.iter().next().map(|e| e.as_ref()),
        }
    }
}

impl fmt::Display for KeyFetchError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            Self::Other(err) => err.fmt(f),
            Self::KeyedError { key, errors } => {
                write!(f, "Key fetch failed {}: {:?}", key, errors)
            }
        }
    }
}

pub(crate) struct FetchErrors {
    /// Errors encountered for specific keys
    pub(crate) fetch_errors: HashMap<Key, Vec<Error>>,

    /// Errors encountered that don't apply to a single key
    pub(crate) other_errors: Vec<Error>,
}

impl FetchErrors {
    pub(crate) fn new() -> Self {
        FetchErrors {
            fetch_errors: HashMap::new(),
            other_errors: Vec::new(),
        }
    }

    pub(crate) fn keyed_error(&mut self, key: Key, err: Error) {
        self.fetch_errors
            .entry(key)
            .or_insert_with(Vec::new)
            .push(err);
    }

    pub(crate) fn other_error(&mut self, err: Error) {
        self.other_errors.push(err);
    }
}

pub struct FetchResults<T> {
    iterator: Box<dyn Iterator<Item = Result<(Key, T), KeyFetchError>>>,
}

impl<T> IntoIterator for FetchResults<T> {
    type Item = Result<(Key, T), KeyFetchError>;
    type IntoIter = Box<dyn Iterator<Item = Result<(Key, T), KeyFetchError>>>;

    fn into_iter(self) -> Self::IntoIter {
        self.iterator
    }
}

impl<T> FetchResults<T> {
    pub fn new(iterator: Box<dyn Iterator<Item = Result<(Key, T), KeyFetchError>>>) -> Self {
        FetchResults { iterator }
    }

    pub fn consume(self) -> (HashMap<Key, T>, HashMap<Key, Vec<Error>>, Vec<Error>) {
        let mut found = HashMap::new();
        let mut missing = HashMap::new();
        let mut errors = vec![];
        for result in self {
            match result {
                Ok((key, value)) => {
                    found.insert(key, value);
                }
                Err(err) => match err {
                    KeyFetchError::KeyedError { key, errors } => {
                        missing.insert(key.clone(), errors);
                    }
                    KeyFetchError::Other(err) => {
                        errors.push(err);
                    }
                },
            };
        }
        (found, missing, errors)
    }

    /// Return the list of keys which could not be fetched, or any errors encountered
    pub fn missing(self) -> Result<Vec<Key>> {
        // Don't use self.consume here since it pends all the found results in memory, which can be
        // expensive.
        let mut missing = vec![];
        for result in self {
            match result {
                Ok(_) => {}
                Err(err) => match err {
                    KeyFetchError::KeyedError { key, .. } => {
                        missing.push(key.clone());
                    }
                    KeyFetchError::Other(err) => {
                        return Err(err);
                    }
                },
            };
        }
        Ok(missing)
    }

    /// Return the single requested file if found, or any errors encountered
    pub fn single(self) -> Result<Option<T>> {
        let mut first = None;
        for result in self {
            let (_, value) = result?;
            if first.is_none() {
                first = Some(value)
            }
        }

        Ok(first)
    }
}

#[cfg(test)]
mod tests {
    use ::types::errors::NetworkError;
    use anyhow::anyhow;

    use super::*;

    #[test]
    fn test_error_chain() {
        {
            let inner_err = anyhow!("inner");
            let outer_err = inner_err.context("context");

            let err: &dyn std::error::Error = &KeyFetchError::Other(outer_err);
            assert_eq!(format!("{}", err.source().unwrap()), "context");
            assert_eq!(
                format!("{}", err.source().unwrap().source().unwrap()),
                "inner"
            );
        }

        {
            let err: &dyn std::error::Error = &KeyFetchError::KeyedError {
                key: Default::default(),
                errors: vec![],
            };
            assert!(err.source().is_none());
        }

        {
            let err: &dyn std::error::Error = &KeyFetchError::KeyedError {
                key: Default::default(),
                errors: vec![anyhow!("one"), anyhow!("two")],
            };
            assert_eq!(format!("{}", err.source().unwrap()), "one");
        }

        {
            let err: anyhow::Error =
                KeyFetchError::Other(NetworkError::wrap(anyhow!("foo"))).into();
            assert!(types::errors::is_network_error(&err));
        }
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::ops::AddAssign;

use crate::indexedlogutil::StoreType;

#[derive(Clone, Debug, Default)]
pub struct FetchMetrics {
    /// Number of requests / batches
    requests: usize,

    /// Numbers of entities requested
    keys: usize,

    /// Number of successfully fetched entities
    hits: usize,

    /// Number of entities which were not found
    misses: usize,

    /// Number of entities which returned a fetch error (including batch errors)
    errors: usize,
}

impl AddAssign for FetchMetrics {
    fn add_assign(&mut self, rhs: Self) {
        self.requests += rhs.requests;
        self.keys += rhs.keys;
        self.hits += rhs.hits;
        self.misses += rhs.misses;
        self.errors += rhs.errors;
    }
}

impl FetchMetrics {
    pub(crate) fn fetch(&mut self, keys: usize) {
        self.requests += 1;
        self.keys += keys;
    }

    pub(crate) fn hit(&mut self, keys: usize) {
        self.hits += keys;
    }

    pub(crate) fn miss(&mut self, keys: usize) {
        self.misses += keys;
    }

    pub(crate) fn err(&mut self, keys: usize) {
        self.errors += keys;
    }

    pub(crate) fn metrics(&self) -> impl Iterator<Item = (&'static str, usize)> {
        [
            ("requests", self.requests),
            ("keys", self.keys),
            ("hits", self.hits),
            ("misses", self.misses),
            ("errors", self.errors),
        ]
        .into_iter()
        .filter(|&(_, v)| v != 0)
    }
}

// TODO(meyer): I don't think this is in any critical paths, but it'd be nicer to rewrite this
// to use `Item = (Vec<&'static str>, usize)` instead of `Item = (String, usize)`, since all
// the fields are indeed statically named right now, or, better, just tree of some sort instead of a
// list of metrics. Probably appropriate for a `SmallVec` too, since the namespace depth is
// limited.
pub(crate) fn namespaced(
    namespace: &'static str,
    metrics: impl Iterator<Item = (impl AsRef<str>, usize)>,
) -> impl Iterator<Item = (String, usize)> {
    metrics.map(move |(k, v)| (namespace.to_string() + "." + k.as_ref(), v))
}

#[derive(Clone, Debug, Default)]
pub struct LocalAndCacheFetchMetrics {
    local: FetchMetrics,
    cache: FetchMetrics,
}

impl LocalAndCacheFetchMetrics {
    pub(crate) fn store(&mut self, typ: StoreType) -> &mut FetchMetrics {
        match typ {
            StoreType::Local => &mut self.local,
            StoreType::Shared => &mut self.cache,
        }
    }

    pub(crate) fn metrics(&self) -> impl Iterator<Item = (String, usize)> {
        namespaced("local", self.local.metrics()).chain(namespaced("cache", self.cache.metrics()))
    }
}

impl AddAssign for LocalAndCacheFetchMetrics {
    fn add_assign(&mut self, rhs: Self) {
        self.local += rhs.local;
        self.cache += rhs.cache;
    }
}

#[derive(Clone, Debug, Default)]
pub struct WriteMetrics {
    /// Numbers of entities we attempted to write
    items: usize,

    /// Number of successfully written entities
    ok: usize,

    /// Number of entities which returned a write error (including batch errors)
    err: usize,
}

impl AddAssign for WriteMetrics {
    fn add_assign(&mut self, rhs: Self) {
        self.items += rhs.items;
        self.ok += rhs.ok;
        self.err += rhs.err;
    }
}

impl WriteMetrics {
    pub(crate) fn item(&mut self, keys: usize) {
        self.items += keys;
    }

    pub(crate) fn ok(&mut self, keys: usize) {
        self.ok += keys;
    }

    pub(crate) fn err(&mut self, keys: usize) {
        self.err += keys;
    }

    pub(crate) fn metrics(&self) -> impl Iterator<Item = (&'static str, usize)> {
        [("items", self.items), ("ok", self.ok), ("err", self.err)]
            .into_iter()
            .filter(|&(_, v)| v != 0)
    }
}

#[derive(Clone, Debug, Default)]
pub struct ApiMetrics {
    /// Number of calls to this API
    calls: usize,

    /// Total number of entities requested across all calls
    keys: usize,

    /// Number of calls for only a single entity
    singles: usize,
}

impl AddAssign for ApiMetrics {
    fn add_assign(&mut self, rhs: Self) {
        self.calls += rhs.calls;
        self.keys += rhs.keys;
        self.singles += rhs.singles;
    }
}

impl ApiMetrics {
    pub(crate) fn call(&mut self, keys: usize) {
        self.calls += 1;
        self.keys += keys;
        if keys == 1 {
            self.singles += 1;
        }
    }

    pub(crate) fn metrics(&self) -> impl Iterator<Item = (&'static str, usize)> {
        [
            ("calls", self.calls),
            ("keys", self.keys),
            ("singles", self.singles),
        ]
        .into_iter()
        .filter(|&(_, v)| v != 0)
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

pub use self::builder::FileStoreBuilder;
pub use self::builder::TreeStoreBuilder;
pub use self::fetch::FetchMode;
pub use self::fetch::KeyFetchError;
pub use self::file::FileAttributes;
pub use self::file::FileAuxData;
pub use self::file::FileStore;
pub use self::file::StoreFile;
pub use self::tree::TreeStore;
pub use self::util::file_to_async_key_stream;

pub mod activitylogger;
pub mod attrs;
pub mod builder;
pub mod file;
pub mod tree;
pub mod util;
pub mod value;

pub(crate) mod fetch;
pub(crate) mod metrics;


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::path::PathBuf;
use std::str::FromStr;

use anyhow::anyhow;
use anyhow::Result;
use futures::future;
use futures::stream::Stream;
use futures::stream::StreamExt;
use tokio::fs::File;
use tokio::io::AsyncBufReadExt;
use tokio::io::BufReader;
use tokio_stream::wrappers::LinesStream;
use tracing::error;
//use async_runtime::{block_on_future as block_on, stream_to_iter as block_on_stream};
use types::HgId;
use types::Key;
use types::RepoPathBuf;

// TODO(meyer): Find a better place for this. testutil? A debug command isn't really a test.
// Maybe refactor so less logic happens in hgcommands / pyrevisionstore, and migrate the actual
// business logic into revisionstore::scmstore::util or something.
pub async fn file_to_async_key_stream(path: PathBuf) -> Result<impl Stream<Item = Key>> {
    let file = BufReader::new(File::open(&path).await?);
    let lines = LinesStream::new(file.lines());
    Ok(lines
        .map(|line| {
            let line = line?;
            let hgid_path: Vec<_> = line.splitn(2, ',').collect();
            let hgid = HgId::from_str(hgid_path[0])?;
            let path = hgid_path
                .get(1)
                .ok_or_else(|| anyhow!("malformed line, no comma found"))?;
            let path = RepoPathBuf::from_string(path.to_string())?;
            anyhow::Ok(Key::new(path, hgid))
        })
        .filter_map(|res| {
            future::ready(match res {
                Ok(key) => Some(key),
                Err(e) => {
                    error!({ error = %e }, "error reading key from line");
                    None
                }
            })
        }))
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::ops::BitOr;

use crate::scmstore::attrs::StoreAttrs;

pub trait StoreValue: BitOr<Output = Self> + Default + Sized {
    type Attrs: StoreAttrs;

    /// Returns the attributes present in the value.
    fn attrs(&self) -> Self::Attrs;

    /// Return only the specified attributes of the value.
    fn mask(self, attrs: Self::Attrs) -> Self;
}