/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashSet;
use std::ops::Deref;
use std::path::PathBuf;
use std::sync::Arc;

use anyhow::Result;
use edenapi_types::FileEntry;
use edenapi_types::TreeEntry;
use minibytes::Bytes;
use regex::Regex;
use serde_derive::Deserialize;
use serde_derive::Serialize;
use types::Key;
use types::RepoPathBuf;

use crate::fetch_logger::FetchLogger;
use crate::localstore::LocalStore;
use crate::types::ContentHash;
use crate::types::StoreKey;
pub use crate::Metadata;
use crate::RepackLocation;

#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct Delta {
    pub data: Bytes,
    #[serde(with = "types::serde_with::key::tuple")]
    pub base: Option<Key>,
    #[serde(with = "types::serde_with::key::tuple")]
    pub key: Key,
}

#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
pub enum StoreResult<T> {
    Found(T),
    NotFound(StoreKey),
}

impl<T> From<StoreResult<T>> for Option<T> {
    fn from(v: StoreResult<T>) -> Self {
        match v {
            StoreResult::Found(v) => Some(v),
            StoreResult::NotFound(_) => None,
        }
    }
}

pub trait HgIdDataStore: LocalStore + Send + Sync {
    fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>>;
    fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>>;
    fn refresh(&self) -> Result<()>;
}

/// The `RemoteDataStore` trait indicates that data can fetched over the network. Care must be
/// taken to avoid serially fetching data and instead data should be fetched in bulk via the
/// `prefetch` API.
pub trait RemoteDataStore: HgIdDataStore + Send + Sync {
    /// Attempt to bring the data corresponding to the passed in keys to a local store.
    ///
    /// When implemented on a pure remote store, like the `EdenApi`, the method will always fetch
    /// everything that was asked. On a higher level store, such as the `ContentStore`, this will
    /// avoid fetching data that is already present locally.
    fn prefetch(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>>;

    /// Send all the blobs referenced by the keys to the remote store.
    fn upload(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>>;
}

pub trait HgIdMutableDeltaStore: HgIdDataStore + Send + Sync {
    fn add(&self, delta: &Delta, metadata: &Metadata) -> Result<()>;
    fn flush(&self) -> Result<Option<Vec<PathBuf>>>;

    fn add_file(&self, entry: &FileEntry) -> Result<()> {
        let delta = Delta {
            data: entry.data()?.into(),
            base: None,
            key: entry.key().clone(),
        };
        self.add(&delta, entry.metadata()?)
    }

    fn add_tree(&self, entry: &TreeEntry) -> Result<()> {
        let delta = Delta {
            data: entry.data()?.into(),
            base: None,
            key: entry.key().clone(),
        };
        self.add(
            &delta,
            &Metadata {
                flags: None,
                size: None,
            },
        )
    }
}

pub trait LegacyStore: HgIdMutableDeltaStore + RemoteDataStore + Send + Sync {
    fn get_file_content(&self, key: &Key) -> Result<Option<Bytes>>;
    fn get_logged_fetches(&self) -> HashSet<RepoPathBuf>;
    fn get_shared_mutable(&self) -> Arc<dyn HgIdMutableDeltaStore>;
    fn add_pending(
        &self,
        key: &Key,
        data: Bytes,
        meta: Metadata,
        location: RepackLocation,
    ) -> Result<()>;
    fn commit_pending(&self, location: RepackLocation) -> Result<Option<Vec<PathBuf>>>;
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ContentMetadata {
    pub size: usize,
    pub hash: ContentHash,
    pub is_binary: bool,
}

/// The `ContentDataStore` is intended for pure content only stores
///
/// Overtime, this new trait will replace the need for the `HgIdDataStore`, for now, only the LFS
/// store can implement it. Non content only stores could implement it, but the cost of the
/// `metadata` method will become linear over the blob size, reducing the benefit. A caching layer
/// will need to be put in place to avoid this.
pub trait ContentDataStore: Send + Sync {
    /// Read the blob from the store, the blob returned is the pure blob and will not contain any
    /// Mercurial copy_from header.
    fn blob(&self, key: StoreKey) -> Result<StoreResult<Bytes>>;

    /// Read the blob metadata from the store.
    fn metadata(&self, key: StoreKey) -> Result<StoreResult<ContentMetadata>>;
    // XXX: Add write operations.
}

/// Implement `HgIdDataStore` for all types that can be `Deref` into a `HgIdDataStore`. This includes all
/// the smart pointers like `Box`, `Rc`, `Arc`.
impl<T: HgIdDataStore + ?Sized, U: Deref<Target = T> + Send + Sync> HgIdDataStore for U {
    fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
        T::get(self, key)
    }

    fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
        T::get_meta(self, key)
    }

    /// Tell the underlying stores that there may be new data on disk.
    fn refresh(&self) -> Result<()> {
        T::refresh(self)
    }
}

/// Implement `RemoteDataStore` for all types that can be `Deref` into a `RemoteDataStore`. This
/// includes all the smart pointers like `Box`, `Rc`, `Arc`.
impl<T: RemoteDataStore + ?Sized, U: Deref<Target = T> + Send + Sync> RemoteDataStore for U {
    fn prefetch(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        T::prefetch(self, keys)
    }

    fn upload(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        T::upload(self, keys)
    }
}

impl<T: HgIdMutableDeltaStore + ?Sized, U: Deref<Target = T> + Send + Sync> HgIdMutableDeltaStore
    for U
{
    fn add(&self, delta: &Delta, metadata: &Metadata) -> Result<()> {
        T::add(self, delta, metadata)
    }

    fn flush(&self) -> Result<Option<Vec<PathBuf>>> {
        T::flush(self)
    }
}

/// Implement `ContentDataStore` for all types that can be `Deref` into a `ContentDataStore`.
impl<T: ContentDataStore + ?Sized, U: Deref<Target = T> + Send + Sync> ContentDataStore for U {
    fn blob(&self, key: StoreKey) -> Result<StoreResult<Bytes>> {
        T::blob(self, key)
    }

    fn metadata(&self, key: StoreKey) -> Result<StoreResult<ContentMetadata>> {
        T::metadata(self, key)
    }
}

pub struct ReportingRemoteDataStore {
    store: Box<dyn RemoteDataStore>,
    logger: FetchLogger,
}

impl ReportingRemoteDataStore {
    pub fn new(store: Box<dyn RemoteDataStore>, filter: Option<Regex>) -> Self {
        Self {
            store,
            logger: FetchLogger::new(filter),
        }
    }

    pub fn take_seen(&self) -> HashSet<RepoPathBuf> {
        self.logger.take_seen()
    }

    fn report_keys(&self, keys: &[StoreKey]) {
        self.logger.report_store_keys(keys.iter())
    }
}

impl LocalStore for ReportingRemoteDataStore {
    fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        Ok(keys.to_vec())
    }
}

impl HgIdDataStore for ReportingRemoteDataStore {
    fn get(&self, key: StoreKey) -> Result<StoreResult<Vec<u8>>> {
        self.report_keys(&[key.clone()]);
        self.store.get(key)
    }
    fn get_meta(&self, key: StoreKey) -> Result<StoreResult<Metadata>> {
        self.report_keys(&[key.clone()]);
        self.store.get_meta(key)
    }

    fn refresh(&self) -> Result<()> {
        self.store.refresh()
    }
}

impl RemoteDataStore for ReportingRemoteDataStore {
    fn prefetch(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        self.report_keys(keys);
        self.store.prefetch(keys)
    }

    fn upload(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        self.store.upload(keys)
    }
}

#[cfg(test)]
mod tests {
    use std::io::Cursor;

    use super::*;

    fn roundtrip_meta_serialize(meta: &Metadata) {
        let mut buf = vec![];
        meta.write(&mut buf).expect("write");
        let read_meta = Metadata::read(&mut Cursor::new(&buf)).expect("meta");
        assert!(*meta == read_meta);
    }

    #[test]
    fn test_metadata_serialize() {
        roundtrip_meta_serialize(&Metadata {
            size: None,
            flags: None,
        });
        roundtrip_meta_serialize(&Metadata {
            size: Some(5),
            flags: None,
        });
        roundtrip_meta_serialize(&Metadata {
            size: Some(0),
            flags: Some(12),
        });
        roundtrip_meta_serialize(&Metadata {
            size: Some(1000),
            flags: Some(12),
        });
        roundtrip_meta_serialize(&Metadata {
            size: Some(234214134),
            flags: Some(9879489),
        });
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::sync::Arc;
use std::time::Duration;

use anyhow::Error;
use http::header::HeaderMap;
use http::status::StatusCode;
use http_client::HttpClientError;
use http_client::Method;
use thiserror::Error;
use url::Url;

#[derive(Debug, Error)]
#[error("Empty Mutable Pack")]
pub struct EmptyMutablePack;

#[derive(Error, Debug)]
#[error("Fetch failed: {} {}", .url, .method)]
pub struct FetchError {
    pub url: Url,
    pub method: Method,
    #[source]
    pub error: TransferError,
}

#[derive(Clone, Debug, Error)]
#[error("{error}")]
pub struct ClonableError {
    #[source]
    pub error: Arc<Error>,
}

impl ClonableError {
    pub fn new(error: Error) -> Self {
        ClonableError {
            error: Arc::new(error),
        }
    }
}

#[derive(Debug, Clone)]
pub struct Advice(Option<String>);

impl std::fmt::Display for Advice {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match &self.0 {
            None => Ok(()),
            Some(advice) => write!(f, "Advice: {}", advice),
        }
    }
}

impl From<Option<String>> for Advice {
    fn from(opt_s: Option<String>) -> Self {
        Advice(opt_s)
    }
}

#[derive(Error, Debug)]
pub enum TransferError {
    #[error("HTTP status {}. Returned headers: {:#?}", .0, .1)]
    HttpStatus(StatusCode, HeaderMap),

    #[error("HTTP transfer failed")]
    HttpClientError(#[from] HttpClientError),

    #[error("Unexpected end of stream for http fetch")]
    EndOfStream,

    #[error("Timed out after waiting {:?}", .0)]
    Timeout(Duration),

    #[error(
        "Timed out after waiting {:?} for a chunk, after having received {} bytes over {}ms. Request ID: {}",
        .timeout, .bytes, .elapsed, .request_id
    )]
    ChunkTimeout {
        timeout: Duration,
        bytes: usize,
        elapsed: u128,
        request_id: String,
    },

    #[error("Unexpected HTTP Status. Expected {}, received {}", .expected, .received)]
    UnexpectedHttpStatus {
        expected: StatusCode,
        received: StatusCode,
    },

    #[error(transparent)]
    InvalidResponse(Error),
}

#[cfg(test)]
mod test {
    use super::*;

    #[test]
    fn test_clonable_source() {
        let clonable: anyhow::Error = ClonableError::new(EmptyMutablePack {}.into()).into();
        assert!(clonable.chain().any(|e| e.is::<EmptyMutablePack>()))
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::io::Cursor;
use std::io::Write;

use anyhow::Result;
use byteorder::BigEndian;
use byteorder::ReadBytesExt;
use byteorder::WriteBytesExt;
use thiserror::Error;
use types::HgId;

const SMALL_FANOUT_FACTOR: u8 = 1;
const LARGE_FANOUT_FACTOR: u8 = 2;
const SMALL_FANOUT_LENGTH: usize = 256; // 2^8
const LARGE_FANOUT_LENGTH: usize = 65536; // 2^16
const SMALL_RAW_SIZE: usize = 1024; // SMALL_FANOUT_LENGTH * sizeof(u32)
const LARGE_RAW_SIZE: usize = 262144; // LARGE_FANOUT_LENGTH * sizeof(u32)

#[derive(Debug, Error)]
#[error("Fanout Table Error: {0:?}")]
struct FanoutTableError(String);

fn get_fanout_index(table_size: usize, hgid: &HgId) -> Result<u64> {
    let mut cursor = Cursor::new(hgid.as_ref());
    match table_size {
        SMALL_RAW_SIZE => Ok(cursor.read_u8()? as u64),
        LARGE_RAW_SIZE => Ok(cursor.read_u16::<BigEndian>()? as u64),
        _ => Err(FanoutTableError(format!("invalid fanout table size ({:?})", table_size)).into()),
    }
}

pub struct FanoutTable {}

impl FanoutTable {
    /// Returns the (start, end) search bounds indicated by the fanout table. If end is None, then
    /// search to the end of the index.
    pub fn get_bounds(table: &[u8], hgid: &HgId) -> Result<(usize, Option<usize>)> {
        // Get the integer equivalent of the first few bytes of the hgid.
        let index = get_fanout_index(table.len(), hgid)?;

        // Read the start bound at the index location.
        let mut cur = Cursor::new(table);
        cur.set_position(index * 4);
        let start = cur.read_u32::<BigEndian>()? as usize;

        // Find the end bound by scanning forward for the first different entry.
        let mut end: Option<usize> = Option::None;
        while cur.position() < table.len() as u64 {
            let candidate = cur.read_u32::<BigEndian>()? as usize;
            if candidate != start {
                end = Option::Some(candidate as usize);
                break;
            }
        }

        Ok((start, end))
    }

    /// Writes a fanout table for the given list of sorted nodes.
    ///
    /// `fanout_factor` - Either '1' or '2', representing how many bytes should be used for the
    /// fanout.
    ///
    /// `hgid_iter` - The nodes that can be looked up in the fanout table. *MUST BE SORTED*
    ///
    /// `entry_size` - The fixed size of each hgid's index value, which is used to compute the
    /// offset in the index of that hgid's value.
    ///
    /// `locations` - A presized, mutable vector where the offset for each hgid index value will be
    /// written.
    pub fn write<'b, T: Write, I: Iterator<Item = &'b HgId>>(
        writer: &mut T,
        fanout_factor: u8,
        hgid_iter: &mut I,
        entry_size: usize,
        mut locations: Option<&mut Vec<u32>>,
    ) -> Result<()> {
        let fanout_raw_size = match fanout_factor {
            SMALL_FANOUT_FACTOR => SMALL_RAW_SIZE,
            LARGE_FANOUT_FACTOR => LARGE_RAW_SIZE,
            _ => {
                return Err(FanoutTableError(format!(
                    "invalid fanout factor ({:?})",
                    fanout_factor
                ))
                .into());
            }
        };
        let fanout_table_length = match fanout_factor {
            SMALL_FANOUT_FACTOR => SMALL_FANOUT_LENGTH,
            LARGE_FANOUT_FACTOR => LARGE_FANOUT_LENGTH,
            _ => {
                return Err(FanoutTableError(format!(
                    "invalid fanout factor ({:?})",
                    fanout_factor
                ))
                .into());
            }
        };

        let mut fanout_table: Vec<Option<u32>> = vec![None; fanout_table_length];

        // Fill in the fanout table with the offset of the first entry for each prefix.
        let mut offset: u32 = 0;
        for (i, hgid) in hgid_iter.enumerate() {
            let fanout_key = get_fanout_index(fanout_raw_size, &hgid)?;
            if fanout_table[fanout_key as usize].is_none() {
                fanout_table[fanout_key as usize] = Some(offset);
            }
            if let Some(locations) = locations.as_mut() {
                locations[i] = offset;
            }
            offset += entry_size as u32;
        }

        // Serialize the fanout table. For fanout keys that have no value, use the previous valid
        // value.
        let mut last_offset = 0;
        for offset in fanout_table.into_iter() {
            let offset = match offset {
                Some(offset) => {
                    last_offset = offset;
                    offset
                }
                None => last_offset,
            };

            writer.write_u32::<BigEndian>(offset)?;
        }

        Ok(())
    }

    pub fn get_size(large: bool) -> usize {
        if large {
            LARGE_RAW_SIZE
        } else {
            SMALL_RAW_SIZE
        }
    }
}

#[cfg(test)]
mod tests {
    use std::mem::size_of;

    use quickcheck::quickcheck;

    use super::*;

    fn make_hgid(first: u8, second: u8, third: u8, fourth: u8) -> HgId {
        let buf = [
            first, second, third, fourth, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        ];
        HgId::from(&buf)
    }

    #[test]
    fn test_small_fanout() {
        let nodes: Vec<HgId> = vec![
            make_hgid(0, 0, 0, 0),
            make_hgid(1, 0, 0, 0),
            make_hgid(1, 0, 0, 5),
            make_hgid(230, 5, 0, 0),
            make_hgid(230, 12, 0, 0),
        ];
        let mut locations = Vec::with_capacity(nodes.len());
        unsafe {
            locations.set_len(nodes.len());
        }
        let mut buf: Vec<u8> = vec![];
        FanoutTable::write(
            &mut buf,
            SMALL_FANOUT_FACTOR,
            &mut nodes.iter(),
            size_of::<u32>() as usize,
            Some(&mut locations),
        )
        .expect("fanout write");
        assert_eq!(SMALL_RAW_SIZE, buf.len());

        let table = buf.as_ref();
        assert_eq!(
            FanoutTable::get_bounds(table, &nodes[0]).expect("bounds0"),
            (0, Some(4))
        );
        assert_eq!(
            FanoutTable::get_bounds(table, &nodes[1]).expect("bounds1"),
            (4, Some(12))
        );
        assert_eq!(
            FanoutTable::get_bounds(table, &nodes[2]).expect("bounds2"),
            (4, Some(12))
        );
        assert_eq!(
            FanoutTable::get_bounds(table, &nodes[3]).expect("bounds3"),
            (12, None)
        );
        assert_eq!(
            FanoutTable::get_bounds(table, &nodes[4]).expect("bounds4"),
            (12, None)
        );
    }

    #[test]
    fn test_large_fanout() {
        let nodes: Vec<HgId> = vec![
            make_hgid(0, 0, 0, 0),
            make_hgid(1, 0, 0, 0),
            make_hgid(1, 0, 0, 5),
            make_hgid(230, 5, 0, 0),
            make_hgid(230, 12, 0, 0),
        ];
        let mut locations = Vec::with_capacity(nodes.len());
        unsafe {
            locations.set_len(nodes.len());
        }
        let mut buf: Vec<u8> = vec![];
        FanoutTable::write(
            &mut buf,
            LARGE_FANOUT_FACTOR,
            &mut nodes.iter(),
            size_of::<u32>() as usize,
            Some(&mut locations),
        )
        .expect("fanout write");
        assert_eq!(LARGE_RAW_SIZE, buf.len());

        let table = buf.as_ref();
        assert_eq!(
            FanoutTable::get_bounds(table, &nodes[0]).expect("bounds0"),
            (0, Some(4))
        );
        assert_eq!(
            FanoutTable::get_bounds(table, &nodes[1]).expect("bounds1"),
            (4, Some(12))
        );
        assert_eq!(
            FanoutTable::get_bounds(table, &nodes[2]).expect("bounds2"),
            (4, Some(12))
        );
        assert_eq!(
            FanoutTable::get_bounds(table, &nodes[3]).expect("bounds3"),
            (12, Some(16))
        );
        assert_eq!(
            FanoutTable::get_bounds(table, &nodes[4]).expect("bounds4"),
            (16, None)
        );
    }

    #[test]
    fn test_empty() {
        let nodes: Vec<HgId> = vec![];
        let mut locations = vec![];
        let mut buf: Vec<u8> = vec![];
        FanoutTable::write(
            &mut buf,
            SMALL_FANOUT_FACTOR,
            &mut nodes.iter(),
            size_of::<u32>() as usize,
            Some(&mut locations),
        )
        .expect("fanout write");
        assert_eq!(SMALL_RAW_SIZE, buf.len());

        let table = buf.as_ref();
        assert_eq!(
            FanoutTable::get_bounds(table, &make_hgid(0, 0, 0, 0)).expect("bounds1"),
            (0, None)
        );
    }

    /// Tests that lookups still work when all nodes in the pack start with the same fanout. To
    /// avoid bugs like in D8131020.
    #[test]
    fn test_same_prefix() {
        let nodes: Vec<HgId> = vec![
            make_hgid(200, 0, 0, 0),
            make_hgid(200, 0, 0, 1),
            make_hgid(200, 0, 1, 5),
            make_hgid(200, 5, 0, 0),
            make_hgid(200, 12, 0, 0),
        ];
        let mut locations = Vec::with_capacity(nodes.len());
        unsafe {
            locations.set_len(nodes.len());
        }
        let mut buf: Vec<u8> = vec![];
        FanoutTable::write(
            &mut buf,
            SMALL_FANOUT_FACTOR,
            &mut nodes.iter(),
            size_of::<u32>() as usize,
            Some(&mut locations),
        )
        .expect("fanout write");
        assert_eq!(SMALL_RAW_SIZE, buf.len());

        let table = buf.as_ref();
        assert_eq!(
            FanoutTable::get_bounds(table, &nodes[0]).expect("bounds0"),
            (0, None)
        );
        assert_eq!(
            FanoutTable::get_bounds(table, &nodes[1]).expect("bounds1"),
            (0, None)
        );
        assert_eq!(
            FanoutTable::get_bounds(table, &nodes[2]).expect("bounds2"),
            (0, None)
        );
        assert_eq!(
            FanoutTable::get_bounds(table, &nodes[3]).expect("bounds3"),
            (0, None)
        );
        assert_eq!(
            FanoutTable::get_bounds(table, &nodes[4]).expect("bounds4"),
            (0, None)
        );
    }

    quickcheck! {
        fn test_random_nodes(fanout: u8, nodes: Vec<HgId>) -> bool {
            let mut nodes = nodes;
            nodes.sort();
            let fanout_factor = (fanout % 2) + 1;
            let mut locations = Vec::with_capacity(nodes.len());
            unsafe {
                locations.set_len(nodes.len());
            }
            let mut buf: Vec<u8> = vec![];
            let hgid_size = HgId::len();
            FanoutTable::write(
                &mut buf,
                fanout_factor,
                &mut nodes.iter(),
                hgid_size,
                Some(&mut locations),
            ).expect("fanout write");

            // Simulate a data file that includes just the nodes
            let data_buf: Vec<u8> = nodes.iter().flat_map(|x| x.as_ref().iter()).map(|x| x.clone()).collect();

            // Validate the locations are correct
            for (i, hgid) in nodes.iter().enumerate() {
                let pos = locations[i] as usize;
                if &data_buf[pos..pos + hgid_size] != hgid.as_ref() {
                    return false;
                }
            }

            // Validate the returned bounds contain each hgid
            let table = buf.as_ref();
            for hgid in nodes.iter() {
                let (start, end) = FanoutTable::get_bounds(table, hgid).expect("bounds");
                let end = end.unwrap_or(data_buf.len());

                // Manually scan for the hgid in the data buffer bounds.
                let mut found = false;
                let mut cur = start;
                while start < end {
                    if &data_buf[cur..cur + hgid_size] == hgid.as_ref() {
                        found = true;
                        break;
                    }

                    cur += hgid_size;
                }
                if !found {
                    return false;
                }
            }

            true
        }
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashSet;

use parking_lot::Mutex;
use regex::Regex;
use types::Key;
use types::RepoPathBuf;

use crate::StoreKey;

// TODO(meyer): This was implemented for ovrsource migration, and shouldn't be needed anymore.
pub struct FetchLogger {
    filter: Option<Regex>,
    seen: Mutex<HashSet<RepoPathBuf>>,
}

impl FetchLogger {
    pub fn new(filter: Option<Regex>) -> Self {
        Self {
            filter,
            seen: Mutex::new(HashSet::new()),
        }
    }

    pub fn take_seen(&self) -> HashSet<RepoPathBuf> {
        let mut seen = self.seen.lock();
        std::mem::take(&mut *seen)
    }

    pub fn report_store_keys<'a>(&self, keys: impl Iterator<Item = &'a StoreKey>) {
        self.report_keys(keys.filter_map(|k| k.maybe_as_key()))
    }

    pub fn report_keys<'a>(&self, keys: impl Iterator<Item = &'a Key>) {
        if let Some(filter) = &self.filter {
            let mut matches = Vec::new();
            for path in keys
                .map(|k| &k.path)
                .filter(|p| filter.is_match(p.as_str()))
            {
                matches.push(path.clone());
            }

            if !matches.is_empty() {
                let mut seen = self.seen.lock();
                seen.extend(matches.into_iter());
            }
        }
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::cmp::Ordering;
use std::collections::HashMap;
use std::collections::HashSet;
use std::fs::File;
use std::io::Cursor;
use std::io::Read;
use std::io::Write;
use std::path::Path;

use anyhow::Result;
use byteorder::BigEndian;
use byteorder::ReadBytesExt;
use byteorder::WriteBytesExt;
use memmap2::Mmap;
use memmap2::MmapOptions;
#[cfg(test)]
use quickcheck_arbitrary_derive::Arbitrary;
use sha1::Digest;
use sha1::Sha1;
use thiserror::Error;
use types::HgId;
use types::Key;
use types::RepoPath;

use crate::fanouttable::FanoutTable;
use crate::historypack::HistoryPackVersion;
use crate::sliceext::SliceExt;

#[derive(Debug, Error)]
#[error("HistoryIndex Error: {0:?}")]
struct HistoryIndexError(String);

const SMALL_FANOUT_CUTOFF: usize = 8192; // 2^16 / 8

#[derive(Debug, PartialEq)]
struct HistoryIndexOptions {
    pub version: HistoryPackVersion,
    // Indicates whether to use the large fanout (2 bytes) or the small (1 byte)
    pub large: bool,
}

impl HistoryIndexOptions {
    pub fn read<T: Read>(reader: &mut T) -> Result<HistoryIndexOptions> {
        let version = reader.read_u8()?;
        let version = match version {
            0 => HistoryPackVersion::Zero,
            1 => HistoryPackVersion::One,
            _ => {
                return Err(
                    HistoryIndexError(format!("unsupported version '{:?}'", version)).into(),
                );
            }
        };

        let raw_config = reader.read_u8()?;
        let large = match raw_config {
            0b10000000 => true,
            0 => false,
            _ => {
                return Err(
                    HistoryIndexError(format!("invalid history index '{:?}'", raw_config)).into(),
                );
            }
        };
        Ok(HistoryIndexOptions { version, large })
    }

    pub fn write<T: Write>(&self, writer: &mut T) -> Result<()> {
        writer.write_u8(match self.version {
            HistoryPackVersion::Zero => 0,
            HistoryPackVersion::One => 1,
        })?;
        writer.write_u8(if self.large { 0b10000000 } else { 0 })?;
        Ok(())
    }
}

#[derive(Clone, Debug)]
#[cfg_attr(test, derive(Arbitrary))]
pub struct FileSectionLocation {
    pub offset: u64,
    pub size: u64,
}

#[derive(Clone, Debug)]
#[cfg_attr(test, derive(Arbitrary))]
pub struct NodeLocation {
    pub offset: u64,
}

#[derive(PartialEq, Debug)]
pub(crate) struct FileIndexEntry {
    pub hgid: HgId,
    pub file_section_offset: u64,
    pub file_section_size: u64,
    pub hgid_index_offset: u32,
    pub hgid_index_size: u32,
}
const FILE_ENTRY_LEN: usize = 44;

impl FileIndexEntry {
    pub fn read(buf: &[u8]) -> Result<Self> {
        let mut cur = Cursor::new(buf);
        cur.set_position(20);
        let hgid_slice: &[u8] = buf.get_err(0..20)?;
        Ok(FileIndexEntry {
            hgid: HgId::from_slice(hgid_slice)?,
            file_section_offset: cur.read_u64::<BigEndian>()?,
            file_section_size: cur.read_u64::<BigEndian>()?,
            hgid_index_offset: cur.read_u32::<BigEndian>()?,
            hgid_index_size: cur.read_u32::<BigEndian>()?,
        })
    }

    fn write<T: Write>(&self, writer: &mut T) -> Result<()> {
        writer.write_all(self.hgid.as_ref())?;
        writer.write_u64::<BigEndian>(self.file_section_offset)?;
        writer.write_u64::<BigEndian>(self.file_section_size)?;
        writer.write_u32::<BigEndian>(self.hgid_index_offset)?;
        writer.write_u32::<BigEndian>(self.hgid_index_size)?;
        Ok(())
    }
}

#[derive(Debug, PartialEq)]
pub(crate) struct NodeIndexEntry {
    pub hgid: HgId,
    pub offset: u64,
}
const NODE_ENTRY_LEN: usize = 28;

impl NodeIndexEntry {
    pub fn read(buf: &[u8]) -> Result<Self> {
        let mut cur = Cursor::new(buf);
        cur.set_position(20);
        let hgid_slice: &[u8] = buf.get_err(0..20)?;
        Ok(NodeIndexEntry {
            hgid: HgId::from_slice(hgid_slice)?,
            offset: cur.read_u64::<BigEndian>()?,
        })
    }

    pub fn write<T: Write>(&self, writer: &mut T) -> Result<()> {
        writer.write_all(self.hgid.as_ref())?;
        writer.write_u64::<BigEndian>(self.offset)?;
        Ok(())
    }
}

pub(crate) struct HistoryIndex {
    mmap: Mmap,
    #[allow(dead_code)]
    version: HistoryPackVersion,
    fanout_size: usize,
    index_start: usize,
    index_end: usize,
}

impl HistoryIndex {
    pub fn new(path: &Path) -> Result<Self> {
        let file = File::open(path)?;
        let len = file.metadata()?.len();
        if len < 1 {
            return Err(HistoryIndexError(format!(
                "empty histidx '{:?}' is invalid",
                path.to_str().unwrap_or("<unknown>")
            ))
            .into());
        }

        let mmap = unsafe { MmapOptions::new().len(len as usize).map(&file)? };
        let options = HistoryIndexOptions::read(&mut Cursor::new(&mmap))?;
        let version = options.version;
        let fanout_size = FanoutTable::get_size(options.large);
        let mut index_start = 2 + fanout_size;

        // Version one records the number of entries in the index
        let index_end = if version == HistoryPackVersion::One {
            let mut cursor = Cursor::new(&mmap);
            cursor.set_position(index_start as u64);
            let file_count = cursor.read_u64::<BigEndian>()? as usize;
            index_start += 8;
            index_start + (file_count * FILE_ENTRY_LEN)
        } else {
            mmap.len()
        };

        Ok(HistoryIndex {
            mmap,
            version,
            fanout_size,
            index_start,
            index_end,
        })
    }

    pub fn write<T: Write>(
        writer: &mut T,
        file_sections: &[(&RepoPath, FileSectionLocation)],
        nodes: &HashMap<&RepoPath, HashMap<Key, NodeLocation>>,
    ) -> Result<()> {
        // Write header
        let options = HistoryIndexOptions {
            version: HistoryPackVersion::One,
            large: file_sections.len() > SMALL_FANOUT_CUTOFF,
        };
        options.write(writer)?;

        let mut file_sections: Vec<(&RepoPath, HgId, FileSectionLocation)> = file_sections
            .iter()
            .map(|e| Ok((e.0, sha1(&e.0.as_byte_slice()), e.1.clone())))
            .collect::<Result<Vec<(&RepoPath, HgId, FileSectionLocation)>>>()?;
        // They must be written in sorted order so they can be bisected.
        file_sections.sort_by_key(|x| x.1);

        // Write the fanout table
        FanoutTable::write(
            writer,
            if options.large { 2 } else { 1 },
            &mut file_sections.iter().map(|x| &x.1),
            FILE_ENTRY_LEN,
            None,
        )?;

        // Write out the number of files in the file portion.
        writer.write_u64::<BigEndian>(file_sections.len() as u64)?;

        <HistoryIndex>::write_file_index(writer, &options, &file_sections, nodes)?;

        // For each file, write a hgid index
        for &(file_name, ..) in file_sections.iter() {
            <HistoryIndex>::write_hgid_section(writer, nodes, file_name)?;
        }

        Ok(())
    }

    fn write_file_index<T: Write>(
        writer: &mut T,
        options: &HistoryIndexOptions,
        file_sections: &[(&RepoPath, HgId, FileSectionLocation)],
        nodes: &HashMap<&RepoPath, HashMap<Key, NodeLocation>>,
    ) -> Result<()> {
        // For each file, keep track of where its hgid index will start.
        // The first ones starts after the header, fanout, file count, file section, and hgid count.
        let mut hgid_offset: usize = 2
            + FanoutTable::get_size(options.large)
            + 8
            + (file_sections.len() * FILE_ENTRY_LEN)
            + 8;
        let mut hgid_count = 0;

        // Write out the file section entries
        let mut seen_files = HashSet::<&RepoPath>::with_capacity(file_sections.len());
        for &(file_name, file_hash, ref section_location) in file_sections.iter() {
            if seen_files.contains(file_name) {
                return Err(HistoryIndexError(format!(
                    "file '{:?}' was specified twice",
                    file_name
                ))
                .into());
            }
            seen_files.insert(&file_name);

            let file_nodes: &HashMap<Key, NodeLocation> =
                nodes.get(file_name).ok_or_else(|| {
                    HistoryIndexError(format!("unable to find nodes for {:?}", file_name))
                })?;
            let hgid_section_size = file_nodes.len() * NODE_ENTRY_LEN;
            FileIndexEntry {
                hgid: file_hash.clone(),
                file_section_offset: section_location.offset,
                file_section_size: section_location.size,
                hgid_index_offset: hgid_offset as u32,
                hgid_index_size: hgid_section_size as u32,
            }
            .write(writer)?;

            // Keep track of the current hgid index offset
            hgid_offset += 2 + file_name.as_byte_slice().len() + hgid_section_size;
            hgid_count += file_nodes.len();
        }

        // Write the total number of nodes
        writer.write_u64::<BigEndian>(hgid_count as u64)?;

        Ok(())
    }

    fn write_hgid_section<T: Write>(
        writer: &mut T,
        nodes: &HashMap<&RepoPath, HashMap<Key, NodeLocation>>,
        file_name: &RepoPath,
    ) -> Result<()> {
        // Write the filename
        let file_name_slice = file_name.as_byte_slice();
        writer.write_u16::<BigEndian>(file_name_slice.len() as u16)?;
        writer.write_all(file_name_slice)?;

        // Write each hgid, in sorted order so the can be bisected
        let file_nodes = nodes.get(file_name).ok_or_else(|| {
            HistoryIndexError(format!("unabled to find nodes for {:?}", file_name))
        })?;
        let mut file_nodes: Vec<(&Key, &NodeLocation)> =
            file_nodes.iter().collect::<Vec<(&Key, &NodeLocation)>>();
        file_nodes.sort_by_key(|x| x.0.hgid);

        for &(key, location) in file_nodes.iter() {
            NodeIndexEntry {
                hgid: key.hgid.clone(),
                offset: location.offset,
            }
            .write(writer)?;
        }

        Ok(())
    }

    pub fn get_file_entry(&self, key: &Key) -> Result<Option<FileIndexEntry>> {
        let filename_hgid = sha1(key.path.as_byte_slice());
        let (start, end) = FanoutTable::get_bounds(self.get_fanout_slice(), &filename_hgid)?;
        let start = start + self.index_start;
        let end = end.map_or(self.index_end, |pos| pos + self.index_start);

        let buf = self.mmap.get_err(start..end)?;
        let entry_offset = match self.binary_search_files(&filename_hgid, buf) {
            None => return Ok(None),
            Some(offset) => offset,
        };
        self.read_file_entry((start + entry_offset) - self.index_start)
            .map(Some)
    }

    pub fn get_hgid_entry(&self, key: &Key) -> Result<Option<NodeIndexEntry>> {
        let file_entry = match self.get_file_entry(&key)? {
            None => return Ok(None),
            Some(entry) => entry,
        };

        let start = file_entry.hgid_index_offset as usize + 2 + key.path.as_byte_slice().len();
        let end = start + file_entry.hgid_index_size as usize;

        let buf = self.mmap.get_err(start..end)?;
        let entry_offset = match self.binary_search_nodes(&key.hgid, &buf) {
            None => return Ok(None),
            Some(offset) => offset,
        };

        self.read_hgid_entry((start + entry_offset) - self.index_start)
            .map(Some)
    }

    fn read_file_entry(&self, offset: usize) -> Result<FileIndexEntry> {
        FileIndexEntry::read(self.read_data(offset, FILE_ENTRY_LEN)?)
    }

    fn read_hgid_entry(&self, offset: usize) -> Result<NodeIndexEntry> {
        NodeIndexEntry::read(self.read_data(offset, NODE_ENTRY_LEN)?)
    }

    fn read_data(&self, offset: usize, size: usize) -> Result<&[u8]> {
        let offset = offset + self.index_start;
        Ok(self.mmap.get_err(offset..offset + size)?)
    }

    // These two binary_search_* functions are very similar, but I couldn't find a way to unify
    // them without using macros.
    fn binary_search_files(&self, key: &HgId, slice: &[u8]) -> Option<usize> {
        let size = slice.len() / FILE_ENTRY_LEN;
        // Cast the slice into an array of entry buffers so we can bisect across them
        let slice: &[[u8; FILE_ENTRY_LEN]] = unsafe {
            ::std::slice::from_raw_parts(slice.as_ptr() as *const [u8; FILE_ENTRY_LEN], size)
        };
        let search_result = slice.binary_search_by(|entry| match entry.get(0..20) {
            Some(buf) => buf.cmp(key.as_ref()),
            None => Ordering::Greater,
        });
        match search_result {
            Ok(offset) => Some(offset * FILE_ENTRY_LEN),
            Err(_offset) => None,
        }
    }

    fn binary_search_nodes(&self, key: &HgId, slice: &[u8]) -> Option<usize> {
        let size = slice.len() / NODE_ENTRY_LEN;
        // Cast the slice into an array of entry buffers so we can bisect across them
        let slice: &[[u8; NODE_ENTRY_LEN]] = unsafe {
            ::std::slice::from_raw_parts(slice.as_ptr() as *const [u8; NODE_ENTRY_LEN], size)
        };
        let search_result = slice.binary_search_by(|entry| match entry.get(0..20) {
            Some(buf) => buf.cmp(key.as_ref()),
            None => Ordering::Greater,
        });
        match search_result {
            Ok(offset) => Some(offset * NODE_ENTRY_LEN),
            Err(_offset) => None,
        }
    }

    fn get_fanout_slice(&self) -> &[u8] {
        self.mmap[2..2 + self.fanout_size].as_ref()
    }
}

fn sha1(value: &[u8]) -> HgId {
    let mut hasher = Sha1::new();
    hasher.update(value);
    let buf: [u8; 20] = hasher.finalize().into();
    (&buf).into()
}

#[cfg(test)]
mod tests {
    use quickcheck::quickcheck;
    use tempfile::NamedTempFile;
    use types::path::RepoPathBuf;

    use super::*;

    fn make_index(
        file_sections: &[(&RepoPath, FileSectionLocation)],
        nodes: &HashMap<&RepoPath, HashMap<Key, NodeLocation>>,
    ) -> HistoryIndex {
        let mut file = NamedTempFile::new().unwrap();
        HistoryIndex::write(&mut file, file_sections, nodes).unwrap();
        let path = file.into_temp_path();

        HistoryIndex::new(&path).unwrap()
    }

    quickcheck! {
        fn test_file_index_entry_roundtrip(
            hgid: HgId,
            file_section_offset: u64,
            file_section_size: u64,
            hgid_index_offset: u32,
            hgid_index_size: u32
        ) -> bool {
            let entry = FileIndexEntry {
                hgid,
                file_section_offset,
                file_section_size,
                hgid_index_offset,
                hgid_index_size,
            };

            let mut buf: Vec<u8> = vec![];
            entry.write(&mut buf).unwrap();
            entry == FileIndexEntry::read(buf.as_ref()).unwrap()
        }

        fn test_hgid_index_entry_roundtrip(hgid: HgId, offset: u64) -> bool {
            let entry = NodeIndexEntry {
                hgid, offset
            };

            let mut buf: Vec<u8> = vec![];
            entry.write(&mut buf).unwrap();
            entry == NodeIndexEntry::read(buf.as_ref()).unwrap()
        }

        fn test_options_serialization(version: u8, large: bool) -> bool {
            let version = if version % 2 == 0 { HistoryPackVersion::Zero } else { HistoryPackVersion::One };
            let options = HistoryIndexOptions { version, large };
            let mut buf: Vec<u8> = vec![];
            options.write(&mut buf).expect("write");
            let parsed_options = HistoryIndexOptions::read(&mut Cursor::new(buf)).expect("read");
            options == parsed_options
        }

        fn test_roundtrip_index(data: Vec<(RepoPathBuf, (FileSectionLocation, HashMap<Key, NodeLocation>))>) -> bool {
            let mut file_sections: Vec<(&RepoPath, FileSectionLocation)> = vec![];
            let mut file_nodes: HashMap<&RepoPath, HashMap<Key, NodeLocation>> = HashMap::new();

            let mut seen_files: HashSet<RepoPathBuf> = HashSet::new();
            for &(ref path, (ref location, ref nodes)) in data.iter() {
                // Don't allow a filename to be used twice
                if seen_files.contains(path) {
                    continue;
                }
                seen_files.insert(path.clone());

                file_sections.push((path, location.clone()));
                let mut hgid_map: HashMap<Key, NodeLocation> = HashMap::new();
                for (key, hgid_location) in nodes.iter() {
                    let key = Key::new(path.clone(), key.hgid.clone());
                    hgid_map.insert(key, hgid_location.clone());
                }
                file_nodes.insert(path, hgid_map);
            }

            let index = make_index(&file_sections, &file_nodes);

            // Lookup each file section
            for (path, location) in file_sections {
                let my_key = Key::new(path.to_owned(), HgId::null_id().clone());
                let entry = index.get_file_entry(&my_key).unwrap().unwrap();
                assert_eq!(location.offset, entry.file_section_offset);
                assert_eq!(location.size, entry.file_section_size);
            }

            // Lookup each hgid
            for (path, hgid_map) in file_nodes {
                for (key, location) in hgid_map {
                    assert_eq!(path, key.path.as_ref());
                    let entry = index.get_hgid_entry(&key).unwrap().unwrap();
                    assert_eq!(key.hgid, entry.hgid);
                    assert_eq!(location.offset, entry.offset);
                }
            }

            true
        }
    }

    // TODO: test write() when duplicate files and duplicate nodes passed
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

//! Classes for constructing and serializing a histpack file and index.
//!
//! A history pack is a pair of files that contain the revision history for
//! various file revisions in Mercurial. It contains only revision history (like
//! parent pointers and linknodes), not any revision content information.
//!
//! It consists of two files, with the following format:
//!
//! ```text
//!
//! .histpack
//!     The pack itself is a series of file revisions with some basic header
//!     information on each.
//!
//!     datapack = <version: 1 byte>
//!                [<filesection>,...]
//!     filesection = <filename len: 2 byte unsigned int>
//!                   <filename>
//!                   <revision count: 4 byte unsigned int>
//!                   [<revision>,...]
//!     revision = <hgid: 20 byte>
//!                <p1node: 20 byte>
//!                <p2node: 20 byte>
//!                <linknode: 20 byte>
//!                <copyfromlen: 2 byte>
//!                <copyfrom>
//!
//!     The revisions within each filesection are stored in topological order
//!     (newest first). If a given entry has a parent from another file (a copy)
//!     then p1node is the hgid from the other file, and copyfrom is the
//!     filepath of the other file.
//!
//! .histidx
//!     The index file provides a mapping from filename to the file section in
//!     the histpack. In V1 it also contains sub-indexes for specific nodes
//!     within each file. It consists of three parts, the fanout, the file index
//!     and the hgid indexes.
//!
//!     The file index is a list of index entries, sorted by filename hash (one
//!     per file section in the pack). Each entry has:
//!
//!     - hgid (The 20 byte hash of the filename)
//!     - pack entry offset (The location of this file section in the histpack)
//!     - pack content size (The on-disk length of this file section's pack
//!                          data)
//!     - hgid index offset (The location of the file's hgid index in the index
//!                          file) [1]
//!     - hgid index size (the on-disk length of this file's hgid index) [1]
//!
//!     The fanout is a quick lookup table to reduce the number of steps for
//!     bisecting the index. It is a series of 4 byte pointers to positions
//!     within the index. It has 2^16 entries, which corresponds to hash
//!     prefixes [00, 01, 02,..., FD, FE, FF]. Example: the pointer in slot 4F
//!     points to the index position of the first revision whose hgid starts
//!     with 4F. This saves log(2^16) bisect steps.
//!
//!     dataidx = <fanouttable>
//!               <file count: 8 byte unsigned> [1]
//!               <fileindex>
//!               <hgid count: 8 byte unsigned> [1]
//!               [<nodeindex>,...] [1]
//!     fanouttable = [<index offset: 4 byte unsigned int>,...] (2^16 entries)
//!
//!     fileindex = [<file index entry>,...]
//!     fileindexentry = <hgid: 20 byte>
//!                      <pack file section offset: 8 byte unsigned int>
//!                      <pack file section size: 8 byte unsigned int>
//!                      <hgid index offset: 4 byte unsigned int> [1]
//!                      <hgid index size: 4 byte unsigned int>   [1]
//!     nodeindex = <filename>[<hgid index entry>,...] [1]
//!     filename = <filename len : 2 byte unsigned int><filename value> [1]
//!     nodeindexentry = <hgid: 20 byte> [1]
//!                      <pack file hgid offset: 8 byte unsigned int> [1]
//!
//! ```
//! [1]: new in version 1.

use std::fs::File;
use std::io::Cursor;
use std::io::Read;
use std::io::Write;
use std::mem::take;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;

use anyhow::format_err;
use anyhow::Result;
use byteorder::BigEndian;
use byteorder::ReadBytesExt;
use byteorder::WriteBytesExt;
use memmap2::Mmap;
use memmap2::MmapOptions;
use thiserror::Error;
use types::HgId;
use types::Key;
use types::NodeInfo;
use types::RepoPath;
use types::RepoPathBuf;
use util::path::remove_file;

use crate::historyindex::HistoryIndex;
use crate::historystore::HgIdHistoryStore;
use crate::localstore::ExtStoredPolicy;
use crate::localstore::LocalStore;
use crate::localstore::StoreFromPath;
use crate::repack::Repackable;
use crate::repack::ToKeys;
use crate::sliceext::SliceExt;
use crate::types::StoreKey;

#[derive(Debug, Error)]
#[error("Historypack Error: {0:?}")]
struct HistoryPackError(String);

#[derive(Clone, Debug, PartialEq)]
pub enum HistoryPackVersion {
    Zero,
    One,
}

impl HistoryPackVersion {
    fn new(value: u8) -> Result<Self> {
        match value {
            0 => Ok(HistoryPackVersion::Zero),
            1 => Ok(HistoryPackVersion::One),
            _ => Err(HistoryPackError(format!(
                "invalid history pack version number '{:?}'",
                value
            ))
            .into()),
        }
    }
}

impl From<HistoryPackVersion> for u8 {
    fn from(version: HistoryPackVersion) -> u8 {
        match version {
            HistoryPackVersion::Zero => 0,
            HistoryPackVersion::One => 1,
        }
    }
}

#[derive(Debug, PartialEq)]
pub(crate) struct FileSectionHeader<'a> {
    pub file_name: &'a RepoPath,
    pub count: u32,
}

#[derive(Debug, PartialEq)]
pub struct HistoryEntry<'a> {
    pub hgid: HgId,
    pub p1: HgId,
    pub p2: HgId,
    pub link_hgid: HgId,
    pub copy_from: Option<&'a RepoPath>,
}

fn read_slice<'a, 'b>(cur: &'a mut Cursor<&[u8]>, buf: &'b [u8], size: usize) -> Result<&'b [u8]> {
    let start = cur.position() as usize;
    let end = start + size;
    let file_name = buf.get_err(start..end)?;
    cur.set_position(end as u64);
    Ok(file_name)
}

impl<'a> FileSectionHeader<'a> {
    pub(crate) fn read(buf: &[u8]) -> Result<FileSectionHeader> {
        let mut cur = Cursor::new(buf);
        let file_name_len = cur.read_u16::<BigEndian>()? as usize;
        let file_name_slice = read_slice(&mut cur, &buf, file_name_len)?;
        let file_name = RepoPath::from_utf8(file_name_slice)?;

        let count = cur.read_u32::<BigEndian>()?;
        Ok(FileSectionHeader { file_name, count })
    }

    pub fn write<T: Write>(&self, writer: &mut T) -> Result<()> {
        let file_name_slice = self.file_name.as_byte_slice();
        writer.write_u16::<BigEndian>(file_name_slice.len() as u16)?;
        writer.write_all(file_name_slice)?;
        writer.write_u32::<BigEndian>(self.count)?;
        Ok(())
    }
}

impl<'a> HistoryEntry<'a> {
    pub(crate) fn read(buf: &[u8]) -> Result<HistoryEntry> {
        let mut cur = Cursor::new(buf);
        let mut hgid_buf: [u8; 20] = Default::default();

        // Node
        cur.read_exact(&mut hgid_buf)?;
        let hgid = HgId::from(&hgid_buf);

        // Parents
        cur.read_exact(&mut hgid_buf)?;
        let p1 = HgId::from(&hgid_buf);
        cur.read_exact(&mut hgid_buf)?;
        let p2 = HgId::from(&hgid_buf);

        // LinkNode
        cur.read_exact(&mut hgid_buf)?;
        let link_hgid = HgId::from(&hgid_buf);

        // Copyfrom
        let copy_from_len = cur.read_u16::<BigEndian>()? as usize;
        let copy_from = if copy_from_len > 0 {
            let slice = read_slice(&mut cur, &buf, copy_from_len)?;
            Some(RepoPath::from_utf8(slice)?)
        } else {
            None
        };

        Ok(HistoryEntry {
            hgid,
            p1,
            p2,
            link_hgid,
            copy_from,
        })
    }

    pub fn write<T: Write>(
        writer: &mut T,
        hgid: &HgId,
        p1: &HgId,
        p2: &HgId,
        linknode: &HgId,
        copy_from: &Option<&RepoPath>,
    ) -> Result<()> {
        writer.write_all(hgid.as_ref())?;
        writer.write_all(p1.as_ref())?;
        writer.write_all(p2.as_ref())?;
        writer.write_all(linknode.as_ref())?;
        match *copy_from {
            Some(file_name) => {
                let file_name_slice = file_name.as_byte_slice();
                writer.write_u16::<BigEndian>(file_name_slice.len() as u16)?;
                writer.write_all(file_name_slice)?;
            }
            None => writer.write_u16::<BigEndian>(0)?,
        };

        Ok(())
    }
}

pub struct HistoryPack {
    mmap: Mmap,
    #[allow(dead_code)]
    version: HistoryPackVersion,
    index: HistoryIndex,
    base_path: Arc<PathBuf>,
    pack_path: PathBuf,
    index_path: PathBuf,
}

impl HistoryPack {
    pub fn new(path: impl AsRef<Path>) -> Result<Self> {
        HistoryPack::with_path(path.as_ref())
    }

    fn with_path(path: &Path) -> Result<Self> {
        let base_path = PathBuf::from(path);
        let pack_path = path.with_extension("histpack");
        let file = File::open(&pack_path)?;
        let len = file.metadata()?.len();
        if len < 1 {
            return Err(format_err!(
                "empty histpack '{:?}' is invalid",
                path.to_str().unwrap_or("<unknown>")
            ));
        }

        let mmap = unsafe { MmapOptions::new().len(len as usize).map(&file)? };
        let version = HistoryPackVersion::new(mmap[0])?;
        if version != HistoryPackVersion::One {
            return Err(HistoryPackError(format!("version {:?} not supported", version)).into());
        }

        let index_path = path.with_extension("histidx");
        Ok(HistoryPack {
            mmap,
            version,
            index: HistoryIndex::new(&index_path)?,
            base_path: Arc::new(base_path),
            pack_path,
            index_path,
        })
    }

    pub fn len(&self) -> usize {
        self.mmap.len()
    }

    pub fn base_path(&self) -> &Path {
        &self.base_path
    }

    pub fn pack_path(&self) -> &Path {
        &self.pack_path
    }

    pub fn index_path(&self) -> &Path {
        &self.index_path
    }

    fn read_file_section_header(&self, offset: u64) -> Result<FileSectionHeader> {
        FileSectionHeader::read(&self.mmap.as_ref().get_err(offset as usize..)?)
    }

    fn read_history_entry(&self, offset: u64) -> Result<HistoryEntry> {
        HistoryEntry::read(&self.mmap.as_ref().get_err(offset as usize..)?)
    }

    fn read_node_info(&self, key: &Key, offset: u64) -> Result<NodeInfo> {
        let entry = self.read_history_entry(offset)?;
        assert_eq!(entry.hgid, key.hgid);
        let p1 = Key::new(
            match entry.copy_from {
                Some(value) => value.to_owned(),
                None => key.path.clone(),
            },
            entry.p1.clone(),
        );
        let p2 = Key::new(key.path.clone(), entry.p2.clone());

        Ok(NodeInfo {
            parents: [p1, p2],
            linknode: entry.link_hgid.clone(),
        })
    }
}

impl HgIdHistoryStore for HistoryPack {
    fn get_node_info(&self, key: &Key) -> Result<Option<NodeInfo>> {
        let hgid_location = match self.index.get_hgid_entry(key)? {
            None => return Ok(None),
            Some(location) => location,
        };
        self.read_node_info(key, hgid_location.offset).map(Some)
    }

    fn refresh(&self) -> Result<()> {
        Ok(())
    }
}

impl StoreFromPath for HistoryPack {
    fn from_path(path: &Path, _extstored_policy: ExtStoredPolicy) -> Result<Self> {
        HistoryPack::new(path)
    }
}

impl LocalStore for HistoryPack {
    fn get_missing(&self, keys: &[StoreKey]) -> Result<Vec<StoreKey>> {
        Ok(keys
            .iter()
            .filter(|k| match k {
                StoreKey::HgId(k) => match self.index.get_hgid_entry(k) {
                    Ok(None) | Err(_) => true,
                    Ok(Some(_)) => false,
                },
                StoreKey::Content(_, _) => true,
            })
            .cloned()
            .collect())
    }
}

impl ToKeys for HistoryPack {
    fn to_keys(&self) -> Vec<Result<Key>> {
        HistoryPackIterator::new(self).collect()
    }
}

impl Repackable for HistoryPack {
    fn delete(mut self) -> Result<()> {
        // On some platforms, removing a file can fail if it's still opened or mapped, let's make
        // sure we close and unmap them before deletion.
        let pack_path = take(&mut self.pack_path);
        let index_path = take(&mut self.index_path);
        drop(self);

        let result1 = remove_file(&pack_path);
        let result2 = remove_file(&index_path);
        // Only check for errors after both have run. That way if pack_path doesn't exist,
        // index_path is still deleted.
        result1?;
        result2?;
        Ok(())
    }

    fn size(&self) -> u64 {
        self.mmap.len() as u64
    }
}

struct HistoryPackIterator<'a> {
    pack: &'a HistoryPack,
    offset: u64,
    current_name: RepoPathBuf,
    current_remaining: u32,
}

impl<'a> HistoryPackIterator<'a> {
    pub fn new(pack: &'a HistoryPack) -> Self {
        HistoryPackIterator {
            pack,
            offset: 1, // Start after the header byte
            current_name: RepoPathBuf::new(),
            current_remaining: 0,
        }
    }
}

impl<'a> Iterator for HistoryPackIterator<'a> {
    type Item = Result<Key>;

    fn next(&mut self) -> Option<Self::Item> {
        while self.current_remaining == 0 && (self.offset as usize) < self.pack.len() {
            let file_header = self.pack.read_file_section_header(self.offset);
            match file_header {
                Ok(header) => {
                    let file_name_slice = header.file_name.as_byte_slice();
                    self.current_name = header.file_name.to_owned();
                    self.current_remaining = header.count;
                    self.offset += 4 + 2 + file_name_slice.len() as u64;
                }
                Err(e) => {
                    self.offset = self.pack.len() as u64;
                    return Some(Err(e));
                }
            };
        }

        if self.offset as usize >= self.pack.len() {
            return None;
        }

        let entry = self.pack.read_history_entry(self.offset);
        self.current_remaining -= 1;
        Some(match entry {
            Ok(ref e) => {
                self.offset += 80;
                self.offset += match e.copy_from {
                    Some(path) => 2 + path.as_byte_slice().len() as u64,
                    None => 2,
                };
                Ok(Key::new(self.current_name.clone(), e.hgid))
            }
            Err(e) => {
                // The entry is corrupted, and we have no way to know where the next one is
                // located, let's forcibly stop the iteration.
                self.offset = self.pack.len() as u64;
                Err(e)
            }
        })
    }
}

#[cfg(test)]
pub mod tests {
    use std::collections::HashMap;
    use std::fs::set_permissions;
    use std::fs::File;
    use std::fs::OpenOptions;

    use quickcheck::quickcheck;
    use rand::SeedableRng;
    use rand_chacha::ChaChaRng;
    use tempfile::TempDir;
    use types::testutil::*;
    use types::RepoPathBuf;

    use super::*;
    use crate::historystore::HgIdMutableHistoryStore;
    use crate::mutablehistorypack::MutableHistoryPack;

    pub fn make_historypack(tempdir: &TempDir, nodes: &HashMap<Key, NodeInfo>) -> HistoryPack {
        let mutpack = MutableHistoryPack::new(tempdir.path(), HistoryPackVersion::One);
        for (ref key, ref info) in nodes.iter() {
            mutpack.add(key.clone(), info.clone()).unwrap();
        }

        let path = &mutpack.flush().unwrap().unwrap()[0];

        HistoryPack::new(&path).unwrap()
    }

    pub fn get_nodes(mut rng: &mut ChaChaRng) -> HashMap<Key, NodeInfo> {
        let file1 = RepoPath::from_str("path").unwrap();
        let file2 = RepoPath::from_str("path/file").unwrap();
        let null = HgId::null_id();
        let node1 = HgId::random(&mut rng);
        let node2 = HgId::random(&mut rng);
        let node3 = HgId::random(&mut rng);
        let node4 = HgId::random(&mut rng);
        let node5 = HgId::random(&mut rng);
        let node6 = HgId::random(&mut rng);

        let mut nodes = HashMap::new();

        // Insert key 1
        let key1 = Key::new(file1.to_owned(), node2.clone());
        let info = NodeInfo {
            parents: [
                Key::new(file1.to_owned(), node1.clone()),
                Key::new(file1.to_owned(), null.clone()),
            ],
            linknode: HgId::random(&mut rng),
        };
        nodes.insert(key1.clone(), info.clone());

        // Insert key 2
        let key2 = Key::new(file2.to_owned(), node3.clone());
        let info = NodeInfo {
            parents: [
                Key::new(file2.to_owned(), node5.clone()),
                Key::new(file2.to_owned(), node6.clone()),
            ],
            linknode: HgId::random(&mut rng),
        };
        nodes.insert(key2.clone(), info.clone());

        // Insert key 3
        let key3 = Key::new(file1.to_owned(), node4.clone());
        let info = NodeInfo {
            parents: [key2.clone(), key1.clone()],
            linknode: HgId::random(&mut rng),
        };
        nodes.insert(key3.clone(), info.clone());

        nodes
    }

    #[test]
    fn test_get_node_info() {
        let mut rng = ChaChaRng::from_seed([0u8; 32]);
        let tempdir = TempDir::new().unwrap();

        let nodes = get_nodes(&mut rng);

        let pack = make_historypack(&tempdir, &nodes);

        for (ref key, ref info) in nodes.iter() {
            let response: NodeInfo = pack.get_node_info(key).unwrap().unwrap();
            assert_eq!(response, **info);
        }
    }

    #[test]
    fn test_get_missing() {
        let mut rng = ChaChaRng::from_seed([0u8; 32]);
        let tempdir = TempDir::new().unwrap();

        let nodes = get_nodes(&mut rng);

        let pack = make_historypack(&tempdir, &nodes);

        let mut test_keys: Vec<StoreKey> = nodes.keys().map(|k| StoreKey::from(k)).collect();
        let missing_key = key("missing", "f0f0f0");
        test_keys.push(StoreKey::from(&missing_key));

        let missing = pack.get_missing(&test_keys[..]).unwrap();
        assert_eq!(vec![StoreKey::from(missing_key)], missing);
    }

    #[test]
    fn test_iter() {
        let mut rng = ChaChaRng::from_seed([0u8; 32]);
        let tempdir = TempDir::new().unwrap();

        let nodes = get_nodes(&mut rng);

        let pack = make_historypack(&tempdir, &nodes);

        let mut keys: Vec<Key> = nodes.keys().map(|k| k.clone()).collect();
        keys.sort_unstable();
        let mut iter_keys = pack
            .to_keys()
            .into_iter()
            .collect::<Result<Vec<Key>>>()
            .unwrap();
        iter_keys.sort_unstable();
        assert_eq!(iter_keys, keys,);
    }

    #[test]
    fn test_open_v0() {
        let mut rng = ChaChaRng::from_seed([0u8; 32]);
        let tempdir = TempDir::new().unwrap();

        let nodes = get_nodes(&mut rng);

        let mutpack = MutableHistoryPack::new(tempdir.path(), HistoryPackVersion::One);
        for (ref key, ref info) in nodes.iter() {
            mutpack.add(key.clone(), info.clone()).unwrap();
        }

        let path = &mutpack.flush().unwrap().unwrap()[0];
        let pack_path = path.with_extension("histpack");

        let mut buf = Vec::new();
        {
            let mut file = File::open(&pack_path).unwrap();
            file.read_to_end(&mut buf).unwrap();

            // After being closed the datapacks are read-only. Since the next part of the test
            // corrupt it, let's make it writable.
            let mut perms = file.metadata().unwrap().permissions();
            perms.set_readonly(false);

            drop(file);
            set_permissions(&pack_path, perms).unwrap();
        }
        buf[0] = 0;
        OpenOptions::new()
            .write(true)
            .open(&pack_path)
            .unwrap()
            .write_all(&buf)
            .unwrap();

        assert!(HistoryPack::new(&pack_path).is_err());
    }

    quickcheck! {
        fn test_file_section_header_serialization(path: RepoPathBuf, count: u32) -> bool {
            let header = FileSectionHeader {
                file_name: path.as_ref(),
                count: count,
            };
            let mut buf = vec![];
            header.write(&mut buf).unwrap();
            header == FileSectionHeader::read(&buf).unwrap()
        }

        fn test_history_entry_serialization(
            hgid: HgId,
            p1: HgId,
            p2: HgId,
            link_hgid: HgId,
            copy_from: Option<RepoPathBuf>
        ) -> bool {
            let mut buf = vec![];
            HistoryEntry::write(
                &mut buf,
                &hgid,
                &p1,
                &p2,
                &link_hgid,
                &copy_from.as_ref().map(|x| x.as_ref()),
            ).unwrap();
            let entry = HistoryEntry::read(&buf).unwrap();
            assert_eq!(hgid, entry.hgid);
            assert_eq!(p1, entry.p1);
            assert_eq!(p2, entry.p2);
            assert_eq!(link_hgid, entry.link_hgid);
            true
        }
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::ops::Deref;
use std::path::PathBuf;

use anyhow::Result;
use edenapi_types::HistoryEntry;
use types::Key;
use types::NodeInfo;

use crate::localstore::LocalStore;
use crate::types::StoreKey;

pub trait HgIdHistoryStore: LocalStore + Send + Sync {
    fn get_node_info(&self, key: &Key) -> Result<Option<NodeInfo>>;
    fn refresh(&self) -> Result<()>;
}

pub trait HgIdMutableHistoryStore: HgIdHistoryStore + Send + Sync {
    fn add(&self, key: &Key, info: &NodeInfo) -> Result<()>;
    fn flush(&self) -> Result<Option<Vec<PathBuf>>>;

    fn add_entry(&self, entry: &HistoryEntry) -> Result<()> {
        self.add(&entry.key, &entry.nodeinfo)
    }
}

/// The `RemoteHistoryStore` trait indicates that data can fetched over the network. Care must be
/// taken to avoid serially fetching data and instead data should be fetched in bulk via the
/// `prefetch` API.
pub trait RemoteHistoryStore: HgIdHistoryStore + Send + Sync {
    /// Attempt to bring the data corresponding to the passed in keys to a local store.
    ///
    /// When implemented on a pure remote store, like the `EdenApi`, the method will always fetch
    /// everything that was asked. On a higher level store, such as the `MetadataStore`, this will
    /// avoid fetching data that is already present locally.
    fn prefetch(&self, keys: &[StoreKey]) -> Result<()>;
}

/// Implement `HgIdHistoryStore` for all types that can be `Deref` into a `HgIdHistoryStore`.
impl<T: HgIdHistoryStore + ?Sized, U: Deref<Target = T> + Send + Sync> HgIdHistoryStore for U {
    fn get_node_info(&self, key: &Key) -> Result<Option<NodeInfo>> {
        T::get_node_info(self, key)
    }

    fn refresh(&self) -> Result<()> {
        T::refresh(self)
    }
}

impl<T: HgIdMutableHistoryStore + ?Sized, U: Deref<Target = T> + Send + Sync>
    HgIdMutableHistoryStore for U
{
    fn add(&self, key: &Key, info: &NodeInfo) -> Result<()> {
        T::add(self, key, info)
    }

    fn flush(&self) -> Result<Option<Vec<PathBuf>>> {
        T::flush(self)
    }
}

impl<T: RemoteHistoryStore + ?Sized, U: Deref<Target = T> + Send + Sync> RemoteHistoryStore for U {
    fn prefetch(&self, keys: &[StoreKey]) -> Result<()> {
        T::prefetch(self, keys)
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::io::Cursor;
use std::io::Read;
use std::io::Write;
use std::path::Path;

use anyhow::bail;
use anyhow::Result;
use byteorder::ReadBytesExt;
use byteorder::WriteBytesExt;
use configmodel::convert::ByteCount;
use configmodel::Config;
use configmodel::ConfigExt;
use edenapi_types::Blake3;
use edenapi_types::ContentId;
use edenapi_types::FileAuxData;
use edenapi_types::Sha1;
use edenapi_types::Sha256;
use indexedlog::log::IndexOutput;
use minibytes::Bytes;
use parking_lot::RwLock;
use types::hgid::ReadHgIdExt;
use types::HgId;
use vlqencoding::VLQDecode;
use vlqencoding::VLQEncode;

use crate::indexedlogutil::Store;
use crate::indexedlogutil::StoreOpenOptions;
use crate::indexedlogutil::StoreType;

/// See edenapi_types::FileAuxData and mononoke_types::ContentMetadataV2
#[derive(Copy, Clone, Debug, Default, Eq, PartialEq)]
pub struct Entry {
    pub(crate) total_size: u64,
    pub(crate) content_id: ContentId,
    pub(crate) content_sha1: Sha1,
    pub(crate) content_sha256: Sha256,
    pub(crate) content_seeded_blake3: Option<Blake3>,
}

impl From<FileAuxData> for Entry {
    fn from(v: FileAuxData) -> Self {
        Entry {
            total_size: v.total_size,
            content_id: v.content_id,
            content_sha1: v.sha1,
            content_sha256: v.sha256,
            content_seeded_blake3: v.seeded_blake3,
        }
    }
}

impl Entry {
    pub fn total_size(&self) -> u64 {
        self.total_size
    }

    pub fn content_id(&self) -> ContentId {
        self.content_id
    }

    pub fn content_sha1(&self) -> Sha1 {
        self.content_sha1
    }

    pub fn content_sha256(&self) -> Sha256 {
        self.content_sha256
    }

    pub fn content_seeded_blake3(&self) -> Option<Blake3> {
        self.content_seeded_blake3.clone()
    }

    /// Serialize the Entry to Bytes.
    ///
    /// The serialization format is as follows:
    /// - HgId <20 bytes>
    /// - Version <1 byte> (for compatibility)
    /// - content_id <32 bytes>
    /// - content sha1 <20 bytes>
    /// - content sha256 <32 bytes>
    /// - total_size <u64 VLQ, 1-9 bytes>
    /// - presence byte for seeded blake3 <1 byte>
    /// - content seeded blake3 <32 OR 0 bytes>
    fn serialize(&self, hgid: HgId) -> Result<Bytes> {
        let mut buf = Vec::new();
        buf.write_all(hgid.as_ref())?;
        buf.write_u8(0)?; // write version
        buf.write_all(self.content_id.as_ref())?;
        buf.write_all(self.content_sha1.as_ref())?;
        buf.write_all(self.content_sha256.as_ref())?;
        buf.write_vlq(self.total_size)?;
        match self.content_seeded_blake3() {
            Some(content_seeded_blake3) => {
                buf.write_u8(1)?; // A value of 1 indicates the blake3 hash is present
                buf.write_all(content_seeded_blake3.as_ref())?;
            }
            None => buf.write_u8(0)?, // A value of 0 indicates the blake3 hash is absent
        };
        Ok(buf.into())
    }

    fn deserialize(bytes: Bytes) -> Result<(HgId, Self)> {
        let data: &[u8] = bytes.as_ref();
        let mut cur = Cursor::new(data);

        let hgid = cur.read_hgid()?;

        let version = cur.read_u8()?;
        if version != 0 {
            bail!("unsupported auxstore entry version {}", version);
        }

        let mut content_id = [0u8; 32];
        cur.read_exact(&mut content_id)?;

        let mut content_sha1 = [0u8; 20];
        cur.read_exact(&mut content_sha1)?;

        let mut content_sha256 = [0u8; 32];
        cur.read_exact(&mut content_sha256)?;

        let total_size: u64 = cur.read_vlq()?;
        let remaining = cur.position() < bytes.len() as u64;
        let content_seeded_blake3 = if remaining && cur.read_u8()? == 1 {
            let mut content_seeded_blake3 = [0u8; 32];
            cur.read_exact(&mut content_seeded_blake3)?;
            Some(content_seeded_blake3.into())
        } else {
            None
        };

        Ok((
            hgid,
            Entry {
                content_id: content_id.into(),
                content_sha1: content_sha1.into(),
                content_sha256: content_sha256.into(),
                total_size,
                content_seeded_blake3,
            },
        ))
    }
}

pub struct AuxStore(RwLock<Store>);

impl AuxStore {
    pub fn new(path: impl AsRef<Path>, config: &dyn Config, store_type: StoreType) -> Result<Self> {
        // TODO(meyer): Eliminate "local" AuxStore - always treat it as shared / cache?
        let open_options = AuxStore::open_options(config)?;

        let log = match store_type {
            StoreType::Local => open_options.local(&path),
            StoreType::Shared => open_options.shared(&path),
        }?;

        Ok(AuxStore(RwLock::new(log)))
    }

    fn open_options(config: &dyn Config) -> Result<StoreOpenOptions> {
        let mut open_options = StoreOpenOptions::new()
            .max_log_count(4)
            .max_bytes_per_log(250 * 1000 * 1000 / 4)
            .auto_sync_threshold(10 * 1024 * 1024)
            .create(true)
            .index("node", |_| {
                vec![IndexOutput::Reference(0..HgId::len() as u64)]
            });

        if let Some(max_log_count) = config.get_opt::<u8>("indexedlog", "aux.max-log-count")? {
            open_options = open_options.max_log_count(max_log_count);
        }
        if let Some(max_bytes_per_log) =
            config.get_opt::<ByteCount>("indexedlog", "aux.max-bytes-per-log")?
        {
            open_options = open_options.max_bytes_per_log(max_bytes_per_log.value());
        } else if let Some(max_bytes_per_log) =
            config.get_opt::<ByteCount>("remotefilelog", "auxlimit")?
        {
            let log_count: u64 = open_options.max_log_count.unwrap_or(1).max(1).into();
            open_options =
                open_options.max_bytes_per_log((max_bytes_per_log.value() / log_count).max(1));
        }
        Ok(open_options)
    }

    pub fn get(&self, hgid: HgId) -> Result<Option<Entry>> {
        let log = self.0.read();
        let mut entries = log.lookup(0, &hgid)?;

        let slice = match entries.next() {
            None => return Ok(None),
            Some(slice) => slice?,
        };
        let bytes = log.slice_to_bytes(slice);
        drop(log);

        Entry::deserialize(bytes).map(|(_hgid, entry)| Some(entry))
    }

    pub fn put(&self, hgid: HgId, entry: &Entry) -> Result<()> {
        let serialized = entry.serialize(hgid)?;
        self.0.write().append(&serialized)
    }

    pub fn flush(&self) -> Result<()> {
        self.0.write().flush()
    }

    #[cfg(test)]
    pub(crate) fn hgids(&self) -> Result<Vec<HgId>> {
        let log = self.0.read();
        log.iter()
            .map(|slice| {
                let bytes = log.slice_to_bytes(slice?);
                Entry::deserialize(bytes).map(|(hgid, _entry)| hgid)
            })
            .collect()
    }
}

#[cfg(test)]
mod tests {
    use std::fs::remove_file;
    use std::str::FromStr;
    use std::sync::Arc;

    use tempfile::TempDir;
    use types::testutil::*;

    use super::*;
    use crate::indexedlogdatastore::IndexedLogHgIdDataStore;
    use crate::indexedlogdatastore::IndexedLogHgIdDataStoreConfig;
    use crate::scmstore::FetchMode;
    use crate::scmstore::FileAttributes;
    use crate::scmstore::FileStore;
    use crate::testutil::*;
    use crate::ExtStoredPolicy;
    use crate::HgIdMutableDeltaStore;

    fn single_byte_sha1(fst: u8) -> Sha1 {
        let mut x: [u8; Sha1::len()] = Default::default();
        x[0] = fst;
        Sha1::from(x)
    }

    #[test]
    fn test_empty() -> Result<()> {
        let tempdir = TempDir::new()?;
        let store = AuxStore::new(&tempdir, &empty_config(), StoreType::Shared)?;
        store.flush()?;
        Ok(())
    }

    #[test]
    fn test_add_get() -> Result<()> {
        let tempdir = TempDir::new().unwrap();
        let store = AuxStore::new(&tempdir, &empty_config(), StoreType::Shared)?;

        let mut entry = Entry::default();
        entry.total_size = 1;
        entry.content_sha1 = single_byte_sha1(1);

        let k = key("a", "1");

        store.put(k.hgid, &entry)?;
        store.flush()?;

        let found = store.get(k.hgid)?;
        assert_eq!(Some(entry), found);
        Ok(())
    }

    #[test]
    fn test_lookup_failure() -> Result<()> {
        let tempdir = TempDir::new().unwrap();
        let store = AuxStore::new(&tempdir, &empty_config(), StoreType::Shared)?;

        let mut entry = Entry::default();
        entry.total_size = 1;
        entry.content_sha1 = single_byte_sha1(1);

        let k = key("a", "1");

        store.put(k.hgid, &entry)?;
        store.flush()?;

        let k2 = key("b", "2");

        let found = store.get(k2.hgid)?;
        assert_eq!(None, found);
        Ok(())
    }

    #[test]
    fn test_corrupted() -> Result<()> {
        let tempdir = TempDir::new()?;
        let store = AuxStore::new(&tempdir, &empty_config(), StoreType::Shared)?;

        let k = key("a", "2");
        let mut entry = Entry::default();
        entry.total_size = 2;
        entry.content_sha1 = single_byte_sha1(2);

        store.put(k.hgid, &entry)?;
        store.flush()?;
        drop(store);

        // Corrupt the log by removing the "log" file.
        let mut rotate_log_path = tempdir.path().to_path_buf();
        rotate_log_path.push("0");
        rotate_log_path.push("log");
        remove_file(rotate_log_path)?;

        let store = AuxStore::new(&tempdir, &empty_config(), StoreType::Shared)?;

        let k = key("a", "3");
        let mut entry = Entry::default();
        entry.total_size = 3;
        entry.content_sha1 = single_byte_sha1(3);

        store.put(k.hgid, &entry)?;
        store.flush()?;

        // There should be only one key in the store.
        assert_eq!(store.hgids().into_iter().count(), 1);
        Ok(())
    }

    #[test]
    fn test_scmstore_read() -> Result<()> {
        let tmp = TempDir::new()?;
        let aux = Arc::new(AuxStore::new(&tmp, &empty_config(), StoreType::Shared)?);

        let mut entry = Entry::default();
        entry.total_size = 1;
        entry.content_sha1 = single_byte_sha1(1);

        let k = key("a", "1");

        aux.put(k.hgid, &entry)?;
        aux.flush()?;

        // Set up local-only FileStore
        let mut store = FileStore::empty();
        store.aux_local = Some(aux.clone());

        // Attempt fetch.
        let fetched = store
            .fetch(
                std::iter::once(k.clone()),
                FileAttributes::AUX,
                FetchMode::AllowRemote,
            )
            .single()?
            .expect("key not found");
        assert_eq!(entry, fetched.aux_data().expect("no aux data found").into());
        Ok(())
    }

    #[test]
    fn test_scmstore_compute_read() -> Result<()> {
        let k = key("a", "def6f29d7b61f9cb70b2f14f79cd5c43c38e21b2");
        let d = delta("1234", None, k.clone());
        let meta = Default::default();

        // Setup local indexedlog
        let tmp = TempDir::new()?;
        let config = IndexedLogHgIdDataStoreConfig {
            max_log_count: None,
            max_bytes_per_log: None,
            max_bytes: None,
        };
        let content = Arc::new(IndexedLogHgIdDataStore::new(
            &tmp,
            ExtStoredPolicy::Ignore,
            &config,
            StoreType::Shared,
        )?);

        content.add(&d, &meta).unwrap();
        content.flush().unwrap();

        let tmp = TempDir::new()?;
        let aux = Arc::new(AuxStore::new(&tmp, &empty_config(), StoreType::Shared)?);

        // Set up local-only FileStore
        let mut store = FileStore::empty();
        store.indexedlog_local = Some(content.clone());
        store.aux_local = Some(aux.clone());

        let mut expected = Entry::default();
        expected.total_size = 4;
        expected.content_id = ContentId::from_str(
            "aa6ab85da77ca480b7624172fe44aa9906b6c3f00f06ff23c3e5f60bfd0c414e",
        )?;
        expected.content_sha1 = Sha1::from_str("7110eda4d09e062aa5e4a390b0a572ac0d2c0220")?;
        expected.content_sha256 =
            Sha256::from_str("03ac674216f3e15c761ee1a5e255f067953623c8b388b4459e13f978d7c846f4")?;
        expected.content_seeded_blake3 = Some(Blake3::from_str(
            "2078b4229b5353de0268efc7f64b68f3c99fb8829e9c052117b4e1e090b2603a",
        )?);
        // Attempt fetch.
        let fetched = store
            .fetch(
                std::iter::once(k.clone()),
                FileAttributes::AUX,
                FetchMode::AllowRemote,
            )
            .single()?
            .expect("key not found");
        assert_eq!(
            expected,
            fetched.aux_data().expect("no aux data found").into()
        );

        // Verify we can read it directly too
        let found = aux.get(k.hgid)?;
        assert_eq!(Some(expected), found);
        Ok(())
    }

    #[test]
    /// Test that we can deserialize non-BLAKE3 entries stored in cache.
    fn test_deserialize_non_blake3_entry() -> Result<()> {
        let k = key("a", "def6f29d7b61f9cb70b2f14f79cd5c43c38e21b2");
        let entry = Entry {
            total_size: 4,
            content_id: ContentId::from_str(
                "aa6ab85da77ca480b7624172fe44aa9906b6c3f00f06ff23c3e5f60bfd0c414e",
            )?,
            content_sha1: Sha1::from_str("7110eda4d09e062aa5e4a390b0a572ac0d2c0220")?,
            content_sha256: Sha256::from_str(
                "03ac674216f3e15c761ee1a5e255f067953623c8b388b4459e13f978d7c846f4",
            )?,
            content_seeded_blake3: Some(Blake3::from_str(
                "2078b4229b5353de0268efc7f64b68f3c99fb8829e9c052117b4e1e090b2603a",
            )?),
        };

        let mut buf = Vec::new();
        buf.write_all(k.hgid.as_ref())?;
        buf.write_u8(0)?; // write version
        buf.write_all(entry.content_id.as_ref())?;
        buf.write_all(entry.content_sha1.as_ref())?;
        buf.write_all(entry.content_sha256.as_ref())?;
        buf.write_vlq(entry.total_size)?;

        // Validate that we can deserialize the entry even when the Blake3 hash has not been written to it.
        Entry::deserialize(buf.into()).expect("Failed to deserialize non-Blake3 entry");
        Ok(())
    }
}