/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::cmp::Ordering;
use std::fmt;
use std::sync::Arc;

use anyhow::bail;
use anyhow::Error;
use bulkops::Direction;
use mononoke_types::RepositoryId;
use mononoke_types::Timestamp;
use slog::info;
use slog::Logger;
use sql_construct::SqlConstruct;
use sql_construct::SqlConstructFromMetadataDatabaseConfig;
use sql_ext::mononoke_queries;
use sql_ext::SqlConnections;

use crate::detail::log;

#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Checkpoint {
    pub lower_bound: u64,
    pub upper_bound: u64,
    pub create_timestamp: Timestamp,
    pub update_timestamp: Timestamp,
    pub update_run_number: u64,
    pub update_chunk_number: u64,
    pub last_finish_timestamp: Option<Timestamp>,
}

impl Checkpoint {
    /// Get the bounds for a catchup stream for new Changesets, plus the main stream for continuing from this checkpoint
    pub fn stream_bounds(
        &self,
        repo_lower: u64,
        repo_upper: u64,
        direction: Direction,
    ) -> Result<(Option<(u64, u64)>, Option<(u64, u64)>), Error> {
        if direction == Direction::NewestFirst {
            // First work out the main bound from checkpoint restart
            let main_bound = match repo_lower.cmp(&self.lower_bound) {
                // Checkpoint didn't get to the end, continue from it
                Ordering::Less => Some((repo_lower, self.lower_bound)),
                Ordering::Greater => bail!(
                    "Repo lower bound reversed from {} to {}",
                    self.lower_bound,
                    repo_lower
                ),
                Ordering::Equal => None,
            };

            // Then if we need to catchup due to new Changesets
            match repo_upper.cmp(&self.upper_bound) {
                // repo has advanced. We'll do the newest part first, then continue from checkpoint
                Ordering::Greater => Ok((Some((self.upper_bound, repo_upper)), main_bound)),
                Ordering::Less => bail!(
                    "Repo upper bound reversed from {} to {}",
                    self.upper_bound,
                    repo_upper
                ),
                // repo upper still the same as checkpoint, no need to catchup for new Changesets
                Ordering::Equal => Ok((None, main_bound)),
            }
        } else {
            // First work out the main bound from checkpoint restart
            let main_bound = match repo_upper.cmp(&self.upper_bound) {
                // Checkpoint didn't get to the end, continue from it
                Ordering::Greater => Some((self.upper_bound, repo_upper)),
                Ordering::Less => bail!(
                    "Repo upper bound reversed from {} to {}",
                    self.upper_bound,
                    repo_upper
                ),
                Ordering::Equal => None,
            };

            // Then if we need to catchup due to lower bound moving (unlikely, but lets cover it)
            match repo_lower.cmp(&self.lower_bound) {
                // repo bounds have widened. We'll do the new wider part first, then continue from checkpoint
                Ordering::Less => Ok((Some((repo_lower, self.lower_bound)), main_bound)),
                Ordering::Greater => bail!(
                    "Repo lower bound reversed from {} to {}",
                    self.lower_bound,
                    repo_lower
                ),
                // repo lower still the same as checkpoint, no need to catchup for wider bounds (expect this to be normal case)
                Ordering::Equal => Ok((None, main_bound)),
            }
        }
    }
}

#[derive(Clone)]
pub struct CheckpointsByName {
    pub checkpoint_name: String,
    sql_checkpoints: Arc<SqlCheckpoints>,
    pub sample_rate: u64,
}

impl CheckpointsByName {
    pub fn new(checkpoint_name: String, sql_checkpoints: SqlCheckpoints, sample_rate: u64) -> Self {
        Self {
            checkpoint_name,
            sql_checkpoints: Arc::new(sql_checkpoints),
            sample_rate,
        }
    }

    pub async fn load(&self, repo_id: RepositoryId) -> Result<Option<Checkpoint>, Error> {
        self.sql_checkpoints
            .load(repo_id, &self.checkpoint_name)
            .await
    }

    pub async fn persist(
        &self,
        logger: &Logger,
        repo_id: RepositoryId,
        chunk_num: u64,
        checkpoint: Option<Checkpoint>,
        lower_bound: u64,
        upper_bound: u64,
    ) -> Result<Checkpoint, Error> {
        let now = Timestamp::now();
        let new_cp = if let Some(mut checkpoint) = checkpoint {
            checkpoint.lower_bound = lower_bound;
            checkpoint.upper_bound = upper_bound;
            checkpoint.update_chunk_number = chunk_num;
            checkpoint.update_timestamp = now;
            info!(logger, #log::CHUNKING, "Chunk {} updating checkpoint to ({}, {})", chunk_num, lower_bound, upper_bound);
            self.update(repo_id, &checkpoint).await?;
            checkpoint
        } else {
            let new_cp = Checkpoint {
                lower_bound,
                upper_bound,
                create_timestamp: now,
                update_timestamp: now,
                update_run_number: 1,
                update_chunk_number: chunk_num,
                last_finish_timestamp: None,
            };
            info!(logger, #log::CHUNKING, "Chunk {} inserting checkpoint ({}, {})", chunk_num, lower_bound, upper_bound);
            self.insert(repo_id, &new_cp).await?;
            new_cp
        };
        Ok(new_cp)
    }

    pub async fn insert(
        &self,
        repo_id: RepositoryId,
        checkpoint: &Checkpoint,
    ) -> Result<(), Error> {
        self.sql_checkpoints
            .insert(repo_id, &self.checkpoint_name, checkpoint)
            .await
    }

    pub async fn update(
        &self,
        repo_id: RepositoryId,
        checkpoint: &Checkpoint,
    ) -> Result<(), Error> {
        self.sql_checkpoints
            .update(repo_id, &self.checkpoint_name, checkpoint)
            .await
    }

    pub async fn finish(
        &self,
        repo_id: RepositoryId,
        checkpoint: &Checkpoint,
    ) -> Result<(), Error> {
        self.sql_checkpoints
            .finish(repo_id, &self.checkpoint_name, checkpoint)
            .await
    }

    pub fn name(&self) -> &str {
        self.checkpoint_name.as_str()
    }
}

impl fmt::Debug for CheckpointsByName {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("CheckpointsByName")
            .field("checkpoint_name", &self.checkpoint_name)
            .finish()
    }
}

pub struct SqlCheckpoints {
    connections: SqlConnections,
}

impl SqlConstruct for SqlCheckpoints {
    const LABEL: &'static str = "walker_checkpoints";

    const CREATION_QUERY: &'static str =
        include_str!("../../schemas/sqlite-walker_checkpoints.sql");

    fn from_sql_connections(connections: SqlConnections) -> Self {
        Self { connections }
    }
}

impl SqlCheckpoints {
    pub async fn load(
        &self,
        repo_id: RepositoryId,
        checkpoint_name: &str,
    ) -> Result<Option<Checkpoint>, Error> {
        let rows = SelectCheckpoint::query(
            &self.connections.read_master_connection,
            &repo_id,
            &checkpoint_name,
        )
        .await?;

        Ok(rows.into_iter().next().map(|row| Checkpoint {
            lower_bound: row.0,
            upper_bound: row.1,
            create_timestamp: row.2,
            update_timestamp: row.3,
            update_run_number: row.4,
            update_chunk_number: row.5,
            last_finish_timestamp: row.6,
        }))
    }

    pub async fn insert(
        &self,
        repo_id: RepositoryId,
        // Query macro wants &String rather than &str
        checkpoint_name: &String,
        checkpoint: &Checkpoint,
    ) -> Result<(), Error> {
        InsertCheckpoint::query(
            &self.connections.write_connection,
            &[(
                &repo_id,
                checkpoint_name,
                &checkpoint.lower_bound,
                &checkpoint.upper_bound,
                &checkpoint.create_timestamp,
                &checkpoint.update_timestamp,
                &checkpoint.update_run_number,
                &checkpoint.update_chunk_number,
            )],
        )
        .await?;
        Ok(())
    }

    pub async fn update(
        &self,
        repo_id: RepositoryId,
        // Query macro wants &String rather than &str
        checkpoint_name: &String,
        checkpoint: &Checkpoint,
    ) -> Result<(), Error> {
        UpdateCheckpoint::query(
            &self.connections.write_connection,
            &repo_id,
            checkpoint_name,
            &checkpoint.lower_bound,
            &checkpoint.upper_bound,
            &checkpoint.create_timestamp,
            &checkpoint.update_timestamp,
            &checkpoint.update_run_number,
            &checkpoint.update_chunk_number,
        )
        .await?;
        Ok(())
    }

    pub async fn finish(
        &self,
        repo_id: RepositoryId,
        // Query macro wants &String rather than &str
        checkpoint_name: &String,
        checkpoint: &Checkpoint,
    ) -> Result<(), Error> {
        FinishCheckpoint::query(
            &self.connections.write_connection,
            &repo_id,
            checkpoint_name,
            &checkpoint.lower_bound,
            &checkpoint.upper_bound,
            &checkpoint.update_timestamp,
            &checkpoint.update_run_number,
            &checkpoint.update_chunk_number,
        )
        .await?;
        Ok(())
    }
}

impl SqlConstructFromMetadataDatabaseConfig for SqlCheckpoints {}

mononoke_queries! {
    read SelectCheckpoint(
        repo_id: RepositoryId,
        checkpoint_name: &str,
    ) -> (u64, u64, Timestamp, Timestamp, u64, u64, Option<Timestamp>) {
        "SELECT lower_bound, upper_bound, create_timestamp, update_timestamp, update_run_number, update_chunk_number, last_finish_timestamp
        FROM walker_checkpoints WHERE repo_id={repo_id} AND checkpoint_name={checkpoint_name}"
    }

    write InsertCheckpoint(
        values: (
            repo_id: RepositoryId,
            checkpoint_name: String,
            lower_bound: u64,
            upper_bound: u64,
            create_timestamp: Timestamp,
            update_timestamp: Timestamp,
            update_run_number: u64,
            update_chunk_number: u64,
        ),
    ) {
        none,
        "INSERT INTO walker_checkpoints
         (repo_id, checkpoint_name, lower_bound, upper_bound, create_timestamp, update_timestamp, update_run_number, update_chunk_number)
         VALUES {values}"
    }

    write UpdateCheckpoint(
        repo_id: RepositoryId,
        checkpoint_name: String,
        lower_bound: u64,
        upper_bound: u64,
        create_timestamp: Timestamp,
        update_timestamp: Timestamp,
        update_run_number: u64,
        update_chunk_number: u64,
    ) {
        none,
        "UPDATE walker_checkpoints
        SET lower_bound={lower_bound}, upper_bound={upper_bound}, create_timestamp={create_timestamp}, update_timestamp={update_timestamp}, update_run_number={update_run_number}, update_chunk_number={update_chunk_number}
        WHERE repo_id={repo_id} AND checkpoint_name={checkpoint_name}"
    }

    write FinishCheckpoint(
        repo_id: RepositoryId,
        checkpoint_name: String,
        lower_bound: u64,
        upper_bound: u64,
        last_finish_timestamp: Timestamp,
        last_finish_run_number: u64,
        last_finish_chunk_number: u64,
    ) {
        none,
        "UPDATE walker_checkpoints
        SET lower_bound={lower_bound}, upper_bound={upper_bound}, last_finish_timestamp={last_finish_timestamp}, last_finish_run_number={last_finish_run_number}, last_finish_chunk_number={last_finish_chunk_number}
        WHERE repo_id={repo_id} AND checkpoint_name={checkpoint_name}"
    }
}

#[cfg(test)]
mod tests {
    use fbinit::FacebookInit;

    use super::*;

    #[fbinit::test]
    async fn test_sql_roundtrip(_fb: FacebookInit) -> Result<(), Error> {
        let checkpoints = CheckpointsByName::new(
            "test_checkpoint".to_string(),
            SqlCheckpoints::with_sqlite_in_memory()?,
            0,
        );

        let repo_id = RepositoryId::new(123);
        let lower_bound = 0;
        let upper_bound = 8;

        // Brand new checkpoint
        let now = Timestamp::now();
        let initial = Checkpoint {
            lower_bound,
            upper_bound,
            create_timestamp: now,
            update_timestamp: now,
            update_run_number: 1,
            update_chunk_number: 1,
            last_finish_timestamp: None,
        };
        checkpoints.insert(repo_id, &initial).await?;

        // Check roundtrip
        let roundtripped = checkpoints.load(repo_id).await?;
        assert_eq!(Some(&initial), roundtripped.as_ref());

        // Update
        let updated = Checkpoint {
            upper_bound: 9,
            ..initial.clone()
        };
        checkpoints.update(repo_id, &updated).await?;

        // Check roundtrip of update
        let roundtripped2 = checkpoints.load(repo_id).await?;
        assert_ne!(Some(&initial), roundtripped2.as_ref());
        assert_eq!(Some(&updated), roundtripped2.as_ref());

        // Finish
        checkpoints.finish(repo_id, &updated).await?;
        let roundtripped3 = checkpoints.load(repo_id).await?;
        assert_eq!(
            roundtripped2.map(|o| o.update_timestamp),
            roundtripped3.and_then(|o| o.last_finish_timestamp),
        );

        Ok(())
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;
use std::collections::HashSet;
use std::io::Write;
use std::path::Path;
use std::path::PathBuf;
use std::sync::atomic::AtomicBool;
use std::sync::Arc;

use anyhow::Error;
use blobstore::BlobstoreGetData;
use cloned::cloned;
use context::CoreContext;
use context::SamplingKey;
use fbinit::FacebookInit;
use filetime::FileTime;
use futures::future;
use futures::future::try_join_all;
use futures::future::FutureExt;
use futures::future::TryFutureExt;
use futures::stream::Stream;
use futures::stream::TryStreamExt;
use maplit::hashset;
use mononoke_types::datetime::DateTime;
use percent_encoding::percent_encode;
use percent_encoding::AsciiSet;
use percent_encoding::CONTROLS;
use regex::Regex;
use repo_identity::RepoIdentityRef;
use samplingblob::SamplingHandler;
use tokio::fs::{self as tkfs};

use crate::commands::JobParams;
use crate::commands::JobWalkParams;
use crate::commands::RepoSubcommandParams;
use crate::commands::CORPUS;
use crate::detail::graph::FileContentData;
use crate::detail::graph::Node;
use crate::detail::graph::NodeData;
use crate::detail::graph::NodeType;
use crate::detail::graph::WrappedPath;
use crate::detail::progress::progress_stream;
use crate::detail::progress::report_state;
use crate::detail::progress::ProgressOptions;
use crate::detail::progress::ProgressReporter;
use crate::detail::progress::ProgressStateCountByType;
use crate::detail::progress::ProgressStateMutex;
use crate::detail::sampling::PathTrackingRoute;
use crate::detail::sampling::SampleTrigger;
use crate::detail::sampling::SamplingOptions;
use crate::detail::sampling::SamplingWalkVisitor;
use crate::detail::sampling::WalkKeyOptPath;
use crate::detail::sampling::WalkPayloadMtime;
use crate::detail::sampling::WalkSampleMapping;
use crate::detail::scrub::ScrubStats;
use crate::detail::tail::walk_exact_tail;
use crate::detail::walk::RepoWalkParams;
use crate::detail::walk::RepoWalkTypeParams;

// https://url.spec.whatwg.org/#fragment-percent-encode-set
const FRAGMENT: &AsciiSet = &CONTROLS.add(b' ').add(b'"').add(b'<').add(b'>').add(b'`');
// https://url.spec.whatwg.org/#path-percent-encode-set plus comma
const PATH: &AsciiSet = &FRAGMENT.add(b'#').add(b'?').add(b'{').add(b'}').add(b',');

// A path not found in repo paths.
// The constant includes a comma so that it is not a valid percent encoding
// so that we can distiguish in-repo paths fron the hierarchy used for the blobs.
const DUMP_DIR: &str = ".mononoke,";

// A subdir used for temp files before they are moved to final location
const INFLIGHT_DIR: &str = "Inflight";

// Force load of leaf data like file contents that graph traversal did not need
// Output is the samples
fn corpus_stream<InStream, SS>(
    scheduled_max: usize,
    output_dir: Option<String>,
    s: InStream,
    sampler: Arc<CorpusSamplingHandler<CorpusSample>>,
) -> impl Stream<Item = Result<(Node, Option<()>, Option<ScrubStats>), Error>>
where
    InStream: Stream<Item = Result<(WalkKeyOptPath<WrappedPath>, WalkPayloadMtime, Option<SS>), Error>>
        + 'static
        + Send,
{
    s.map_ok(move |(walk_key, payload, _progress_stats)| {
        let mtime = payload.mtime;
        match payload.data {
            Some(NodeData::FileContent(FileContentData::ContentStream(file_bytes_stream))) => {
                cloned!(sampler);
                file_bytes_stream
                    // Force file chunks to be loaded
                    .try_fold(0, |acc, file_bytes| future::ok(acc + file_bytes.size()))
                    // We take the size from the sample rather than the stream as it
                    // includes thrift wrapper overhead so more closely matches store
                    .map_ok(move |_num_bytes| {
                        let sample = sampler.complete_step(&walk_key);
                        let size = ScrubStats::from(sample.as_ref());
                        (walk_key, sample, mtime, Some(size))
                    })
                    .left_future()
            }
            _ => {
                let sample = sampler.complete_step(&walk_key);
                let size = ScrubStats::from(sample.as_ref());
                future::ready(Ok((walk_key, sample, mtime, Some(size)))).right_future()
            }
        }
    })
    .try_buffer_unordered(scheduled_max)
    // Dump the data to disk
    .map_ok(move |(walk_key, sample, mtime, stats)| {
        let node = walk_key.node;
        match sample {
            Some(sample) => move_node_files(
                output_dir.clone(),
                node.clone(),
                walk_key.path,
                mtime,
                sample,
            )
            .map_ok(move |()| (node, Some(()), stats))
            .left_future(),
            None => future::ok((node, Some(()), stats)).right_future(),
        }
    })
    .try_buffer_unordered(scheduled_max)
}

// Disk directory layout is of the form NodeType/root/<repo_path>/.mononoke,/aa/bb/cc/blob_key
// where the repo_path and blob_key are both percent_encoded and
// aa/bb/cc etc is a subset of the hash used to prevent any one directory becoming too large.
// For types without any in-repo path (e.g. `Changeset`) the repo_path component is omitted.
fn disk_node_dir(
    base_for_type: &Path,
    path: Option<&WrappedPath>,
    hash_subset: &[u8],
    dump_extension: bool,
) -> PathBuf {
    let mut o = base_for_type.to_path_buf();
    match path {
        Some(WrappedPath::NonRoot(path)) => {
            let path = PathBuf::from(percent_encode(&path.mpath().to_vec(), PATH).to_string());
            if dump_extension {
                match path.extension() {
                    Some(ext) => {
                        o.push("byext");
                        o.push(ext);
                    }
                    None => {
                        o.push("noext");
                    }
                }
            }
            o.push("root");
            o.push(path);
        }
        // This is content directly for the root, e.g. a root manifest
        Some(WrappedPath::Root) => o.push("root"),
        // Not path associated in any way, e.g. a Changeset
        None => {}
    };

    // Separate the dumped data from the repo dir structure
    o.push(DUMP_DIR);

    // 16777216 directories per path should be enough
    for d in 0..3 {
        o.push(hex::encode(&hash_subset[d..(d + 1)]));
    }
    o
}

fn dump_with_extension(node_type: NodeType) -> bool {
    match node_type {
        NodeType::Root => false,
        // Bonsai
        NodeType::Bookmark => false,
        NodeType::Changeset => false,
        NodeType::BonsaiHgMapping => false,
        NodeType::PhaseMapping => false,
        NodeType::PublishedBookmarks => false,
        // Hg
        NodeType::HgBonsaiMapping => false,
        NodeType::HgChangeset => false,
        NodeType::HgChangesetViaBonsai => false,
        NodeType::HgManifest => false,
        NodeType::HgFileEnvelope => true,
        NodeType::HgFileNode => true,
        NodeType::HgManifestFileNode => false,
        // Content
        NodeType::FileContent => true,
        NodeType::FileContentMetadataV2 => true,
        NodeType::AliasContentMapping => true,
        // Derived Data
        NodeType::Blame => false,
        NodeType::ChangesetInfo => false,
        NodeType::ChangesetInfoMapping => false,
        NodeType::DeletedManifestV2 => false,
        NodeType::DeletedManifestV2Mapping => false,
        NodeType::FastlogBatch => false,
        NodeType::FastlogDir => false,
        NodeType::FastlogFile => false,
        NodeType::Fsnode => false,
        NodeType::FsnodeMapping => false,
        NodeType::SkeletonManifest => false,
        NodeType::SkeletonManifestMapping => false,
        NodeType::BasenameSuffixSkeletonManifest => false,
        NodeType::BasenameSuffixSkeletonManifestMapping => false,
        NodeType::UnodeFile => false,
        NodeType::UnodeManifest => false,
        NodeType::UnodeMapping => false,
    }
}

async fn move_node_files(
    output_dir: Option<String>,
    node: Node,
    repo_path: Option<WrappedPath>,
    mtime: Option<DateTime>,
    sample: CorpusSample,
) -> Result<(), Error> {
    let hash_subset = node
        .sampling_fingerprint()
        .unwrap_or_default()
        .to_le_bytes();

    let output_dir = match output_dir {
        Some(output_dir) => output_dir,
        None => return Ok(()),
    };

    let mut base = PathBuf::from(output_dir);
    base.push(node.get_type().to_string());

    let inflight_dir = match sample.inflight_dir {
        Some(inflight_dir) => inflight_dir,
        None => return Ok(()),
    };

    let dump_extension = dump_with_extension(node.get_type());

    for (k, _) in sample.data {
        let mut dest_path = disk_node_dir(&base, repo_path.as_ref(), &hash_subset, dump_extension);
        tkfs::create_dir_all(&dest_path).await?;

        let key_file = percent_encode(k.as_bytes(), PATH).to_string();
        dest_path.push(&key_file);
        let mut source_path = inflight_dir.clone();
        source_path.push(&key_file);

        if let Some(mtime) = mtime {
            filetime::set_file_mtime(
                source_path.clone(),
                FileTime::from_unix_time(mtime.timestamp_secs(), 0),
            )?;
        }

        tkfs::rename(source_path, dest_path).await?;
    }
    tkfs::remove_dir(inflight_dir).await?;
    Ok(())
}

#[derive(Debug)]
pub struct CorpusSamplingHandler<T> {
    inner: WalkSampleMapping<WalkKeyOptPath<WrappedPath>, T>,
    output_dir: Option<String>,
}

impl<T> SampleTrigger<WalkKeyOptPath<WrappedPath>> for CorpusSamplingHandler<T>
where
    T: Default,
{
    fn map_keys(&self, sample_key: SamplingKey, walk_key: WalkKeyOptPath<WrappedPath>) {
        self.inner.map_keys(sample_key, walk_key);
    }
}

// This exists so we can track output_dir
impl<T> CorpusSamplingHandler<T> {
    pub fn new(output_dir: Option<String>) -> Self {
        Self {
            inner: WalkSampleMapping::new(),
            output_dir,
        }
    }

    pub fn complete_step(&self, walk_key: &WalkKeyOptPath<WrappedPath>) -> Option<T> {
        self.inner.complete_step(walk_key)
    }
}

#[derive(Debug)]
pub struct CorpusSample {
    pub inflight_dir: Option<PathBuf>,
    pub data: HashMap<String, u64>,
}

impl Default for CorpusSample {
    fn default() -> Self {
        Self {
            inflight_dir: None,
            data: HashMap::with_capacity(1),
        }
    }
}

impl From<Option<&CorpusSample>> for ScrubStats {
    fn from(sample: Option<&CorpusSample>) -> Self {
        sample
            .map(|sample| ScrubStats {
                blobstore_keys: sample.data.values().len() as u64,
                blobstore_bytes: sample.data.values().by_ref().sum(),
            })
            .unwrap_or_default()
    }
}

impl SamplingHandler for CorpusSamplingHandler<CorpusSample> {
    fn sample_get(
        &self,
        ctx: &CoreContext,
        key: &str,
        value: Option<&BlobstoreGetData>,
    ) -> Result<(), Error> {
        let output_dir = match &self.output_dir {
            Some(d) => d,
            None => return Ok(()),
        };

        let sampling_key = match ctx.sampling_key() {
            Some(s) => s,
            None => return Ok(()),
        };

        let mut inflight_path = PathBuf::from(output_dir);
        inflight_path.push(INFLIGHT_DIR);
        inflight_path.push(sampling_key.inner().to_string());
        if let Some(mut guard) = self.inner.inflight().get_mut(sampling_key) {
            std::fs::create_dir_all(&inflight_path)?;
            guard.inflight_dir = Some(inflight_path.clone());
            inflight_path.push(percent_encode(key.as_bytes(), PATH).to_string());
            let mut f = std::fs::File::create(inflight_path)?;
            if let Some(value) = value {
                f.write_all(value.as_bytes().as_bytes())?;
            }
            guard.data.insert(
                key.to_owned(),
                value.map_or(0, |v| v.as_bytes().len()) as u64,
            );
        }

        Ok(())
    }
}

#[derive(Clone)]
pub struct CorpusCommand {
    pub output_dir: Option<String>,
    pub progress_options: ProgressOptions,
    pub sampling_options: SamplingOptions,
    pub sampling_path_regex: Option<Regex>,
    pub sampler: Arc<CorpusSamplingHandler<CorpusSample>>,
}

impl CorpusCommand {
    fn apply_repo(&mut self, repo_params: &RepoWalkParams) {
        self.sampling_options
            .retain_or_default(&repo_params.include_node_types);
    }
}

// Subcommand entry point for dumping a corpus of blobs to disk
pub async fn corpus(
    fb: FacebookInit,
    job_params: JobParams,
    command: CorpusCommand,
    cancellation_requested: Arc<AtomicBool>,
) -> Result<(), Error> {
    let JobParams {
        walk_params,
        per_repo,
    } = job_params;

    let mut all_walks = Vec::new();
    for (sub_params, repo_params) in per_repo {
        cloned!(mut command, walk_params);

        command.apply_repo(&repo_params);

        let walk = run_one(
            fb,
            walk_params,
            sub_params,
            repo_params,
            command,
            Arc::clone(&cancellation_requested),
        );
        all_walks.push(walk);
    }
    try_join_all(all_walks).await.map(|_| ())
}

async fn run_one(
    fb: FacebookInit,
    job_params: JobWalkParams,
    sub_params: RepoSubcommandParams,
    repo_params: RepoWalkParams,
    command: CorpusCommand,
    cancellation_requested: Arc<AtomicBool>,
) -> Result<(), Error> {
    let sizing_progress_state =
        ProgressStateMutex::new(ProgressStateCountByType::<ScrubStats, ScrubStats>::new(
            fb,
            repo_params.logger.clone(),
            CORPUS,
            repo_params.repo.repo_identity().name().to_string(),
            command.sampling_options.node_types.clone(),
            command.progress_options,
        ));

    let make_sink = {
        cloned!(command, job_params.quiet, sub_params.progress_state,);
        move |ctx: &CoreContext, repo_params: &RepoWalkParams| {
            cloned!(ctx, repo_params.scheduled_max);
            async move |walk_output, _run_start, _chunk_num, _checkpoint_name| {
                cloned!(ctx, sizing_progress_state);
                let walk_progress = progress_stream(quiet, &progress_state, walk_output);

                let corpus = corpus_stream(
                    scheduled_max,
                    command.output_dir,
                    walk_progress,
                    command.sampler,
                );
                let report_sizing = progress_stream(quiet, &sizing_progress_state, corpus);
                report_state(ctx, report_sizing).await?;
                sizing_progress_state.report_progress();
                progress_state.report_progress();
                Ok(())
            }
        }
    };

    let walk_state = SamplingWalkVisitor::new(
        repo_params.include_node_types.clone(),
        repo_params.include_edge_types.clone(),
        command.sampling_options,
        command.sampling_path_regex,
        command.sampler,
        job_params.enable_derive,
        sub_params
            .tail_params
            .chunking
            .as_ref()
            .map(|v| v.direction),
    );

    let type_params = RepoWalkTypeParams {
        required_node_data_types: hashset![NodeType::FileContent],
        always_emit_edge_types: HashSet::new(),
        keep_edge_paths: true,
    };

    walk_exact_tail::<_, _, _, _, _, PathTrackingRoute<WrappedPath>>(
        fb,
        job_params,
        repo_params,
        type_params,
        sub_params.tail_params,
        walk_state,
        make_sink,
        cancellation_requested,
    )
    .await
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::fmt;
use std::hash::Hash;
use std::hash::Hasher;
use std::str::FromStr;

use ahash::RandomState;
use anyhow::format_err;
use anyhow::Error;
use basename_suffix_skeleton_manifest::RootBasenameSuffixSkeletonManifest;
use bitflags::bitflags;
use blame::RootBlameV2;
use blobrepo::BlobRepo;
use blobstore_factory::SqlTierInfo;
use bookmarks::BookmarkKey;
use changeset_info::ChangesetInfo;
use context::CoreContext;
use deleted_manifest::RootDeletedManifestV2Id;
use derived_data_manager::BonsaiDerivable as NewBonsaiDerivable;
use fastlog::unode_entry_to_fastlog_batch_key;
use fastlog::RootFastlog;
use filenodes::FilenodeInfo;
use filenodes_derivation::FilenodesOnlyPublic;
use filestore::Alias;
use fsnodes::RootFsnodeId;
use futures::compat::Future01CompatExt;
use futures::future::BoxFuture;
use futures::stream;
use futures::stream::BoxStream;
use futures::FutureExt;
use futures::StreamExt;
use futures::TryStreamExt;
use hash_memo::EagerHashMemoizer;
use internment::ArcIntern;
use manifest::Entry;
use mercurial_derivation::MappedHgChangesetId;
use mercurial_types::blobs::HgBlobChangeset;
use mercurial_types::blobs::HgBlobManifest;
use mercurial_types::calculate_hg_node_id_stream;
use mercurial_types::FileBytes;
use mercurial_types::HgChangesetId;
use mercurial_types::HgFileEnvelope;
use mercurial_types::HgFileEnvelopeMut;
use mercurial_types::HgFileNodeId;
use mercurial_types::HgManifestId;
use mercurial_types::HgParents;
use mononoke_types::basename_suffix_skeleton_manifest::BasenameSuffixSkeletonManifest;
use mononoke_types::blame_v2::BlameV2;
use mononoke_types::deleted_manifest_v2::DeletedManifestV2;
use mononoke_types::fastlog_batch::FastlogBatch;
use mononoke_types::fsnode::Fsnode;
use mononoke_types::skeleton_manifest::SkeletonManifest;
use mononoke_types::unode::FileUnode;
use mononoke_types::unode::ManifestUnode;
use mononoke_types::BasenameSuffixSkeletonManifestId;
use mononoke_types::BlameV2Id;
use mononoke_types::BlobstoreKey;
use mononoke_types::BlobstoreValue;
use mononoke_types::BonsaiChangeset;
use mononoke_types::ChangesetId;
use mononoke_types::ContentId;
use mononoke_types::ContentMetadataV2;
use mononoke_types::DeletedManifestV2Id;
use mononoke_types::FastlogBatchId;
use mononoke_types::FileUnodeId;
use mononoke_types::FsnodeId;
use mononoke_types::MPath;
use mononoke_types::MPathHash;
use mononoke_types::ManifestUnodeId;
use mononoke_types::MononokeId;
use mononoke_types::RepoPath;
use mononoke_types::SkeletonManifestId;
use newfilenodes::PathHash;
use once_cell::sync::OnceCell;
use phases::Phase;
use repo_blobstore::RepoBlobstoreRef;
use skeleton_manifest::RootSkeletonManifestId;
use thiserror::Error;
use unodes::RootUnodeManifestId;

use crate::detail::walk::OutgoingEdge;

#[derive(Error, Debug)]
pub enum HashValidationError {
    #[error("Error while computing hash validation")]
    Error(#[from] Error),
    #[error("failed to validate filenode hash: expected {expected_hash} actual {actual_hash}")]
    HashMismatch {
        actual_hash: String,
        expected_hash: String,
    },
    #[error("hash validation for {0} is not supported")]
    NotSupported(String),
}

// Helper to save repetition for the type enums
macro_rules! define_type_enum {
     (enum $enum_name:ident {
         $($variant:ident),* $(,)?
     }) => {
         #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, strum::AsRefStr,
         strum::EnumCount, strum::EnumIter, strum::EnumString,
         strum::EnumVariantNames, strum::IntoStaticStr)]
         pub enum $enum_name {
             $($variant),*
         }
    }
}

macro_rules! incoming_type {
    ($nodetypeenum:ident, $edgetypeenum:tt, Root) => {
        None
    };
    ($nodetypeenum:ident, $edgetypeenum:tt, $sourcetype:tt) => {
        Some($nodetypeenum::$sourcetype)
    };
}

macro_rules! outgoing_type {
    // Edge isn't named exactly same as the target node type
    ($nodetypeenum:ident, $edgetypeenum:tt, $targetlabel:ident($targettype:ident)) => {
        $nodetypeenum::$targettype
    };
    // In most cases its the same
    ($nodetypeenum:ident, $edgetypeenum:tt, $targetlabel:ident) => {
        $nodetypeenum::$targetlabel
    };
}

#[doc(hidden)]
macro_rules! create_graph_impl {
    ($nodetypeenum:ident, $nodekeyenum:ident, $edgetypeenum:ident,
        {$($nodetype:tt)*} {$($nodekeys:tt)*} {$(($edgetype:tt, $edgesourcetype:tt, $($edgetargetdef:tt)+)),+ $(,)?} ) => {
        define_type_enum!{
            enum $nodetypeenum {$($nodetype)*}
        }

        #[derive(Clone, Debug, PartialEq, Eq, Hash)]
        pub enum $nodekeyenum {$($nodekeys)*}

        define_type_enum!{
            enum $edgetypeenum {$($edgetype),*}
        }

        impl $edgetypeenum {
            pub fn incoming_type(&self) -> Option<$nodetypeenum> {
                match self {
                    $($edgetypeenum::$edgetype => incoming_type!($nodetypeenum, $edgetypeenum, $edgesourcetype)),*
                }
            }
        }

        impl $edgetypeenum {
            pub fn outgoing_type(&self) -> $nodetypeenum {
                match self {
                    $($edgetypeenum::$edgetype => outgoing_type!($nodetypeenum, $edgetypeenum, $($edgetargetdef)+)),*
                }
            }
        }
    };
    ($nodetypeenum:ident, $nodekeyenum:ident, $edgetypeenum:ident,
        {$($nodetype:tt)*} {$($nodekeys:tt)*} {$(($edgetype:tt, $edgesourcetype:tt, $($edgetargetdef:tt)+)),* $(,)?}
            ($source:ident, $sourcekey:ty, [$($target:ident$(($targettype:ident))?),*]) $($rest:tt)*) => {
        paste::item!{
            create_graph_impl! {
                $nodetypeenum, $nodekeyenum, $edgetypeenum,
                {$($nodetype)* $source,}
                {$($nodekeys)* $source($sourcekey),}
                {
                    $(($edgetype, $edgesourcetype, $($edgetargetdef)+),)*
                    $(([<$source To $target>], $source, $target$(($targettype))? ),)*
                }
                $($rest)*
            }
        }
    };
}

macro_rules! root_edge_type {
    ($edgetypeenum:ident, Root) => {
        None
    };
    ($edgetypeenum:ident, $target:ident) => {
        Some(paste::item! {$edgetypeenum::[<RootTo $target>]})
    };
}

macro_rules! create_graph {
    ($nodetypeenum:ident, $nodekeyenum:ident, $edgetypeenum:ident,
        $(($source:ident, $sourcekey:ty, [$($target:ident$(($targettype:ident))?),*])),* $(,)?) => {
        create_graph_impl! {
            $nodetypeenum, $nodekeyenum, $edgetypeenum,
            {}
            {}
            {}
            $(($source, $sourcekey, [$($target$(($targettype))?),*]))*
        }

        impl $nodetypeenum {
            pub fn root_edge_type(&self) -> Option<$edgetypeenum> {
                match self {
                    $($nodetypeenum::$source => root_edge_type!($edgetypeenum, $source)),*
                }
            }
            pub fn parse_node(&self, s: &str) -> Result<$nodekeyenum, Error> {
                match self {
                    $($nodetypeenum::$source => Ok($nodekeyenum::$source(<$sourcekey>::from_str(s)?))),*
                }
            }
        }
        impl $nodekeyenum {
            pub fn get_type(&self) -> $nodetypeenum {
                match self {
                    $($nodekeyenum::$source(_) => $nodetypeenum::$source),*
                }
            }
        }
    }
}

#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct UnitKey();

#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct PathKey<T: fmt::Debug + Clone + PartialEq + Eq + Hash> {
    pub id: T,
    pub path: WrappedPath,
}
impl<T: fmt::Debug + Clone + PartialEq + Eq + Hash> PathKey<T> {
    pub fn new(id: T, path: WrappedPath) -> Self {
        Self { id, path }
    }
}

#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct AliasKey(pub Alias);

/// Used for both Bonsai and HgChangesets to track if filenode data is present
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct ChangesetKey<T> {
    pub inner: T,
    pub filenode_known_derived: bool,
}

impl ChangesetKey<ChangesetId> {
    fn blobstore_key(&self) -> String {
        self.inner.blobstore_key()
    }

    fn sampling_fingerprint(&self) -> u64 {
        self.inner.sampling_fingerprint()
    }
}

impl ChangesetKey<HgChangesetId> {
    fn blobstore_key(&self) -> String {
        self.inner.blobstore_key()
    }

    fn sampling_fingerprint(&self) -> u64 {
        self.inner.sampling_fingerprint()
    }
}

bitflags! {
    /// Some derived data needs unodes as precondition, flags represent what is available in a compact way
    #[derive(Default)]
    pub struct UnodeFlags: u8 {
        const NONE = 0b00000000;
        const BLAME = 0b00000001;
        const FASTLOG = 0b00000010;
    }
}

/// Not all unodes should attempt to traverse blame or fastlog
/// e.g. a unode for non-public commit is not expected to have it
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct UnodeKey<T> {
    pub inner: T,
    pub flags: UnodeFlags,
}

impl<T: MononokeId> UnodeKey<T>
where
    T::Value: BlobstoreValue<Key = T>,
{
    fn blobstore_key(&self) -> String {
        self.inner.blobstore_key()
    }

    fn sampling_fingerprint(&self) -> u64 {
        self.inner.sampling_fingerprint()
    }
}

pub type UnodeManifestEntry = Entry<ManifestUnodeId, FileUnodeId>;

/// newtype so we can implement blobstore_key()
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct FastlogKey<T> {
    pub inner: T,
}

impl<T: MononokeId> FastlogKey<T>
where
    T::Value: BlobstoreValue<Key = T>,
{
    fn sampling_fingerprint(&self) -> u64 {
        self.inner.sampling_fingerprint()
    }

    pub fn new(inner: T) -> Self {
        Self { inner }
    }
}

impl FastlogKey<FileUnodeId> {
    fn blobstore_key(&self) -> String {
        unode_entry_to_fastlog_batch_key(&UnodeManifestEntry::Leaf(self.inner))
    }
}

impl FastlogKey<ManifestUnodeId> {
    fn blobstore_key(&self) -> String {
        unode_entry_to_fastlog_batch_key(&UnodeManifestEntry::Tree(self.inner))
    }
}

create_graph!(
    NodeType,
    Node,
    EdgeType,
    (
        Root,
        UnitKey,
        [
            // Bonsai
            Bookmark,
            Changeset,
            BonsaiHgMapping,
            PhaseMapping,
            PublishedBookmarks,
            // Hg
            HgBonsaiMapping,
            HgChangeset,
            HgChangesetViaBonsai,
            HgManifest,
            HgFileEnvelope,
            HgFileNode,
            HgManifestFileNode,
            // Content
            FileContent,
            FileContentMetadataV2,
            AliasContentMapping,
            // Derived
            Blame,
            ChangesetInfo,
            ChangesetInfoMapping,
            DeletedManifestV2,
            DeletedManifestV2Mapping,
            FastlogBatch,
            FastlogDir,
            FastlogFile,
            Fsnode,
            FsnodeMapping,
            SkeletonManifest,
            SkeletonManifestMapping,
            BasenameSuffixSkeletonManifest,
            BasenameSuffixSkeletonManifestMapping,
            UnodeFile,
            UnodeManifest,
            UnodeMapping
        ]
    ),
    // Bonsai
    (Bookmark, BookmarkKey, [Changeset, BonsaiHgMapping]),
    (
        Changeset,
        ChangesetKey<ChangesetId>,
        [
            FileContent,
            BonsaiParent(Changeset),
            BonsaiHgMapping,
            PhaseMapping,
            ChangesetInfo,
            ChangesetInfoMapping,
            DeletedManifestV2Mapping,
            FsnodeMapping,
            SkeletonManifestMapping,
            BasenameSuffixSkeletonManifestMapping,
            UnodeMapping
        ]
    ),
    (BonsaiHgMapping, ChangesetKey<ChangesetId>, [HgBonsaiMapping, HgChangesetViaBonsai]),
    (PhaseMapping, ChangesetId, []),
    (
        PublishedBookmarks,
        UnitKey,
        [Changeset, BonsaiHgMapping]
    ),
    // Hg
    (HgBonsaiMapping, ChangesetKey<HgChangesetId>, [Changeset]),
    (
        HgChangeset,
        ChangesetKey<HgChangesetId>,
        [HgParent(HgChangesetViaBonsai), HgManifest, HgManifestFileNode]
    ),
    (HgChangesetViaBonsai, ChangesetKey<HgChangesetId>, [HgChangeset]),
    (
        HgManifest,
        PathKey<HgManifestId>,
        [HgFileEnvelope, HgFileNode, HgManifestFileNode, ChildHgManifest(HgManifest)]
    ),
    (HgFileEnvelope, HgFileNodeId, [FileContent]),
    (
        HgFileNode,
        PathKey<HgFileNodeId>,
        [
            LinkedHgBonsaiMapping(HgBonsaiMapping),
            LinkedHgChangeset(HgChangesetViaBonsai),
            HgParentFileNode(HgFileNode),
            HgCopyfromFileNode(HgFileNode)
        ]
    ),
    (
        HgManifestFileNode,
        PathKey<HgFileNodeId>,
        [
            LinkedHgBonsaiMapping(HgBonsaiMapping),
            LinkedHgChangeset(HgChangesetViaBonsai),
            HgParentFileNode(HgManifestFileNode),
            HgCopyfromFileNode(HgManifestFileNode)
        ]
    ),
    // Content
    (FileContent, ContentId, [FileContentMetadataV2]),
    (
        FileContentMetadataV2,
        ContentId,
        [
            Sha1Alias(AliasContentMapping),
            Sha256Alias(AliasContentMapping),
            GitSha1Alias(AliasContentMapping),
            SeededBlake3Alias(AliasContentMapping)
        ]
    ),
    (AliasContentMapping, AliasKey, [FileContent]),
    // Derived data
    (
        Blame,
        BlameV2Id,
        [Changeset]
    ),
    (
        ChangesetInfo,
        ChangesetId,
        [ChangesetInfoParent(ChangesetInfo)]
    ),
    (
        ChangesetInfoMapping,
        ChangesetId,
        [ChangesetInfo]
    ),
    (
        DeletedManifestV2,
        DeletedManifestV2Id,
        [DeletedManifestV2Child(DeletedManifestV2), LinkedChangeset(Changeset)]
    ),
    (DeletedManifestV2Mapping, ChangesetId, [RootDeletedManifestV2(DeletedManifestV2)]),
    (
        Fsnode,
        FsnodeId,
        [ChildFsnode(Fsnode), FileContent]
    ),
    (
        FastlogBatch,
        FastlogBatchId,
        [Changeset, PreviousBatch(FastlogBatch)]
    ),
    (
        FastlogDir,
        FastlogKey<ManifestUnodeId>,
        [Changeset, PreviousBatch(FastlogBatch)]
    ),
    (
        FastlogFile,
        FastlogKey<FileUnodeId>,
        [Changeset, PreviousBatch(FastlogBatch)]
    ),
    (FsnodeMapping, ChangesetId, [RootFsnode(Fsnode)]),
    (
        SkeletonManifest,
        SkeletonManifestId,
        [SkeletonManifestChild(SkeletonManifest)]
    ),
    (SkeletonManifestMapping, ChangesetId, [RootSkeletonManifest(SkeletonManifest)]),
    (
        BasenameSuffixSkeletonManifest,
        BasenameSuffixSkeletonManifestId,
        [BasenameSuffixSkeletonManifestChild(BasenameSuffixSkeletonManifest)]
    ),
    (BasenameSuffixSkeletonManifestMapping, ChangesetId, [RootBasenameSuffixSkeletonManifest(BasenameSuffixSkeletonManifest)]),
    (
        UnodeFile,
        UnodeKey<FileUnodeId>,
        [Blame, FastlogFile, FileContent, LinkedChangeset(Changeset), UnodeFileParent(UnodeFile)]
    ),
    (
        UnodeManifest,
        UnodeKey<ManifestUnodeId>,
        [FastlogDir, UnodeFileChild(UnodeFile), UnodeManifestChild(UnodeManifest), UnodeManifestParent(UnodeManifest), LinkedChangeset(Changeset)]
    ),
    (UnodeMapping, ChangesetId, [RootUnodeManifest(UnodeManifest)]),
);

impl fmt::Display for NodeType {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{:?}", self)
    }
}

impl NodeType {
    /// Derived data types are keyed by their statically defined NAME
    pub fn derived_data_name(&self) -> Option<&'static str> {
        match self {
            NodeType::Root => None,
            // Bonsai
            NodeType::Bookmark => None,
            NodeType::Changeset => None,
            // from filenodes/lib.rs: If hg changeset is not generated, then root filenode can't possible be generated
            // therefore this is the same as MappedHgChangesetId + FilenodesOnlyPublic
            NodeType::BonsaiHgMapping => Some(FilenodesOnlyPublic::NAME),
            NodeType::PhaseMapping => None,
            NodeType::PublishedBookmarks => None,
            // Hg
            NodeType::HgBonsaiMapping => Some(MappedHgChangesetId::NAME),
            NodeType::HgChangeset => Some(MappedHgChangesetId::NAME),
            NodeType::HgChangesetViaBonsai => Some(MappedHgChangesetId::NAME),
            NodeType::HgManifest => Some(MappedHgChangesetId::NAME),
            NodeType::HgFileEnvelope => Some(MappedHgChangesetId::NAME),
            NodeType::HgFileNode => Some(FilenodesOnlyPublic::NAME),
            NodeType::HgManifestFileNode => Some(FilenodesOnlyPublic::NAME),
            // Content
            NodeType::FileContent => None,
            NodeType::FileContentMetadataV2 => None,
            NodeType::AliasContentMapping => None,
            // Derived data
            NodeType::Blame => Some(RootBlameV2::NAME),
            NodeType::ChangesetInfo => Some(ChangesetInfo::NAME),
            NodeType::ChangesetInfoMapping => Some(ChangesetInfo::NAME),
            NodeType::DeletedManifestV2 => Some(RootDeletedManifestV2Id::NAME),
            NodeType::DeletedManifestV2Mapping => Some(RootDeletedManifestV2Id::NAME),
            NodeType::FastlogBatch => Some(RootFastlog::NAME),
            NodeType::FastlogDir => Some(RootFastlog::NAME),
            NodeType::FastlogFile => Some(RootFastlog::NAME),
            NodeType::Fsnode => Some(RootFsnodeId::NAME),
            NodeType::FsnodeMapping => Some(RootFsnodeId::NAME),
            NodeType::SkeletonManifest => Some(RootSkeletonManifestId::NAME),
            NodeType::SkeletonManifestMapping => Some(RootSkeletonManifestId::NAME),
            NodeType::BasenameSuffixSkeletonManifest => {
                Some(RootBasenameSuffixSkeletonManifest::NAME)
            }
            NodeType::BasenameSuffixSkeletonManifestMapping => {
                Some(RootBasenameSuffixSkeletonManifest::NAME)
            }
            NodeType::UnodeFile => Some(RootUnodeManifestId::NAME),
            NodeType::UnodeManifest => Some(RootUnodeManifestId::NAME),
            NodeType::UnodeMapping => Some(RootUnodeManifestId::NAME),
        }
    }

    // Only certain node types can have repo paths associated
    pub fn allow_repo_path(&self) -> bool {
        match self {
            NodeType::Root => false,
            // Bonsai
            NodeType::Bookmark => false,
            NodeType::Changeset => false,
            NodeType::BonsaiHgMapping => false,
            NodeType::PhaseMapping => false,
            NodeType::PublishedBookmarks => false,
            // Hg
            NodeType::HgBonsaiMapping => false,
            NodeType::HgChangeset => false,
            NodeType::HgChangesetViaBonsai => false,
            NodeType::HgManifest => true,
            NodeType::HgFileEnvelope => true,
            NodeType::HgFileNode => true,
            NodeType::HgManifestFileNode => true,
            // Content
            NodeType::FileContent => true,
            NodeType::FileContentMetadataV2 => true,
            NodeType::AliasContentMapping => true,
            // Derived Data
            NodeType::Blame => false,
            NodeType::ChangesetInfo => false,
            NodeType::ChangesetInfoMapping => false,
            NodeType::DeletedManifestV2 => true,
            NodeType::DeletedManifestV2Mapping => false,
            NodeType::FastlogBatch => true,
            NodeType::FastlogDir => true,
            NodeType::FastlogFile => true,
            NodeType::Fsnode => true,
            NodeType::FsnodeMapping => false,
            NodeType::SkeletonManifest => true,
            NodeType::SkeletonManifestMapping => false,
            NodeType::BasenameSuffixSkeletonManifest => true,
            NodeType::BasenameSuffixSkeletonManifestMapping => false,
            NodeType::UnodeFile => true,
            NodeType::UnodeManifest => true,
            NodeType::UnodeMapping => false,
        }
    }
}

const ROOT_FINGERPRINT: u64 = 0;

// Can represent Path and PathHash
pub trait WrappedPathLike {
    fn sampling_fingerprint(&self) -> u64;
    fn evolve_path<'a>(
        from_route: Option<&'a Self>,
        walk_item: &'a OutgoingEdge,
    ) -> Option<&'a Self>;
}

/// Represent root or non root path hash.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum WrappedPathHash {
    Root,
    NonRoot(MPathHash),
}

impl WrappedPathHash {
    pub fn as_ref(&self) -> Option<&MPathHash> {
        match self {
            Self::Root => None,
            Self::NonRoot(mpath_hash) => Some(mpath_hash),
        }
    }
}

impl WrappedPathLike for WrappedPathHash {
    fn sampling_fingerprint(&self) -> u64 {
        match self {
            WrappedPathHash::Root => ROOT_FINGERPRINT,
            WrappedPathHash::NonRoot(path_hash) => path_hash.sampling_fingerprint(),
        }
    }
    fn evolve_path<'a>(
        from_route: Option<&'a Self>,
        walk_item: &'a OutgoingEdge,
    ) -> Option<&'a Self> {
        match walk_item.path.as_ref() {
            // Step has set explicit path, e.g. bonsai file
            Some(from_step) => Some(from_step.get_path_hash()),
            None => match walk_item.target.stats_path() {
                // Path is part of node identity
                Some(from_node) => Some(from_node.get_path_hash()),
                // No per-node path, so use the route, filtering out nodes that can't have repo paths
                None => {
                    if walk_item.target.get_type().allow_repo_path() {
                        from_route
                    } else {
                        None
                    }
                }
            },
        }
    }
}

impl fmt::Display for WrappedPathHash {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            Self::Root => write!(f, ""),
            Self::NonRoot(mpath_hash) => write!(f, "{}", mpath_hash.to_hex()),
        }
    }
}

// Memoize the hash of the path as it is used frequently
#[derive(Debug)]
pub struct MPathWithHashMemo {
    mpath: MPath,
    memoized_hash: OnceCell<WrappedPathHash>,
}

impl MPathWithHashMemo {
    fn new(mpath: MPath) -> Self {
        Self {
            mpath,
            memoized_hash: OnceCell::new(),
        }
    }

    pub fn get_path_hash_memo(&self) -> &WrappedPathHash {
        self.memoized_hash
            .get_or_init(|| WrappedPathHash::NonRoot(self.mpath.get_path_hash()))
    }

    pub fn mpath(&self) -> &MPath {
        &self.mpath
    }
}

impl PartialEq for MPathWithHashMemo {
    fn eq(&self, other: &Self) -> bool {
        self.mpath == other.mpath
    }
}

impl Eq for MPathWithHashMemo {}

impl Hash for MPathWithHashMemo {
    fn hash<H: Hasher>(&self, state: &mut H) {
        self.mpath.hash(state);
    }
}

#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub enum WrappedPath {
    Root,
    NonRoot(ArcIntern<EagerHashMemoizer<MPathWithHashMemo>>),
}

impl WrappedPath {
    pub fn as_ref(&self) -> Option<&MPath> {
        match self {
            WrappedPath::Root => None,
            WrappedPath::NonRoot(path) => Some(path.mpath()),
        }
    }

    pub fn get_path_hash(&self) -> &WrappedPathHash {
        match self {
            WrappedPath::Root => &WrappedPathHash::Root,
            WrappedPath::NonRoot(path) => path.get_path_hash_memo(),
        }
    }
}

impl WrappedPathLike for WrappedPath {
    fn sampling_fingerprint(&self) -> u64 {
        self.get_path_hash().sampling_fingerprint()
    }
    fn evolve_path<'a>(
        from_route: Option<&'a Self>,
        walk_item: &'a OutgoingEdge,
    ) -> Option<&'a Self> {
        match walk_item.path.as_ref() {
            // Step has set explicit path, e.g. bonsai file
            Some(from_step) => Some(from_step),
            None => match walk_item.target.stats_path() {
                // Path is part of node identity
                Some(from_node) => Some(from_node),
                // No per-node path, so use the route, filtering out nodes that can't have repo paths
                None => {
                    if walk_item.target.get_type().allow_repo_path() {
                        from_route
                    } else {
                        None
                    }
                }
            },
        }
    }
}

impl fmt::Display for WrappedPath {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            WrappedPath::Root => write!(f, ""),
            WrappedPath::NonRoot(path) => write!(f, "{}", path.mpath()),
        }
    }
}

static PATH_HASHER_FACTORY: OnceCell<RandomState> = OnceCell::new();

impl From<Option<MPath>> for WrappedPath {
    fn from(mpath: Option<MPath>) -> Self {
        let hasher_fac = PATH_HASHER_FACTORY.get_or_init(RandomState::default);
        match mpath {
            Some(mpath) => WrappedPath::NonRoot(ArcIntern::new(EagerHashMemoizer::new(
                MPathWithHashMemo::new(mpath),
                hasher_fac,
            ))),
            None => WrappedPath::Root,
        }
    }
}

define_type_enum! {
    enum AliasType {
        GitSha1,
        Sha1,
        Sha256,
        SeededBlake3,
    }
}

impl fmt::Display for EdgeType {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{:?}", self)
    }
}

/// File content gets a special two-state content so we can chose when to read the data
pub enum FileContentData {
    ContentStream(BoxStream<'static, Result<FileBytes, Error>>),
    Consumed(usize),
}

impl fmt::Debug for FileContentData {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match self {
            FileContentData::ContentStream(_s) => write!(f, "FileContentData::ContentStream(_)"),
            FileContentData::Consumed(s) => write!(f, "FileContentData::Consumed({})", s),
        }
    }
}

/// The data from the walk - this is the "full" form but not necessarily fully loaded.
/// e.g. file content streams are passed to you to read, they aren't pre-loaded to bytes.
#[derive(Debug)]
pub enum NodeData {
    ErrorAsData(Node),
    // Weren't able to find node
    MissingAsData(Node),
    // Node has an invalid hash
    HashValidationFailureAsData(Node),
    NotRequired,
    OutsideChunk,
    // Bonsai
    Bookmark(ChangesetId),
    Changeset(BonsaiChangeset),
    BonsaiHgMapping(Option<HgChangesetId>),
    PhaseMapping(Option<Phase>),
    PublishedBookmarks,
    // Hg
    HgBonsaiMapping(Option<ChangesetId>),
    HgChangeset(HgBlobChangeset),
    HgChangesetViaBonsai(HgChangesetId),
    HgManifest(HgBlobManifest),
    HgFileEnvelope(HgFileEnvelope),
    HgFileNode(Option<FilenodeInfo>),
    HgManifestFileNode(Option<FilenodeInfo>),
    // Content
    FileContent(FileContentData),
    FileContentMetadataV2(Option<ContentMetadataV2>),
    AliasContentMapping(ContentId),
    // Derived data
    Blame(Option<BlameV2>),
    ChangesetInfo(Option<ChangesetInfo>),
    ChangesetInfoMapping(Option<ChangesetId>),
    DeletedManifestV2(Option<DeletedManifestV2>),
    DeletedManifestV2Mapping(Option<DeletedManifestV2Id>),
    FastlogBatch(Option<FastlogBatch>),
    FastlogDir(Option<FastlogBatch>),
    FastlogFile(Option<FastlogBatch>),
    Fsnode(Fsnode),
    FsnodeMapping(Option<FsnodeId>),
    SkeletonManifest(Option<SkeletonManifest>),
    SkeletonManifestMapping(Option<SkeletonManifestId>),
    BasenameSuffixSkeletonManifest(Option<BasenameSuffixSkeletonManifest>),
    BasenameSuffixSkeletonManifestMapping(Option<BasenameSuffixSkeletonManifestId>),
    UnodeFile(FileUnode),
    UnodeManifest(ManifestUnode),
    UnodeMapping(Option<ManifestUnodeId>),
}

#[derive(Clone)]
pub struct SqlShardInfo {
    pub filenodes: SqlTierInfo,
    pub active_keys_per_shard: Option<usize>,
}

// Which type of non-blobstore Mononoke sql shard this node needs access to
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
pub enum SqlShard {
    Metadata,
    HgFileNode(usize),
}

impl Node {
    /// Map node to an SqlShard if any
    pub fn sql_shard(&self, shard_info: &SqlShardInfo) -> Option<SqlShard> {
        // Only report shards if there is a limit of keys per shard
        shard_info.active_keys_per_shard?;

        match self {
            Node::Root(_) => None,
            // Bonsai
            Node::Bookmark(_) => Some(SqlShard::Metadata),
            Node::Changeset(_) => None,
            Node::BonsaiHgMapping(_) => Some(SqlShard::Metadata),
            Node::PhaseMapping(_) => Some(SqlShard::Metadata),
            Node::PublishedBookmarks(_) => Some(SqlShard::Metadata),
            // Hg
            Node::HgBonsaiMapping(_) => Some(SqlShard::Metadata),
            Node::HgChangeset(_) => None,
            Node::HgChangesetViaBonsai(_) => Some(SqlShard::Metadata),
            Node::HgManifest(PathKey { id: _, path: _ }) => None,
            Node::HgFileEnvelope(_) => None,
            Node::HgFileNode(PathKey { id: _, path }) => {
                let path = path
                    .as_ref()
                    .map_or(RepoPath::RootPath, |p| RepoPath::FilePath(p.clone()));
                let path_hash = PathHash::from_repo_path(&path);
                let shard_num = path_hash.shard_number(shard_info.filenodes.shard_num.unwrap_or(1));
                Some(SqlShard::HgFileNode(shard_num))
            }
            Node::HgManifestFileNode(PathKey { id: _, path }) => {
                let path = path
                    .as_ref()
                    .map_or(RepoPath::RootPath, |p| RepoPath::DirectoryPath(p.clone()));
                let path_hash = PathHash::from_repo_path(&path);
                let shard_num = path_hash.shard_number(shard_info.filenodes.shard_num.unwrap_or(1));
                Some(SqlShard::HgFileNode(shard_num))
            }
            // Content
            Node::FileContent(_) => None,
            Node::FileContentMetadataV2(_) => None,
            Node::AliasContentMapping(_) => None,
            // Derived data
            Node::Blame(_) => None,
            Node::ChangesetInfo(_) => None,
            Node::ChangesetInfoMapping(_) => None,
            Node::DeletedManifestV2(_) => None,
            Node::DeletedManifestV2Mapping(_) => None,
            Node::FastlogBatch(_) => None,
            Node::FastlogDir(_) => None,
            Node::FastlogFile(_) => None,
            Node::Fsnode(_) => None,
            Node::FsnodeMapping(_) => None,
            Node::SkeletonManifest(_) => None,
            Node::SkeletonManifestMapping(_) => None,
            Node::BasenameSuffixSkeletonManifest(_) => None,
            Node::BasenameSuffixSkeletonManifestMapping(_) => None,
            Node::UnodeFile(_) => None,
            Node::UnodeManifest(_) => None,
            Node::UnodeMapping(_) => None,
        }
    }

    pub fn stats_key(&self) -> String {
        match self {
            Node::Root(_) => "root".to_string(),
            // Bonsai
            Node::Bookmark(k) => k.to_string(),
            Node::Changeset(k) => k.blobstore_key(),
            Node::BonsaiHgMapping(k) => k.blobstore_key(),
            Node::PhaseMapping(k) => k.blobstore_key(),
            Node::PublishedBookmarks(_) => "published_bookmarks".to_string(),
            // Hg
            Node::HgBonsaiMapping(k) => k.blobstore_key(),
            Node::HgChangeset(k) => k.blobstore_key(),
            Node::HgChangesetViaBonsai(k) => k.blobstore_key(),
            Node::HgManifest(PathKey { id, path: _ }) => id.blobstore_key(),
            Node::HgFileEnvelope(k) => k.blobstore_key(),
            Node::HgFileNode(PathKey { id, path: _ }) => id.blobstore_key(),
            Node::HgManifestFileNode(PathKey { id, path: _ }) => id.blobstore_key(),
            // Content
            Node::FileContent(k) => k.blobstore_key(),
            Node::FileContentMetadataV2(k) => k.blobstore_key(),
            Node::AliasContentMapping(k) => k.0.blobstore_key(),
            // Derived data
            Node::Blame(k) => k.blobstore_key(),
            Node::ChangesetInfo(k) => k.blobstore_key(),
            Node::ChangesetInfoMapping(k) => k.blobstore_key(),
            Node::DeletedManifestV2(k) => k.blobstore_key(),
            Node::DeletedManifestV2Mapping(k) => k.blobstore_key(),
            Node::FastlogBatch(k) => k.blobstore_key(),
            Node::FastlogDir(k) => k.blobstore_key(),
            Node::FastlogFile(k) => k.blobstore_key(),
            Node::Fsnode(k) => k.blobstore_key(),
            Node::FsnodeMapping(k) => k.blobstore_key(),
            Node::SkeletonManifest(k) => k.blobstore_key(),
            Node::SkeletonManifestMapping(k) => k.blobstore_key(),
            Node::BasenameSuffixSkeletonManifest(k) => k.blobstore_key(),
            Node::BasenameSuffixSkeletonManifestMapping(k) => k.blobstore_key(),
            Node::UnodeFile(k) => k.blobstore_key(),
            Node::UnodeManifest(k) => k.blobstore_key(),
            Node::UnodeMapping(k) => k.blobstore_key(),
        }
    }

    pub fn stats_path(&self) -> Option<&WrappedPath> {
        match self {
            Node::Root(_) => None,
            // Bonsai
            Node::Bookmark(_) => None,
            Node::Changeset(_) => None,
            Node::BonsaiHgMapping(_) => None,
            Node::PhaseMapping(_) => None,
            Node::PublishedBookmarks(_) => None,
            // Hg
            Node::HgBonsaiMapping(_) => None,
            Node::HgChangeset(_) => None,
            Node::HgChangesetViaBonsai(_) => None,
            Node::HgManifest(PathKey { id: _, path }) => Some(path),
            Node::HgFileEnvelope(_) => None,
            Node::HgFileNode(PathKey { id: _, path }) => Some(path),
            Node::HgManifestFileNode(PathKey { id: _, path }) => Some(path),
            // Content
            Node::FileContent(_) => None,
            Node::FileContentMetadataV2(_) => None,
            Node::AliasContentMapping(_) => None,
            // Derived data
            Node::Blame(_) => None,
            Node::ChangesetInfo(_) => None,
            Node::ChangesetInfoMapping(_) => None,
            Node::DeletedManifestV2(_) => None,
            Node::DeletedManifestV2Mapping(_) => None,
            Node::FastlogBatch(_) => None,
            Node::FastlogDir(_) => None,
            Node::FastlogFile(_) => None,
            Node::Fsnode(_) => None,
            Node::FsnodeMapping(_) => None,
            Node::SkeletonManifest(_) => None,
            Node::SkeletonManifestMapping(_) => None,
            Node::BasenameSuffixSkeletonManifest(_) => None,
            Node::BasenameSuffixSkeletonManifestMapping(_) => None,
            Node::UnodeFile(_) => None,
            Node::UnodeManifest(_) => None,
            Node::UnodeMapping(_) => None,
        }
    }

    /// None means not hash based
    pub fn sampling_fingerprint(&self) -> Option<u64> {
        match self {
            Node::Root(_) => None,
            // Bonsai
            Node::Bookmark(_k) => None,
            Node::Changeset(k) => Some(k.sampling_fingerprint()),
            Node::BonsaiHgMapping(k) => Some(k.sampling_fingerprint()),
            Node::PhaseMapping(k) => Some(k.sampling_fingerprint()),
            Node::PublishedBookmarks(_) => None,
            // Hg
            Node::HgBonsaiMapping(k) => Some(k.sampling_fingerprint()),
            Node::HgChangeset(k) => Some(k.sampling_fingerprint()),
            Node::HgChangesetViaBonsai(k) => Some(k.sampling_fingerprint()),
            Node::HgManifest(PathKey { id, path: _ }) => Some(id.sampling_fingerprint()),
            Node::HgFileEnvelope(k) => Some(k.sampling_fingerprint()),
            Node::HgFileNode(PathKey { id, path: _ }) => Some(id.sampling_fingerprint()),
            Node::HgManifestFileNode(PathKey { id, path: _ }) => Some(id.sampling_fingerprint()),
            // Content
            Node::FileContent(k) => Some(k.sampling_fingerprint()),
            Node::FileContentMetadataV2(k) => Some(k.sampling_fingerprint()),
            Node::AliasContentMapping(k) => Some(k.0.sampling_fingerprint()),
            // Derived data
            Node::Blame(k) => Some(k.sampling_fingerprint()),
            Node::ChangesetInfo(k) => Some(k.sampling_fingerprint()),
            Node::ChangesetInfoMapping(k) => Some(k.sampling_fingerprint()),
            Node::DeletedManifestV2(k) => Some(k.sampling_fingerprint()),
            Node::DeletedManifestV2Mapping(k) => Some(k.sampling_fingerprint()),
            Node::FastlogBatch(k) => Some(k.sampling_fingerprint()),
            Node::FastlogDir(k) => Some(k.sampling_fingerprint()),
            Node::FastlogFile(k) => Some(k.sampling_fingerprint()),
            Node::Fsnode(k) => Some(k.sampling_fingerprint()),
            Node::FsnodeMapping(k) => Some(k.sampling_fingerprint()),
            Node::SkeletonManifest(k) => Some(k.sampling_fingerprint()),
            Node::SkeletonManifestMapping(k) => Some(k.sampling_fingerprint()),
            Node::BasenameSuffixSkeletonManifest(k) => Some(k.sampling_fingerprint()),
            Node::BasenameSuffixSkeletonManifestMapping(k) => Some(k.sampling_fingerprint()),
            Node::UnodeFile(k) => Some(k.sampling_fingerprint()),
            Node::UnodeManifest(k) => Some(k.sampling_fingerprint()),
            Node::UnodeMapping(k) => Some(k.sampling_fingerprint()),
        }
    }

    pub fn validate_hash(
        &self,
        ctx: CoreContext,
        repo: BlobRepo,
        node_data: &NodeData,
    ) -> BoxFuture<Result<(), HashValidationError>> {
        match (&self, node_data) {
            (Node::HgFileEnvelope(hg_filenode_id), NodeData::HgFileEnvelope(envelope)) => {
                let hg_filenode_id = hg_filenode_id.clone();
                let envelope = envelope.clone();
                async move {
                    let content_id = envelope.content_id();
                    let file_bytes =
                        filestore::fetch(repo.repo_blobstore(), ctx, &envelope.content_id().into())
                            .await?;

                    let file_bytes = file_bytes.ok_or_else(|| {
                        format_err!(
                            "content {} not found for filenode {}",
                            content_id,
                            hg_filenode_id
                        )
                    })?;
                    let HgFileEnvelopeMut {
                        p1, p2, metadata, ..
                    } = envelope.into_mut();
                    let p1 = p1.map(|p| p.into_nodehash());
                    let p2 = p2.map(|p| p.into_nodehash());
                    let actual = calculate_hg_node_id_stream(
                        stream::once(async { Ok(metadata) })
                            .chain(file_bytes)
                            .boxed()
                            .compat(),
                        &HgParents::new(p1, p2),
                    )
                    .compat()
                    .await?;
                    let actual = HgFileNodeId::new(actual);

                    if actual != hg_filenode_id {
                        return Err(HashValidationError::HashMismatch {
                            actual_hash: format!("{}", actual),
                            expected_hash: format!("{}", hg_filenode_id),
                        });
                    }
                    Ok(())
                }
                .boxed()
            }
            _ => {
                let ty = self.get_type();
                let s: &str = ty.into();
                async move { Err(HashValidationError::NotSupported(s.to_string())) }.boxed()
            }
        }
    }
}

#[cfg(test)]
mod tests {
    use std::collections::HashSet;
    use std::mem::size_of;

    use strum::EnumCount;
    use strum::IntoEnumIterator;

    use super::*;

    #[test]
    fn test_node_size() {
        // Node size is important as we have lots of them, add a test to check for accidental changes
        assert_eq!(40, size_of::<Node>());
    }

    #[test]
    fn test_node_type_max_ordinal() {
        // Check the macros worked consistently
        for t in NodeType::iter() {
            assert!((t as usize) < NodeType::COUNT)
        }
    }

    #[test]
    fn test_small_graphs() -> Result<(), Error> {
        create_graph!(
            Test1NodeType,
            Test1Node,
            Test1EdgeType,
            (Root, UnitKey, [Foo]),
            (Foo, u32, []),
        );
        assert_eq!(Test1NodeType::Root, Test1Node::Root(UnitKey()).get_type());
        assert_eq!(Test1NodeType::Foo, Test1Node::Foo(42).get_type());
        assert_eq!(Test1EdgeType::RootToFoo.incoming_type(), None);
        assert_eq!(Test1EdgeType::RootToFoo.outgoing_type(), Test1NodeType::Foo);
        assert_eq!(Test1NodeType::Foo.parse_node("123")?, Test1Node::Foo(123));

        // Make sure type names don't clash
        create_graph!(
            Test2NodeType,
            Test2Node,
            Test2EdgeType,
            (Root, UnitKey, [Foo, Bar]),
            (Foo, u32, [Bar]),
            (Bar, u32, []),
        );
        assert_eq!(Test2NodeType::Root, Test2Node::Root(UnitKey()).get_type());
        assert_eq!(Test2NodeType::Foo, Test2Node::Foo(42).get_type());
        assert_eq!(Test2NodeType::Bar, Test2Node::Bar(42).get_type());
        assert_eq!(Test2EdgeType::RootToFoo.incoming_type(), None);
        assert_eq!(Test2EdgeType::RootToFoo.outgoing_type(), Test2NodeType::Foo);
        assert_eq!(Test2EdgeType::RootToBar.incoming_type(), None);
        assert_eq!(Test2EdgeType::RootToBar.outgoing_type(), Test2NodeType::Bar);
        assert_eq!(
            Test2EdgeType::FooToBar.incoming_type(),
            Some(Test2NodeType::Foo)
        );
        assert_eq!(Test2EdgeType::FooToBar.outgoing_type(), Test2NodeType::Bar);
        assert_eq!(Test2NodeType::Bar.parse_node("123")?, Test2Node::Bar(123));
        Ok(())
    }

    #[test]
    fn test_all_derived_data_types_supported() {
        // All types blobrepo can support
        let a = test_repo_factory::default_test_repo_config()
            .derived_data_config
            .get_active_config()
            .expect("No enabled derived data types config")
            .types
            .clone();

        // supported in graph
        let mut s = HashSet::new();
        for t in NodeType::iter() {
            if let Some(d) = t.derived_data_name() {
                assert!(
                    a.contains(d),
                    "graph derived data type {} for {} is not known by default_test_repo_config()",
                    d,
                    t
                );
                s.insert(d);
            }
        }

        // If you are adding a new derived data type, please add it to the walker graph rather than to this
        // list, otherwise it won't get scrubbed and thus you would be unaware of different representation
        // in different stores
        let grandfathered: HashSet<&'static str> =
            HashSet::from_iter(vec!["git_trees"].into_iter());
        let mut missing = HashSet::new();
        for t in a {
            if s.contains(t.as_str()) {
                assert!(
                    !grandfathered.contains(t.as_str()),
                    "You've added support for {}, please remove it from the grandfathered missing set",
                    t
                );
            } else if !grandfathered.contains(t.as_str()) {
                missing.insert(t);
            }
        }
        assert!(
            missing.is_empty(),
            "blobrepo derived data types {:?} not supported by walker graph",
            missing,
        );
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

/// Tags for slog usage
pub const CHUNKING: &str = "chunking";
pub const GRAPH: &str = "graph";
pub const LOADED: &str = "loaded";
pub const SIZING: &str = "sizing";
pub const VALIDATE: &str = "validate";
pub const SUPPRESS: &str = "suppress";


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

pub mod blobstore;
pub mod checkpoint;
#[macro_use]
pub mod graph;
pub mod corpus;
pub mod log;
pub mod pack;
pub mod parse_node;
pub mod progress;
pub mod sampling;
pub mod scrub;
pub mod sizing;
pub mod state;
pub mod tail;
pub mod validate;
pub mod walk;


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashSet;

use blobstore::SizeMetadata;
use metaconfig_types::BlobstoreId;
use mononoke_types::Timestamp;
use scuba_ext::MononokeScubaSampleBuilder;

use crate::detail::graph::NodeType;
use crate::detail::validate::REPO;

const RUN_START: &str = "run_start";
const CHUNK_NUM: &str = "chunk_num";
const BLOBSTORE_ID: &str = "blobstore_id";
const BLOBSTORE_KEY: &str = "blobstore_key";
const NODE_TYPE: &str = "node_type";
const NODE_FINGERPRINT: &str = "node_fingerprint";
const SIMILARITY_KEY: &str = "similarity_key";
const MTIME: &str = "mtime";
const UNCOMPRESSED_SIZE: &str = "uncompressed_size";
const UNIQUE_COMPRESSED_SIZE: &str = "unique_compressed_size";
const PACK_KEY: &str = "pack_key";
const RELEVANT_UNCOMPRESSED_SIZE: &str = "relevant_uncompressed_size";
const RELEVANT_COMPRESSED_SIZE: &str = "relevant_compressed_size";
const CHECKPOINT_NAME: &str = "checkpoint_name";
pub const CTIME: &str = "ctime";

/// What do we log for each blobstore key
pub struct PackInfo<'a> {
    pub blobstore_id: Option<BlobstoreId>, // We only have blobstore ids when running against a multiplex
    pub blobstore_key: &'a str,
    pub node_type: NodeType, // Might be different from type implied by blobstore_key's prefix string, e.g. if loading a node does multiple blobstore accesses
    pub node_fingerprint: Option<u64>, // the short hash of the graph level node
    pub similarity_key: Option<u64>, // typically the hash of associated repo path
    pub mtime: Option<u64>, // typically the commit time of Changeset from which this item was reached
    pub uncompressed_size: u64, // How big is the value for this key, in bytes.
    pub sizes: Option<SizeMetadata>,
    pub ctime: Option<i64>,
}

pub trait PackInfoLogger {
    fn log(&self, info: PackInfo<'_>);
}

/// What to log for packing and where to send it
#[derive(Clone)]
pub struct PackInfoLogOptions {
    pub log_node_types: HashSet<NodeType>,
    pub log_dest: MononokeScubaSampleBuilder, // TODO(ahornby), add an enum once alternate destinations possible
}

impl PackInfoLogOptions {
    pub fn make_logger(
        &self,
        repo_name: String,
        run_start: Timestamp,
        chunk_num: u64,
        checkpoint_name: Option<String>,
    ) -> ScubaPackInfoLogger {
        let mut scuba = self.log_dest.clone();
        scuba
            .add(REPO, repo_name)
            .add_opt(CHECKPOINT_NAME, checkpoint_name);
        ScubaPackInfoLogger::new(scuba, run_start, chunk_num)
    }
}

// Used for logging to scuba and to JSON
pub struct ScubaPackInfoLogger {
    scuba: MononokeScubaSampleBuilder,
    run_start: Timestamp, // So we can distinguish runs of the logger. If checkpointing then this is the checkpoint creation timestamp
    chunk_num: u64, // What chunk if is in this sequence from master, lower is closer to master. If not chunking its 1.
}

impl ScubaPackInfoLogger {
    pub fn new(scuba: MononokeScubaSampleBuilder, run_start: Timestamp, chunk_num: u64) -> Self {
        Self {
            scuba,
            run_start,
            chunk_num,
        }
    }
}

impl PackInfoLogger for ScubaPackInfoLogger {
    fn log(&self, info: PackInfo<'_>) {
        let mut scuba = self.scuba.clone();
        scuba
            .add_opt(BLOBSTORE_ID, info.blobstore_id)
            .add(BLOBSTORE_KEY, info.blobstore_key)
            .add(RUN_START, self.run_start.timestamp_seconds())
            .add(CHUNK_NUM, self.chunk_num)
            .add(NODE_TYPE, info.node_type.as_ref())
            .add_opt(NODE_FINGERPRINT, info.node_fingerprint)
            .add_opt(SIMILARITY_KEY, info.similarity_key)
            .add_opt(MTIME, info.mtime)
            .add(UNCOMPRESSED_SIZE, info.uncompressed_size)
            .add_opt(CTIME, info.ctime);

        if let Some(sizes) = info.sizes {
            scuba.add(UNIQUE_COMPRESSED_SIZE, sizes.unique_compressed_size);
            if let Some(pack_meta) = sizes.pack_meta {
                scuba.add(PACK_KEY, pack_meta.pack_key);
                scuba.add(
                    RELEVANT_UNCOMPRESSED_SIZE,
                    pack_meta.relevant_uncompressed_size,
                );
                scuba.add(RELEVANT_COMPRESSED_SIZE, pack_meta.relevant_compressed_size);
            }
        }

        scuba.log();
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::str::FromStr;

use anyhow::format_err;
use anyhow::Error;
use filestore::Alias;
use mercurial_types::HgFileNodeId;
use mercurial_types::HgManifestId;
use mononoke_types::hash::Blake3;
use mononoke_types::hash::GitSha1;
use mononoke_types::hash::Sha1;
use mononoke_types::hash::Sha256;
use mononoke_types::FileUnodeId;
use mononoke_types::MPath;
use mononoke_types::ManifestUnodeId;
use strum::IntoEnumIterator;

use crate::detail::graph::AliasKey;
use crate::detail::graph::AliasType;
use crate::detail::graph::ChangesetKey;
use crate::detail::graph::FastlogKey;
use crate::detail::graph::Node;
use crate::detail::graph::NodeType;
use crate::detail::graph::PathKey;
use crate::detail::graph::UnitKey;
use crate::detail::graph::UnodeFlags;
use crate::detail::graph::UnodeKey;
use crate::detail::graph::WrappedPath;

const NODE_SEP: &str = ":";

fn check_and_build_path(node_type: NodeType, parts: &[&str]) -> Result<WrappedPath, Error> {
    if parts.len() < 2 {
        return Err(format_err!(
            "parse_node requires a path and key for {}",
            node_type
        ));
    }
    let mpath = match parts[1..].join(NODE_SEP).as_str() {
        "/" => None,
        p => Some(MPath::new(p)?),
    };
    Ok(WrappedPath::from(mpath))
}

impl FromStr for UnitKey {
    type Err = Error;
    fn from_str(s: &str) -> Result<Self, Self::Err> {
        if s.is_empty() {
            Ok(Self())
        } else {
            Err(format_err!("Expected empty string for UnitKey"))
        }
    }
}

impl FromStr for PathKey<HgManifestId> {
    type Err = Error;
    fn from_str(s: &str) -> Result<Self, Self::Err> {
        let parts: Vec<_> = s.split(NODE_SEP).collect();
        let path = check_and_build_path(NodeType::HgManifest, &parts)?;
        let id = HgManifestId::from_str(parts[0])?;
        Ok(Self { id, path })
    }
}

impl FromStr for PathKey<HgFileNodeId> {
    type Err = Error;
    fn from_str(s: &str) -> Result<Self, Self::Err> {
        let parts: Vec<_> = s.split(NODE_SEP).collect();
        let path = check_and_build_path(NodeType::HgFileNode, &parts)?;
        let id = HgFileNodeId::from_str(parts[0])?;
        Ok(Self { id, path })
    }
}

impl<T> FromStr for ChangesetKey<T>
where
    T: FromStr,
{
    type Err = <T as FromStr>::Err;
    fn from_str(s: &str) -> Result<Self, Self::Err> {
        let inner = T::from_str(s)?;
        Ok(ChangesetKey {
            inner,
            filenode_known_derived: false,
        })
    }
}

impl FromStr for FastlogKey<ManifestUnodeId> {
    type Err = Error;
    fn from_str(s: &str) -> Result<Self, Self::Err> {
        let inner = ManifestUnodeId::from_str(s)?;
        Ok(FastlogKey { inner })
    }
}

impl FromStr for FastlogKey<FileUnodeId> {
    type Err = Error;
    fn from_str(s: &str) -> Result<Self, Self::Err> {
        let inner = FileUnodeId::from_str(s)?;
        Ok(FastlogKey { inner })
    }
}

impl FromStr for UnodeFlags {
    type Err = Error;
    fn from_str(s: &str) -> Result<Self, Self::Err> {
        let bits = u8::from_str_radix(s, 2)?;
        UnodeFlags::from_bits(bits).ok_or_else(|| format_err!("Bad bit flags: {:b}", bits))
    }
}

impl FromStr for UnodeKey<ManifestUnodeId> {
    type Err = Error;
    fn from_str(s: &str) -> Result<Self, Self::Err> {
        let parts: Vec<_> = s.split(NODE_SEP).collect();
        let inner = ManifestUnodeId::from_str(parts[0])?;
        let flags = UnodeFlags::from_str(parts[1])?;
        Ok(UnodeKey { inner, flags })
    }
}

impl FromStr for UnodeKey<FileUnodeId> {
    type Err = Error;
    fn from_str(s: &str) -> Result<Self, Self::Err> {
        let parts: Vec<_> = s.split(NODE_SEP).collect();
        let inner = FileUnodeId::from_str(parts[0])?;
        let flags = UnodeFlags::from_str(parts[1])?;
        Ok(UnodeKey { inner, flags })
    }
}

impl FromStr for AliasKey {
    type Err = Error;
    fn from_str(s: &str) -> Result<Self, Self::Err> {
        let parts: Vec<_> = s.split(NODE_SEP).collect();
        if parts.len() < 2 {
            return Err(format_err!(
                "parse_node requires an alias type from {:?} and key for AliasKey",
                Vec::from_iter(AliasType::iter()),
            ));
        }
        let alias_type = AliasType::from_str(parts[0])?;
        let id = &parts[1..].join(NODE_SEP);
        let alias = match alias_type {
            AliasType::GitSha1 => Alias::GitSha1(GitSha1::from_str(id)?),
            AliasType::Sha1 => Alias::Sha1(Sha1::from_str(id)?),
            AliasType::Sha256 => Alias::Sha256(Sha256::from_str(id)?),
            AliasType::SeededBlake3 => Alias::SeededBlake3(Blake3::from_str(id)?),
        };
        Ok(Self(alias))
    }
}

pub fn parse_node(s: &str) -> Result<Node, Error> {
    let parts: Vec<_> = s.split(NODE_SEP).collect();
    if parts.is_empty() {
        return Err(format_err!("parse_node requires at least NodeType"));
    }
    let node_type = NodeType::from_str(parts[0])?;
    match (node_type, parts.len()) {
        (NodeType::Root, 1) | (NodeType::PublishedBookmarks, 1) => {}
        (NodeType::Root, _) | (NodeType::PublishedBookmarks, _) => {
            return Err(format_err!(
                "parse_node expects {} not to be followed by any parts",
                node_type
            ));
        }
        (_, l) if l < 2 => {
            return Err(format_err!(
                "parse_node for {} requires at least NodeType:node_key",
                node_type
            ));
        }
        _ => {}
    }

    let parts = &parts[1..];
    let node = node_type.parse_node(&parts.join(NODE_SEP))?;
    Ok(node)
}

#[cfg(test)]
mod tests {
    use bookmarks::BookmarkKey;

    use super::*;

    const SAMPLE_BLAKE2: &str = "b847b8838bfe3ae13ea6f8ce2e341c51193587b8392494f6dbab7224b3b116bf";
    const SAMPLE_SHA1: &str = "e797dcabdd6d16ec4ae614165178b60d7054305b";
    const SAMPLE_SHA256: &str = "332ff483aaf1bbc241314576b399f81675a6f81aba205bd3b80b05a4ffda44d4";
    const SAMPLE_SEEDED_BLAKE3: &str =
        "8537f0e61caaa8f1aca4eac89bb9f27ba2562dbdd1b1a2d987999c37f159bf18";
    const SAMPLE_PATH: &str = "/foo/bar/baz";

    fn test_node_type(node_type: &NodeType) -> Result<(), Error> {
        let v = match node_type {
            NodeType::Root => {
                assert_eq!(Node::Root(UnitKey()), parse_node("Root")?);
                assert_eq!(
                    "Err(parse_node expects Root not to be followed by any parts)",
                    format!("{:?}", parse_node("Root:garbage"))
                );
            }
            NodeType::Bookmark => assert_eq!(
                Node::Bookmark(BookmarkKey::new("foo")?),
                parse_node(&format!("Bookmark{}foo", NODE_SEP))?
            ),
            NodeType::Changeset => assert_eq!(
                node_type,
                &parse_node(&format!("Changeset{}{}", NODE_SEP, SAMPLE_BLAKE2))?.get_type()
            ),
            NodeType::BonsaiHgMapping => assert_eq!(
                node_type,
                &parse_node(&format!("BonsaiHgMapping{}{}", NODE_SEP, SAMPLE_BLAKE2))?.get_type()
            ),
            NodeType::PhaseMapping => assert_eq!(
                node_type,
                &parse_node(&format!("PhaseMapping{}{}", NODE_SEP, SAMPLE_BLAKE2))?.get_type()
            ),
            NodeType::PublishedBookmarks => {
                assert_eq!(
                    Node::PublishedBookmarks(UnitKey()),
                    parse_node("PublishedBookmarks")?
                );
                assert_eq!(
                    "Err(parse_node expects PublishedBookmarks not to be followed by any parts)",
                    format!("{:?}", parse_node("PublishedBookmarks:garbage"))
                );
            }
            // Hg
            NodeType::HgBonsaiMapping => assert_eq!(
                node_type,
                &parse_node(&format!("HgBonsaiMapping{}{}", NODE_SEP, SAMPLE_SHA1))?.get_type()
            ),
            NodeType::HgChangeset => assert_eq!(
                node_type,
                &parse_node(&format!("HgChangeset{}{}", NODE_SEP, SAMPLE_SHA1))?.get_type()
            ),
            NodeType::HgChangesetViaBonsai => assert_eq!(
                node_type,
                &parse_node(&format!("HgChangesetViaBonsai{}{}", NODE_SEP, SAMPLE_SHA1))?
                    .get_type()
            ),
            NodeType::HgManifest => assert_eq!(
                node_type,
                &parse_node(&format!(
                    "HgManifest{}{}{}{}",
                    NODE_SEP, SAMPLE_SHA1, NODE_SEP, SAMPLE_PATH
                ))?
                .get_type()
            ),
            NodeType::HgFileEnvelope => assert_eq!(
                node_type,
                &parse_node(&format!("HgFileEnvelope{}{}", NODE_SEP, SAMPLE_SHA1))?.get_type()
            ),
            NodeType::HgFileNode => assert_eq!(
                node_type,
                &parse_node(&format!(
                    "HgFileNode{}{}{}{}",
                    NODE_SEP, SAMPLE_SHA1, NODE_SEP, SAMPLE_PATH
                ))?
                .get_type()
            ),
            NodeType::HgManifestFileNode => assert_eq!(
                node_type,
                &parse_node(&format!(
                    "HgManifestFileNode{}{}{}{}",
                    NODE_SEP, SAMPLE_SHA1, NODE_SEP, SAMPLE_PATH
                ))?
                .get_type()
            ),
            // Content
            NodeType::FileContent => assert_eq!(
                node_type,
                &parse_node(&format!("FileContent{}{}", NODE_SEP, SAMPLE_BLAKE2))?.get_type()
            ),
            NodeType::FileContentMetadataV2 => assert_eq!(
                node_type,
                &parse_node(&format!(
                    "FileContentMetadataV2{}{}",
                    NODE_SEP, SAMPLE_BLAKE2
                ))?
                .get_type()
            ),
            NodeType::AliasContentMapping => {
                assert_eq!(
                    node_type,
                    &parse_node(&format!(
                        "AliasContentMapping{}{}{}{}",
                        NODE_SEP, "Sha1", NODE_SEP, SAMPLE_SHA1
                    ))?
                    .get_type()
                );
                assert_eq!(
                    node_type,
                    &parse_node(&format!(
                        "AliasContentMapping{}{}{}{}",
                        NODE_SEP, "Sha256", NODE_SEP, SAMPLE_SHA256
                    ))?
                    .get_type()
                );
                assert_eq!(
                    node_type,
                    &parse_node(&format!(
                        "AliasContentMapping{}{}{}{}",
                        NODE_SEP, "SeededBlake3", NODE_SEP, SAMPLE_SEEDED_BLAKE3
                    ))?
                    .get_type()
                );
            }
            NodeType::Blame => {
                assert_eq!(
                    node_type,
                    &parse_node(&format!("Blame{}{}", NODE_SEP, SAMPLE_BLAKE2))?.get_type()
                );
            }
            NodeType::ChangesetInfo => {
                assert_eq!(
                    node_type,
                    &parse_node(&format!("ChangesetInfo{}{}", NODE_SEP, SAMPLE_BLAKE2))?.get_type()
                );
            }
            NodeType::ChangesetInfoMapping => {
                assert_eq!(
                    node_type,
                    &parse_node(&format!(
                        "ChangesetInfoMapping{}{}",
                        NODE_SEP, SAMPLE_BLAKE2
                    ))?
                    .get_type()
                );
            }
            NodeType::DeletedManifestV2 => {
                assert_eq!(
                    node_type,
                    &parse_node(&format!("DeletedManifestV2{}{}", NODE_SEP, SAMPLE_BLAKE2))?
                        .get_type()
                );
            }
            NodeType::DeletedManifestV2Mapping => {
                assert_eq!(
                    node_type,
                    &parse_node(&format!(
                        "DeletedManifestV2Mapping{}{}",
                        NODE_SEP, SAMPLE_BLAKE2
                    ))?
                    .get_type()
                );
            }
            NodeType::FastlogBatch => {
                assert_eq!(
                    node_type,
                    &parse_node(&format!("FastlogBatch{}{}", NODE_SEP, SAMPLE_BLAKE2))?.get_type()
                );
            }
            NodeType::FastlogDir => {
                assert_eq!(
                    node_type,
                    &parse_node(&format!("FastlogDir{}{}", NODE_SEP, SAMPLE_BLAKE2))?.get_type()
                );
            }
            NodeType::FastlogFile => {
                assert_eq!(
                    node_type,
                    &parse_node(&format!("FastlogFile{}{}", NODE_SEP, SAMPLE_BLAKE2))?.get_type()
                );
            }
            NodeType::Fsnode => {
                assert_eq!(
                    node_type,
                    &parse_node(&format!("Fsnode{}{}", NODE_SEP, SAMPLE_BLAKE2))?.get_type()
                );
            }
            NodeType::FsnodeMapping => {
                assert_eq!(
                    node_type,
                    &parse_node(&format!("FsnodeMapping{}{}", NODE_SEP, SAMPLE_BLAKE2))?.get_type()
                );
            }
            NodeType::SkeletonManifest => {
                assert_eq!(
                    node_type,
                    &parse_node(&format!("SkeletonManifest{}{}", NODE_SEP, SAMPLE_BLAKE2))?
                        .get_type()
                );
            }
            NodeType::SkeletonManifestMapping => {
                assert_eq!(
                    node_type,
                    &parse_node(&format!(
                        "SkeletonManifestMapping{}{}",
                        NODE_SEP, SAMPLE_BLAKE2
                    ))?
                    .get_type()
                );
            }
            NodeType::BasenameSuffixSkeletonManifest => {
                assert_eq!(
                    node_type,
                    &parse_node(&format!(
                        "BasenameSuffixSkeletonManifest{}{}",
                        NODE_SEP, SAMPLE_BLAKE2
                    ))?
                    .get_type()
                );
            }
            NodeType::BasenameSuffixSkeletonManifestMapping => {
                assert_eq!(
                    node_type,
                    &parse_node(&format!(
                        "BasenameSuffixSkeletonManifestMapping{}{}",
                        NODE_SEP, SAMPLE_BLAKE2
                    ))?
                    .get_type()
                );
            }
            NodeType::UnodeFile => {
                assert_eq!(
                    node_type,
                    &parse_node(&format!(
                        "UnodeFile{}{}{}{:b}",
                        NODE_SEP, SAMPLE_BLAKE2, NODE_SEP, 0b00000011
                    ))?
                    .get_type()
                );
            }
            NodeType::UnodeManifest => {
                assert_eq!(
                    node_type,
                    &parse_node(&format!(
                        "UnodeManifest{}{}{}{:b}",
                        NODE_SEP, SAMPLE_BLAKE2, NODE_SEP, 0b00000011
                    ))?
                    .get_type()
                );
            }
            NodeType::UnodeMapping => {
                assert_eq!(
                    node_type,
                    &parse_node(&format!("UnodeMapping{}{}", NODE_SEP, SAMPLE_BLAKE2))?.get_type()
                );
            }
        };
        Ok(v)
    }

    #[test]
    fn parse_all_node_types() -> Result<(), Error> {
        for t in NodeType::iter() {
            test_node_type(&t)?;
        }
        Ok(())
    }
}