/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

//! SQL Commit Graph Storage
//!
//! Database-backed implementation of the commit graph storage.
#![allow(unused)]

use std::collections::HashMap;
use std::collections::HashSet;
use std::sync::Arc;

use anyhow::anyhow;
use anyhow::Context;
use anyhow::Result;
use async_trait::async_trait;
use commit_graph_types::edges::ChangesetEdges;
use commit_graph_types::edges::ChangesetNode;
use commit_graph_types::edges::ChangesetNodeParents;
use commit_graph_types::edges::ChangesetParents;
use commit_graph_types::storage::CommitGraphStorage;
use commit_graph_types::storage::Prefetch;
use commit_graph_types::storage::PrefetchEdge;
use context::CoreContext;
use context::PerfCounterType;
use mononoke_types::ChangesetId;
use mononoke_types::ChangesetIdPrefix;
use mononoke_types::ChangesetIdsResolvedFromPrefix;
use mononoke_types::Generation;
use mononoke_types::RepositoryId;
use rendezvous::RendezVous;
use rendezvous::RendezVousOptions;
use rendezvous::RendezVousStats;
use rendezvous::TunablesRendezVousController;
use sql::Connection;
use sql::SqlConnections;
use sql_construct::SqlConstruct;
use sql_construct::SqlConstructFromMetadataDatabaseConfig;
use sql_ext::mononoke_queries;
use tunables::tunables;
use vec1::vec1;
use vec1::Vec1;

#[cfg(test)]
mod tests;

/// Maximum number of recursive steps to take when prefetching commits.
///
/// The configured maximum number of recursive steps in MySQL is 1000.
const DEFAULT_PREFETCH_STEP_LIMIT: i64 = 1000;

pub struct SqlCommitGraphStorageBuilder {
    connections: SqlConnections,
}

impl SqlConstruct for SqlCommitGraphStorageBuilder {
    const LABEL: &'static str = "commit_graph";

    const CREATION_QUERY: &'static str = include_str!("../schemas/sqlite-commit-graph.sql");

    fn from_sql_connections(connections: SqlConnections) -> Self {
        Self { connections }
    }
}

impl SqlConstructFromMetadataDatabaseConfig for SqlCommitGraphStorageBuilder {}

impl SqlCommitGraphStorageBuilder {
    pub fn build(self, opts: RendezVousOptions, repo_id: RepositoryId) -> SqlCommitGraphStorage {
        SqlCommitGraphStorage {
            repo_id,
            read_connection: RendezVousConnection::new(
                self.connections.read_connection,
                "read",
                opts,
            ),
            read_master_connection: RendezVousConnection::new(
                self.connections.read_master_connection,
                "read_master",
                opts,
            ),
            write_connection: self.connections.write_connection,
        }
    }
}

#[derive(Clone)]
struct RendezVousConnection {
    fetch_single: RendezVous<ChangesetId, ChangesetEdges>,
    conn: Connection,
}

impl RendezVousConnection {
    fn new(conn: Connection, name: &str, opts: RendezVousOptions) -> Self {
        Self {
            conn,
            fetch_single: RendezVous::new(
                TunablesRendezVousController::new(opts),
                Arc::new(RendezVousStats::new(format!(
                    "commit_graph.fetch_single.{}",
                    name
                ))),
            ),
        }
    }
}

pub struct SqlCommitGraphStorage {
    repo_id: RepositoryId,
    write_connection: Connection,
    read_connection: RendezVousConnection,
    read_master_connection: RendezVousConnection,
}

mononoke_queries! {
    write InsertChangeset(
        repo_id: RepositoryId,
        cs_id: ChangesetId,
        gen: u64,
        skip_tree_depth: u64,
        p1_linear_depth: u64,
        parent_count: usize,
        p1_parent: Option<ChangesetId>,
        merge_ancestor: Option<ChangesetId>,
        skip_tree_parent: Option<ChangesetId>,
        skip_tree_skew_ancestor: Option<ChangesetId>,
        p1_linear_skew_ancestor: Option<ChangesetId>
    ) {
        insert_or_ignore,
        "
        {insert_or_ignore} INTO commit_graph_edges (
            repo_id,
            cs_id,
            gen,
            skip_tree_depth,
            p1_linear_depth,
            parent_count,
            p1_parent,
            merge_ancestor,
            skip_tree_parent,
            skip_tree_skew_ancestor,
            p1_linear_skew_ancestor
        ) VALUES (
            {repo_id},
            {cs_id},
            {gen},
            {skip_tree_depth},
            {p1_linear_depth},
            {parent_count},
            (SELECT cs.id FROM commit_graph_edges cs WHERE cs.repo_id = {repo_id} AND cs.cs_id = {p1_parent}),
            (SELECT cs.id FROM commit_graph_edges cs WHERE cs.repo_id = {repo_id} AND cs.cs_id = {merge_ancestor}),
            (SELECT cs.id FROM commit_graph_edges cs WHERE cs.repo_id = {repo_id} AND cs.cs_id = {skip_tree_parent}),
            (SELECT cs.id FROM commit_graph_edges cs WHERE cs.repo_id = {repo_id} AND cs.cs_id = {skip_tree_skew_ancestor}),
            (SELECT cs.id FROM commit_graph_edges cs WHERE cs.repo_id = {repo_id} AND cs.cs_id = {p1_linear_skew_ancestor})
        )
        "
    }

    write InsertChangesetsNoEdges(values: (
        repo_id: RepositoryId,
        cs_id: ChangesetId,
        gen: u64,
        skip_tree_depth: u64,
        p1_linear_depth: u64,
        parent_count: usize,
    )) {
        insert_or_ignore,
        "
        {insert_or_ignore} INTO commit_graph_edges (
            repo_id,
            cs_id,
            gen,
            skip_tree_depth,
            p1_linear_depth,
            parent_count
        ) VALUES {values}
        "
    }

    // Fix edges for changesets previously added with InsertChangesetsNoEdges
    write FixEdges(values: (
        repo_id: RepositoryId,
        cs_id: ChangesetId,
        // We need the depths otherwise we get an error on sqlite. Though this won't be used because we
        // always replace the edges only.
        gen: u64,
        skip_tree_depth: u64,
        p1_linear_depth: u64,
        parent_count: usize,
        p1_parent: Option<u64>,
        merge_ancestor: Option<u64>,
        skip_tree_parent: Option<u64>,
        skip_tree_skew_ancestor: Option<u64>,
        p1_linear_skew_ancestor: Option<u64>
    )) {
        none,
        mysql("INSERT INTO commit_graph_edges
            (repo_id, cs_id, gen, skip_tree_depth, p1_linear_depth, parent_count,
                p1_parent, merge_ancestor, skip_tree_parent, skip_tree_skew_ancestor, p1_linear_skew_ancestor)
        VALUES {values}
        ON DUPLICATE KEY UPDATE
            p1_parent = VALUES(p1_parent),
            merge_ancestor = VALUES(merge_ancestor),
            skip_tree_parent = VALUES(skip_tree_parent),
            skip_tree_skew_ancestor = VALUES(skip_tree_skew_ancestor),
            p1_linear_skew_ancestor = VALUES(p1_linear_skew_ancestor)")
        sqlite("INSERT INTO commit_graph_edges
            (repo_id, cs_id, gen, skip_tree_depth, p1_linear_depth, parent_count,
                p1_parent, merge_ancestor, skip_tree_parent, skip_tree_skew_ancestor, p1_linear_skew_ancestor)
        VALUES {values}
        ON CONFLICT(repo_id, cs_id) DO UPDATE SET
            p1_parent = excluded.p1_parent,
            merge_ancestor = excluded.merge_ancestor,
            skip_tree_parent = excluded.skip_tree_parent,
            skip_tree_skew_ancestor = excluded.skip_tree_skew_ancestor,
            p1_linear_skew_ancestor = excluded.p1_linear_skew_ancestor")
    }

    read SelectManyIds(repo_id: RepositoryId, >list cs_ids: ChangesetId) -> (ChangesetId, u64) {
        "SELECT cs.cs_id, cs.id FROM commit_graph_edges cs WHERE cs.repo_id = {repo_id} AND cs.cs_id IN {cs_ids}"
    }

    write InsertMergeParents(values: (id: u64, parent_num: usize, parent: u64)) {
        insert_or_ignore,
        "{insert_or_ignore} INTO commit_graph_merge_parents (id, parent_num, parent) VALUES {values}"
    }

    read SelectManyChangesets(repo_id: RepositoryId, >list cs_ids: ChangesetId) -> (
        ChangesetId, // cs_id
        Option<u64>, // gen
        Option<u64>, // skip_tree_depth
        Option<u64>, // p1_linear_depth
        Option<usize>, // parent_count
        Option<ChangesetId>, // merge_ancestor
        Option<u64>, // merge_ancestor_gen
        Option<u64>, // merge_ancestor_skip_tree_depth
        Option<u64>, // merge_ancestor_p1_linear_depth
        Option<ChangesetId>, // skip_tree_parent
        Option<u64>, // skip_tree_parent_gen
        Option<u64>, // skip_tree_parent_skip_tree_depth
        Option<u64>, // skip_tree_parent_p1_linear_depth
        Option<ChangesetId>, // skip_tree_skew_ancestor
        Option<u64>, // skip_tree_skew_ancestor_gen
        Option<u64>, // skip_tree_skew_ancestor_skip_tree_depth
        Option<u64>, // skip_tree_skew_ancestor_p1_linear_depth
        Option<ChangesetId>, // p1_linear_skew_ancestor
        Option<u64>, // p1_linear_skew_ancestor_gen
        Option<u64>, // p1_linear_skew_ancestor_skip_tree_depth
        Option<u64>, // p1_linear_skew_ancestor_p1_linear_depth
        usize, // parent_num
        Option<ChangesetId>, // parent
        Option<u64>, // parent_gen
        Option<u64>, // parent_skip_tree_depth
        Option<u64>, // parent_p1_linear_depth
    ) {
        "
        SELECT
            cs0.cs_id AS cs_id,
            NULL AS gen,
            NULL AS skip_tree_depth,
            NULL AS p1_linear_depth,
            NULL AS parent_count,
            NULL AS merge_ancestor,
            NULL AS merge_ancestor_gen,
            NULL AS merge_ancestor_skip_tree_depth,
            NULL AS merge_ancestor_p1_linear_depth,
            NULL AS skip_tree_parent,
            NULL AS skip_tree_parent_gen,
            NULL AS skip_tree_parent_skip_tree_depth,
            NULL AS skip_tree_parent_p1_linear_depth,
            NULL AS skip_tree_skew_ancestor,
            NULL AS skip_tree_skew_ancestor_gen,
            NULL AS skip_tree_skew_ancestor_skip_tree_depth,
            NULL AS skip_tree_skew_ancestor_p1_linear_depth,
            NULL AS p1_linear_skew_ancestor,
            NULL AS p1_linear_skew_ancestor_gen,
            NULL AS p1_linear_skew_ancestor_skip_tree_depth,
            NULL AS p1_linear_skew_ancestor_p1_linear_depth,
            commit_graph_merge_parents.parent_num AS parent_num,
            cs1.cs_id AS parent,
            cs1.gen AS parent_gen,
            cs1.skip_tree_depth AS parent_skip_tree_depth,
            cs1.p1_linear_depth AS parent_p1_linear_depth
        FROM commit_graph_merge_parents
        INNER JOIN commit_graph_edges cs0 ON cs0.id = commit_graph_merge_parents.id
        INNER JOIN commit_graph_edges cs1 ON cs1.id = commit_graph_merge_parents.parent
        WHERE cs0.repo_id = {repo_id} AND cs0.cs_id IN {cs_ids} AND cs1.repo_id = {repo_id} AND cs0.parent_count >= 2

        UNION

        SELECT
            cs0.cs_id AS cs_id,
            cs0.gen AS gen,
            cs0.skip_tree_depth AS skip_tree_depth,
            cs0.p1_linear_depth AS p1_linear_depth,
            cs0.parent_count AS parent_count,
            cs_merge_ancestor.cs_id AS merge_ancestor,
            cs_merge_ancestor.gen AS merge_ancestor_gen,
            cs_merge_ancestor.skip_tree_depth AS merge_ancestor_skip_tree_depth,
            cs_merge_ancestor.p1_linear_depth AS merge_ancestor_p1_linear_depth,
            cs_skip_tree_parent.cs_id AS skip_tree_parent,
            cs_skip_tree_parent.gen AS skip_tree_parent_gen,
            cs_skip_tree_parent.skip_tree_depth AS skip_tree_parent_skip_tree_depth,
            cs_skip_tree_parent.p1_linear_depth AS skip_tree_parent_p1_linear_depth,
            cs_skip_tree_skew_ancestor.cs_id AS skip_tree_skew_ancestor,
            cs_skip_tree_skew_ancestor.gen AS skip_tree_skew_ancestor_gen,
            cs_skip_tree_skew_ancestor.skip_tree_depth AS skip_tree_skew_ancestor_skip_tree_depth,
            cs_skip_tree_skew_ancestor.p1_linear_depth AS skip_tree_skew_ancestor_p1_linear_depth,
            cs_p1_linear_skew_ancestor.cs_id AS p1_linear_skew_ancestor,
            cs_p1_linear_skew_ancestor.gen AS p1_linear_skew_ancestor_gen,
            cs_p1_linear_skew_ancestor.skip_tree_depth AS p1_linear_skew_ancestor_skip_tree_depth,
            cs_p1_linear_skew_ancestor.p1_linear_depth AS p1_linear_skew_ancestor_p1_linear_depth,
            0 AS parent_num,
            cs_p1_parent.cs_id AS parent,
            cs_p1_parent.gen AS parent_gen,
            cs_p1_parent.skip_tree_depth AS parent_skip_tree_depth,
            cs_p1_parent.p1_linear_depth AS parent_p1_linear_depth
        FROM commit_graph_edges cs0
        LEFT JOIN commit_graph_edges cs_p1_parent ON cs_p1_parent.id = cs0.p1_parent
        LEFT JOIN commit_graph_edges cs_merge_ancestor ON cs_merge_ancestor.id = cs0.merge_ancestor
        LEFT JOIN commit_graph_edges cs_skip_tree_parent ON cs_skip_tree_parent.id = cs0.skip_tree_parent
        LEFT JOIN commit_graph_edges cs_skip_tree_skew_ancestor ON cs_skip_tree_skew_ancestor.id = cs0.skip_tree_skew_ancestor
        LEFT JOIN commit_graph_edges cs_p1_linear_skew_ancestor ON cs_p1_linear_skew_ancestor.id = cs0.p1_linear_skew_ancestor
        WHERE cs0.repo_id = {repo_id} and cs0.cs_id IN {cs_ids}

        ORDER BY parent_num ASC
        "
    }

    read SelectManyChangesetsWithFirstParentPrefetch(repo_id: RepositoryId, step_limit: u64, prefetch_gen: u64, >list cs_ids: ChangesetId) -> (
        ChangesetId, // cs_id
        Option<u64>, // gen
        Option<u64>, // skip_tree_depth
        Option<u64>, // p1_linear_depth
        Option<usize>, // parent_count
        Option<ChangesetId>, // merge_ancestor
        Option<u64>, // merge_ancestor_gen
        Option<u64>, // merge_ancestor_skip_tree_depth
        Option<u64>, // merge_ancestor_p1_linear_depth
        Option<ChangesetId>, // skip_tree_parent
        Option<u64>, // skip_tree_parent_gen
        Option<u64>, // skip_tree_parent_skip_tree_depth
        Option<u64>, // skip_tree_parent_p1_linear_depth
        Option<ChangesetId>, // skip_tree_skew_ancestor
        Option<u64>, // skip_tree_skew_ancestor_gen
        Option<u64>, // skip_tree_skew_ancestor_skip_tree_depth
        Option<u64>, // skip_tree_skew_ancestor_p1_linear_depth
        Option<ChangesetId>, // p1_linear_skew_ancestor
        Option<u64>, // p1_linear_skew_ancestor_gen
        Option<u64>, // p1_linear_skew_ancestor_skip_tree_depth
        Option<u64>, // p1_linear_skew_ancestor_p1_linear_depth
        usize, // parent_num
        Option<ChangesetId>, // parent
        Option<u64>, // parent_gen
        Option<u64>, // parent_skip_tree_depth
        Option<u64>, // parent_p1_linear_depth
    ) {
        "
        WITH RECURSIVE csp AS (
            SELECT
                cs.id, 1 AS step, cs.p1_parent AS next
            FROM commit_graph_edges cs
            WHERE cs.repo_id = {repo_id} AND cs.cs_id IN {cs_ids}
            UNION ALL
            SELECT
                cs.id, csp.step + 1, cs.p1_parent AS next
            FROM csp
            INNER JOIN commit_graph_edges cs ON cs.id = csp.next
            WHERE csp.step < {step_limit} AND cs.gen >= {prefetch_gen}
        )

        SELECT
            cs0.cs_id AS cs_id,
            NULL AS gen,
            NULL AS skip_tree_depth,
            NULL AS p1_linear_depth,
            NULL AS parent_count,
            NULL AS merge_ancestor,
            NULL AS merge_ancestor_gen,
            NULL AS merge_ancestor_skip_tree_depth,
            NULL AS merge_ancestor_p1_linear_depth,
            NULL AS skip_tree_parent,
            NULL AS skip_tree_parent_gen,
            NULL AS skip_tree_parent_skip_tree_depth,
            NULL AS skip_tree_parent_p1_linear_depth,
            NULL AS skip_tree_skew_ancestor,
            NULL AS skip_tree_skew_ancestor_gen,
            NULL AS skip_tree_skew_ancestor_skip_tree_depth,
            NULL AS skip_tree_skew_ancestor_p1_linear_depth,
            NULL AS p1_linear_skew_ancestor,
            NULL AS p1_linear_skew_ancestor_gen,
            NULL AS p1_linear_skew_ancestor_skip_tree_depth,
            NULL AS p1_linear_skew_ancestor_p1_linear_depth,
            cgmp.parent_num AS parent_num,
            cs1.cs_id AS parent,
            cs1.gen AS parent_gen,
            cs1.skip_tree_depth AS parent_skip_tree_depth,
            cs1.p1_linear_depth AS parent_p1_linear_depth
        FROM csp
        INNER JOIN commit_graph_merge_parents cgmp ON csp.id = cgmp.id
        INNER JOIN commit_graph_edges cs0 ON cs0.id = cgmp.id
        INNER JOIN commit_graph_edges cs1 ON cs1.id = cgmp.parent
        WHERE cs0.parent_count >= 2

        UNION

        SELECT
            cs0.cs_id AS cs_id,
            cs0.gen AS gen,
            cs0.skip_tree_depth AS skip_tree_depth,
            cs0.p1_linear_depth AS p1_linear_depth,
            cs0.parent_count AS parent_count,
            cs_merge_ancestor.cs_id AS merge_ancestor,
            cs_merge_ancestor.gen AS merge_ancestor_gen,
            cs_merge_ancestor.skip_tree_depth AS merge_ancestor_skip_tree_depth,
            cs_merge_ancestor.p1_linear_depth AS merge_ancestor_p1_linear_depth,
            cs_skip_tree_parent.cs_id AS skip_tree_parent,
            cs_skip_tree_parent.gen AS skip_tree_parent_gen,
            cs_skip_tree_parent.skip_tree_depth AS skip_tree_parent_skip_tree_depth,
            cs_skip_tree_parent.p1_linear_depth AS skip_tree_parent_p1_linear_depth,
            cs_skip_tree_skew_ancestor.cs_id AS skip_tree_skew_ancestor,
            cs_skip_tree_skew_ancestor.gen AS skip_tree_skew_ancestor_gen,
            cs_skip_tree_skew_ancestor.skip_tree_depth AS skip_tree_skew_ancestor_skip_tree_depth,
            cs_skip_tree_skew_ancestor.p1_linear_depth AS skip_tree_skew_ancestor_p1_linear_depth,
            cs_p1_linear_skew_ancestor.cs_id AS p1_linear_skew_ancestor,
            cs_p1_linear_skew_ancestor.gen AS p1_linear_skew_ancestor_gen,
            cs_p1_linear_skew_ancestor.skip_tree_depth AS p1_linear_skew_ancestor_skip_tree_depth,
            cs_p1_linear_skew_ancestor.p1_linear_depth AS p1_linear_skew_ancestor_p1_linear_depth,
            0 AS parent_num,
            cs_p1_parent.cs_id AS parent,
            cs_p1_parent.gen AS parent_gen,
            cs_p1_parent.skip_tree_depth AS parent_skip_tree_depth,
            cs_p1_parent.p1_linear_depth AS parent_p1_linear_depth
        FROM csp
        INNER JOIN commit_graph_edges cs0 ON csp.id = cs0.id
        LEFT JOIN commit_graph_edges cs_p1_parent ON cs_p1_parent.id = cs0.p1_parent
        LEFT JOIN commit_graph_edges cs_merge_ancestor ON cs_merge_ancestor.id = cs0.merge_ancestor
        LEFT JOIN commit_graph_edges cs_skip_tree_parent ON cs_skip_tree_parent.id = cs0.skip_tree_parent
        LEFT JOIN commit_graph_edges cs_skip_tree_skew_ancestor ON cs_skip_tree_skew_ancestor.id = cs0.skip_tree_skew_ancestor
        LEFT JOIN commit_graph_edges cs_p1_linear_skew_ancestor ON cs_p1_linear_skew_ancestor.id = cs0.p1_linear_skew_ancestor
        ORDER BY parent_num ASC
        "
    }

    read SelectManyChangesetsWithSkipTreeSkewAncestorPrefetch(repo_id: RepositoryId, step_limit: u64, prefetch_gen: u64, >list cs_ids: ChangesetId) -> (
        ChangesetId, // cs_id
        Option<u64>, // gen
        Option<u64>, // skip_tree_depth
        Option<u64>, // p1_linear_depth
        Option<usize>, // parent_count
        Option<ChangesetId>, // merge_ancestor
        Option<u64>, // merge_ancestor_gen
        Option<u64>, // merge_ancestor_skip_tree_depth
        Option<u64>, // merge_ancestor_p1_linear_depth
        Option<ChangesetId>, // skip_tree_parent
        Option<u64>, // skip_tree_parent_gen
        Option<u64>, // skip_tree_parent_skip_tree_depth
        Option<u64>, // skip_tree_parent_p1_linear_depth
        Option<ChangesetId>, // skip_tree_skew_ancestor
        Option<u64>, // skip_tree_skew_ancestor_gen
        Option<u64>, // skip_tree_skew_ancestor_skip_tree_depth
        Option<u64>, // skip_tree_skew_ancestor_p1_linear_depth
        Option<ChangesetId>, // p1_linear_skew_ancestor
        Option<u64>, // p1_linear_skew_ancestor_gen
        Option<u64>, // p1_linear_skew_ancestor_skip_tree_depth
        Option<u64>, // p1_linear_skew_ancestor_p1_linear_depth
        usize, // parent_num
        Option<ChangesetId>, // parent
        Option<u64>, // parent_gen
        Option<u64>, // parent_skip_tree_depth
        Option<u64>, // parent_p1_linear_depth
    ) {
        "
        WITH RECURSIVE csp AS (
            SELECT
                cs.id, 1 AS step, COALESCE(cs.skip_tree_skew_ancestor, cs.p1_parent) AS next
            FROM commit_graph_edges cs
            WHERE cs.repo_id = {repo_id} AND cs.cs_id IN {cs_ids}
            UNION ALL
            SELECT
                cs.id, csp.step + 1, COALESCE(cs.skip_tree_skew_ancestor, cs.p1_parent) AS next
            FROM csp
            INNER JOIN commit_graph_edges cs ON cs.id = csp.next
            WHERE csp.step < {step_limit} AND cs.gen >= {prefetch_gen}
        )

        SELECT
            cs0.cs_id AS cs_id,
            NULL AS gen,
            NULL AS skip_tree_depth,
            NULL AS p1_linear_depth,
            NULL AS parent_count,
            NULL AS merge_ancestor,
            NULL AS merge_ancestor_gen,
            NULL AS merge_ancestor_skip_tree_depth,
            NULL AS merge_ancestor_p1_linear_depth,
            NULL AS skip_tree_parent,
            NULL AS skip_tree_parent_gen,
            NULL AS skip_tree_parent_skip_tree_depth,
            NULL AS skip_tree_parent_p1_linear_depth,
            NULL AS skip_tree_skew_ancestor,
            NULL AS skip_tree_skew_ancestor_gen,
            NULL AS skip_tree_skew_ancestor_skip_tree_depth,
            NULL AS skip_tree_skew_ancestor_p1_linear_depth,
            NULL AS p1_linear_skew_ancestor,
            NULL AS p1_linear_skew_ancestor_gen,
            NULL AS p1_linear_skew_ancestor_skip_tree_depth,
            NULL AS p1_linear_skew_ancestor_p1_linear_depth,
            cgmp.parent_num AS parent_num,
            cs1.cs_id AS parent,
            cs1.gen AS parent_gen,
            cs1.skip_tree_depth AS parent_skip_tree_depth,
            cs1.p1_linear_depth AS parent_p1_linear_depth
        FROM csp
        INNER JOIN commit_graph_merge_parents cgmp ON csp.id = cgmp.id
        INNER JOIN commit_graph_edges cs0 ON cs0.id = cgmp.id
        INNER JOIN commit_graph_edges cs1 ON cs1.id = cgmp.parent
        WHERE cs0.parent_count >= 2

        UNION

        SELECT
            cs0.cs_id AS cs_id,
            cs0.gen AS gen,
            cs0.skip_tree_depth AS skip_tree_depth,
            cs0.p1_linear_depth AS p1_linear_depth,
            cs0.parent_count AS parent_count,
            cs_merge_ancestor.cs_id AS merge_ancestor,
            cs_merge_ancestor.gen AS merge_ancestor_gen,
            cs_merge_ancestor.skip_tree_depth AS merge_ancestor_skip_tree_depth,
            cs_merge_ancestor.p1_linear_depth AS merge_ancestor_p1_linear_depth,
            cs_skip_tree_parent.cs_id AS skip_tree_parent,
            cs_skip_tree_parent.gen AS skip_tree_parent_gen,
            cs_skip_tree_parent.skip_tree_depth AS skip_tree_parent_skip_tree_depth,
            cs_skip_tree_parent.p1_linear_depth AS skip_tree_parent_p1_linear_depth,
            cs_skip_tree_skew_ancestor.cs_id AS skip_tree_skew_ancestor,
            cs_skip_tree_skew_ancestor.gen AS skip_tree_skew_ancestor_gen,
            cs_skip_tree_skew_ancestor.skip_tree_depth AS skip_tree_skew_ancestor_skip_tree_depth,
            cs_skip_tree_skew_ancestor.p1_linear_depth AS skip_tree_skew_ancestor_p1_linear_depth,
            cs_p1_linear_skew_ancestor.cs_id AS p1_linear_skew_ancestor,
            cs_p1_linear_skew_ancestor.gen AS p1_linear_skew_ancestor_gen,
            cs_p1_linear_skew_ancestor.skip_tree_depth AS p1_linear_skew_ancestor_skip_tree_depth,
            cs_p1_linear_skew_ancestor.p1_linear_depth AS p1_linear_skew_ancestor_p1_linear_depth,
            0 AS parent_num,
            cs_p1_parent.cs_id AS parent,
            cs_p1_parent.gen AS parent_gen,
            cs_p1_parent.skip_tree_depth AS parent_skip_tree_depth,
            cs_p1_parent.p1_linear_depth AS parent_p1_linear_depth
        FROM csp
        INNER JOIN commit_graph_edges cs0 ON csp.id = cs0.id
        LEFT JOIN commit_graph_edges cs_p1_parent ON cs_p1_parent.id = cs0.p1_parent
        LEFT JOIN commit_graph_edges cs_merge_ancestor ON cs_merge_ancestor.id = cs0.merge_ancestor
        LEFT JOIN commit_graph_edges cs_skip_tree_parent ON cs_skip_tree_parent.id = cs0.skip_tree_parent
        LEFT JOIN commit_graph_edges cs_skip_tree_skew_ancestor ON cs_skip_tree_skew_ancestor.id = cs0.skip_tree_skew_ancestor
        LEFT JOIN commit_graph_edges cs_p1_linear_skew_ancestor ON cs_p1_linear_skew_ancestor.id = cs0.p1_linear_skew_ancestor
        ORDER BY parent_num ASC
        "
    }

    // The only difference between mysql and sqlite is the FORCE INDEX
    read SelectManyChangesetsInIdRange(repo_id: RepositoryId, start_id: u64, end_id: u64, limit: u64) -> (
        ChangesetId, // cs_id
        Option<u64>, // gen
        Option<u64>, // skip_tree_depth
        Option<u64>, // p1_linear_depth
        Option<usize>, // parent_count
        Option<ChangesetId>, // merge_ancestor
        Option<u64>, // merge_ancestor_gen
        Option<u64>, // merge_ancestor_skip_tree_depth
        Option<u64>, // merge_ancestor_p1_linear_depth
        Option<ChangesetId>, // skip_tree_parent
        Option<u64>, // skip_tree_parent_gen
        Option<u64>, // skip_tree_parent_skip_tree_depth
        Option<u64>, // skip_tree_parent_p1_linear_depth
        Option<ChangesetId>, // skip_tree_skew_ancestor
        Option<u64>, // skip_tree_skew_ancestor_gen
        Option<u64>, // skip_tree_skew_ancestor_skip_tree_depth
        Option<u64>, // skip_tree_skew_ancestor_p1_linear_depth
        Option<ChangesetId>, // p1_linear_skew_ancestor
        Option<u64>, // p1_linear_skew_ancestor_gen
        Option<u64>, // p1_linear_skew_ancestor_skip_tree_depth
        Option<u64>, // p1_linear_skew_ancestor_p1_linear_depth
        usize, // parent_num
        Option<ChangesetId>, // parent
        Option<u64>, // parent_gen
        Option<u64>, // parent_skip_tree_depth
        Option<u64>, // parent_p1_linear_depth
    ) {
        mysql("WITH csp AS (
            SELECT cs.id
            FROM commit_graph_edges cs FORCE INDEX(repo_id_id)
            WHERE cs.repo_id = {repo_id} AND cs.id >= {start_id} AND cs.id <= {end_id}
            ORDER BY cs.id ASC
            LIMIT {limit}
        )

        SELECT
            cs0.cs_id AS cs_id,
            NULL AS gen,
            NULL AS skip_tree_depth,
            NULL AS p1_linear_depth,
            NULL AS parent_count,
            NULL AS merge_ancestor,
            NULL AS merge_ancestor_gen,
            NULL AS merge_ancestor_skip_tree_depth,
            NULL AS merge_ancestor_p1_linear_depth,
            NULL AS skip_tree_parent,
            NULL AS skip_tree_parent_gen,
            NULL AS skip_tree_parent_skip_tree_depth,
            NULL AS skip_tree_parent_p1_linear_depth,
            NULL AS skip_tree_skew_ancestor,
            NULL AS skip_tree_skew_ancestor_gen,
            NULL AS skip_tree_skew_ancestor_skip_tree_depth,
            NULL AS skip_tree_skew_ancestor_p1_linear_depth,
            NULL AS p1_linear_skew_ancestor,
            NULL AS p1_linear_skew_ancestor_gen,
            NULL AS p1_linear_skew_ancestor_skip_tree_depth,
            NULL AS p1_linear_skew_ancestor_p1_linear_depth,
            cgmp.parent_num AS parent_num,
            cs1.cs_id AS parent,
            cs1.gen AS parent_gen,
            cs1.skip_tree_depth AS parent_skip_tree_depth,
            cs1.p1_linear_depth AS parent_p1_linear_depth
        FROM csp
        INNER JOIN commit_graph_merge_parents cgmp ON csp.id = cgmp.id
        INNER JOIN commit_graph_edges cs0 ON cs0.id = cgmp.id
        INNER JOIN commit_graph_edges cs1 ON cs1.id = cgmp.parent
        WHERE cs0.parent_count >= 2

        UNION

        SELECT
            cs0.cs_id AS cs_id,
            cs0.gen AS gen,
            cs0.skip_tree_depth AS skip_tree_depth,
            cs0.p1_linear_depth AS p1_linear_depth,
            cs0.parent_count AS parent_count,
            cs_merge_ancestor.cs_id AS merge_ancestor,
            cs_merge_ancestor.gen AS merge_ancestor_gen,
            cs_merge_ancestor.skip_tree_depth AS merge_ancestor_skip_tree_depth,
            cs_merge_ancestor.p1_linear_depth AS merge_ancestor_p1_linear_depth,
            cs_skip_tree_parent.cs_id AS skip_tree_parent,
            cs_skip_tree_parent.gen AS skip_tree_parent_gen,
            cs_skip_tree_parent.skip_tree_depth AS skip_tree_parent_skip_tree_depth,
            cs_skip_tree_parent.p1_linear_depth AS skip_tree_parent_p1_linear_depth,
            cs_skip_tree_skew_ancestor.cs_id AS skip_tree_skew_ancestor,
            cs_skip_tree_skew_ancestor.gen AS skip_tree_skew_ancestor_gen,
            cs_skip_tree_skew_ancestor.skip_tree_depth AS skip_tree_skew_ancestor_skip_tree_depth,
            cs_skip_tree_skew_ancestor.p1_linear_depth AS skip_tree_skew_ancestor_p1_linear_depth,
            cs_p1_linear_skew_ancestor.cs_id AS p1_linear_skew_ancestor,
            cs_p1_linear_skew_ancestor.gen AS p1_linear_skew_ancestor_gen,
            cs_p1_linear_skew_ancestor.skip_tree_depth AS p1_linear_skew_ancestor_skip_tree_depth,
            cs_p1_linear_skew_ancestor.p1_linear_depth AS p1_linear_skew_ancestor_p1_linear_depth,
            0 AS parent_num,
            cs_p1_parent.cs_id AS parent,
            cs_p1_parent.gen AS parent_gen,
            cs_p1_parent.skip_tree_depth AS parent_skip_tree_depth,
            cs_p1_parent.p1_linear_depth AS parent_p1_linear_depth
        FROM csp
        INNER JOIN commit_graph_edges cs0 ON csp.id = cs0.id
        LEFT JOIN commit_graph_edges cs_p1_parent ON cs_p1_parent.id = cs0.p1_parent
        LEFT JOIN commit_graph_edges cs_merge_ancestor ON cs_merge_ancestor.id = cs0.merge_ancestor
        LEFT JOIN commit_graph_edges cs_skip_tree_parent ON cs_skip_tree_parent.id = cs0.skip_tree_parent
        LEFT JOIN commit_graph_edges cs_skip_tree_skew_ancestor ON cs_skip_tree_skew_ancestor.id = cs0.skip_tree_skew_ancestor
        LEFT JOIN commit_graph_edges cs_p1_linear_skew_ancestor ON cs_p1_linear_skew_ancestor.id = cs0.p1_linear_skew_ancestor
        ORDER BY parent_num ASC")
        sqlite("WITH csp AS (
            SELECT cs.id
            FROM commit_graph_edges cs
            WHERE cs.repo_id = {repo_id} AND cs.id >= {start_id} AND cs.id <= {end_id}
            ORDER BY cs.id ASC
            LIMIT {limit}
        )

        SELECT
            cs0.cs_id AS cs_id,
            NULL AS gen,
            NULL AS skip_tree_depth,
            NULL AS p1_linear_depth,
            NULL AS parent_count,
            NULL AS merge_ancestor,
            NULL AS merge_ancestor_gen,
            NULL AS merge_ancestor_skip_tree_depth,
            NULL AS merge_ancestor_p1_linear_depth,
            NULL AS skip_tree_parent,
            NULL AS skip_tree_parent_gen,
            NULL AS skip_tree_parent_skip_tree_depth,
            NULL AS skip_tree_parent_p1_linear_depth,
            NULL AS skip_tree_skew_ancestor,
            NULL AS skip_tree_skew_ancestor_gen,
            NULL AS skip_tree_skew_ancestor_skip_tree_depth,
            NULL AS skip_tree_skew_ancestor_p1_linear_depth,
            NULL AS p1_linear_skew_ancestor,
            NULL AS p1_linear_skew_ancestor_gen,
            NULL AS p1_linear_skew_ancestor_skip_tree_depth,
            NULL AS p1_linear_skew_ancestor_p1_linear_depth,
            cgmp.parent_num AS parent_num,
            cs1.cs_id AS parent,
            cs1.gen AS parent_gen,
            cs1.skip_tree_depth AS parent_skip_tree_depth,
            cs1.p1_linear_depth AS parent_p1_linear_depth
        FROM csp
        INNER JOIN commit_graph_merge_parents cgmp ON csp.id = cgmp.id
        INNER JOIN commit_graph_edges cs0 ON cs0.id = cgmp.id
        INNER JOIN commit_graph_edges cs1 ON cs1.id = cgmp.parent
        WHERE cs0.parent_count >= 2

        UNION

        SELECT
            cs0.cs_id AS cs_id,
            cs0.gen AS gen,
            cs0.skip_tree_depth AS skip_tree_depth,
            cs0.p1_linear_depth AS p1_linear_depth,
            cs0.parent_count AS parent_count,
            cs_merge_ancestor.cs_id AS merge_ancestor,
            cs_merge_ancestor.gen AS merge_ancestor_gen,
            cs_merge_ancestor.skip_tree_depth AS merge_ancestor_skip_tree_depth,
            cs_merge_ancestor.p1_linear_depth AS merge_ancestor_p1_linear_depth,
            cs_skip_tree_parent.cs_id AS skip_tree_parent,
            cs_skip_tree_parent.gen AS skip_tree_parent_gen,
            cs_skip_tree_parent.skip_tree_depth AS skip_tree_parent_skip_tree_depth,
            cs_skip_tree_parent.p1_linear_depth AS skip_tree_parent_p1_linear_depth,
            cs_skip_tree_skew_ancestor.cs_id AS skip_tree_skew_ancestor,
            cs_skip_tree_skew_ancestor.gen AS skip_tree_skew_ancestor_gen,
            cs_skip_tree_skew_ancestor.skip_tree_depth AS skip_tree_skew_ancestor_skip_tree_depth,
            cs_skip_tree_skew_ancestor.p1_linear_depth AS skip_tree_skew_ancestor_p1_linear_depth,
            cs_p1_linear_skew_ancestor.cs_id AS p1_linear_skew_ancestor,
            cs_p1_linear_skew_ancestor.gen AS p1_linear_skew_ancestor_gen,
            cs_p1_linear_skew_ancestor.skip_tree_depth AS p1_linear_skew_ancestor_skip_tree_depth,
            cs_p1_linear_skew_ancestor.p1_linear_depth AS p1_linear_skew_ancestor_p1_linear_depth,
            0 AS parent_num,
            cs_p1_parent.cs_id AS parent,
            cs_p1_parent.gen AS parent_gen,
            cs_p1_parent.skip_tree_depth AS parent_skip_tree_depth,
            cs_p1_parent.p1_linear_depth AS parent_p1_linear_depth
        FROM csp
        INNER JOIN commit_graph_edges cs0 ON csp.id = cs0.id
        LEFT JOIN commit_graph_edges cs_p1_parent ON cs_p1_parent.id = cs0.p1_parent
        LEFT JOIN commit_graph_edges cs_merge_ancestor ON cs_merge_ancestor.id = cs0.merge_ancestor
        LEFT JOIN commit_graph_edges cs_skip_tree_parent ON cs_skip_tree_parent.id = cs0.skip_tree_parent
        LEFT JOIN commit_graph_edges cs_skip_tree_skew_ancestor ON cs_skip_tree_skew_ancestor.id = cs0.skip_tree_skew_ancestor
        LEFT JOIN commit_graph_edges cs_p1_linear_skew_ancestor ON cs_p1_linear_skew_ancestor.id = cs0.p1_linear_skew_ancestor
        ORDER BY parent_num ASC")
    }

    // The only difference between mysql and sqlite is the FORCE INDEX
    read SelectManyChangesetsIdsInIdRange(repo_id: RepositoryId, start_id: u64, end_id: u64, limit: u64) -> (ChangesetId) {
        mysql("SELECT cs.cs_id
        FROM commit_graph_edges cs FORCE INDEX(repo_id_id)
        WHERE cs.repo_id = {repo_id} AND cs.id >= {start_id} AND cs.id <= {end_id}
        ORDER BY cs.id ASC
        LIMIT {limit}")
        sqlite("SELECT cs.cs_id
        FROM commit_graph_edges cs
        WHERE cs.repo_id = {repo_id} AND cs.id >= {start_id} AND cs.id <= {end_id}
        ORDER BY cs.id ASC
        LIMIT {limit}")
    }

    // The only difference between mysql and sqlite is the FORCE INDEX
    read SelectMaxIdInRange(repo_id: RepositoryId, start_id: u64, end_id: u64, limit: u64) -> (u64) {
        mysql("SELECT MAX(id)
        FROM (
            SELECT id
            FROM commit_graph_edges FORCE INDEX(repo_id_id)
            WHERE repo_id = {repo_id} AND id >= {start_id} AND id <= {end_id}
            ORDER BY id ASC
            LIMIT {limit}
        ) AS ids")
        sqlite("SELECT MAX(id)
        FROM (
            SELECT id
            FROM commit_graph_edges
            WHERE repo_id = {repo_id} AND id >= {start_id} AND id <= {end_id}
            ORDER BY id ASC
            LIMIT {limit}
        ) AS ids")
    }

    read SelectMaxId(repo_id: RepositoryId) -> (u64) {
        "
        SELECT MAX(id)
        FROM commit_graph_edges
        WHERE repo_id = {repo_id}
        "
    }

    read SelectChangesetsInRange(repo_id: RepositoryId, min_id: ChangesetId, max_id: ChangesetId, limit: usize) -> (ChangesetId) {
        "
        SELECT cs_id
        FROM commit_graph_edges
        WHERE repo_id = {repo_id} AND {min_id} <= cs_id AND cs_id <= {max_id}
        ORDER BY cs_id ASC
        LIMIT {limit}
        "
    }

    read SelectChildren(repo_id: RepositoryId, cs_id: ChangesetId) -> (ChangesetId) {
        "
        SELECT
            cs.cs_id
        FROM commit_graph_edges cs
        INNER JOIN commit_graph_edges cs_p1_parent
            ON cs_p1_parent.id = cs.p1_parent
        WHERE
            cs_p1_parent.repo_id = {repo_id}
            AND cs_p1_parent.cs_id = {cs_id}

        UNION

        SELECT
            cs.cs_id
        FROM commit_graph_edges cs
        INNER JOIN commit_graph_merge_parents cgmp
            ON cgmp.id = cs.id
        INNER JOIN commit_graph_edges cs_merge_parent
            ON cgmp.parent = cs_merge_parent.id
        WHERE
            cs_merge_parent.repo_id = {repo_id}
            AND cs_merge_parent.cs_id
                = {cs_id};
        "
    }
}

impl SqlCommitGraphStorage {
    fn collect_changeset_edges(
        fetched_edges: &[(
            ChangesetId,         // cs_id
            Option<u64>,         // gen
            Option<u64>,         // skip_tree_depth
            Option<u64>,         // p1_linear_depth
            Option<usize>,       // parent_count
            Option<ChangesetId>, // merge_ancestor
            Option<u64>,         // merge_ancestor_gen
            Option<u64>,         // merge_ancestor_skip_tree_depth
            Option<u64>,         // merge_ancestor_p1_linear_depth
            Option<ChangesetId>, // skip_tree_parent
            Option<u64>,         // skip_tree_parent_gen
            Option<u64>,         // skip_tree_parent_skip_tree_depth
            Option<u64>,         // skip_tree_parent_p1_linear_depth
            Option<ChangesetId>, // skip_tree_skew_ancestor
            Option<u64>,         // skip_tree_skew_ancestor_gen
            Option<u64>,         // skip_tree_skew_ancestor_skip_tree_depth
            Option<u64>,         // skip_tree_skew_ancestor_p1_linear_depth
            Option<ChangesetId>, // p1_linear_skew_ancestor
            Option<u64>,         // p1_linear_skew_ancestor_gen
            Option<u64>,         // p1_linear_skew_ancestor_skip_tree_depth
            Option<u64>,         // p1_linear_skew_ancestor_p1_linear_depth
            usize,               // parent_num
            Option<ChangesetId>, // parent
            Option<u64>,         // parent_gen
            Option<u64>,         // parent_skip_tree_depth
            Option<u64>,         // parent_p1_linear_depth
        )],
    ) -> HashMap<ChangesetId, ChangesetEdges> {
        let option_fields_to_option_node =
            |cs_id, generation, skip_tree_depth, p1_linear_depth| match (
                cs_id,
                generation,
                skip_tree_depth,
                p1_linear_depth,
            ) {
                (Some(cs_id), Some(generation), Some(skip_tree_depth), Some(p1_linear_depth)) => {
                    Some(ChangesetNode {
                        cs_id,
                        generation: Generation::new(generation),
                        skip_tree_depth,
                        p1_linear_depth,
                    })
                }
                _ => None,
            };
        let mut cs_id_to_cs_edges = HashMap::new();
        for row in fetched_edges.iter() {
            match *row {
                (
                    cs_id,
                    Some(gen),
                    Some(skip_tree_depth),
                    Some(p1_linear_depth),
                    Some(parent_count),
                    merge_ancestor,
                    merge_ancestor_gen,
                    merge_ancestor_skip_tree_depth,
                    merge_ancestor_p1_linear_depth,
                    skip_tree_parent,
                    skip_tree_parent_gen,
                    skip_tree_parent_skip_tree_depth,
                    skip_tree_parent_p1_linear_depth,
                    skip_tree_skew_ancestor,
                    skip_tree_skew_ancestor_gen,
                    skip_tree_skew_ancestor_skip_tree_depth,
                    skip_tree_skew_ancestor_p1_linear_depth,
                    p1_linear_skew_ancestor,
                    p1_linear_skew_ancestor_gen,
                    p1_linear_skew_ancestor_skip_tree_depth,
                    p1_linear_skew_ancestor_p1_linear_depth,
                    ..,
                ) => {
                    cs_id_to_cs_edges
                        .entry(cs_id)
                        .or_insert_with(|| ChangesetEdges {
                            node: ChangesetNode {
                                cs_id,
                                generation: Generation::new(gen),
                                skip_tree_depth,
                                p1_linear_depth,
                            },
                            parents: ChangesetNodeParents::new(),
                            merge_ancestor: option_fields_to_option_node(
                                merge_ancestor,
                                merge_ancestor_gen,
                                merge_ancestor_skip_tree_depth,
                                merge_ancestor_p1_linear_depth,
                            ),
                            skip_tree_parent: option_fields_to_option_node(
                                skip_tree_parent,
                                skip_tree_parent_gen,
                                skip_tree_parent_skip_tree_depth,
                                skip_tree_parent_p1_linear_depth,
                            ),
                            skip_tree_skew_ancestor: option_fields_to_option_node(
                                skip_tree_skew_ancestor,
                                skip_tree_skew_ancestor_gen,
                                skip_tree_skew_ancestor_skip_tree_depth,
                                skip_tree_skew_ancestor_p1_linear_depth,
                            ),
                            p1_linear_skew_ancestor: option_fields_to_option_node(
                                p1_linear_skew_ancestor,
                                p1_linear_skew_ancestor_gen,
                                p1_linear_skew_ancestor_skip_tree_depth,
                                p1_linear_skew_ancestor_p1_linear_depth,
                            ),
                        });
                }
                _ => continue,
            }
        }

        for row in fetched_edges {
            match *row {
                (
                    cs_id,
                    ..,
                    parent_num,
                    Some(parent),
                    Some(parent_gen),
                    Some(parent_skip_tree_depth),
                    Some(parent_p1_linear_depth),
                ) => {
                    if let Some(edge) = cs_id_to_cs_edges.get_mut(&cs_id) {
                        // Only insert if we have the correct next parent
                        if edge.parents.len() == parent_num {
                            edge.parents.push(ChangesetNode {
                                cs_id: parent,
                                generation: Generation::new(parent_gen),
                                skip_tree_depth: parent_skip_tree_depth,
                                p1_linear_depth: parent_p1_linear_depth,
                            })
                        }
                    }
                }
                _ => continue,
            }
        }

        cs_id_to_cs_edges
    }

    async fn fetch_many_edges_impl(
        &self,
        ctx: &CoreContext,
        cs_ids: &[ChangesetId],
        prefetch: Prefetch,
        rendezvous: &RendezVousConnection,
    ) -> Result<HashMap<ChangesetId, ChangesetEdges>> {
        if cs_ids.is_empty() {
            // This is actually NECESSARY, because SQL doesn't deal well with
            // querying empty arrays
            return Ok(HashMap::new());
        }

        if let Some(target) = prefetch.target() {
            let steps = std::cmp::min(
                target.steps,
                tunables()
                    .commit_graph_prefetch_step_limit()
                    .unwrap_or(DEFAULT_PREFETCH_STEP_LIMIT) as u64,
            );
            let fetched_edges = match target.edge {
                PrefetchEdge::FirstParent => {
                    SelectManyChangesetsWithFirstParentPrefetch::query(
                        &self.read_connection.conn,
                        &self.repo_id,
                        &steps,
                        &target.generation.value(),
                        cs_ids,
                    )
                    .await?
                }
                PrefetchEdge::SkipTreeSkewAncestor => {
                    SelectManyChangesetsWithSkipTreeSkewAncestorPrefetch::query(
                        &self.read_connection.conn,
                        &self.repo_id,
                        &steps,
                        &target.generation.value(),
                        cs_ids,
                    )
                    .await?
                }
            };
            Ok(Self::collect_changeset_edges(&fetched_edges))
        } else {
            let ret = rendezvous
                .fetch_single
                .dispatch(ctx.fb.clone(), cs_ids.iter().copied().collect(), || {
                    let conn = rendezvous.conn.clone();
                    let repo_id = self.repo_id.clone();

                    move |cs_ids| async move {
                        let cs_ids = cs_ids.into_iter().collect::<Vec<_>>();

                        let fetched_edges =
                            SelectManyChangesets::query(&conn, &repo_id, cs_ids.as_slice()).await?;
                        Ok(Self::collect_changeset_edges(&fetched_edges))
                    }
                })
                .await?;
            Ok(ret
                .into_iter()
                .filter_map(|(cs_id, cs_edge)| cs_edge.map(|cs_edge| (cs_id, cs_edge)))
                .collect())
        }
    }

    // Lower level APIs for quickly iterating over all changeset edges

    fn read_conn(&self, read_from_master: bool) -> &Connection {
        match read_from_master {
            true => &self.read_master_connection.conn,
            false => &self.read_connection.conn,
        }
    }

    /// Fetch a maximum of `limit` changeset edges for changesets having
    /// auto-increment ids between `start_id` and `end_id`.
    pub async fn fetch_many_edges_in_id_range(
        &self,
        ctx: &CoreContext,
        start_id: u64,
        end_id: u64,
        limit: u64,
        read_from_master: bool,
    ) -> Result<HashMap<ChangesetId, ChangesetEdges>> {
        Ok(Self::collect_changeset_edges(
            &SelectManyChangesetsInIdRange::query(
                self.read_conn(read_from_master),
                &self.repo_id,
                &start_id,
                &end_id,
                &limit,
            )
            .await?,
        ))
    }

    /// Fetch a maximum of `limit` changeset ids for changesets having
    /// auto-increment ids between `start_id` and `end_id`.
    pub async fn fetch_many_cs_ids_in_id_range(
        &self,
        ctx: &CoreContext,
        start_id: u64,
        end_id: u64,
        limit: u64,
        read_from_master: bool,
    ) -> Result<Vec<ChangesetId>> {
        Ok(SelectManyChangesetsIdsInIdRange::query(
            self.read_conn(read_from_master),
            &self.repo_id,
            &start_id,
            &end_id,
            &limit,
        )
        .await?
        .into_iter()
        .map(|(cs_id,)| cs_id)
        .collect())
    }

    /// Returns the maximum auto-increment id for any changeset in the repo,
    /// or `None` if there are no changesets.
    pub async fn max_id(&self, ctx: &CoreContext, read_from_master: bool) -> Result<Option<u64>> {
        Ok(
            SelectMaxId::query(self.read_conn(read_from_master), &self.repo_id)
                .await?
                .first()
                .map(|(id,)| *id),
        )
    }

    /// Returns the maximum auto-increment id of changesets having auto-increment
    /// ids between `start_id` and `end_id`, or `None` if there are no such changesets.
    pub async fn max_id_in_range(
        &self,
        ctx: &CoreContext,
        start_id: u64,
        end_id: u64,
        limit: u64,
        read_from_master: bool,
    ) -> Result<Option<u64>> {
        Ok(SelectMaxIdInRange::query(
            self.read_conn(read_from_master),
            &self.repo_id,
            &start_id,
            &end_id,
            &limit,
        )
        .await?
        .first()
        .map(|(id,)| *id))
    }
}

#[async_trait]
impl CommitGraphStorage for SqlCommitGraphStorage {
    fn repo_id(&self) -> RepositoryId {
        self.repo_id
    }

    async fn add_many(&self, ctx: &CoreContext, many_edges: Vec1<ChangesetEdges>) -> Result<usize> {
        // We need to be careful because there might be dependencies among the edges
        // Part 1 - Add all nodes without any edges, so we generate ids for them
        let transaction = self.write_connection.start_transaction().await?;
        let cs_no_edges = many_edges
            .iter()
            .map(|e| {
                (
                    self.repo_id,
                    e.node.cs_id,
                    e.node.generation.value(),
                    e.node.skip_tree_depth,
                    e.node.p1_linear_depth,
                    e.parents.len(),
                )
            })
            .collect::<Vec<_>>();
        let (transaction, result) = InsertChangesetsNoEdges::query_with_transaction(
            transaction,
            cs_no_edges
                .iter()
                // This does &(TypeA, TypeB, ...) -> (&TypeA, &TypeB, ...)
                .map(|(a, b, c, d, e, f)| (a, b, c, d, e, f))
                .collect::<Vec<_>>()
                .as_slice(),
        )
        .await?;
        let modified = result.affected_rows();
        if modified == 0 {
            // Early return, everything is already stored
            return Ok(0);
        }
        // Part 2 - Collect all changesets we need the ids from, and query them
        // using the same transaction
        let mut need_ids = HashSet::new();
        for edges in &many_edges {
            need_ids.insert(edges.node.cs_id);
            edges.merge_ancestor.map(|u| need_ids.insert(u.cs_id));
            edges.skip_tree_parent.map(|u| need_ids.insert(u.cs_id));
            edges
                .skip_tree_skew_ancestor
                .map(|u| need_ids.insert(u.cs_id));
            edges
                .p1_linear_skew_ancestor
                .map(|u| need_ids.insert(u.cs_id));
            for u in &edges.parents {
                need_ids.insert(u.cs_id);
            }
        }
        let (transaction, cs_to_ids) = if !need_ids.is_empty() {
            // Use the same transaction to make sure we see the new values
            let (transaction, result) = SelectManyIds::query_with_transaction(
                transaction,
                &self.repo_id,
                need_ids.into_iter().collect::<Vec<_>>().as_slice(),
            )
            .await?;
            (transaction, result.into_iter().collect())
        } else {
            (transaction, HashMap::new())
        };
        // Part 3 - Fix edges on all changesets we previously inserted
        let get_id = |node: &ChangesetNode| {
            cs_to_ids
                .get(&node.cs_id)
                .copied()
                .with_context(|| format!("Failed to fetch id for changeset {}", node.cs_id))
        };
        let maybe_get_id = |maybe_node: Option<&ChangesetNode>| maybe_node.map(get_id).transpose();
        let rows = match many_edges
            .iter()
            .map(|e| {
                Ok((
                    self.repo_id,
                    e.node.cs_id,
                    e.node.generation.value(),
                    e.node.skip_tree_depth,
                    e.node.p1_linear_depth,
                    e.parents.len(),
                    maybe_get_id(e.parents.first())?,
                    maybe_get_id(e.merge_ancestor.as_ref())?,
                    maybe_get_id(e.skip_tree_parent.as_ref())?,
                    maybe_get_id(e.skip_tree_skew_ancestor.as_ref())?,
                    maybe_get_id(e.p1_linear_skew_ancestor.as_ref())?,
                ))
            })
            .collect::<Result<Vec<_>>>()
        {
            Ok(rows) => rows,
            Err(err) => {
                transaction.rollback().await?;
                return Err(err);
            }
        };

        let (transaction, _) = FixEdges::query_with_transaction(
            transaction,
            rows.iter()
                .map(|(a, b, c, d, e, f, g, h, i, j, k)| (a, b, c, d, e, f, g, h, i, j, k))
                .collect::<Vec<_>>()
                .as_slice(),
        )
        .await?;

        let merge_parent_rows = many_edges
            .iter()
            .flat_map(|edges| {
                edges
                    .parents
                    .iter()
                    .enumerate()
                    .skip(1)
                    .map(|(parent_num, node)| Ok((get_id(&edges.node)?, parent_num, get_id(node)?)))
            })
            .collect::<Result<Vec<_>>>()?;

        let (transaction, result) = InsertMergeParents::query_with_transaction(
            transaction,
            merge_parent_rows
                .iter()
                .map(|(a, b, c)| (a, b, c))
                .collect::<Vec<_>>()
                .as_slice(),
        )
        .await?;

        // All good, nodes were added and correctly updated, let's commit.
        transaction.commit().await?;
        ctx.perf_counters()
            .increment_counter(PerfCounterType::SqlWrites);

        Ok(modified.try_into()?)
    }

    async fn add(&self, ctx: &CoreContext, edges: ChangesetEdges) -> Result<bool> {
        let merge_parent_cs_id_to_id: HashMap<ChangesetId, u64> = if edges.parents.len() >= 2 {
            ctx.perf_counters()
                .increment_counter(PerfCounterType::SqlReadsReplica);
            SelectManyIds::query(
                &self.read_connection.conn,
                &self.repo_id,
                &edges
                    .parents
                    .iter()
                    .map(|node| node.cs_id)
                    .collect::<Vec<_>>(),
            )
            .await?
            .into_iter()
            .collect()
        } else {
            Default::default()
        };

        let transaction = self.write_connection.start_transaction().await?;

        let (transaction, result) = InsertChangeset::query_with_transaction(
            transaction,
            &self.repo_id,
            &edges.node.cs_id,
            &edges.node.generation.value(),
            &edges.node.skip_tree_depth,
            &edges.node.p1_linear_depth,
            &edges.parents.len(),
            &edges.parents.get(0).map(|node| node.cs_id),
            &edges.merge_ancestor.map(|node| node.cs_id),
            &edges.skip_tree_parent.map(|node| node.cs_id),
            &edges.skip_tree_skew_ancestor.map(|node| node.cs_id),
            &edges.p1_linear_skew_ancestor.map(|node| node.cs_id),
        )
        .await?;

        match result.last_insert_id() {
            Some(last_insert_id) if result.affected_rows() == 1 => {
                let merge_parent_rows = edges
                    .parents
                    .iter()
                    .enumerate()
                    .skip(1)
                    .map(|(parent_num, node)| {
                        Ok((
                            last_insert_id,
                            parent_num,
                            *merge_parent_cs_id_to_id
                                .get(&node.cs_id)
                                .ok_or_else(|| anyhow!("Failed to fetch id for {}", node.cs_id))?,
                        ))
                    })
                    .collect::<Result<Vec<_>>>()?;

                let (transaction, result) = InsertMergeParents::query_with_transaction(
                    transaction,
                    merge_parent_rows
                        .iter()
                        .map(|(a, b, c)| (a, b, c))
                        .collect::<Vec<_>>()
                        .as_slice(),
                )
                .await?;

                transaction.commit().await?;
                ctx.perf_counters()
                    .increment_counter(PerfCounterType::SqlWrites);

                Ok(true)
            }
            _ => {
                transaction.rollback().await?;
                Ok(false)
            }
        }
    }

    async fn fetch_edges(
        &self,
        ctx: &CoreContext,
        cs_id: ChangesetId,
    ) -> Result<Option<ChangesetEdges>> {
        Ok(self
            .fetch_many_edges(ctx, &[cs_id], Prefetch::None)
            .await?
            .remove(&cs_id))
    }

    async fn fetch_edges_required(
        &self,
        ctx: &CoreContext,
        cs_id: ChangesetId,
    ) -> Result<ChangesetEdges> {
        self.fetch_many_edges_required(ctx, &[cs_id], Prefetch::None)
            .await?
            .remove(&cs_id)
            .ok_or_else(|| anyhow!("Missing changeset from sql commit graph storage: {}", cs_id))
    }

    async fn fetch_many_edges(
        &self,
        ctx: &CoreContext,
        cs_ids: &[ChangesetId],
        prefetch: Prefetch,
    ) -> Result<HashMap<ChangesetId, ChangesetEdges>> {
        ctx.perf_counters()
            .increment_counter(PerfCounterType::SqlReadsReplica);
        self.fetch_many_edges_impl(ctx, cs_ids, prefetch, &self.read_connection)
            .await
    }

    async fn fetch_many_edges_required(
        &self,
        ctx: &CoreContext,
        cs_ids: &[ChangesetId],
        prefetch: Prefetch,
    ) -> Result<HashMap<ChangesetId, ChangesetEdges>> {
        ctx.perf_counters()
            .increment_counter(PerfCounterType::SqlReadsReplica);
        let mut edges = self
            .fetch_many_edges_impl(ctx, cs_ids, prefetch, &self.read_connection)
            .await?;
        let unfetched_ids: Vec<ChangesetId> = cs_ids
            .iter()
            .filter(|id| !edges.contains_key(id))
            .copied()
            .collect();
        let unfetched_ids = if !unfetched_ids.is_empty() {
            // Let's go to master with the remaining edges
            ctx.perf_counters()
                .increment_counter(PerfCounterType::SqlReadsMaster);
            let extra_edges = self
                .fetch_many_edges_impl(ctx, &unfetched_ids, prefetch, &self.read_master_connection)
                .await?;
            edges.extend(extra_edges);
            cs_ids
                .iter()
                .filter(|id| !edges.contains_key(id))
                .copied()
                .collect()
        } else {
            unfetched_ids
        };
        if !unfetched_ids.is_empty() {
            anyhow::bail!(
                "Missing changesets from sql commit graph storage: {}",
                unfetched_ids
                    .into_iter()
                    .map(|id| format!("{}, ", id))
                    .collect::<String>()
            );
        }
        Ok(edges)
    }

    async fn find_by_prefix(
        &self,
        ctx: &CoreContext,
        cs_prefix: ChangesetIdPrefix,
        limit: usize,
    ) -> Result<ChangesetIdsResolvedFromPrefix> {
        ctx.perf_counters()
            .increment_counter(PerfCounterType::SqlReadsReplica);
        let mut fetched_ids = SelectChangesetsInRange::query(
            &self.read_connection.conn,
            &self.repo_id,
            &cs_prefix.min_bound(),
            &cs_prefix.max_bound(),
            &(limit + 1),
        )
        .await?
        .into_iter()
        .map(|(cs_id,)| cs_id)
        .collect::<Vec<_>>();

        Ok(ChangesetIdsResolvedFromPrefix::from_vec_and_limit(
            fetched_ids,
            limit,
        ))
    }

    async fn fetch_children(
        &self,
        ctx: &CoreContext,
        cs_id: ChangesetId,
    ) -> Result<Vec<ChangesetId>> {
        Ok(
            SelectChildren::query(&self.read_master_connection.conn, &self.repo_id, &cs_id)
                .await?
                .into_iter()
                .map(|(cs_id,)| cs_id)
                .collect(),
        )
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;
use std::collections::HashSet;
use std::future::Future;
use std::sync::Arc;

use anyhow::Result;
use commit_graph_testlib::utils::from_dag;
use commit_graph_testlib::utils::name_cs_id;
use commit_graph_testlib::*;
use commit_graph_types::storage::CommitGraphStorage;
use commit_graph_types::storage::Prefetch;
use context::CoreContext;
use fbinit::FacebookInit;
use mononoke_types::RepositoryId;
use rendezvous::RendezVousOptions;
use sql_construct::SqlConstruct;

use crate::SqlCommitGraphStorageBuilder;

async fn run_test<Fut>(
    fb: FacebookInit,
    test_function: impl FnOnce(CoreContext, Arc<dyn CommitGraphStorage>) -> Fut,
) -> Result<()>
where
    Fut: Future<Output = Result<()>>,
{
    let ctx = CoreContext::test_mock(fb);
    let storage = Arc::new(
        SqlCommitGraphStorageBuilder::with_sqlite_in_memory()
            .unwrap()
            .build(RendezVousOptions::for_test(), RepositoryId::new(1)),
    );
    test_function(ctx, storage).await
}

impl_commit_graph_tests!(run_test);

#[fbinit::test]
pub async fn test_lower_level_api(fb: FacebookInit) -> Result<()> {
    let ctx = CoreContext::test_mock(fb);
    let storage = Arc::new(
        SqlCommitGraphStorageBuilder::with_sqlite_in_memory()
            .unwrap()
            .build(RendezVousOptions::for_test(), RepositoryId::new(1)),
    );

    let graph = from_dag(
        &ctx,
        r##"
             A-B-C-D-E-F-G-H-I-J
         "##,
        storage.clone(),
    )
    .await?;

    assert_eq!(storage.max_id(&ctx, false).await?, Some(10));
    assert_eq!(storage.max_id(&ctx, true).await?, Some(10));

    assert_eq!(
        storage.max_id_in_range(&ctx, 1, 10, 10, false).await?,
        Some(10),
    );
    assert_eq!(
        storage.max_id_in_range(&ctx, 2, 10, 5, false).await?,
        Some(6),
    );
    assert_eq!(
        storage.max_id_in_range(&ctx, 4, 7, 100, false).await?,
        Some(7),
    );

    assert_eq!(
        storage
            .fetch_many_cs_ids_in_id_range(&ctx, 1, 10, 10, false)
            .await?,
        ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"]
            .into_iter()
            .map(name_cs_id)
            .collect::<Vec<_>>(),
    );
    assert_eq!(
        storage
            .fetch_many_cs_ids_in_id_range(&ctx, 1, 10, 5, false)
            .await?,
        ["A", "B", "C", "D", "E"]
            .into_iter()
            .map(name_cs_id)
            .collect::<Vec<_>>(),
    );
    assert_eq!(
        storage
            .fetch_many_cs_ids_in_id_range(&ctx, 4, 6, 100, false)
            .await?,
        ["D", "E", "F"]
            .into_iter()
            .map(name_cs_id)
            .collect::<Vec<_>>(),
    );

    let all_edges = storage
        .fetch_many_edges_required(
            &ctx,
            &["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"]
                .into_iter()
                .map(name_cs_id)
                .collect::<Vec<_>>(),
            Prefetch::None,
        )
        .await?;

    assert_eq!(
        storage
            .fetch_many_edges_in_id_range(&ctx, 1, 10, 10, false)
            .await?,
        ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"]
            .into_iter()
            .map(|id| {
                let cs_id = name_cs_id(id);
                (cs_id, all_edges.get(&cs_id).unwrap().clone())
            })
            .collect::<HashMap<_, _>>(),
    );
    assert_eq!(
        storage
            .fetch_many_edges_in_id_range(&ctx, 1, 10, 5, false)
            .await?,
        ["A", "B", "C", "D", "E"]
            .into_iter()
            .map(|id| {
                let cs_id = name_cs_id(id);
                (cs_id, all_edges.get(&cs_id).unwrap().clone())
            })
            .collect::<HashMap<_, _>>(),
    );
    assert_eq!(
        storage
            .fetch_many_edges_in_id_range(&ctx, 4, 6, 100, false)
            .await?,
        ["D", "E", "F"]
            .into_iter()
            .map(|id| {
                let cs_id = name_cs_id(id);
                (cs_id, all_edges.get(&cs_id).unwrap().clone())
            })
            .collect::<HashMap<_, _>>(),
    );

    Ok(())
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashSet;
use std::str::FromStr;
use std::string::ToString;

use anyhow::anyhow;
use anyhow::Result;
use futures::stream;
use futures::stream::StreamExt;
use futures::stream::TryStreamExt;
use metaconfig_types::RemoteDatabaseConfig;
use metaconfig_types::RemoteMetadataDatabaseConfig;
use mononoke_types::ChangesetId;
use mononoke_types::RepositoryId;
use mononoke_types::Timestamp;
use sql::Connection;
use sql_construct::SqlConstruct;
use sql_construct::SqlConstructFromMetadataDatabaseConfig;
use sql_ext::mononoke_queries;
use sql_ext::SqlConnections;
use strum::Display;
use strum::EnumString;

pub const MYSQL_INSERT_CHUNK_SIZE: usize = 1000;

mononoke_queries! {
    write InsertCandidate(values: (repo_id: RepositoryId, cs_id: ChangesetId, blob_key: String, reason: String, stage: String, timestamp: Timestamp)) {
    insert_or_ignore,
    "{insert_or_ignore} INTO deletion_plan (repo_id, cs_id, blob_key, reason, stage, timestamp) VALUES {values}"
    }

    write UpdateCandidate(values: (repo_id: RepositoryId, cs_id: ChangesetId, blob_key: String, reason: String, stage: String, timestamp: Timestamp)) {
        none,
        mysql("INSERT INTO deletion_plan
            (repo_id, cs_id, blob_key, reason, stage, timestamp)
        VALUES {values}
        ON DUPLICATE KEY UPDATE
            stage = VALUES(stage),
            timestamp = VALUES(timestamp)")
        sqlite("INSERT INTO deletion_plan
            (repo_id, cs_id, blob_key, reason, stage, timestamp)
        VALUES {values}
        ON CONFLICT(repo_id, cs_id, blob_key, reason) DO UPDATE SET
            stage = excluded.stage,
            timestamp = excluded.timestamp")
    }

    read GetBlobKeysForRequest(repo_id: RepositoryId, reason: String) -> (ChangesetId, String, String) {
        "SELECT cs_id, blob_key, stage FROM deletion_plan WHERE repo_id = {repo_id} AND reason = {reason}"
    }

    read GetBlobKeysForRequestAndStage(repo_id: RepositoryId, reason: String, stage: String) -> (ChangesetId, String) {
        "SELECT cs_id, blob_key FROM deletion_plan WHERE repo_id = {repo_id} AND reason = {reason} AND stage = {stage}"
    }
}

#[derive(Clone, Display, Debug, EnumString, PartialEq, Eq, Hash)]
pub enum DeletionStage {
    Planned,
    Staged,
    Deleted,
    Cancelled,
}

impl DeletionStage {
    pub fn allowed_transition_from(self, from_stage: Self) -> bool {
        match (from_stage, self) {
            (Self::Planned, Self::Staged) => true,
            (Self::Staged, Self::Deleted) => true,
            (Self::Staged, Self::Cancelled) => true,
            (_, _) => false,
        }
    }
}

pub trait TransitionState {
    fn from() -> DeletionStage;
    fn to() -> DeletionStage;
}

pub struct PlannedToStaged {}

impl TransitionState for PlannedToStaged {
    fn from() -> DeletionStage {
        DeletionStage::Planned
    }
    fn to() -> DeletionStage {
        DeletionStage::Staged
    }
}
pub struct StagedToDeleted {}

impl TransitionState for StagedToDeleted {
    fn from() -> DeletionStage {
        DeletionStage::Staged
    }
    fn to() -> DeletionStage {
        DeletionStage::Deleted
    }
}
pub struct StagedToCancelled {}

impl TransitionState for StagedToCancelled {
    fn from() -> DeletionStage {
        DeletionStage::Staged
    }
    fn to() -> DeletionStage {
        DeletionStage::Cancelled
    }
}

#[facet::facet]
pub struct DeletionLog {
    pub sql_deletion_log: SqlDeletionLog,
}

impl DeletionLog {
    pub async fn insert_candidates(
        &self,
        repo_id: RepositoryId,
        reason: String,
        stage: DeletionStage,
        cs_to_blobs: Vec<(ChangesetId, String)>,
    ) -> Result<u64> {
        self.sql_deletion_log
            .insert_candidates(repo_id, reason, stage.to_string(), cs_to_blobs)
            .await
    }

    pub async fn update_candidates<T: TransitionState>(
        &self,
        repo_id: RepositoryId,
        reason: String,
        cs_to_blobs: Vec<(ChangesetId, String)>,
    ) -> Result<u64> {
        self.sql_deletion_log
            .update_candidates::<T>(repo_id, reason, cs_to_blobs)
            .await
    }

    pub async fn get_blob_keys_for_request(
        &self,
        repo_id: RepositoryId,
        reason: String,
        stage: DeletionStage,
    ) -> Result<Vec<(ChangesetId, String)>> {
        self.sql_deletion_log
            .get_blob_keys_for_request_and_stage(repo_id, reason, stage.to_string())
            .await
    }
}

pub struct SqlDeletionLog {
    write_connection: Connection,
    read_connection: Connection,
}

impl SqlConstruct for SqlDeletionLog {
    const LABEL: &'static str = "deletion_plan";

    const CREATION_QUERY: &'static str = include_str!("../schemas/sqlite-deletion-log.sql");

    fn from_sql_connections(connections: SqlConnections) -> Self {
        Self {
            write_connection: connections.write_connection,
            read_connection: connections.read_connection,
        }
    }
}

impl SqlConstructFromMetadataDatabaseConfig for SqlDeletionLog {
    fn remote_database_config(
        remote: &RemoteMetadataDatabaseConfig,
    ) -> Option<&RemoteDatabaseConfig> {
        remote.deletion_log.as_ref()
    }
}

impl SqlDeletionLog {
    async fn insert_candidates(
        &self,
        repo_id: RepositoryId,
        reason: String,
        stage: String,
        cs_to_blobs: Vec<(ChangesetId, String)>,
    ) -> Result<u64> {
        if cs_to_blobs.is_empty() {
            return Ok(0);
        }
        stream::iter(cs_to_blobs.chunks(MYSQL_INSERT_CHUNK_SIZE))
            .then(|values| {
                let reason = reason.clone();
                let stage = stage.clone();
                let timestamp = Timestamp::now();
                async move {
                    let v = values
                        .iter()
                        .map(|(cs_id, blob_key)| {
                            (&repo_id, cs_id, blob_key, &reason, &stage, &timestamp)
                        })
                        .collect::<Vec<_>>();
                    InsertCandidate::query(&self.write_connection, v.as_slice()).await
                }
            })
            .try_fold(0, |acc, res| async move { Ok(acc + res.affected_rows()) })
            .await
    }

    async fn update_candidates<T: TransitionState>(
        &self,
        repo_id: RepositoryId,
        reason: String,
        cs_to_blobs: Vec<(ChangesetId, String)>,
    ) -> Result<u64> {
        let blobs_count = cs_to_blobs.len();
        if blobs_count == 0 {
            return Ok(0);
        }
        let from_stage = T::from();
        let to_stage = T::to();
        let rows = self
            .get_blob_keys_for_request(repo_id, reason.clone())
            .await?;
        let (in_target_state, not_in_target_state): (Vec<_>, Vec<_>) = rows
            .into_iter()
            .partition(|(_, _, stage)| *stage == to_stage);
        if blobs_count == in_target_state.len() {
            return Ok(0);
        }
        let (from_state, other_state): (HashSet<_>, HashSet<_>) = not_in_target_state
            .into_iter()
            .partition(|(_, _, stage)| *stage == from_stage);
        if !other_state.is_empty() {
            return Err(anyhow!(
                "While transition from {} to {} found blobs in the incorrect state {:?}",
                from_stage,
                to_stage,
                other_state
            ));
        }

        // collect the blobs which are requested and has not been processed yet
        let blobs_to_update = cs_to_blobs
            .into_iter()
            .map(|(cs_id, blob)| (cs_id, blob, from_stage.clone()))
            .collect::<HashSet<_>>();
        let remaining: Vec<_> = blobs_to_update.intersection(&from_state).collect();

        stream::iter(remaining.chunks(MYSQL_INSERT_CHUNK_SIZE))
            .then(|values| {
                let reason = reason.clone();
                let stage = to_stage.to_string();
                let timestamp = Timestamp::now();
                async move {
                    let v = values
                        .iter()
                        .map(|(cs_id, blob_key, _)| {
                            (&repo_id, cs_id, blob_key, &reason, &stage, &timestamp)
                        })
                        .collect::<Vec<_>>();
                    UpdateCandidate::query(&self.write_connection, v.as_slice()).await
                }
            })
            .try_fold(0, |acc, res| async move { Ok(acc + res.affected_rows()) })
            .await
    }

    async fn get_blob_keys_for_request(
        &self,
        repo_id: RepositoryId,
        reason: String,
    ) -> Result<Vec<(ChangesetId, String, DeletionStage)>> {
        let blobs = GetBlobKeysForRequest::query(&self.read_connection, &repo_id, &reason).await?;
        blobs
            .into_iter()
            .map(|(cs_id, blob, stage)| Ok((cs_id, blob, DeletionStage::from_str(&stage)?)))
            .collect::<Result<Vec<_>>>()
    }

    async fn get_blob_keys_for_request_and_stage(
        &self,
        repo_id: RepositoryId,
        reason: String,
        stage: String,
    ) -> Result<Vec<(ChangesetId, String)>> {
        GetBlobKeysForRequestAndStage::query(&self.read_connection, &repo_id, &reason, &stage).await
    }
}

#[cfg(test)]
mod test {
    use mononoke_types_mocks::changesetid::ONES_CSID;
    use mononoke_types_mocks::changesetid::THREES_CSID;
    use mononoke_types_mocks::changesetid::TWOS_CSID;
    use sql_construct::SqlConstruct;

    use super::*;

    #[tokio::test]
    async fn test_read_write() -> Result<()> {
        let sql = SqlDeletionLog::with_sqlite_in_memory()?;
        let repo_id = RepositoryId::new(1);
        let reason = "my_reason".to_string();
        let one = vec![
            (ONES_CSID, "blob1".to_string()),
            (ONES_CSID, "blob2".to_string()),
            (ONES_CSID, "blob3".to_string()),
        ];
        let two = vec![
            (TWOS_CSID, "blob4".to_string()),
            (TWOS_CSID, "blob5".to_string()),
        ];
        let three = vec![(THREES_CSID, "blob6".to_string())];
        let res = sql
            .insert_candidates(
                repo_id,
                reason.clone(),
                DeletionStage::Deleted.to_string(),
                one.clone(),
            )
            .await?;
        assert_eq!(res, 3);
        let res = sql
            .insert_candidates(
                repo_id,
                reason.clone(),
                DeletionStage::Staged.to_string(),
                two,
            )
            .await?;
        assert_eq!(res, 2);
        let res = sql
            .insert_candidates(
                repo_id,
                reason.clone(),
                DeletionStage::Staged.to_string(),
                three,
            )
            .await?;
        assert_eq!(res, 1);

        let res = sql
            .get_blob_keys_for_request_and_stage(
                repo_id,
                reason,
                DeletionStage::Deleted.to_string(),
            )
            .await?;
        assert_eq!(res, one);
        Ok(())
    }

    #[tokio::test]
    async fn test_update() -> Result<()> {
        let sql = SqlDeletionLog::with_sqlite_in_memory()?;
        let deletion_log = DeletionLog {
            sql_deletion_log: sql,
        };
        let repo_id = RepositoryId::new(1);
        let reason = "my_reason".to_string();
        let one = vec![
            (ONES_CSID, "blob1".to_string()),
            (ONES_CSID, "blob2".to_string()),
            (ONES_CSID, "blob3".to_string()),
        ];

        let res = deletion_log
            .insert_candidates(repo_id, reason.clone(), DeletionStage::Planned, one.clone())
            .await?;
        assert_eq!(res, 3);

        let update = vec![(ONES_CSID, "blob1".to_string())];

        let res = deletion_log
            .update_candidates::<PlannedToStaged>(repo_id, reason.clone(), update.clone())
            .await?;
        assert_eq!(res, 1);
        let res = deletion_log
            .get_blob_keys_for_request(repo_id, reason.clone(), DeletionStage::Staged)
            .await?;
        assert_eq!(res, update);
        let res = deletion_log
            .get_blob_keys_for_request(repo_id, reason, DeletionStage::Planned)
            .await?;
        assert_eq!(res, one.into_iter().skip(1).collect::<Vec<_>>());
        Ok(())
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

//! Repo bookmark attributes
//!
//! Stores configuration and permission checkers for bookmarks

use anyhow::bail;
use anyhow::Result;
use bookmarks_types::BookmarkKey;
use context::CoreContext;
use metaconfig_types::BookmarkParams;
use permission_checker::AclProvider;
use permission_checker::BoxMembershipChecker;

/// Repository bookmark attributes.
#[facet::facet]
pub struct RepoBookmarkAttrs {
    bookmark_attrs: Vec<BookmarkAttr>,
}

impl RepoBookmarkAttrs {
    /// Construct a new RepoBookmarkAttrs.
    pub async fn new(
        acl_provider: &dyn AclProvider,
        bookmark_params: impl IntoIterator<Item = BookmarkParams>,
    ) -> Result<RepoBookmarkAttrs> {
        let mut bookmark_attrs = Vec::new();
        for params in bookmark_params {
            let attr = BookmarkAttr::new(acl_provider, params).await?;
            bookmark_attrs.push(attr);
        }
        Ok(RepoBookmarkAttrs { bookmark_attrs })
    }

    /// Construct a new RepoBookmarkAttrs for testing.
    pub fn new_test(
        bookmark_params: impl IntoIterator<Item = BookmarkParams>,
    ) -> Result<RepoBookmarkAttrs> {
        let mut bookmark_attrs = Vec::new();
        for params in bookmark_params {
            let attr = BookmarkAttr::new_test(params)?;
            bookmark_attrs.push(attr);
        }
        Ok(RepoBookmarkAttrs { bookmark_attrs })
    }

    /// Select bookmark params matching provided bookmark
    pub fn select<'a>(
        &'a self,
        bookmark: &'a BookmarkKey,
    ) -> impl Iterator<Item = &'a BookmarkAttr> {
        self.bookmark_attrs
            .iter()
            .filter(move |attr| attr.params().bookmark.matches(bookmark))
    }

    /// Check if provided bookmark is fast-forward only
    pub fn is_fast_forward_only(&self, bookmark: &BookmarkKey) -> bool {
        self.select(bookmark)
            .any(|attr| attr.params().only_fast_forward)
    }

    /// Check if a bookmark config overrides whether date should be rewritten during pushrebase.
    /// Return None if there are no bookmark config overriding rewrite_dates.
    pub fn should_rewrite_dates(&self, bookmark: &BookmarkKey) -> Option<bool> {
        for attr in self.select(bookmark) {
            // NOTE: If there are multiple patterns matching the bookmark, the first match
            // overrides others.
            if let Some(rewrite_dates) = attr.params().rewrite_dates {
                return Some(rewrite_dates);
            }
        }
        None
    }

    /// Check if the user is allowed to move the specified bookmark
    pub async fn is_allowed_user(
        &self,
        ctx: &CoreContext,
        unixname: &str,
        bookmark: &BookmarkKey,
    ) -> bool {
        for attr in self.select(bookmark) {
            let maybe_allowed = attr
                .params()
                .allowed_users
                .as_ref()
                .map(|re| re.is_match(unixname));

            let maybe_member = if let Some(membership) = &attr.membership {
                Some(membership.is_member(ctx.metadata().identities()).await)
            } else {
                None
            };

            // Check if the user is either allowed to access it or that they
            // are a member of the allowed hipster group.
            //
            // If there is no allowlist and no configured group, then
            // everyone is permitted access.
            let allowed = match (maybe_allowed, maybe_member) {
                (Some(x), Some(y)) => x || y,
                (Some(x), None) | (None, Some(x)) => x,
                (None, None) => true,
            };
            if !allowed {
                return false;
            }
        }
        true
    }
}

/// Single set of attributes for a bookmark or bookmark pattern.
pub struct BookmarkAttr {
    params: BookmarkParams,
    membership: Option<BoxMembershipChecker>,
}

impl BookmarkAttr {
    async fn new(acl_provider: &dyn AclProvider, params: BookmarkParams) -> Result<BookmarkAttr> {
        let membership = match &params.allowed_hipster_group {
            Some(hipster_group) => Some(acl_provider.group(hipster_group).await?),
            None => None,
        };
        Ok(BookmarkAttr { params, membership })
    }

    fn new_test(params: BookmarkParams) -> Result<BookmarkAttr> {
        if params.allowed_hipster_group.is_some() {
            bail!("Bookmark hipster groups are not supported in tests");
        }
        Ok(BookmarkAttr {
            params,
            membership: None,
        })
    }

    /// Bookmark parameters from config
    pub fn params(&self) -> &BookmarkParams {
        &self.params
    }

    /// Membership checker
    pub fn membership(&self) -> Option<&BoxMembershipChecker> {
        self.membership.as_ref()
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

//! Repo Cross Repo.
//!
//! Manages cross-repo interactions for this repo

use std::sync::Arc;

use cacheblob::LeaseOps;
use live_commit_sync_config::LiveCommitSyncConfig;
use synced_commit_mapping::ArcSyncedCommitMapping;

/// Repository identity information.
#[facet::facet]
pub struct RepoCrossRepo {
    /// The mapping of synced commits.
    synced_commit_mapping: ArcSyncedCommitMapping,

    /// The commit sync config that is currently live.
    live_commit_sync_config: Arc<dyn LiveCommitSyncConfig>,

    /// Lease for cross-repo syncing, to prevent multiple servers stampeding
    /// to sync the same commits.
    sync_lease: Arc<dyn LeaseOps>,
}

impl RepoCrossRepo {
    /// Construct a new RepoCrossRepo.
    pub fn new(
        synced_commit_mapping: ArcSyncedCommitMapping,
        live_commit_sync_config: Arc<dyn LiveCommitSyncConfig>,
        sync_lease: Arc<dyn LeaseOps>,
    ) -> RepoCrossRepo {
        RepoCrossRepo {
            synced_commit_mapping,
            live_commit_sync_config,
            sync_lease,
        }
    }

    pub fn synced_commit_mapping(&self) -> &ArcSyncedCommitMapping {
        &self.synced_commit_mapping
    }

    pub fn live_commit_sync_config(&self) -> &Arc<dyn LiveCommitSyncConfig> {
        &self.live_commit_sync_config
    }

    pub fn sync_lease(&self) -> &Arc<dyn LeaseOps> {
        &self.sync_lease
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

//! Repo derived data
//!
//! Stores configuration and state for data derivation.

use std::sync::Arc;

use anyhow::anyhow;
use anyhow::Result;
use bonsai_hg_mapping::BonsaiHgMapping;
use cacheblob::LeaseOps;
use changesets::Changesets;
use commit_graph::CommitGraph;
use context::CoreContext;
use derived_data_manager::BonsaiDerivable;
use derived_data_manager::DerivationError;
use derived_data_manager::DerivedDataManager;
use derived_data_remote::DerivationClient;
use filenodes::Filenodes;
use metaconfig_types::DerivedDataConfig;
use metaconfig_types::DerivedDataTypesConfig;
use mononoke_types::ChangesetId;
use mononoke_types::RepositoryId;
use repo_blobstore::RepoBlobstore;
use scuba_ext::MononokeScubaSampleBuilder;

/// Repository derived data management.
#[facet::facet]
pub struct RepoDerivedData {
    /// Configuration for derived data.
    config: DerivedDataConfig,

    /// Derived data manager for the enabled types on this repo.
    manager: DerivedDataManager,
}

impl RepoDerivedData {
    /// Construct a new RepoDerivedData.
    pub fn new(
        repo_id: RepositoryId,
        repo_name: String,
        changesets: Arc<dyn Changesets>,
        commit_graph: Arc<CommitGraph>,
        bonsai_hg_mapping: Arc<dyn BonsaiHgMapping>,
        filenodes: Arc<dyn Filenodes>,
        repo_blobstore: RepoBlobstore,
        lease: Arc<dyn LeaseOps>,
        scuba: MononokeScubaSampleBuilder,
        config: DerivedDataConfig,
        derivation_service_client: Option<Arc<dyn DerivationClient>>,
    ) -> Result<RepoDerivedData> {
        let config_name = config.enabled_config_name.clone();
        let manager = DerivedDataManager::new(
            repo_id,
            repo_name,
            changesets,
            commit_graph,
            bonsai_hg_mapping,
            filenodes,
            repo_blobstore,
            lease,
            scuba,
            config_name.clone(),
            config
                .get_config(&config_name)
                .ok_or_else(|| {
                    anyhow!(
                        "Requested config name: {} is not in the available configs",
                        config_name
                    )
                })?
                .clone(),
            derivation_service_client,
        );
        Ok(RepoDerivedData { config, manager })
    }

    // For dangerous-override: allow replacement of lease-ops
    pub fn with_replaced_lease(&self, lease: Arc<dyn LeaseOps>) -> Self {
        Self {
            config: self.config.clone(),
            manager: self.manager.with_replaced_lease(lease),
        }
    }

    // For dangerous-override: allow replacement of blobstore
    pub fn with_replaced_blobstore(&self, repo_blobstore: RepoBlobstore) -> Self {
        Self {
            config: self.config.clone(),
            manager: self.manager.with_replaced_blobstore(repo_blobstore),
        }
    }

    // For dangerous-override: allow replacement of changesets
    pub fn with_replaced_changesets(&self, changesets: Arc<dyn Changesets>) -> Self {
        Self {
            config: self.config.clone(),
            manager: self.manager.with_replaced_changesets(changesets),
        }
    }

    // For dangerous-override: allow replacement of bonsai-hg-mapping
    pub fn with_replaced_bonsai_hg_mapping(
        &self,
        bonsai_hg_mapping: Arc<dyn BonsaiHgMapping>,
    ) -> Self {
        Self {
            config: self.config.clone(),
            manager: self
                .manager
                .with_replaced_bonsai_hg_mapping(bonsai_hg_mapping),
        }
    }

    // For dangerous-override: allow replacement of filenodes
    pub fn with_replaced_filenodes(&self, filenodes: Arc<dyn Filenodes>) -> Self {
        Self {
            config: self.config.clone(),
            manager: self.manager.with_replaced_filenodes(filenodes),
        }
    }

    pub fn with_manager(&self, manager: DerivedDataManager) -> Self {
        Self {
            config: self.config.clone(),
            manager,
        }
    }

    /// Current derived data configuration for this repo.
    pub fn config(&self) -> &DerivedDataConfig {
        &self.config
    }

    /// Config for the currently active derived data.
    pub fn active_config(&self) -> &DerivedDataTypesConfig {
        self.manager.config()
    }

    /// Derived data lease for this repo.
    pub fn lease(&self) -> &Arc<dyn LeaseOps> {
        self.manager.lease().lease_ops()
    }

    /// Default manager for derivation.
    pub fn manager(&self) -> &DerivedDataManager {
        &self.manager
    }

    /// Count the number of ancestors of a commit that are underived.
    pub async fn count_underived<Derivable>(
        &self,
        ctx: &CoreContext,
        csid: ChangesetId,
        limit: Option<u64>,
    ) -> Result<u64, DerivationError>
    where
        Derivable: BonsaiDerivable,
    {
        self.manager
            .count_underived::<Derivable>(ctx, csid, limit, None)
            .await
    }

    /// Derive a derived data type using the default manager.
    pub async fn derive<Derivable>(
        &self,
        ctx: &CoreContext,
        csid: ChangesetId,
    ) -> Result<Derivable, DerivationError>
    where
        Derivable: BonsaiDerivable,
    {
        self.manager.derive::<Derivable>(ctx, csid, None).await
    }

    /// Fetch an already derived derived data type using the default manager.
    pub async fn fetch_derived<Derivable>(
        &self,
        ctx: &CoreContext,
        csid: ChangesetId,
    ) -> Result<Option<Derivable>, DerivationError>
    where
        Derivable: BonsaiDerivable,
    {
        self.manager
            .fetch_derived::<Derivable>(ctx, csid, None)
            .await
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

use std::collections::HashMap;
use std::sync::Arc;

use anyhow::Result;
use bonsai_hg_mapping::BonsaiHgMapping;
use cacheblob::LeaseOps;
use changesets::Changesets;
use commit_graph::CommitGraph;
use derived_data_manager::DerivedDataManager;
use derived_data_remote::DerivationClient;
use filenodes::Filenodes;
use metaconfig_types::DerivedDataConfig;
use metaconfig_types::DerivedDataTypesConfig;
use mononoke_types::RepositoryId;
use repo_blobstore::RepoBlobstore;
use scuba_ext::MononokeScubaSampleBuilder;
use slog::Logger;

#[facet::facet]
pub struct DerivedDataManagerSet {
    repo_id: RepositoryId,
    logger: Logger,
    configs: HashMap<String, DerivedDataManager>,
    changesets: Arc<dyn Changesets>,
}

impl DerivedDataManagerSet {
    pub fn get_manager(&self, config_name: impl Into<String>) -> Option<&DerivedDataManager> {
        self.configs.get(&config_name.into())
    }

    pub fn logger(&self) -> &Logger {
        &self.logger
    }

    pub fn changesets(&self) -> Arc<dyn Changesets> {
        self.changesets.clone()
    }

    pub fn repo_id(&self) -> RepositoryId {
        self.repo_id
    }

    pub fn iter(&self) -> impl Iterator<Item = (&str, &DerivedDataManager)> {
        self.configs.iter().map(|(s, ddm)| (s.as_str(), ddm))
    }
}

#[facet::container]
#[derive(Clone)]
pub struct DerivedDataServiceRepo {
    #[facet]
    pub manager_set: DerivedDataManagerSet,
}

impl DerivedDataManagerSet {
    pub fn new(
        repo_id: RepositoryId,
        repo_name: String,
        changesets: Arc<dyn Changesets>,
        commit_graph: Arc<CommitGraph>,
        bonsai_hg_mapping: Arc<dyn BonsaiHgMapping>,
        filenodes: Arc<dyn Filenodes>,
        repo_blobstore: RepoBlobstore,
        lease: Arc<dyn LeaseOps>,
        logger: Logger,
        derived_data_scuba: MononokeScubaSampleBuilder,
        config: DerivedDataConfig,
        derivation_service_client: Option<Arc<dyn DerivationClient>>,
    ) -> Result<Self> {
        let manager = DerivedDataManager::new(
            repo_id,
            repo_name,
            changesets.clone(),
            commit_graph,
            bonsai_hg_mapping,
            filenodes,
            repo_blobstore,
            lease,
            derived_data_scuba,
            String::default(),
            DerivedDataTypesConfig::default(),
            derivation_service_client,
        );
        let configs = config
            .available_configs
            .into_iter()
            .map(|(config_name, config)| {
                (
                    config_name.clone(),
                    manager.with_replaced_config(config_name, config),
                )
            })
            .collect();

        Ok(Self {
            repo_id,
            logger,
            configs,
            changesets,
        })
    }
}


/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This software may be used and distributed according to the terms of the
 * GNU General Public License version 2.
 */

//! Repo Identity.
//!
//! Stores the id and name of the repository.
use std::hash::Hash;

use mononoke_types::RepositoryId;

/// Repository identity information.
#[facet::facet]
#[derive(Hash, PartialEq, Eq)]
pub struct RepoIdentity {
    /// The ID of the repository.
    id: RepositoryId,

    /// The name of the repository.
    name: String,
}

impl RepoIdentity {
    /// Construct a new RepoIdentity.
    pub fn new(id: RepositoryId, name: String) -> RepoIdentity {
        RepoIdentity { id, name }
    }

    /// The ID of the repository.
    pub fn id(&self) -> RepositoryId {
        self.id
    }

    /// The name of the repository.
    pub fn name(&self) -> &str {
        self.name.as_str()
    }
}