use crate::{
    common::divector::DiVector,
    pointstore::PointStore,
    samplerplustree::{boundingbox::BoundingBox, nodestore::NodeStore},
    visitor::visitor::VisitorInfo,
};

pub trait UpdatableNodeView<NS, PS>
where
    NS: NodeStore,
    PS: PointStore,
{
    fn create(root: usize, node_store: &NS) -> Self;
    fn update_at_leaf(
        &mut self,
        point: &[f32],
        index: usize,
        node_store: &NS,
        point_store: &PS,
        visitor_info: &VisitorInfo,
    );
    fn update_from_node_traversing_down(
        &mut self,
        point: &[f32],
        index: usize,
        node_store: &NS,
        point_store: &PS,
        visitor_info: &VisitorInfo,
    );
    fn update_from_node_traversing_up(
        &mut self,
        point: &[f32],
        index: usize,
        node_store: &NS,
        point_store: &PS,
        visitor_info: &VisitorInfo,
    );
    fn get_current_node(&self) -> usize;
    fn set_use_shadow_box(&mut self, node_store: &NS, point_store: &PS);
}

pub trait UpdatableMultiNodeView<NS, PS>: UpdatableNodeView<NS, PS>
where
    NS: NodeStore,
    PS: PointStore,
{
    fn create(root: usize, node_store: &NS) -> Self;
    fn set_trigger_traversing_down(
        &mut self,
        point: &[f32],
        parent: usize,
        node_store: &NS,
        point_store: &PS,
        visitor_info: &VisitorInfo,
    );
    fn update_view_to_parent_with_missing_coordinates(
        &mut self,
        parent: usize,
        point: &[f32],
        missing_coordinates: &[bool],
        node_store: &NS,
        point_store: &PS,
        visitor_info: &VisitorInfo,
    );
    fn set_current_node(&mut self, index: usize);
    fn get_bounding_box(&self) -> Option<BoundingBox>;
    fn merge_paths(
        &mut self,
        parent: usize,
        saved_box: Option<BoundingBox>,
        point: &[f32],
        missing_coordinates: &[bool],
        node_store: &NS,
        point_store: &PS,
    );
}

#[repr(C)]
pub struct SmallNodeView {
    current_node: usize,
    probability_of_cut: f64,
    shadow_box_probablity_of_cut: f64,
    mass: usize,
    depth: usize,
    leaf_index: usize,
    leaf_duplicate: bool,
    use_shadow_box: bool,
    current_box: Option<BoundingBox>,
    shadow_box: Option<BoundingBox>,
}

impl SmallNodeView {
    pub fn get_probability_of_cut(&self) -> f64 {
        self.probability_of_cut
    }
    pub fn get_shadow_box_probability_of_cut(&self) -> f64 {
        self.shadow_box_probablity_of_cut
    }
    pub fn get_mass(&self) -> usize {
        self.mass
    }
    pub fn get_depth(&self) -> usize {
        self.depth
    }
    pub fn get_leaf_index(&self) -> usize {
        self.leaf_index
    }
    pub fn is_duplicate(&self) -> bool {
        self.leaf_duplicate
    }
    pub fn new<NS: NodeStore>(root: usize, node_store: &NS) -> Self {
        SmallNodeView {
            current_node: root,
            probability_of_cut: f64::MAX, // not feasible; but that is the point!
            shadow_box_probablity_of_cut: f64::MAX,
            mass: 0,
            depth: 0,
            leaf_index: usize::MAX,
            leaf_duplicate: false,
            use_shadow_box: false,
            current_box: None,
            shadow_box: None,
        }
    }
}

impl<NS, PS> UpdatableNodeView<NS, PS> for SmallNodeView
where
    NS: NodeStore,
    PS: PointStore,
{
    fn create(root: usize, _node_store: &NS) -> Self {
        SmallNodeView::new(root, _node_store)
    }

    fn update_at_leaf(
        &mut self,
        point: &[f32],
        index: usize,
        node_store: &NS,
        point_store: &PS,
        visito_info: &VisitorInfo,
    ) {
        self.leaf_index = node_store.get_leaf_point_index(index);
        self.mass = node_store.get_mass(index);
        self.probability_of_cut = if point_store.is_equal(point, self.leaf_index) {
            self.leaf_duplicate = true;
            0.0
        } else {
            self.leaf_duplicate = false;
            1.0f64
        };
        if node_store.use_path_for_box() {
            self.current_box = Some(node_store.get_box(self.current_node, point_store));
        }
    }

    fn update_from_node_traversing_down(
        &mut self,
        point: &[f32],
        index: usize,
        node_store: &NS,
        point_store: &PS,
        visitor_info: &VisitorInfo,
    ) {
        if node_store.is_left_of(self.current_node, point) {
            self.current_node = node_store.get_left_index(self.current_node);
        } else {
            self.current_node = node_store.get_right_index(self.current_node);
        }
        self.depth += 1;
    }

    fn update_from_node_traversing_up(
        &mut self,
        point: &[f32],
        index: usize,
        node_store: &NS,
        point_store: &PS,
        visitor_info: &VisitorInfo,
    ) {
        self.probability_of_cut = match &mut self.current_box {
            Some(x) => {
                let sibling = node_store.get_sibling(self.current_node, index);
                if self.use_shadow_box {
                    let z = node_store.get_box(sibling, point_store);
                    x.add_box(&z);
                    match &mut self.shadow_box {
                        Some(y) => y.add_box(&z),
                        None => self.shadow_box = Some(z),
                    }
                    self.shadow_box_probablity_of_cut =
                        self.shadow_box.as_ref().unwrap().probability_of_cut(point);
                } else {
                    node_store.grow_node_box(x, point_store, index, sibling);
                };
                x.probability_of_cut(point)
            }
            None => node_store.get_probability_of_cut(index, point, point_store),
        };
        self.current_node = index;
        self.mass = node_store.get_mass(index);
        self.depth -= 1;
    }

    fn get_current_node(&self) -> usize {
        self.current_node
    }

    fn set_use_shadow_box(&mut self, node_store: &NS, point_store: &PS) {
        self.use_shadow_box = true;
        // we will maintain a current box since we haver to maintain a shadow box in any case
        // the shadow box is not set; it can only be set at the next level
        // when update_from_node_up() is invoked
        // we will maintain the invariant that *if the shadow is present then the current box also is present*
        self.current_box = Some(node_store.get_box(self.current_node, point_store));
    }
}

#[repr(C)]
pub struct MediumNodeView {
    current_node: usize,
    sibling: usize,
    probability_of_cut: f64,
    shadow_box_probablity_of_cut: f64,
    mass: usize,
    depth: usize,
    leaf_index: usize,
    leaf_duplicate: bool,
    use_shadow_box: bool,
    current_box: Option<BoundingBox>,
    shadow_box: Option<BoundingBox>,
    cut_dimension: usize,
    cut_value: f32,
    point_at_leaf: Vec<f32>,
}

impl MediumNodeView {
    pub fn get_probability_of_cut(&self) -> f64 {
        self.probability_of_cut
    }
    pub fn get_shadow_box_probability_of_cut(&self) -> f64 {
        self.shadow_box_probablity_of_cut
    }
    pub fn get_mass(&self) -> usize {
        self.mass
    }
    pub fn get_depth(&self) -> usize {
        self.depth
    }
    pub fn get_leaf_index(&self) -> usize {
        self.leaf_index
    }
    pub fn is_duplicate(&self) -> bool {
        self.leaf_duplicate
    }
    pub fn get_cut_dimension(&self) -> usize {
        self.cut_dimension
    }
    pub fn get_cut_value(&self) -> f32 {
        self.cut_value
    }
    pub fn get_leaf_point(&self) -> Vec<f32> {
        self.point_at_leaf.clone()
    }
    pub fn new<NS: NodeStore>(root: usize, node_store: &NS) -> Self {
        let (cut_dimension, cut_value, left_child, right_child) =
            node_store.get_cut_and_children(root);
        let mass = node_store.get_mass(root);
        Self {
            current_node: root,
            sibling: usize::MAX,
            probability_of_cut: f64::MAX,
            shadow_box_probablity_of_cut: f64::MAX,
            mass,
            depth: 0,
            leaf_index: usize::MAX,
            leaf_duplicate: false,
            use_shadow_box: false,
            current_box: None,
            shadow_box: None,
            cut_dimension,
            cut_value,
            point_at_leaf: Vec::new(),
        }
    }
}

impl<NS, PS> UpdatableNodeView<NS, PS> for MediumNodeView
where
    NS: NodeStore,
    PS: PointStore,
{
    fn create(root: usize, node_store: &NS) -> Self {
        MediumNodeView::new(root, node_store)
    }

    fn update_at_leaf(
        &mut self,
        point: &[f32],
        index: usize,
        node_store: &NS,
        point_store: &PS,
        visitor_info: &VisitorInfo,
    ) {
        self.leaf_index = node_store.get_leaf_point_index(index);
        self.point_at_leaf = point_store.get_copy(self.leaf_index);
        self.mass = node_store.get_mass(index);
        self.probability_of_cut = if self.point_at_leaf.eq(point) {
            self.leaf_duplicate = true;
            0.0
        } else {
            self.leaf_duplicate = false;
            1.0f64
        };
        if node_store.use_path_for_box() {
            self.current_box = Some(BoundingBox::new(&self.point_at_leaf, &self.point_at_leaf));
        }
    }

    fn update_from_node_traversing_down(
        &mut self,
        point: &[f32],
        index: usize,
        node_store: &NS,
        point_store: &PS,
        visitor_info: &VisitorInfo,
    ) {
        if node_store.is_left_of(self.current_node, point) {
            self.current_node = node_store.get_left_index(self.current_node);
        } else {
            self.current_node = node_store.get_right_index(self.current_node);
        }
        self.depth += 1;
    }

    fn update_from_node_traversing_up(
        &mut self,
        point: &[f32],
        index: usize,
        node_store: &NS,
        point_store: &PS,
        visitor_info: &VisitorInfo,
    ) {
        self.probability_of_cut = match &mut self.current_box {
            Some(x) => {
                self.sibling = node_store.get_sibling(self.current_node, index);
                if self.use_shadow_box {
                    let z = node_store.get_box(self.sibling, point_store);
                    x.add_box(&z);
                    match &mut self.shadow_box {
                        Some(y) => y.add_box(&z),
                        None => self.shadow_box = Some(z),
                    }
                    self.shadow_box_probablity_of_cut =
                        self.shadow_box.as_ref().unwrap().probability_of_cut(point);
                } else {
                    node_store.grow_node_box(x, point_store, index, self.sibling);
                };
                x.probability_of_cut(point)
            }
            None => node_store.get_probability_of_cut(index, point, point_store),
        };
        self.current_node = index;
        let (cut_dimension, cut_value, left_child, right_child) =
            node_store.get_cut_and_children(self.current_node);
        self.cut_dimension = cut_dimension;
        self.cut_value = cut_value;
        self.mass = node_store.get_mass(index);
        self.depth -= 1;
    }

    fn get_current_node(&self) -> usize {
        self.current_node
    }

    fn set_use_shadow_box(&mut self, node_store: &NS, point_store: &PS) {
        self.use_shadow_box = true;
        // we will maintain a current box since we haver to maintain a shadow box in any case
        // the shadow box is not set; it can only be set at the next level
        // when update_from_node_up() is invoked
        // we will maintain the invariant that *if the shadow is present then the current box also is present*
        self.current_box = Some(node_store.get_box(self.current_node, point_store));
    }
}

impl<NS, PS> UpdatableMultiNodeView<NS, PS> for MediumNodeView
where
    NS: NodeStore,
    PS: PointStore,
{
    fn create(root: usize, node_store: &NS) -> Self {
        MediumNodeView::new(root, node_store)
    }

    fn set_trigger_traversing_down(
        &mut self,
        point: &[f32],
        parent: usize,
        node_store: &NS,
        point_store: &PS,
        visitor_info: &VisitorInfo,
    ) {
        let (cut_dimension, cut_value, left_child, right_child) =
            node_store.get_cut_and_children(self.current_node);
        self.cut_dimension = cut_dimension;
        self.cut_value = cut_value;
    }

    fn update_view_to_parent_with_missing_coordinates(
        &mut self,
        parent: usize,
        point: &[f32],
        missing_coordinates: &[bool],
        node_store: &NS,
        point_store: &PS,
        visitor_info: &VisitorInfo,
    ) {
        if node_store.use_path_for_box() {
            let sibling = node_store.get_sibling(self.current_node, parent);
            node_store.grow_node_box(
                self.current_box.as_mut().unwrap(),
                point_store,
                parent,
                sibling,
            );
            self.probability_of_cut = self
                .current_box
                .as_ref()
                .unwrap()
                .probability_of_cut_with_missing_coordinates(point, missing_coordinates);
        } else {
            self.probability_of_cut = node_store.get_probability_of_cut_with_missing_coordinates(
                parent,
                point,
                missing_coordinates,
                point_store,
            );
        }
        self.current_node = parent;
    }

    fn set_current_node(&mut self, index: usize) {
        self.current_node = index;
    }

    fn get_bounding_box(&self) -> Option<BoundingBox> {
        match &self.current_box {
            Some(x) => Some(x.copy()),
            None => None,
        }
    }

    fn merge_paths(
        &mut self,
        parent: usize,
        saved_box: Option<BoundingBox>,
        point: &[f32],
        missing_coordinates: &[bool],
        node_store: &NS,
        point_store: &PS,
    ) {
        if node_store.use_path_for_box() {
            // both boxes, current and saved, should be present as invariant
            self.current_box
                .as_mut()
                .unwrap()
                .add_box(saved_box.as_ref().unwrap());
            self.probability_of_cut = self
                .current_box
                .as_ref()
                .unwrap()
                .probability_of_cut_with_missing_coordinates(point, missing_coordinates);
        } else {
            self.probability_of_cut = node_store.get_probability_of_cut_with_missing_coordinates(
                parent,
                point,
                missing_coordinates,
                point_store,
            );
        }
        self.current_node = parent;
    }
}

#[repr(C)]
pub struct LargeNodeView {
    current_node: usize,
    sibling: usize,
    probability_of_cut: f64,
    shadow_box_probablity_of_cut: f64,
    mass: usize,
    depth: usize,
    leaf_index: usize,
    leaf_duplicate: bool,
    use_shadow_box: bool,
    current_box: Option<BoundingBox>,
    shadow_box: Option<BoundingBox>,
    cut_dimension: usize,
    cut_value: f32,
    left_child: usize,
    right_child: usize,
    point_at_leaf: Vec<f32>,
}

impl LargeNodeView {
    pub fn get_probability_of_cut(&self) -> f64 {
        self.probability_of_cut
    }
    pub fn get_shadow_box_probability_of_cut(&self) -> f64 {
        self.shadow_box_probablity_of_cut
    }
    pub fn get_mass(&self) -> usize {
        self.mass
    }
    pub fn get_depth(&self) -> usize {
        self.depth
    }
    pub fn get_leaf_index(&self) -> usize {
        self.leaf_index
    }
    pub fn is_duplicate(&self) -> bool {
        self.leaf_duplicate
    }
    pub fn get_cut_dimension(&self) -> usize {
        self.cut_dimension
    }
    pub fn get_cut_value(&self) -> f32 {
        self.cut_value
    }
    pub fn get_leaf_point(&self) -> Vec<f32> {
        self.point_at_leaf.clone()
    }
    pub fn bounding_box(&self) -> BoundingBox {
        self.current_box.as_ref().unwrap().copy()
    }
    pub fn shadow_box(&self) -> BoundingBox {
        self.shadow_box.as_ref().unwrap().copy()
    }
    pub fn assign_probability_of_cut(&self, di_vector: &mut DiVector, point: &[f32]) {
        di_vector.assign_as_probability_of_cut(self.current_box.as_ref().unwrap(), point)
    }
    pub fn assign_probability_of_cut_shadow_box(&self, di_vector: &mut DiVector, point: &[f32]) {
        assert!(self.use_shadow_box, "shadow box not in use");
        di_vector.assign_as_probability_of_cut(self.shadow_box.as_ref().unwrap(), point)
    }
    pub fn new<NS: NodeStore>(root: usize, node_store: &NS) -> Self {
        let (cut_dimension, cut_value, left_child, right_child) =
            node_store.get_cut_and_children(root);
        let mass = node_store.get_mass(root);
        Self {
            current_node: root,
            sibling: usize::MAX,
            probability_of_cut: f64::MAX,
            shadow_box_probablity_of_cut: f64::MAX,
            mass,
            depth: 0,
            leaf_index: usize::MAX,
            leaf_duplicate: false,
            use_shadow_box: false,
            current_box: None,
            shadow_box: None,
            cut_dimension,
            cut_value,
            left_child,
            right_child,
            point_at_leaf: Vec::new(),
        }
    }
}

impl<NS, PS> UpdatableNodeView<NS, PS> for LargeNodeView
where
    NS: NodeStore,
    PS: PointStore,
{
    fn create(root: usize, node_store: &NS) -> Self {
        LargeNodeView::new(root, node_store)
    }

    fn update_at_leaf(
        &mut self,
        point: &[f32],
        index: usize,
        node_store: &NS,
        point_store: &PS,
        visitor_info: &VisitorInfo,
    ) {
        self.leaf_index = node_store.get_leaf_point_index(index);
        self.point_at_leaf = point_store.get_copy(self.leaf_index);
        self.mass = node_store.get_mass(index);
        self.probability_of_cut = if self.point_at_leaf.eq(point) {
            self.leaf_duplicate = true;
            0.0
        } else {
            self.leaf_duplicate = false;
            1.0f64
        };
        self.current_box = Some(BoundingBox::new(&self.point_at_leaf, &self.point_at_leaf));
    }

    fn update_from_node_traversing_down(
        &mut self,
        point: &[f32],
        index: usize,
        node_store: &NS,
        point_store: &PS,
        visitor_info: &VisitorInfo,
    ) {
        if point[self.cut_dimension] <= self.cut_value {
            self.current_node = self.left_child;
        } else {
            self.current_node = self.right_child;
        }
        let (cut_dimension, cut_value, left_child, right_child) =
            node_store.get_cut_and_children(self.current_node);
        self.cut_dimension = cut_dimension;
        self.cut_value = cut_value;
        self.left_child = left_child;
        self.right_child = right_child;
        self.depth += 1;
        self.mass = node_store.get_mass(self.current_node);
    }

    fn update_from_node_traversing_up(
        &mut self,
        point: &[f32],
        index: usize,
        node_store: &NS,
        point_store: &PS,
        visitor_info: &VisitorInfo,
    ) {
        self.sibling = node_store.get_sibling(self.current_node, index);
        if self.use_shadow_box {
            let z = node_store.get_box(self.sibling, point_store);
            self.current_box.as_mut().unwrap().add_box(&z);
            match &mut self.shadow_box {
                Some(y) => y.add_box(&z),
                None => self.shadow_box = Some(z),
            }
            self.shadow_box_probablity_of_cut =
                self.shadow_box.as_ref().unwrap().probability_of_cut(point);
        } else {
            node_store.grow_node_box(
                self.current_box.as_mut().unwrap(),
                point_store,
                index,
                self.sibling,
            );
        };
        self.probability_of_cut = self.current_box.as_ref().unwrap().probability_of_cut(point);

        self.current_node = index;
        let (cut_dimension, cut_value, left_child, right_child) =
            node_store.get_cut_and_children(self.current_node);
        self.cut_dimension = cut_dimension;
        self.cut_value = cut_value;
        self.left_child = left_child;
        self.right_child = right_child;
        self.mass = node_store.get_mass(index);
        self.depth -= 1;
    }

    fn get_current_node(&self) -> usize {
        self.current_node
    }

    fn set_use_shadow_box(&mut self, node_store: &NS, point_store: &PS) {
        self.use_shadow_box = true;
        // note that the current box is always maintained
    }
}

impl<NS, PS> UpdatableMultiNodeView<NS, PS> for LargeNodeView
where
    NS: NodeStore,
    PS: PointStore,
{
    fn create(root: usize, node_store: &NS) -> Self {
        LargeNodeView::new(root, node_store)
    }

    fn set_trigger_traversing_down(
        &mut self,
        point: &[f32],
        parent: usize,
        node_store: &NS,
        point_store: &PS,
        visitor_info: &VisitorInfo,
    ) {
        let (cut_dimension, cut_value, left_child, right_child) =
            node_store.get_cut_and_children(self.current_node);
        self.cut_dimension = cut_dimension;
        self.cut_value = cut_value;
    }

    fn update_view_to_parent_with_missing_coordinates(
        &mut self,
        parent: usize,
        point: &[f32],
        missing_coordinates: &[bool],
        node_store: &NS,
        point_store: &PS,
        visitor_info: &VisitorInfo,
    ) {
        if node_store.use_path_for_box() {
            let sibling = node_store.get_sibling(self.current_node, parent);
            node_store.grow_node_box(
                self.current_box.as_mut().unwrap(),
                point_store,
                parent,
                sibling,
            );
            self.probability_of_cut = self
                .current_box
                .as_ref()
                .unwrap()
                .probability_of_cut_with_missing_coordinates(point, missing_coordinates);
        } else {
            self.probability_of_cut = node_store.get_probability_of_cut_with_missing_coordinates(
                parent,
                point,
                missing_coordinates,
                point_store,
            );
        }
        self.current_node = parent;
    }

    fn set_current_node(&mut self, index: usize) {
        self.current_node = index;
    }

    fn get_bounding_box(&self) -> Option<BoundingBox> {
        match &self.current_box {
            Some(x) => Some(x.copy()),
            None => None,
        }
    }

    fn merge_paths(
        &mut self,
        parent: usize,
        saved_box: Option<BoundingBox>,
        point: &[f32],
        missing_coordinates: &[bool],
        node_store: &NS,
        point_store: &PS,
    ) {
        if node_store.use_path_for_box() {
            // both boxes, current and saved, should be present as invariant
            self.current_box
                .as_mut()
                .unwrap()
                .add_box(saved_box.as_ref().unwrap());
            self.probability_of_cut = self
                .current_box
                .as_ref()
                .unwrap()
                .probability_of_cut_with_missing_coordinates(point, missing_coordinates);
        } else {
            self.probability_of_cut = node_store.get_probability_of_cut_with_missing_coordinates(
                parent,
                point,
                missing_coordinates,
                point_store,
            );
        }
        self.current_node = parent;
    }
}


use std::fmt::Debug;

use rand::{Rng, SeedableRng};
use rand_chacha::ChaCha20Rng;
use rand_core::RngCore;

use crate::{
    pointstore::PointStore,
    samplerplustree::{
        boundingbox::BoundingBox,
        cut::Cut,
        nodestore::{NodeStore, VectorNodeStore},
        nodeview::{MediumNodeView, UpdatableMultiNodeView, UpdatableNodeView},
    },
    types::Location,
    visitor::{
        imputevisitor::ImputeVisitor,
        visitor::{SimpleMultiVisitor, Visitor, VisitorInfo},
    },
};

extern crate rand;
extern crate rand_chacha;

#[repr(C)]
pub struct RCFTree<C, P, N>
where
    C: Location,
    usize: From<C>,
    P: Location,
    usize: From<P>,
    N: Location,
    usize: From<N>,
{
    dimensions: usize,
    capacity: usize,
    node_store: VectorNodeStore<C, P, N>,
    random_seed: u64,
    root: usize,
    tree_mass: usize,
    using_transforms: bool,
}

impl<C, P, N> RCFTree<C, P, N>
where
    C: Location,
    usize: From<C>,
    P: Location,
    usize: From<P>,
    N: Location,
    usize: From<N>,
    <C as TryFrom<usize>>::Error: Debug,
    <P as TryFrom<usize>>::Error: Debug,
    <N as TryFrom<usize>>::Error: Debug,
{
    pub fn new(
        dimensions: usize,
        capacity: usize,
        using_transforms: bool,
        bounding_box_cache_fraction: f64,
        random_seed: u64,
    ) -> Self {
        let project_to_tree: fn(Vec<f32>) -> Vec<f32> = { |x| x };
        let node_store = VectorNodeStore::<C, P, N>::new(
            capacity,
            dimensions,
            using_transforms,
            project_to_tree,
            bounding_box_cache_fraction,
        );
        let root = node_store.null_node();
        RCFTree {
            dimensions,
            capacity,
            using_transforms,
            random_seed,
            node_store,
            root,
            tree_mass: 0,
        }
    }

    pub fn add(
        &mut self,
        point_index: usize,
        _point_attribute: usize,
        point_store: &dyn PointStore,
    ) -> usize {
        if self.root == self.node_store.null_node() {
            self.root = self.node_store.leaf_index(point_index);
            self.tree_mass = 1;
            point_index
        } else {
            let point = &point_store.get_copy(point_index);
            let mut path_to_root = Vec::new();
            self.node_store
                .set_path(&mut path_to_root, self.root, point);
            let (mut node, mut sibling) = path_to_root.pop().unwrap();

            let leaf_point_index = self.node_store.get_point_index(node);
            let old_point = &point_store.get_copy(leaf_point_index);

            self.tree_mass += 1;
            if point.eq(old_point) {
                self.node_store.increase_leaf_mass(node);
                self.node_store
                    .manage_ancestors_add(&mut path_to_root, point, point_store, true);
                return leaf_point_index;
            } else {
                let mut saved_parent = if path_to_root.len() != 0 {
                    path_to_root.last().unwrap().0
                } else {
                    self.node_store.null_node()
                };
                let mut saved_node = node;
                let mut current_box = BoundingBox::new(old_point, old_point);
                let mut saved_box = current_box.copy();
                let mut parent_path: Vec<(usize, usize)> = Vec::new();
                let mut rng = ChaCha20Rng::seed_from_u64(self.random_seed);
                self.random_seed = rng.next_u64();

                let mut parent = saved_parent;
                let mut saved_cut = Cut::new(usize::MAX, 0.0);
                /* the loop has the execute once */
                loop {
                    let factor: f64 = rng.gen();
                    let (new_cut, separation) =
                        Cut::random_cut_and_separation(&current_box, factor, point);
                    if separation {
                        saved_cut = new_cut;
                        saved_parent = parent;
                        saved_node = node;
                        saved_box = current_box.copy();
                        parent_path.clear();
                    } else {
                        parent_path.push((node, sibling));
                    }
                    assert!(saved_cut.dimension != usize::MAX);

                    if parent == self.node_store.null_node() {
                        break;
                    } else {
                        self.node_store.grow_node_box(
                            &mut current_box,
                            point_store,
                            parent,
                            sibling,
                        );
                        let (a, b) = path_to_root.pop().unwrap();
                        node = a;
                        sibling = b;
                        parent = if path_to_root.len() != 0 {
                            path_to_root.last().unwrap().0
                        } else {
                            self.node_store.null_node()
                        };
                    }
                }

                if saved_parent != self.node_store.null_node() {
                    while !parent_path.is_empty() {
                        path_to_root.push(parent_path.pop().unwrap());
                    }
                    assert!(path_to_root.last().unwrap().0 == saved_parent);
                } else {
                    assert!(path_to_root.len() == 0);
                }
                let merged_node = self.node_store.add_node(
                    saved_parent,
                    point,
                    saved_node,
                    point_index,
                    saved_cut,
                    &saved_box,
                );

                if saved_parent != self.node_store.null_node() {
                    self.node_store.manage_ancestors_add(
                        &mut path_to_root,
                        point,
                        point_store,
                        false,
                    );
                } else {
                    self.root = merged_node;
                }
            }
            point_index
        }
    }

    pub fn delete(
        &mut self,
        point_index: usize,
        _point_attribute: usize,
        point_store: &dyn PointStore,
    ) -> usize {
        if self.root == self.node_store.null_node() {
            println!(" deleting from an empty tree");
            panic!();
        }
        self.tree_mass = self.tree_mass - 1;
        let point = &point_store.get_copy(point_index);
        let mut leaf_path = Vec::new();
        self.node_store.set_path(&mut leaf_path, self.root, point);
        let (leaf_node, leaf_saved_sibling) = leaf_path.pop().unwrap();

        let leaf_point_index = self.node_store.get_point_index(leaf_node);

        if leaf_point_index != point_index {
            if !point_store.is_equal(point, leaf_point_index) {
                println!(
                    " deleting wrong node; looking for {} found {}",
                    point_index, leaf_point_index
                );
                let old_point = point_store.get_copy(leaf_point_index);
                for j in 0..self.dimensions.into() {
                    println!("want {} found {}", point[j], old_point[j]);
                }
                panic!();
            }
        }

        if self.node_store.decrease_leaf_mass(leaf_node) == 0 {
            if leaf_path.len() == 0 {
                self.root = self.node_store.null_node();
            } else {
                let (parent, _sibling) = leaf_path.pop().unwrap();
                let grand_parent = if leaf_path.len() == 0 {
                    self.node_store.null_node()
                } else {
                    leaf_path.last().unwrap().0
                };

                if grand_parent == self.node_store.null_node() {
                    self.root = leaf_saved_sibling;
                    self.node_store.set_root(self.root);
                } else {
                    self.node_store
                        .replace_node(grand_parent, parent, leaf_saved_sibling);
                    self.node_store.manage_ancestors_delete(
                        &mut leaf_path,
                        point,
                        point_store,
                        false,
                    );
                }

                self.node_store.delete_internal_node(parent);
            }
        } else {
            self.node_store
                .manage_ancestors_delete(&mut leaf_path, point, point_store, true);
        }
        leaf_point_index
    }

    pub fn conditional_field<PS: PointStore>(
        &self,
        missing: &[usize],
        point: &[f32],
        point_store: &PS,
        centrality: f64,
        seed: u64,
        visitor_info: &VisitorInfo,
    ) -> (f64, usize, f64) {
        if self.root == self.node_store.null_node() {
            return (0.0, usize::MAX, 0.0);
        }
        let mut visitor = ImputeVisitor::new(missing, centrality, self.tree_mass, seed);
        let mut node_view = MediumNodeView::new(self.root, &self.node_store);
        let mut missing_coordinates = vec![false; self.dimensions];
        for i in missing.iter() {
            missing_coordinates[*i] = true;
        }
        self.traverse_multi_with_missing_coordinates(
            &mut node_view,
            &mut visitor,
            visitor_info,
            point,
            &missing_coordinates,
            point_store,
        );
        visitor.result(&visitor_info)
    }

    pub fn traverse_multi_with_missing_coordinates<V, NodeView, PS, R>(
        &self,
        node_view: &mut NodeView,
        visitor: &mut V,
        visitor_info: &VisitorInfo,
        point: &[f32],
        missing_coordinates: &[bool],
        point_store: &PS,
    ) where
        V: SimpleMultiVisitor<NodeView, R>,
        NodeView: UpdatableMultiNodeView<VectorNodeStore<C, P, N>, PS>,
        PS: PointStore,
    {
        let node = node_view.get_current_node();
        if self.node_store.is_leaf(node) {
            node_view.update_at_leaf(point, node, &self.node_store, &point_store, &visitor_info);
            visitor.accept_leaf(point, visitor_info, node_view);
        } else {
            let parent = node;
            node_view.set_trigger_traversing_down(
                point,
                parent,
                &self.node_store,
                point_store,
                visitor_info,
            );
            if missing_coordinates[self.node_store.get_cut_dimension(parent)] {
                let right = self.node_store.get_left_index(parent);
                let left = self.node_store.get_right_index(parent);
                node_view.set_current_node(left);
                self.traverse_multi_with_missing_coordinates(
                    node_view,
                    visitor,
                    visitor_info,
                    point,
                    missing_coordinates,
                    point_store,
                );
                let saved_box = node_view.get_bounding_box();
                node_view.set_current_node(right);
                self.traverse_multi_with_missing_coordinates(
                    node_view,
                    visitor,
                    visitor_info,
                    point,
                    missing_coordinates,
                    point_store,
                );
                visitor.combine_branches(point, &node_view, visitor_info);
                if !visitor.is_converged() {
                    node_view.merge_paths(
                        parent,
                        saved_box,
                        point,
                        missing_coordinates,
                        &self.node_store,
                        point_store,
                    );
                }
            } else {
                node_view.update_from_node_traversing_down(
                    point,
                    parent,
                    &self.node_store,
                    point_store,
                    &visitor_info,
                );
                self.traverse_multi_with_missing_coordinates(
                    node_view,
                    visitor,
                    visitor_info,
                    point,
                    missing_coordinates,
                    point_store,
                );
                if !visitor.is_converged() {
                    node_view.update_view_to_parent_with_missing_coordinates(
                        parent,
                        point,
                        missing_coordinates,
                        &self.node_store,
                        point_store,
                        &visitor_info,
                    );
                }
            }
            if !visitor.is_converged() {
                visitor.accept(point, visitor_info, node_view);
            }
        }
    }

    pub fn get_size(&self) -> usize {
        self.node_store.get_size(self.dimensions.into()) + std::mem::size_of::<RCFTree<C, P, N>>()
    }

    fn traverse_recursive<R, PS, NodeView, V>(
        &self,
        point: &[f32],
        node_view: &mut NodeView,
        visitor: &mut V,
        visitor_info: &VisitorInfo,
        point_store: &PS,
    ) where
        PS: PointStore,
        V: Visitor<NodeView, R>,
        R: Clone,
        NodeView: UpdatableNodeView<VectorNodeStore<C, P, N>, PS>,
    {
        let current_node = node_view.get_current_node();
        if self.node_store.is_leaf(current_node) {
            node_view.update_at_leaf(
                point,
                current_node,
                &self.node_store,
                &point_store,
                &visitor_info,
            );
            visitor.accept_leaf(point, visitor_info, &node_view);
            if visitor.use_shadow_box() {
                node_view.set_use_shadow_box(&self.node_store, point_store);
            }
        } else {
            node_view.update_from_node_traversing_down(
                point,
                current_node,
                &self.node_store,
                point_store,
                visitor_info,
            );
            self.traverse_recursive(point, node_view, visitor, visitor_info, point_store);
            if !visitor.is_converged() {
                node_view.update_from_node_traversing_up(
                    point,
                    current_node,
                    &self.node_store,
                    &point_store,
                    &visitor_info,
                );
                visitor.accept(point, visitor_info, &node_view);
            }
        }
    }
}

pub trait Traversable<NodeView, N, PS, V, R>
where
    N: Location,
    <N as TryFrom<usize>>::Error: Debug,
    usize: From<N>,
    PS: PointStore,
    V: Visitor<NodeView, R>,
{
    fn traverse(
        &self,
        point: &[f32],
        parameters: &[usize],
        visitor_factory: fn(usize, &[usize], &VisitorInfo) -> V,
        visitor_info: &VisitorInfo,
        point_store: &PS,
        default: &R,
    ) -> R;
}

impl<C, P, N, PS, NodeView, V, R> Traversable<NodeView, N, PS, V, R> for RCFTree<C, P, N>
where
    C: Location,
    <C as TryFrom<usize>>::Error: Debug,
    usize: From<C>,
    P: Location,
    <P as TryFrom<usize>>::Error: Debug,
    usize: From<P>,
    N: Location,
    <N as TryFrom<usize>>::Error: Debug,
    usize: From<N>,
    PS: PointStore,
    NodeView: UpdatableNodeView<VectorNodeStore<C, P, N>, PS>,
    V: Visitor<NodeView, R>,
    R: Clone,
{
    fn traverse(
        &self,
        point: &[f32],
        parameters: &[usize],
        visitor_factory: fn(usize, &[usize], &VisitorInfo) -> V,
        visitor_info: &VisitorInfo,
        point_store: &PS,
        default: &R,
    ) -> R {
        if self.root == self.node_store.null_node() {
            return default.clone();
        }
        let mut visitor = visitor_factory(self.tree_mass, parameters, &visitor_info);
        let mut node_view = NodeView::create(self.root, &self.node_store);
        self.traverse_recursive(
            point,
            &mut node_view,
            &mut visitor,
            &visitor_info,
            &point_store,
        );
        visitor.result(visitor_info)
    }
}


use std::fmt::Debug;

use crate::types::Max;

#[repr(C)]
pub struct Sampler<P> {
    capacity: usize,
    weights: Vec<f32>,
    points: Vec<P>,
    store_attributes: bool,
    point_attributes: Vec<P>,
    current_size: usize,
    accepted_state: (f32, usize, usize),
}

#[repr(C)]
pub struct SamplerAcceptState {
    pub(crate) eviction_occurred: bool,
    pub(crate) point_index: usize,
    pub(crate) evicted_weight: f32,
    pub(crate) point_attribute: usize,
}

impl<P: Max + Copy + std::cmp::PartialEq> Sampler<P>
where
    P: std::convert::TryFrom<usize>,
    usize: From<P>,
{
    pub fn new(capacity: usize, store_attributes: bool) -> Self {
        let attrib_vec: Vec<P> = if store_attributes {
            vec![P::MAX; capacity]
        } else {
            Vec::new()
        };
        Sampler {
            store_attributes,
            capacity,
            weights: vec![0.0; capacity],
            points: vec![P::MAX; capacity],
            point_attributes: attrib_vec,
            accepted_state: (0.0, usize::MAX, usize::MAX),
            current_size: 0,
        }
    }

    pub fn get_references(&self) -> &[P] {
        &self.points[0..self.current_size]
    }

    fn swap_down(&mut self, start_index: usize, validate: bool) {
        let mut current: usize = start_index;
        while 2 * current + 1 < self.current_size {
            let mut max_index: usize = 2 * current + 1;
            if 2 * current + 2 < self.current_size
                && self.weights[2 * current + 2] > self.weights[max_index]
            {
                max_index = 2 * current + 2;
            }
            if self.weights[max_index] > self.weights[current] {
                if validate {
                    println!("the heap property is not satisfied at index '{}' ", current);
                    panic!();
                }
                self.swap_weights(current, max_index);
                current = max_index;
            } else {
                break;
            }
        }
    }

    pub fn reheap(&mut self, validate: bool) {
        for i in ((self.current_size + 1) / 2)..=0 {
            self.swap_down(i, validate);
        }
    }

    fn swap_weights(&mut self, a: usize, b: usize) {
        if self.points[a] == P::MAX || self.points[b] == P::MAX {
            panic!();
        }

        let tmp: P = self.points[a];
        self.points[a] = self.points[b];
        self.points[b] = tmp;

        let tmp_weight: f32 = self.weights[a];
        self.weights[a] = self.weights[b];
        self.weights[b] = tmp_weight;

        if self.store_attributes {
            let tmp_attrib: P = self.point_attributes[a];
            self.point_attributes[a] = self.point_attributes[b];
            self.point_attributes[b] = tmp_attrib;
        }
    }

    pub fn add_point(&mut self, point_index: usize)
    where
        <P as TryFrom<usize>>::Error: Debug,
    {
        if point_index != usize::MAX {
            assert!(self.current_size < self.capacity.into(), "sampler full");
            assert_ne!(
                self.accepted_state.1,
                usize::MAX,
                "this method should only be called after a successful call to accept_sample(long)"
            );

            self.weights[self.current_size] = self.accepted_state.0;
            self.points[self.current_size] = point_index.try_into().unwrap();
            // note, not self.accepted_state.1, even though we want that to not
            // P::MAX This corresponds to the change in the index value via
            // duplicates in the trees
            if self.store_attributes {
                self.point_attributes[self.current_size] = if self.accepted_state.2 != usize::MAX {
                    self.accepted_state.2.try_into().unwrap()
                } else {
                    P::MAX
                }
            };

            let mut current = self.current_size;
            self.current_size += 1;

            while current > 0 {
                let tmp = (current - 1) / 2;
                if self.weights[tmp] < self.weights[current] {
                    self.swap_weights(current, tmp);
                    current = tmp;
                } else {
                    break;
                }
            }
            // resetting the state
            self.accepted_state = (0.0, usize::MAX, usize::MAX);
        }
    }

    pub fn accept_point(
        &mut self,
        initial: bool,
        weight: f32,
        point_index: usize,
        attribute: usize,
    ) -> SamplerAcceptState {
        let mut return_val = (true, usize::MAX, weight, usize::MAX);
        if initial || (weight < self.weights[0]) {
            self.accepted_state = (weight, point_index, attribute);
            if !initial {
                let partial = self.evict_max();
                return_val = (true, partial.0, partial.1, partial.2);
            }
        } else {
            return_val.0 = false;
        }
        return SamplerAcceptState {
            eviction_occurred: return_val.0,
            point_index: return_val.1,
            evicted_weight: return_val.2,
            point_attribute: return_val.3,
        };
    }

    /**
     * evicts the maximum weight point from the sampler. can be used repeatedly to
     * change the size of the sampler and associated tree
     */

    pub fn evict_max(&mut self) -> (usize, f32, usize) {
        let evicted_attribute_index: usize = if self.store_attributes {
            self.point_attributes[0].into()
        } else {
            usize::MAX
        };

        let evicted_point = (
            self.points[0].into(),
            self.weights[0],
            evicted_attribute_index,
        );
        self.current_size -= 1;
        let current: usize = self.current_size.into();
        self.weights[0] = self.weights[current];
        self.points[0] = self.points[current];
        if self.store_attributes {
            self.point_attributes[0] = self.point_attributes[current];
        }
        self.swap_down(0, false);

        evicted_point
    }

    pub fn is_full(&self) -> bool {
        self.current_size == self.capacity
    }

    pub fn get_fill_fraction(&self) -> f64 {
        if self.current_size == self.capacity {
            return 1.0;
        }
        let fill_fraction: f64 = self.current_size as f64 / self.capacity as f64;
        fill_fraction
    }

    pub fn get_size(&self) -> usize {
        (self.weights.len()) * std::mem::size_of::<f32>()
            + (self.points.len()) * std::mem::size_of::<P>()
            + std::mem::size_of::<Sampler<P>>()
    }
}


extern crate rand;
extern crate rand_chacha;

use std::fmt::Debug;

use rand::{Rng, SeedableRng};
use rand_chacha::ChaCha20Rng;
use rand_core::RngCore;

use crate::{
    pointstore::PointStore,
    samplerplustree::{
        nodestore::VectorNodeStore,
        nodeview::UpdatableNodeView,
        randomcuttree::{RCFTree, Traversable},
        sampler::Sampler,
    },
    types::Location,
    visitor::visitor::{Visitor, VisitorInfo},
};

#[repr(C)]
pub struct SamplerPlusTree<C, P, N>
where
    C: Location,
    usize: From<C>,
    P: Location,
    usize: From<P>,
    N: Location,
    usize: From<N>,
{
    tree: RCFTree<C, P, N>,
    sampler: Sampler<P>,
    using_transforms: bool,
    time_decay: f64,
    entries_seen: usize,
    initial_accept_fraction: f64,
    random_seed: u64,
}

impl<C, P, N> SamplerPlusTree<C, P, N>
where
    C: Location,
    usize: From<C>,
    P: Location,
    usize: From<P>,
    N: Location,
    usize: From<N>,
    <C as TryFrom<usize>>::Error: Debug,
    <P as TryFrom<usize>>::Error: Debug,
    <N as TryFrom<usize>>::Error: Debug,
{
    pub fn new(
        dimensions: usize,
        capacity: usize,
        using_transforms: bool,
        random_seed: u64,
        store_attributes: bool,
        time_decay: f64,
        initial_accept_fraction: f64,
        bounding_box_cache_fraction: f64,
    ) -> Self {
        let mut rng = ChaCha20Rng::seed_from_u64(random_seed);
        let self_seed = rng.next_u64();

        SamplerPlusTree {
            time_decay,
            initial_accept_fraction,
            using_transforms,
            tree: RCFTree::<C, P, N>::new(
                dimensions,
                capacity,
                using_transforms,
                bounding_box_cache_fraction,
                rng.next_u64(),
            ),
            sampler: Sampler::new(capacity, store_attributes),
            entries_seen: 0,
            random_seed: self_seed,
        }
    }

    pub fn update(
        &mut self,
        point_index: usize,
        point_attribute: usize,
        point_store: &dyn PointStore,
    ) -> (usize, usize) {
        if point_index != usize::MAX {
            let mut initial = false;
            let mut rng = ChaCha20Rng::seed_from_u64(self.random_seed);
            self.random_seed = rng.next_u64();
            let random_number: f64 = rng.gen();
            let weight: f64 =
                f64::ln(-f64::ln(random_number)) - ((self.entries_seen as f64) * self.time_decay);
            if !self.sampler.is_full() {
                let other_random: f64 = rng.gen();
                let fill_fraction: f64 = self.sampler.get_fill_fraction();
                initial = other_random < self.initial_accept_probability(fill_fraction);
            }
            let accept_state =
                self.sampler
                    .accept_point(initial, weight as f32, point_index, point_attribute);

            self.entries_seen += 1;
            if accept_state.eviction_occurred {
                let delete_ref = if accept_state.point_index != usize::MAX {
                    self.tree.delete(
                        accept_state.point_index,
                        accept_state.point_attribute,
                        point_store,
                    )
                } else {
                    usize::MAX
                };

                // the tree may choose to return a reference to an existing point
                // whose value is equal to `point`
                let added_ref = self.tree.add(point_index, point_attribute, point_store);

                self.sampler.add_point(added_ref);
                return (added_ref, delete_ref);
            }
        }
        (usize::MAX, usize::MAX)
    }

    fn initial_accept_probability(&self, fill_fraction: f64) -> f64 {
        return if fill_fraction < self.initial_accept_fraction {
            1.0
        } else if self.initial_accept_fraction >= 1.0 {
            0.0
        } else {
            1.0 - (fill_fraction - self.initial_accept_fraction)
                / (1.0 - self.initial_accept_fraction)
        };
    }

    pub fn simple_traversal<NodeView, V, R, PS>(
        &self,
        point: &[f32],
        point_store: &PS,
        parameters: &[usize],
        visitor_info: &VisitorInfo,
        visitor_factory: fn(usize, &[usize], &VisitorInfo) -> V,
        default: &R,
    ) -> R
    where
        NodeView: UpdatableNodeView<VectorNodeStore<C, P, N>, PS>,
        V: Visitor<NodeView, R>,
        PS: PointStore,
        R: Clone,
    {
        self.tree.traverse(
            point,
            parameters,
            visitor_factory,
            visitor_info,
            point_store,
            default,
        )
    }

    pub fn conditional_field<PS>(
        &self,
        positions: &[usize],
        centrality: f64,
        point: &[f32],
        point_store: &PS,
        visitor_info: &VisitorInfo,
    ) -> (f64, usize, f64)
    where
        PS: PointStore,
    {
        self.tree.conditional_field(
            positions,
            point,
            point_store,
            centrality,
            self.random_seed,
            visitor_info,
        )
    }

    pub fn get_size(&self) -> usize {
        self.tree.get_size()
            + self.sampler.get_size()
            + std::mem::size_of::<SamplerPlusTree<C, P, N>>()
    }
}


use crate::common::deviation::Deviation;

const DEFAULT_ELASTICITY : f32= 0.01;
const DEFAULT_HORIZON : f32 = 0.5;
const DEFAULT_HORIZON_ONED : f32 = 0.75;
const DEFAULT_MINIMUM_SCORES : i32 = 10;
const DEFAULT_ABSOLUTE_THRESHOLD : f32= 0.8;
const DEFAULT_ABSOLUTE_SCORE_FRACTION :f32 = 0.5;
const DEFAULT_UPPER_THRESHOLD :f32= 2.0;
const DEFAULT_LOWER_THRESHOLD :f32 = 1.0;
const DEFAULT_LOWER_THRESHOLD_ONED :f32 = 1.1;
const DEFAULT_LOWER_THRESHOLD_NORMALIZED : f32 = 0.9;
const DEFAULT_INITIAL_THRESHOLD :f32= 1.5;
const DEFAULT_Z_FACTOR :f32 = 2.5;
const DEFAULT_UPPER_FACTOR :f32 = 5.0;
const DEFAULT_AUTO_ADJUST_LOWER_THRESHOLD :bool = false;
const DEFAULT_THRESHOLD_STEP :f32 = 0.1;

#[repr(C)]
#[derive(Clone)]
pub struct BasicThresholder {
    elasticity: f32,
    count: i32,
    horizon: f32,
    last_score: f32,
    primary_deviation: Deviation,
    secondary_deviation: Deviation,
    threshold_deviation: Deviation,
    auto_threshold: bool,
    absolute_threshold:f32,
    absolute_score_fraction: f32,
    upper_threshold: f32,
    lower_threshold: f32,
    initial_threshold: f32,
    z_factor: f32,
    upper_z_factor: f32,
    in_potential_anomaly: bool,
    minimum_scores: i32,
}

impl BasicThresholder {
    pub fn new_adjustible(discount : f32, adjust:bool) -> Self {
        BasicThresholder{
            elasticity: DEFAULT_ELASTICITY,
            count: 0,
            horizon: DEFAULT_HORIZON,
            last_score: 0.0,
            primary_deviation: Deviation::new(discount as f64),
            secondary_deviation: Deviation::new(discount as f64),
            threshold_deviation: Deviation::new(discount as f64/2.0),
            auto_threshold: adjust,
            absolute_threshold: DEFAULT_ABSOLUTE_THRESHOLD,
            absolute_score_fraction: DEFAULT_ABSOLUTE_SCORE_FRACTION,
            upper_threshold: DEFAULT_UPPER_THRESHOLD,
            lower_threshold: DEFAULT_LOWER_THRESHOLD,
            initial_threshold: DEFAULT_INITIAL_THRESHOLD,
            z_factor: DEFAULT_Z_FACTOR,
            upper_z_factor: DEFAULT_UPPER_FACTOR,
            in_potential_anomaly: false,
            minimum_scores: DEFAULT_MINIMUM_SCORES
        }
    }

    pub fn new(discount : f32) -> Self{
        BasicThresholder::new_adjustible(discount,false)
    }

    pub fn is_deviation_ready(&self) -> bool {
        if self.count < self.minimum_scores {
            return false;
        }

        if self.horizon == 0.0 {
            return self.secondary_deviation.count() >= self.minimum_scores;
        } else if self.horizon == 1.0 {
            return self.primary_deviation.count() >= self.minimum_scores;
        } else {
            return self.secondary_deviation.count() >= self.minimum_scores
                && self.primary_deviation.count() >= self.minimum_scores;
        }
    }

    fn intermediate_fraction(&self) -> f32 {
        if self.count < self.minimum_scores {
            return 0.0;
        } else if self.count > 2 * self.minimum_scores {
            return 1.0;
        } else {
            return (self.count - self.minimum_scores) as f32 * 1.0 / self.minimum_scores as f32;
        }
    }

    fn short_term_threshold(&self, factor : f32, intermediate_fraction : f32) -> f32{
        if !self.is_deviation_ready() { // count < minimumScore is this branch
            f32::max(self.initial_threshold,self.lower_threshold)
        } else {
            let t = intermediate_fraction *self.longterm_threshold(factor) +
                (1.0 -intermediate_fraction)*self.initial_threshold;
            f32::max(t,self.lower_threshold)
        }
    }

    pub fn threshold(&self) -> f32{
        return self.longterm_threshold(self.z_factor);
    }

    pub fn longterm_threshold(&self, factor:f32) -> f32 {
        let t= (self.primary_deviation.mean() as f32) + factor * self.longterm_deviation();
        f32::min(t,self.lower_threshold)
    }

    fn longterm_deviation(&self) -> f32{
        self.horizon * (self.primary_deviation.deviation() as f32) + (1.0 - self.horizon) * (self.secondary_deviation.deviation() as f32)
    }

    pub fn anomaly_grade_with_factor(&self, score : f32, previous : bool, factor: f32) -> f32{
        assert!(factor >= self.z_factor, "incorrect call");

        let elastic_addition = if previous {self.elasticity} else {0.0};
        let intermediate_fraction = self.intermediate_fraction();
        if intermediate_fraction == 1.0 {
            if score < self.longterm_threshold(factor) - elastic_addition {
                return 0.0;
            }
            let mut t_factor = self.upper_z_factor;
            let longterm_deviation = self.longterm_deviation();
            if longterm_deviation > 0.0 {
                let t =  (score - self.primary_deviation.mean() as f32) / longterm_deviation;
                if (t as f32) < t_factor {
                    t_factor = t as f32;
                }
            }
            return (t_factor - self.z_factor) / (self.upper_z_factor - self.z_factor);
        } else {
            let t = self.short_term_threshold(factor, intermediate_fraction);
            if score < t - elastic_addition {
                return 0.0;
            }
            let upper = if self.upper_threshold > 2.0 * t {self.upper_threshold} else {2.0*t};
            let quasi_score = if score <upper {score} else {upper};
            return (quasi_score - t) / (upper - t);
        }
    }

    pub fn anomaly_grade(&self, score : f32, previous : bool) -> f32{
        return self.anomaly_grade_with_factor(score, previous, self.z_factor);
    }

    pub fn update_threshold(&mut self, score:f32) {
        let gap : f32 = if score > self.lower_threshold {1.0} else {0.0};
        self.threshold_deviation.update(gap as f64);
        if self.auto_threshold && self.threshold_deviation.count() > self.minimum_scores {
            // note the rate is set at half the anomaly rate
            if self.threshold_deviation.mean() > self.threshold_deviation.discount() {
                let t = self.lower_threshold + DEFAULT_THRESHOLD_STEP;
                self.set_lower_threshold(t, self.auto_threshold);
                self.threshold_deviation.set_count(0);
            } else if self.threshold_deviation.mean() < self.threshold_deviation.discount() / 4.0 {
                let t = self.lower_threshold - DEFAULT_THRESHOLD_STEP;
                if t > self.absolute_threshold {
                    self.set_lower_threshold(t, self.auto_threshold);
                    self.threshold_deviation.set_count(0);
                }
            }
        }
    }

    pub fn update_primary(&mut self, score : f64) {
        self.last_score = score as f32;
        self.primary_deviation.update(score);
        self.update_threshold(score as f32);
        self.count += 1;
    }

    pub fn update_both(&mut self, primary : f32, secondary : f32) {
        self.last_score = primary;
        self.primary_deviation.update(primary as f64);
        self.secondary_deviation.update(secondary as f64);
        self.update_threshold(primary as f32);
        self.count += 1;
    }

    pub fn update(&mut self, primary: f32, secondary: f32, last_score: f32, previous : bool){
        self.update_both(primary,secondary - last_score);
        self.in_potential_anomaly = previous;
    }
    // the next set of functions maintain the invariant that
    // DEFAULT_Z_FACTOR <= z_FACTOR < 2.0 * z_factor <= upper_z_factor

    pub fn set_z_factor(&mut self, factor : f32) {
        let z_factor = if factor > DEFAULT_Z_FACTOR {factor} else {DEFAULT_Z_FACTOR};
        self.z_factor = z_factor;
        if self.upper_z_factor < 2.0 * z_factor {
            self.upper_z_factor = 2.0 * z_factor;
        }
    }

    pub fn set_upper_z_factor(&mut self, factor : f32) {
        let t = if factor> 2.0*self.z_factor {factor} else {2.0*self.z_factor};
        self.upper_z_factor = t
    }

    pub fn z_factor(&self) -> f32 {
        self.z_factor
    }

    // the next set of functions maintain the invariant that
    // absolute_threshold <= lower_threshold < initial_threshold <= upper_threshold
    // absolute_threshold <= lower_threshold < 2.0 *lower_threshold <= upper_threshold
    // to increase proceed as upper_threshold, initial_threshold, lower_threshold, absolute_threshold
    // to decrease proceed in reverse of the above order

    pub fn set_lower_threshold(&mut self, lower : f32, adjust: bool) {
        let lower_t = if lower>self.absolute_threshold {lower} else {self.absolute_threshold};
        self.lower_threshold = lower_t;
        self.auto_threshold = adjust;
        if self.initial_threshold < lower_t {
            self.initial_threshold = lower_t;
        };
        if self.upper_threshold < 2.0 * lower_t {
            self.upper_threshold = 2.0 * lower_t;
        }
    }

    pub fn set_absolute_threshold(&mut self, value:f32) {
        self.absolute_threshold = value;
        self.set_lower_threshold(self.absolute_threshold, self.auto_threshold);
    }

    pub fn set_initial_threshold(&mut self, initial : f32) {
        self.initial_threshold = if initial< self.lower_threshold {self.lower_threshold} else {initial};
        if self.upper_threshold < initial {
            self.upper_threshold = initial;
        }
    }

    pub fn set_upper_threshold(&mut self, upper : f32) {
        self.upper_threshold = if upper < self.initial_threshold {self.initial_threshold} else {upper};
        if self.upper_threshold < 2.0 * self.lower_threshold {
            self.upper_threshold = 2.0 * self.lower_threshold;
        }
    }

    pub fn in_potential_anomaly(&self) -> bool {
        self.in_potential_anomaly
    }

    pub fn set_horizon(&mut self, horizon:f32) {
        assert!(horizon >= 0.0 && horizon <= 1.0, "incorrect horizon parameter");
        self.horizon = horizon;
    }

    pub fn last_score(&self) -> f32{
        self.last_score
    }

    pub fn primary_mean(&self) -> f64 {
        self.primary_deviation.mean()
    }

    pub fn primary_deviation(&self) -> f64 {
        self.primary_deviation.deviation()
    }

}


use crate::common::anomalydescriptor::AnomalyDescriptor;
use crate::common::rangevector::RangeVector;
use crate::rcf::{create_rcf, RCF};
use crate::types::Result;
use crate::trcf::predictorcorrector::PredictorCorrector;

pub struct BasicTRCF {
    dimensions: usize,
    capacity: usize,
    number_of_trees: usize,
    rcf : Box<dyn RCF>,
    predictor_corrector: PredictorCorrector,
    shingle_size: usize,
    internal_timestamp : usize,
    bounding_box_cache_fraction: f64,
    last_anomaly_descriptor : AnomalyDescriptor,
}

impl BasicTRCF {
    pub fn new(
        dimensions: usize,
        shingle_size: usize,
        capacity: usize,
        number_of_trees: usize,
        random_seed: u64,
        parallel_enabled: bool,
        time_decay: f64,
        anomaly_rate: f32,
        initial_accept_fraction: f64,
        bounding_box_cache_fraction: f64,
    ) -> Self {
        BasicTRCF{
            dimensions,
            capacity,
            number_of_trees,
            rcf: create_rcf(dimensions,shingle_size,capacity,number_of_trees,random_seed,false,parallel_enabled,true,false,time_decay,initial_accept_fraction,bounding_box_cache_fraction),
            predictor_corrector : PredictorCorrector::new(anomaly_rate),
            shingle_size,
            bounding_box_cache_fraction,
            internal_timestamp : 0,
            last_anomaly_descriptor: AnomalyDescriptor::new(&Vec::new(),0),
        }
    }

    pub fn process(&mut self, point: &[f32], timestamp: usize) -> Result<AnomalyDescriptor>{
        let mut result = AnomalyDescriptor::new(point,timestamp);
        result.internal_timestamp = self.internal_timestamp;
        self.internal_timestamp += 1;
        result.rcf_point = Some(self.rcf.shingled_point(point));
        result.forecast_reasonable = self.rcf.entries_seen() > 200;
        self.predictor_corrector.detect_and_modify(&mut result,&self.last_anomaly_descriptor,self.shingle_size, &self.rcf,false,false);
        if result.anomaly_grade > 0.0 {
            if let Some(x) = &result.expected_rcf_point {
                let base_dimension = self.dimensions / self.shingle_size;
                let start: usize = (self.shingle_size as i32 + result.relative_index.unwrap() - 1) as usize * base_dimension;
                result.expected_values_list = Some(vec![Vec::from(&x[start..start+base_dimension])]);
                result.past_values = Some(Vec::from(&result.rcf_point.as_ref().unwrap()[start..start+base_dimension]));
                result.likelihood_of_values=Some(vec![1.0f32]);
            }
            self.last_anomaly_descriptor = result.clone();
        }

        self.rcf.update(point,timestamp as u64)?;
        Ok(result)
    }

    pub fn extrapolate(&self, look_ahead: usize) -> Result<RangeVector> {
        self.rcf.extrapolate(look_ahead)
    }

}


mod predictorcorrector;
pub(crate) mod basicthresholder;
pub mod basictrcf;


use crate::common::divector::DiVector;
use crate::rcf::RCF;
use crate::trcf::basicthresholder::BasicThresholder;
use crate::common::anomalydescriptor::AnomalyDescriptor;

const DEFAULT_NUMBER_OF_MAX_ATTRIBUTORS: usize = 5;
const DEFAULT_REPEAT_ANOMALY_Z_FACTOR : f32 = 3.5;
const DEFAULT_IGNORE_SIMILAR_FACTOR :f32 = 0.3;
const DEFAULT_IGNORE_SIMILAR : bool = false;

#[repr(C)]
#[derive(Clone)]
pub struct PredictorCorrector {
    basic_thresholder: BasicThresholder,
    ignore_similar : f32,
    trigger_factor : f32,
    max_attributors: usize
}

impl PredictorCorrector {
    // for mappers
    pub fn new(discount: f32) -> Self {
        PredictorCorrector {
            basic_thresholder: BasicThresholder::new_adjustible(discount, false),
            ignore_similar: DEFAULT_IGNORE_SIMILAR_FACTOR,
            trigger_factor: DEFAULT_REPEAT_ANOMALY_Z_FACTOR,
            max_attributors: DEFAULT_NUMBER_OF_MAX_ATTRIBUTORS
        }
    }

    pub fn expected_point(di_vector: &DiVector, max_attributors: usize, position: usize, base_dimension: usize, point: &[f32],
                          forest: &Box<dyn RCF>) -> Vec<f32> {
        let mut likely_missing_indices: Vec<usize>;

        if base_dimension == 1 {
            likely_missing_indices = vec![position; 1];
        } else {
            let mut sum = 0.0;
            let mut values = vec![0.0; base_dimension];
            for i in 0..base_dimension {
                values[i] = di_vector.high_low_sum(i + position);
                sum += values[i];
            }
            // sort decreasing
            values.sort_by(|a, b| b.partial_cmp(a).unwrap());

            let mut pick = 0;
            while pick < base_dimension && values[pick] >= (sum * 0.5 / max_attributors as f64) {
                pick += 1;
            }
            likely_missing_indices = Vec::new();

            if pick != 0 && pick <= max_attributors {
                let cutoff = values[pick - 1];
                for i in 0..base_dimension {
                    if di_vector.high_low_sum(i + position) >= cutoff && likely_missing_indices.len() < max_attributors {
                        likely_missing_indices.push(position + i);
                    }
                }
            }
        }

        let mut answer = Vec::from(point);
        if likely_missing_indices.len() != 0 && (2 * likely_missing_indices.len()  < forest.dimensions()) {
            let prediction = forest.conditional_field(&likely_missing_indices, point, 1.0, false, 0).unwrap().median;
            for i in likely_missing_indices{
                answer[i] = prediction[i];
            }
        }
        answer
    }

    fn trigger(&self, candidate: &DiVector, gap: usize, base_dimension: usize, ideal: Option<DiVector>,
               previous: bool, last_anomaly_descriptor: &AnomalyDescriptor) -> bool {
        match &last_anomaly_descriptor.attribution {
            None => { return true; },
            Some(x) => {
                let last_score = last_anomaly_descriptor.score;
                let dimensions = candidate.dimensions();
                let difference = gap * base_dimension;
                if difference < dimensions {
                    match ideal {
                        None => {
                            let mut remainder = 0.0;
                            for i in (dimensions - difference)..dimensions {
                                remainder += candidate.high_low_sum(i);
                            }
                            return self.basic_thresholder.anomaly_grade_with_factor((remainder as f32) * dimensions as f32 / difference as f32, previous, self.trigger_factor) > 0.0;
                        },
                        Some(ref y) => {
                            let mut differential_remainder = 0.0;
                            for i in (dimensions - difference)..dimensions {
                                let low_diff = candidate.low[i] - ideal.as_ref().unwrap().low[i];
                                differential_remainder += if low_diff > 0.0 { low_diff } else { -low_diff };
                                let high_diff = candidate.high[i] - ideal.as_ref().unwrap().high[i];
                                differential_remainder += if high_diff > 0.0 { high_diff } else { -high_diff };
                            }
                            return (differential_remainder > self.ignore_similar as f64 * last_score as f64) &&
                                self.basic_thresholder.anomaly_grade_with_factor((differential_remainder as f32) * (dimensions as f32) / difference as f32, previous, self.trigger_factor) > 0.0;
                        }
                    }
                } else {
                    return true;
                }
            }
        }
    }

    pub fn apply_basic_corrector(point: &[f32], gap: usize, shingle_size: usize, base_dimension: usize,
                                 last_anomaly_descriptor: &AnomalyDescriptor, use_difference: bool, time_augmented: bool) -> Vec<f32> {
        assert!(gap <= shingle_size, "incorrect invocation");

        let mut corrected_point = Vec::from(point);
        let last_expected_point = last_anomaly_descriptor.expected_rcf_point.as_ref().unwrap();
        let last_anomaly_point = last_anomaly_descriptor.rcf_point.as_ref().unwrap();
        let last_relative_index = last_anomaly_descriptor.relative_index.unwrap();
        if gap < shingle_size {
            for i in gap * base_dimension..point.len() {
                corrected_point[i - gap * base_dimension] = last_expected_point[i];
            }
        }
        if last_relative_index == 0 { // is is possible to fix other cases, but is more complicated
            if use_difference {
                for y in 0..base_dimension {
                    corrected_point[point.len() - gap * base_dimension + y] +=
                        last_anomaly_point[point.len() - base_dimension + y] -
                            last_expected_point[point.len() - base_dimension + y];
                }
            } else if time_augmented {
                // definitely correct the time dimension which is always differenced
                // this applies to the non-differenced cases
                corrected_point[point.len() - (gap - 1) * base_dimension - 1] +=
                    last_anomaly_point[point.len() - 1]
                        - last_expected_point[point.len() - 1];
            }
        }
        return corrected_point;
    }

    pub fn detect_and_modify(&mut self, result: &mut AnomalyDescriptor, last_anomaly_descriptor : &AnomalyDescriptor, shingle_size: usize, forest : &Box<dyn RCF>, use_difference: bool, time_augmented: bool) {
        match &result.rcf_point {
            None => return,
            Some(point) => {
                let score = forest.score(point).unwrap() as f32;
                result.score = score;
                if score == 0.0 {
                    return;
                }
                let internal_timestamp = result.internal_timestamp;
                let base_dimension = forest.dimensions() / shingle_size;
                result.threshold = self.basic_thresholder.threshold();
                let previous = self.basic_thresholder.in_potential_anomaly();

                if self.basic_thresholder.anomaly_grade(score, previous) == 0.0 {
                    result.anomaly_grade = 0.0;
                    result.in_high_score_region = Some(false);
                    self.basic_thresholder.update(score, score, 0.0, false);
                    return;
                }

                // the score is now high enough to be considered an anomaly
                result.in_high_score_region = Some(true);

                let gap = internal_timestamp - last_anomaly_descriptor.internal_timestamp;

                let reasonable_forecast = result.forecast_reasonable;

                if reasonable_forecast && gap > 0 && gap <= shingle_size {
                    if let Some(_expected) = &last_anomaly_descriptor.expected_rcf_point {
                        let corrected_point = Self::apply_basic_corrector(point, gap, shingle_size, base_dimension,
                                                                          last_anomaly_descriptor, use_difference, time_augmented);
                        let corrected_score = forest.score(&corrected_point).unwrap() as f32;
                        // we know we are looking previous anomalies
                        if self.basic_thresholder.anomaly_grade(corrected_score, true) == 0.0 {
                            // fixing the past makes this anomaly go away; nothing to do but process the
                            // score
                            // we will not change inHighScoreRegion however, because the score has been
                            // larger
                            self.basic_thresholder.update(score, corrected_score, 0.0, false);
                            result.expected_rcf_point = Some(corrected_point);
                            result.anomaly_grade = 0.0;
                            return;
                        }
                    }
                }

                let attribution = forest.attribution(point).unwrap();
                // index is 0 .. (shingle_size - 1); this is a departure from java version
                let index = attribution.max_contribution(base_dimension);

                if !previous && self.trigger(&attribution, gap, base_dimension, None, false, last_anomaly_descriptor) {
                    result.anomaly_grade = self.basic_thresholder.anomaly_grade(score, false);
                    result.start_of_anomaly = Some(true);
                    self.basic_thresholder.update(score, score, 0.0, true);
                    let start_position = base_dimension * index;
                    if reasonable_forecast {
                        let new_point = Self::expected_point(&attribution, self.max_attributors, start_position, base_dimension, point, &forest);
                        result.expected_rcf_point = Some(new_point);
                    }
                    result.relative_index = Some(index as i32 - shingle_size as i32 + 1);
                    result.attribution = Some(attribution);
                    return;
                } else if reasonable_forecast {
                    let start_position = base_dimension * index;
                    let new_point = Self::expected_point(&attribution, self.max_attributors, start_position, base_dimension, point, &forest);
                    let new_attribution = forest.attribution(&new_point).unwrap();
                    let new_score = forest.score(&new_point).unwrap() as f32;

                    if self.trigger(&attribution, gap, base_dimension, Some(new_attribution), previous,
                                    &last_anomaly_descriptor) {
                        result.anomaly_grade = self.basic_thresholder.anomaly_grade(score, previous);
                        result.expected_rcf_point = Some(new_point);
                        result.relative_index = Some(0);
                        result.attribution = Some(attribution);
                        self.basic_thresholder.update(score, new_score, 0.0, true);
                    } else {
                        // previousIsPotentialAnomaly is true now, but not calling it anomaly either
                        self.basic_thresholder.update(score, new_score, 0.0, true);
                        result.anomaly_grade = 0.0;
                        return;
                    }
                }
                // previousIsPotentialAnomaly is true now, but not calling it anomaly either
                self.basic_thresholder.update(score, score, 0.0, true);
                result.anomaly_grade = 0.0;
                return;
            },
        }
    }

    pub fn set_z_factor(&mut self, factor: f32) {
        self.basic_thresholder.set_z_factor(factor);
        if factor > self.trigger_factor {
            self.trigger_factor = factor;
        }
    }

    pub fn set_lower_threshold(&mut self, lower :f32) {
        self.basic_thresholder.set_absolute_threshold(lower);
    }

    pub fn set_horizon(&mut self, horizon : f32) {
        self.basic_thresholder.set_horizon(horizon);
    }

    pub fn set_initial_threshold(&mut self, initial : f32) {
        self.basic_thresholder.set_initial_threshold(initial);
    }
}


use num::abs;

use crate::{
    common::divector::DiVector,
    samplerplustree::nodeview::LargeNodeView,
    visitor::visitor::{Visitor, VisitorInfo},
};

#[repr(C)]
pub struct AttributionVisitor {
    converged: bool,
    leaf_index: usize,
    score: f64,
    tree_mass: usize,
    hit_duplicate: bool,
    use_shadow_box: bool,
    attribution: DiVector,
    probability: DiVector,
}

impl AttributionVisitor {
    pub fn new(tree_mass: usize, dimension: usize, visitor_info: &VisitorInfo) -> Self {
        AttributionVisitor {
            tree_mass,
            leaf_index: usize::MAX,
            converged: false,
            score: 0.0,
            hit_duplicate: false,
            use_shadow_box: false,
            attribution: DiVector::empty(dimension),
            probability: DiVector::empty(dimension),
        }
    }

    pub fn create_visitor(
        tree_mass: usize,
        parameters: &[usize],
        visitor_info: &VisitorInfo,
    ) -> AttributionVisitor {
        let dimension = parameters[0];
        AttributionVisitor::new(tree_mass, dimension, visitor_info)
    }
}

impl Visitor<LargeNodeView, DiVector> for AttributionVisitor {
    fn accept_leaf(
        &mut self,
        point: &[f32],
        visitor_info: &VisitorInfo,
        node_view: &LargeNodeView,
    ) {
        let mass = node_view.get_mass();
        self.leaf_index = node_view.get_leaf_index();
        if mass > visitor_info.ignore_mass {
            if node_view.is_duplicate() {
                self.score = (visitor_info.damp)(mass, self.tree_mass)
                    * (visitor_info.score_seen)(node_view.get_depth(), mass);
                self.hit_duplicate = true;
                self.use_shadow_box = true;
            } else {
                self.score = (visitor_info.score_unseen)(node_view.get_depth(), mass);
                node_view.assign_probability_of_cut(&mut self.probability, point);
                assert!(abs(self.probability.total() - 1.0) < 1e-6);
                self.attribution.add_from(&self.probability, self.score);
            }
        } else {
            self.score = (visitor_info.score_unseen)(node_view.get_depth(), mass);
            self.use_shadow_box = true;
        }
    }

    fn accept(&mut self, point: &[f32], visitor_info: &VisitorInfo, node_view: &LargeNodeView) {
        if !self.converged {
            if !self.use_shadow_box {
                node_view.assign_probability_of_cut(&mut self.probability, point);
            } else {
                node_view.assign_probability_of_cut_shadow_box(&mut self.probability, point);
            };
            let prob = self.probability.total();
            if prob == 0.0 {
                self.converged = true;
            } else {
                let new_value =
                    (visitor_info.score_unseen)(node_view.get_depth(), node_view.get_mass());
                if !self.hit_duplicate {
                    self.score = (1.0 - prob) * self.score + prob * new_value;
                }
                self.attribution.scale(1.0 - prob);
                self.attribution.add_from(&self.probability, new_value);
            }
        }
    }

    fn result(&self, visitor_info: &VisitorInfo) -> DiVector {
        let t = (visitor_info.normalizer)(self.score, self.tree_mass);
        let mut answer = self.attribution.clone();
        answer.normalize(t);
        answer
    }

    fn is_converged(&self) -> bool {
        self.converged
    }

    fn use_shadow_box(&self) -> bool {
        self.use_shadow_box
    }
}


use num::abs;
use rand::{Rng, SeedableRng};
use rand_chacha::ChaCha20Rng;

use crate::{
    samplerplustree::nodeview::MediumNodeView,
    visitor::visitor::{SimpleMultiVisitor, Visitor, VisitorInfo},
};

#[repr(C)]
pub struct ImputeVisitor {
    centrality: f64,
    tree_mass: usize,
    rng: ChaCha20Rng,
    missing: Vec<usize>,
    stack: Vec<ImputeVisitorStackElement>,
    use_shadow_box: bool,
}

#[repr(C)]
struct ImputeVisitorStackElement {
    converged: bool,
    score: f64,
    random: f32,
    index: usize,
    distance: f64,
}

impl ImputeVisitor {
    pub fn new(missing: &[usize], centrality: f64, tree_mass: usize, seed: u64) -> Self {
        ImputeVisitor {
            tree_mass,
            centrality,
            rng: ChaCha20Rng::seed_from_u64(seed),
            missing: Vec::from(missing),
            stack: Vec::new(),
            use_shadow_box: false,
        }
    }

    pub fn create_nbr_visitor(
        tree_mass: usize,
        parameters: &[usize],
        _visitor_info: &VisitorInfo,
    ) -> Self {
        let percentile = if parameters.len() > 0 {
            parameters[0]
        } else {
            50
        };
        let seed = if parameters.len() > 1 {
            parameters[1]
        } else {
            0
        };
        let centrality = if percentile < 5 || percentile > 95 {
            0.0
        } else {
            1.0 - abs(1.0 - percentile as f64 / 50.0)
        };
        ImputeVisitor::new(&Vec::new(), centrality, tree_mass, seed as u64)
    }

    /// the following function allows the score to vary between the score used in
    /// anomaly detection and fully random sample based on the parameter centrality
    /// these two cases correspond to centrality = 1 and centrality = 0 respectively

    fn adjusted_score(&self, e: &ImputeVisitorStackElement, visitor_info: &VisitorInfo) -> f64 {
        self.centrality * (visitor_info.normalizer)(e.score, self.tree_mass)
            + (1.0 - self.centrality) * e.random as f64
    }
}

impl Visitor<MediumNodeView, (f64, usize, f64)> for ImputeVisitor {
    fn accept_leaf(
        &mut self,
        point: &[f32],
        visitor_info: &VisitorInfo,
        node_view: &MediumNodeView,
    ) {
        let mass = node_view.get_mass();
        let leaf_point = node_view.get_leaf_point();
        let mut new_point = Vec::from(point);
        for i in self.missing.iter() {
            new_point[*i] = leaf_point[*i];
        }

        let mut converged = false;
        let score: f64;
        if mass > visitor_info.ignore_mass || self.missing.len() != 0 {
            if node_view.is_duplicate() {
                score = (visitor_info.damp)(mass, self.tree_mass)
                    * (visitor_info.score_seen)(node_view.get_depth(), mass);
                converged = true;
            } else {
                score = (visitor_info.score_unseen)(node_view.get_depth(), mass);
            }
        } else {
            // shadow box is undefined for missing values
            // for not missing values, this block corresponds to exact same evaluation
            // in score and attribution visitor
            // note that multi-visitors ignore the shadow box anyways
            score = (visitor_info.score_unseen)(node_view.get_depth(), mass);
            self.use_shadow_box = true;
        }
        let dist = (visitor_info.distance)(&new_point, &leaf_point);
        self.stack.push(ImputeVisitorStackElement {
            converged,
            score,
            index: node_view.get_leaf_index(),
            random: self.rng.gen::<f32>(),
            distance: dist,
        });
    }

    fn accept(&mut self, point: &[f32], visitor_info: &VisitorInfo, node_view: &MediumNodeView) {
        assert!(
            self.stack.len() > 0,
            " there should have been an accept_leaf call which would have created a non-null stack"
        );
        let mut top_of_stack = self.stack.pop().unwrap();
        if !top_of_stack.converged {
            let prob = if !self.use_shadow_box {
                // note that this probablity ignores any missing coordinates
                // which would be accurate since the value used is inside the box
                node_view.get_probability_of_cut()
            } else {
                node_view.get_shadow_box_probability_of_cut()
            };
            if prob == 0.0 {
                top_of_stack.converged = true;
            } else {
                let new_score = (1.0 - prob) * top_of_stack.score
                    + prob
                        * (visitor_info.score_unseen)(node_view.get_depth(), node_view.get_mass());
                top_of_stack.converged = false;
                top_of_stack.score = new_score;
            }
            self.stack.push(top_of_stack);
        }
    }

    fn result(&self, visitor_info: &VisitorInfo) -> (f64, usize, f64) {
        assert_eq!(
            self.stack.len(),
            1,
            "incorrect state, stack length should be 1"
        );
        let top_of_stack = self.stack.last().unwrap();
        let t = (visitor_info.normalizer)(top_of_stack.score, self.tree_mass);
        (t, top_of_stack.index, top_of_stack.distance)
    }

    fn is_converged(&self) -> bool {
        self.stack.len() != 0 && self.stack.last().unwrap().converged
    }

    fn use_shadow_box(&self) -> bool {
        self.use_shadow_box
    }
}

impl SimpleMultiVisitor<MediumNodeView, (f64, usize, f64)> for ImputeVisitor {
    fn combine_branches(
        &mut self,
        _point: &[f32],
        _node_view: &MediumNodeView,
        visitor_info: &VisitorInfo,
    ) {
        assert!(self.stack.len() >= 2, "incorrect state");
        let mut top_of_stack = self.stack.pop().unwrap();
        let mut next_of_stack = self.stack.pop().unwrap();

        if self.adjusted_score(&top_of_stack, &visitor_info)
            < self.adjusted_score(&next_of_stack, &visitor_info)
        {
            top_of_stack.converged = top_of_stack.converged || next_of_stack.converged;
            self.stack.push(top_of_stack);
        } else {
            next_of_stack.converged = top_of_stack.converged || next_of_stack.converged;
            self.stack.push(next_of_stack);
        }
    }
}


use crate::{
    common::{directionaldensity::InterpolationMeasure},
    samplerplustree::nodeview::LargeNodeView,
    visitor::visitor::{Visitor, VisitorInfo},
};

#[repr(C)]
pub struct InterpolationVisitor {
    converged: bool,
    leaf_index: usize,
    score: f64,
    tree_mass: usize,
    hit_duplicate: bool,
    use_shadow_box: bool,
    interpolation_measure: InterpolationMeasure,
}

impl InterpolationVisitor {
    pub fn new(tree_mass: usize, dimension: usize, visitor_info: &VisitorInfo) -> Self {
        InterpolationVisitor {
            tree_mass,
            leaf_index: usize::MAX,
            converged: false,
            score: 0.0,
            hit_duplicate: false,
            use_shadow_box: false,
            interpolation_measure: InterpolationMeasure::empty(dimension, tree_mass as f32),
        }
    }

    pub fn create_visitor(
        tree_mass: usize,
        parameters: &[usize],
        visitor_info: &VisitorInfo,
    ) -> InterpolationVisitor {
        let dimension = parameters[0];
        InterpolationVisitor::new(tree_mass, dimension, visitor_info)
    }
}

impl Visitor<LargeNodeView, InterpolationMeasure> for InterpolationVisitor {
    fn accept_leaf(
        &mut self,
        point: &[f32],
        visitor_info: &VisitorInfo,
        node_view: &LargeNodeView,
    ) {
        let mass = node_view.get_mass();
        self.leaf_index = node_view.get_leaf_index();
        if mass > visitor_info.ignore_mass {
            if node_view.is_duplicate() {
                self.score = (visitor_info.damp)(mass, self.tree_mass)
                    * (visitor_info.score_seen)(node_view.get_depth(), mass);
                self.hit_duplicate = true;
                self.use_shadow_box = true;
            } else {
                let t = (visitor_info.score_unseen)(node_view.get_depth(), mass);
                self.score = t;
                let bounding_box = node_view.bounding_box();
                self.interpolation_measure.update(point, &bounding_box, t);
            }
        } else {
            self.score = (visitor_info.score_unseen)(node_view.get_depth(), mass);
            self.use_shadow_box = true;
        }
    }

    fn accept(&mut self, point: &[f32], visitor_info: &VisitorInfo, node_view: &LargeNodeView) {
        if !self.converged {
            let bounding_box = if !self.use_shadow_box {
                node_view.bounding_box()
            } else {
                node_view.shadow_box()
            };
            let new_value =
                (visitor_info.score_unseen)(node_view.get_depth(), node_view.get_mass());
            let prob = self
                .interpolation_measure
                .update(point, &bounding_box, new_value);
            if prob == 0.0 {
                self.converged = true;
            } else {
                if !self.hit_duplicate {
                    self.score = (1.0 - prob) * self.score + prob * new_value;
                }
            }
        }
    }

    fn result(&self, visitor_info: &VisitorInfo) -> InterpolationMeasure {
        let t = (visitor_info.normalizer)(self.score, self.tree_mass);
        let mut answer = self.interpolation_measure.clone();
        answer.measure.normalize(t);
        answer
    }

    fn is_converged(&self) -> bool {
        self.converged
    }

    fn use_shadow_box(&self) -> bool {
        self.use_shadow_box
    }
}


pub mod attributionvisitor;
pub mod imputevisitor;
pub mod interpolationvisitor;
pub mod scalarscorevisitor;
pub mod visitor;


use crate::{
    samplerplustree::nodeview::SmallNodeView,
    visitor::visitor::{Visitor, VisitorInfo},
};

#[repr(C)]
pub struct ScalarScoreVisitor {
    converged: bool,
    leaf_index: usize,
    score: f64,
    tree_mass: usize,
    use_shadow_box: bool,
}

impl ScalarScoreVisitor {
    pub fn default(tree_mass: usize, _parameters: &[usize], _visitor_info: &VisitorInfo) -> Self {
        ScalarScoreVisitor {
            tree_mass,
            leaf_index: usize::MAX,
            converged: false,
            score: 0.0,
            use_shadow_box: false,
        }
    }
}

impl Visitor<SmallNodeView, f64> for ScalarScoreVisitor {
    fn accept_leaf(
        &mut self,
        _point: &[f32],
        visitor_info: &VisitorInfo,
        node_view: &SmallNodeView,
    ) {
        let mass = node_view.get_mass();
        self.leaf_index = node_view.get_leaf_index();
        if mass > visitor_info.ignore_mass {
            if node_view.is_duplicate() {
                self.score = (visitor_info.damp)(mass, self.tree_mass)
                    * (visitor_info.score_seen)(node_view.get_depth(), mass);
                self.converged = true;
            } else {
                self.score = (visitor_info.score_unseen)(node_view.get_depth(), mass);
            }
        } else {
            self.score = (visitor_info.score_unseen)(node_view.get_depth(), mass);
            self.use_shadow_box = true;
        }
    }

    fn accept(&mut self, point: &[f32], visitor_info: &VisitorInfo, node_view: &SmallNodeView) {
        if !self.converged {
            let prob = if !self.use_shadow_box {
                node_view.get_probability_of_cut()
            } else {
                node_view.get_shadow_box_probability_of_cut()
            };
            if prob == 0.0 {
                self.converged = true;
            } else {
                self.score = (1.0 - prob) * self.score
                    + prob
                        * (visitor_info.score_unseen)(node_view.get_depth(), node_view.get_mass());
            }
        }
    }

    fn result(&self, visitor_info: &VisitorInfo) -> f64 {
        (visitor_info.normalizer)(self.score, self.tree_mass)
    }

    fn is_converged(&self) -> bool {
        self.converged
    }

    fn use_shadow_box(&self) -> bool {
        self.use_shadow_box
    }
}


use crate::{
    l1distance,
    rcf::{
        damp, displacement_normalizer, identity, normalizer, score_seen, score_seen_displacement,
        score_unseen, score_unseen_displacement,
    },
};

#[repr(C)]
pub struct VisitorInfo {
    pub ignore_mass: usize,
    pub score_seen: fn(usize, usize) -> f64,
    pub score_unseen: fn(usize, usize) -> f64,
    pub damp: fn(usize, usize) -> f64,
    pub normalizer: fn(f64, usize) -> f64,
    pub distance: fn(&[f32], &[f32]) -> f64,
}

pub trait Visitor<NodeView, Result> {
    fn accept(&mut self, point: &[f32], visitor_info: &VisitorInfo, node_view: &NodeView);
    fn accept_leaf(&mut self, point: &[f32], visitor_info: &VisitorInfo, node_view: &NodeView);
    fn is_converged(&self) -> bool;
    fn result(&self, visitor_info: &VisitorInfo) -> Result;
    fn use_shadow_box(&self) -> bool;
}

pub trait SimpleMultiVisitor<NodeView, Result>: Visitor<NodeView, Result> {
    fn combine_branches(
        &mut self,
        point: &[f32],
        _node_view: &NodeView,
        visitor_info: &VisitorInfo,
    );
}

pub trait UniqueMultiVisitor<NodeView, Result>: SimpleMultiVisitor<NodeView, Result> {
    fn trigger(&self, point: &[f32], node_view: &NodeView, visitor_info: &VisitorInfo) -> bool;
    fn unique_answer(&self, visitor_info: &VisitorInfo) -> Vec<f32>;
}

pub trait StreamingMultiVisitor<NodeView, Result>: UniqueMultiVisitor<NodeView, Result> {
    fn initialize_branch_split(
        &mut self,
        point: &[f32],
        node_view: &NodeView,
        visitor_info: &VisitorInfo,
    );
    fn second_branch(&mut self, point: &[f32], node_view: &NodeView, visitor_info: &VisitorInfo);
}

impl VisitorInfo {
    pub fn default() -> Self {
        VisitorInfo {
            ignore_mass: 0,
            score_seen,
            score_unseen,
            damp,
            normalizer,
            distance: l1distance,
        }
    }
    pub fn displacement() -> Self {
        VisitorInfo {
            ignore_mass: 0,
            score_seen: score_seen_displacement,
            score_unseen: score_unseen_displacement,
            damp,
            normalizer: displacement_normalizer,
            distance: l1distance,
        }
    }
    pub fn density() -> Self {
        VisitorInfo {
            ignore_mass: 0,
            score_seen: score_unseen_displacement,
            score_unseen: score_unseen_displacement,
            damp,
            normalizer: identity,
            distance: l1distance,
        }
    }
    pub fn use_score(
        ignore_mass: usize,
        score_seen: fn(usize, usize) -> f64,
        score_unseen: fn(usize, usize) -> f64,
        damp: fn(usize, usize) -> f64,
        normalizer: fn(f64, usize) -> f64,
    ) -> Self {
        VisitorInfo {
            ignore_mass,
            score_seen,
            score_unseen,
            damp,
            normalizer,
            distance: l1distance,
        }
    }
    pub fn use_distance(distance: fn(&[f32], &[f32]) -> f64) -> Self {
        VisitorInfo {
            ignore_mass: 0,
            score_seen,
            score_unseen,
            damp,
            normalizer,
            distance,
        }
    }
}


extern crate rand;
extern crate rand_chacha;
extern crate rcflib;

use num::abs;
use rand::{Rng, SeedableRng};
use rand_chacha::ChaCha20Rng;
use rcflib::{
    common::multidimdatawithkey::MultiDimDataWithKey,
    rcf::{create_rcf, RCF},
};

/// try cargo test --release
/// these tests are designed to be longish

#[test]
fn anomalydetection_attribution_and_update() {
    let shingle_size = 8;
    let base_dimension = 5;
    let data_size = 100000;
    let number_of_trees = 30;
    let capacity = 256;
    let initial_accept_fraction = 0.1;
    let dimensions = shingle_size * base_dimension;
    let _point_store_capacity = capacity * number_of_trees + 1;
    let time_decay = 0.1 / capacity as f64;
    let bounding_box_cache_fraction = 1.0;
    let random_seed = 17;
    let parallel_enabled: bool = false;
    let store_attributes: bool = false;
    let internal_shingling: bool = true;
    let internal_rotation = false;
    let noise = 5.0;

    let mut forest: Box<dyn RCF> = create_rcf(
        dimensions,
        shingle_size,
        capacity,
        number_of_trees,
        random_seed,
        store_attributes,
        parallel_enabled,
        internal_shingling,
        internal_rotation,
        time_decay,
        initial_accept_fraction,
        bounding_box_cache_fraction,
    );
    let mut rng = ChaCha20Rng::seed_from_u64(42);
    let mut amplitude = Vec::new();
    for _i in 0..base_dimension {
        amplitude.push((1.0 + 0.2 * rng.gen::<f32>()) * 100.0);
    }
    let data_with_key = MultiDimDataWithKey::multi_cosine(
        data_size,
        &vec![60; base_dimension],
        &amplitude,
        noise,
        0,
        base_dimension.into(),
    );

    let mut score: f64 = 0.0;
    let _next_index = 0;

    for i in 0..data_with_key.data.len() {
        let attribution = forest.attribution(&data_with_key.data[i]).unwrap();
        let new_score = forest.score(&data_with_key.data[i]).unwrap();
        assert!(abs(new_score - attribution.total()) < 1e-6);

        /*
        if next_index < data_with_key.change_indices.len() && data_with_key.change_indices[next_index] == i {
            println!(" score at change {} position {} ", new_score, i);
            next_index += 1;
        }
        */

        score += attribution.total();
        forest.update(&data_with_key.data[i], 0).unwrap();
    }

    println!(
        "Average score {} ",
        (score / data_with_key.data.len() as f64)
    );
    assert!(
        score < data_with_key.data.len() as f64,
        " average score is above 1"
    );
    println!("Success! {}", forest.entries_seen());
    println!("PointStore Size {} ", forest.point_store_size());
    println!("Total size {} bytes (approx)", forest.size());
}