use crate::{
    common::samplesummary::{summarize, SampleSummary},
    pointstore::PointStore,
};

fn project_missing(point: &Vec<f32>, position: &[usize]) -> Vec<f32> {
    position.iter().map(|i| point[*i]).collect()
}

const CONDITIONAL_UPPER_FRACTION : f64 = 0.9;

const CONDITIONAL_LOWER_FRACTION : f64 = 0.1;

/// the following function is a conduit that summarizes the conditional samples derived from the trees
/// The samples are denoted by (PointIndex, f32) where the PointIndex(usize) corresponds to the point identifier
/// in the point store and the f32 associated with a scalar value (corresponding to weight)
/// the field missing corresponds to the list of missing fields in the space of the full (potentially shingled) points
/// centrality corresponds to the parameter which was used to derive the samples, and thus provides a mechanism for
/// refined interpretation in summarization
/// project corresponds to a boolean flag, determining whether we wish to focus on the missing fields only (project = true)
/// or we focus on the entire space of (potentially shingled) points (in case of project = false) which have different
/// and complementary uses.
/// max_number corresponds to a parameter that controls the summarization -- in the current version this corresponds to
/// an upper bound on the number of summary points in the SampleSummary construct
///
/// Note that the global, mean and median do not perform any weighting/pruning; whereas the summarize() performs on
/// somewhat denoised data to provide a list of summary. Note further that summarize() is redundant (and skipped)
/// when max_number = 0
/// The combination appears to provide the best of all worlds with little performance overhead and can be
/// used and reconfigured easily. In the fullness of time, it is possible to leverage a dynamic Kernel, since
/// the entire PointStore is present and the PointStore is dynamic.
#[repr(C)]
pub struct FieldSummarizer {
    centrality: f64,
    project: bool,
    max_number: usize,
    distance: fn(&[f32], &[f32]) -> f64,
}

impl FieldSummarizer {
    pub fn new(
        centrality: f64,
        project: bool,
        max_number: usize,
        distance: fn(&[f32], &[f32]) -> f64,
    ) -> Self {
        FieldSummarizer {
            centrality,
            project,
            max_number,
            distance,
        }
    }

    pub fn summarize_list(
        &self,
        pointstore: &dyn PointStore,
        point_list_with_distance: &[(f64, usize, f64)],
        missing: &[usize]
    ) -> SampleSummary {
        let mut distance_list: Vec<f64> = point_list_with_distance.iter().map(|a| a.2).collect();
        distance_list.sort_by(|a, b| a.partial_cmp(&b).unwrap());
        let mut threshold = 0.0;
        if self.centrality > 0.0 {
            let mut always_include = 0;
            while always_include < point_list_with_distance.len() - 1
                && distance_list[always_include] == 0.0
            {
                always_include += 1;
            }
            threshold = self.centrality
                * (distance_list[always_include + (distance_list.len() - always_include) / 3]
                    + distance_list[always_include + (distance_list.len() - always_include) / 2])
                    as f64;
        }
        threshold +=
            (1.0 - self.centrality) * distance_list[point_list_with_distance.len() - 1] as f64;

        let total_weight = point_list_with_distance.len() as f64;
        let dimensions = if !self.project || missing.len() == 0 {
            pointstore.get_copy(point_list_with_distance[0].1).len()
        } else {
            missing.len()
        };
        let mut mean = vec![0.0f32; dimensions];
        let mut deviation = vec![0.0f32; dimensions];
        let mut sum_values_sq = vec![0.0f64; dimensions];
        let mut sum_values = vec![0.0f64; dimensions];
        let mut vec = Vec::new();
        for i in 0..point_list_with_distance.len() {
            let point = if !self.project || missing.len() == 0 {
                pointstore.get_copy(point_list_with_distance[i].1)
            } else {
                project_missing(
                    &pointstore.get_copy(point_list_with_distance[i].1),
                    &missing,
                )
            };
            for j in 0..dimensions {
                sum_values[j] += point[j] as f64;
                sum_values_sq[j] += point[j] as f64 * point[j] as f64;
            }
            // the else can be filtered further
            let weight: f32 = if point_list_with_distance[i].2 <= threshold {
                1.0
            } else {
                (threshold / point_list_with_distance[i].2) as f32
            };

            vec.push((point, weight));
        }

        for j in 0..dimensions {
            mean[j] = (sum_values[j] / total_weight as f64) as f32;
            let t: f64 = sum_values_sq[j] / total_weight as f64
                - sum_values[j] * sum_values[j] / (total_weight as f64 * total_weight as f64);
            deviation[j] = f64::sqrt(if t > 0.0 { t } else { 0.0 }) as f32;
        }
        let vec_weight : f64 = vec.iter().map(|x| x.1 as f64).sum();
        let num = vec_weight/2.0;
        let lower_fraction = vec_weight *CONDITIONAL_LOWER_FRACTION;
        let upper_fraction = vec_weight*CONDITIONAL_UPPER_FRACTION;
        let mut median = vec![0.0f32; dimensions];
        let mut upper = vec![0.0f32; dimensions];
        let mut lower = vec![0.0f32; dimensions];

        for j in 0..dimensions {
            let mut y: Vec<(f32,f32)> = vec.iter().map(|x| (x.0[j],x.1)).collect();
            y.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap());
            let first = SampleSummary::pick(&y,lower_fraction,0,0.0);
            lower[j] = y[first.0].0;
            let second = SampleSummary::pick(&y,num,first.0,first.1);
            median[j] = y[second.0].0;
            let third = SampleSummary::pick(&y,upper_fraction,second.0,second.1);
            upper[j] = y[third.0].0;
        }

        let summary = summarize(&vec, self.distance, self.max_number, false).unwrap();
        SampleSummary {
            summary_points: summary.summary_points.clone(),
            relative_weight: summary.relative_weight.clone(),
            total_weight: summary.total_weight,
            mean,
            median,
            upper,
            lower,
            deviation,
        }
    }
}


/**
 * This class maintains a simple discounted statistics. Setters are avoided
 * except for discount rate which is useful as initialization from raw scores
 */
#[repr(C)]
#[derive(Clone)]
pub struct Deviation {
    pub discount: f64,
    pub weight: f64,
    pub sum_squared:f64,
    pub sum: f64,
    pub count: i32
}

impl Deviation {
    pub fn new(discount: f64) -> Self {
        assert!(discount>=0.0 && discount < 1.0, "incorrect discount value");
        Deviation {
            discount,
            weight: 0.0,
            sum:0.0,
            sum_squared:0.0,
            count:0
        }
    }

    pub fn default() -> Self {
        Deviation {
            discount: 0.0,
            weight: 0.0,
            sum:0.0,
            sum_squared:0.0,
            count:0
        }
    }

    pub fn create(discount:f64,weight:f64,sum:f64,sum_squared:f64,count:i32) -> Self{
        Deviation{
            discount,
            weight,
            sum,
            sum_squared,
            count
        }
    }

    pub fn mean(&self) -> f64 {
        if self.is_empty() {
            0.0
        } else {
            self.sum / self.weight
        }
    }

    pub fn update(&mut self, score: f64) {
        let factor = if self.discount == 0.0 {1.0} else {
            let a = 1.0 - self.discount;
            let b= 1.0 - 1.0 / (self.count + 2) as f64;
            if a<b {a} else {b}
        };
        self.sum = self.sum * factor + score;
        self.sum_squared = self.sum_squared * factor + score * score;
        self.weight = self.weight * factor + 1.0;
        self.count += 1;
    }

    pub fn deviation(&self) -> f64{
        if self.is_empty() {
            return 0.0;
        }
        let temp = self.sum / self.weight;
        let answer = self.sum_squared / self.weight - temp * temp;
        if answer > 0.0 {
            f64::sqrt(answer)
        } else {
            0.0
        }
    }

    pub fn is_empty(&self) -> bool{
        self.weight <= 0.0
    }

    pub fn discount(&self) -> f64 {
        self.discount
    }

    pub fn set_discount(&mut self, discount:f64) {
        self.discount = discount;
    }

    pub fn sum(&self) -> f64 {
        self.sum
    }

    pub fn sum_squared(&self) -> f64{
        self.sum_squared
    }

    pub fn weight(&self) -> f64{
        self.weight
    }

    pub fn count(&self) -> i32{
        self.count
    }

    pub fn set_count(&mut self,count:i32) {
        self.count = count;
    }
}


use crate::{common::divector::DiVector, samplerplustree::boundingbox::BoundingBox};

#[repr(C)]
#[derive(Clone)]
pub struct InterpolationMeasure {
    pub measure: DiVector,
    pub distance: DiVector,
    pub probability_mass: DiVector,
    pub sample_size: f32,
}

impl InterpolationMeasure {
    pub fn empty(dimension: usize, sample_size: f32) -> Self {
        InterpolationMeasure {
            measure: DiVector::empty(dimension),
            distance: DiVector::empty(dimension),
            probability_mass: DiVector::empty(dimension),
            sample_size,
        }
    }

    pub fn new(
        measure: DiVector,
        distance: DiVector,
        prob_mass: DiVector,
        sample_size: f32,
    ) -> Self {
        assert!(
            measure.dimensions() == distance.dimensions(),
            " incorrect lengths"
        );
        assert!(
            measure.dimensions() == prob_mass.dimensions(),
            " incorrect lengths"
        );
        InterpolationMeasure {
            measure: measure,
            distance: distance,
            probability_mass: prob_mass,
            sample_size,
        }
    }

    pub fn add_to(&self, other: &mut InterpolationMeasure) {
        self.probability_mass.add_to(&mut other.probability_mass);
        self.distance.add_to(&mut other.distance);
        self.measure.add_to(&mut other.measure);
        other.sample_size += self.sample_size;
    }

    pub fn divide(&mut self, num: usize) {
        self.scale(1.0 / num as f64);
        self.scale_samples(1.0 / num as f64);
    }

    pub fn scale(&mut self, factor: f64) {
        self.distance.scale(factor);
        self.probability_mass.scale(factor);
        self.measure.scale(factor);
    }

    pub fn scale_samples(&mut self, factor: f64) {
        self.sample_size = (self.sample_size as f64 * factor) as f32;
    }

    pub fn update(&mut self, point: &[f32], bounding_box: &BoundingBox, measure: f64) -> f64 {
        let min_values = bounding_box.get_min_values();
        let max_values = bounding_box.get_max_values();
        let minsum: f32 = min_values
            .iter()
            .zip(point)
            .map(|(&x, &y)| if x - y > 0.0 { x - y } else { 0.0 })
            .sum();
        let maxsum: f32 = point
            .iter()
            .zip(max_values)
            .map(|(&x, &y)| if x - y > 0.0 { x - y } else { 0.0 })
            .sum();
        let sum = maxsum + minsum;
        let new_range = sum as f64 + bounding_box.get_range_sum();
        let prob = sum as f64 / (new_range);
        if prob > 0.0 {
            self.scale(1.0 - prob);
            for i in 0..point.len() {
                if point[i] > max_values[i] {
                    let t = (point[i] - max_values[i]) as f64 / new_range;
                    self.distance.high[i] += t * (point[i] - min_values[i]) as f64;
                    self.probability_mass.high[i] += t;
                    self.measure.high[i] += measure * t;
                } else if point[i] < min_values[i] {
                    let t = (min_values[i] - point[i]) as f64 / new_range;
                    self.distance.low[i] += t * (max_values[i] - point[i]) as f64;
                    self.probability_mass.low[i] += t;
                    self.measure.low[i] += measure * t;
                }
            }
        }
        prob
    }

    pub fn directional_measure(&self, threshold: f64, manifold_dimension: f64) -> DiVector {
        assert!(
            self.sample_size >= 0.0 && self.measure.total() >= 0.0,
            " cannot have negative samples or measure"
        );
        if self.sample_size == 0.0f32 || self.measure.total() == 0.0 {
            return DiVector::empty(self.measure.dimensions());
        }

        let mut sum_of_factors = 0.0;

        for i in 0..self.measure.dimensions() {
            let mut t = if self.probability_mass.high_low_sum(i) > 0.0 {
                self.distance.high_low_sum(i) / self.probability_mass.high_low_sum(i)
            } else {
                0.0
            };

            if t > 0.0 {
                t = f64::exp(f64::ln(t) * manifold_dimension)
                    * self.probability_mass.high_low_sum(i);
            }
            sum_of_factors += t;
        }

        let density_factor = 1.0 / (threshold + sum_of_factors);
        let mut answer = self.measure.clone();
        answer.scale(density_factor);
        answer
    }

    pub fn directional_density(&self) -> DiVector {
        self.directional_measure(1e-3, self.measure.dimensions() as f64)
    }

    pub fn density(&self) -> f64 {
        self.directional_density().total()
    }
}


use crate::samplerplustree::boundingbox::BoundingBox;

#[repr(C)]
#[derive(Clone)]
pub struct DiVector {
    pub high: Vec<f64>,
    pub low: Vec<f64>,
}

impl DiVector {
    pub fn empty(dimension: usize) -> Self {
        DiVector {
            high: vec![0.0; dimension],
            low: vec![0.0; dimension],
        }
    }

    pub fn new(high: &[f64], low: &[f64]) -> Self {
        assert!(high.len() == low.len(), " incorrect lengths");
        DiVector {
            high: Vec::from(high),
            low: Vec::from(low),
        }
    }

    pub fn assign_as_probability_of_cut(&mut self, bounding_box: &BoundingBox, point: &[f32]) {
        let minsum: f64 = self
            .low
            .iter_mut()
            .zip(bounding_box.get_min_values())
            .zip(point)
            .map(|((x, &y), &z)| {
                if y - z > 0.0 {
                    *x = (y - z) as f64;
                    *x
                } else {
                    *x = 0.0;
                    *x
                }
            })
            .sum();
        let maxsum: f64 = self
            .high
            .iter_mut()
            .zip(point)
            .zip(bounding_box.get_max_values())
            .map(|((x, &y), &z)| {
                if y - z > 0.0 {
                    *x = (y - z) as f64;
                    *x
                } else {
                    *x = 0.0;
                    *x
                }
            })
            .sum();

        let sum = minsum + maxsum;
        if sum != 0.0 {
            self.scale(1.0 / (bounding_box.get_range_sum() + sum));
        }
    }

    pub fn assign_as_probability_of_cut_with_missing_coordinates(
        &mut self,
        bounding_box: &BoundingBox,
        point: &[f32],
        missing_coordinates: &[bool],
    ) {
        let minsum: f64 = self
            .low
            .iter_mut()
            .zip(bounding_box.get_min_values())
            .zip(point)
            .zip(missing_coordinates)
            .map(|(((x, &y), &z), &b)| {
                if !b && y - z > 0.0 {
                    *x = (y - z) as f64;
                    *x
                } else {
                    *x = 0.0;
                    *x
                }
            })
            .sum();
        let maxsum: f64 = self
            .high
            .iter_mut()
            .zip(point)
            .zip(bounding_box.get_max_values())
            .zip(missing_coordinates)
            .map(|(((x, &y), &z), &b)| {
                if !b && y - z > 0.0 {
                    *x = (y - z) as f64;
                    *x
                } else {
                    *x = 0.0;
                    *x
                }
            })
            .sum();

        let sum = minsum + maxsum;
        if sum != 0.0 {
            self.scale(1.0 / (bounding_box.get_range_sum() + sum));
        }
    }

    pub fn assign(&mut self, other: &DiVector) {
        for (x, &y) in self.high.iter_mut().zip(&other.high) {
            *x = y;
        }
        for (x, &y) in self.low.iter_mut().zip(&other.low) {
            *x = y;
        }
    }

    pub fn add_from(&mut self, other: &DiVector, factor: f64) {
        other.add_to_scaled(self, factor);
    }

    pub fn add_to(&self, other: &mut DiVector) {
        for (x, &y) in other.high.iter_mut().zip(&self.high) {
            *x += y;
        }
        for (x, &y) in other.low.iter_mut().zip(&self.low) {
            *x += y;
        }
    }

    pub fn add_to_scaled(&self, other: &mut DiVector, factor: f64) {
        for (x, &y) in other.high.iter_mut().zip(&self.high) {
            *x += y * factor;
        }
        for (x, &y) in other.low.iter_mut().zip(&self.low) {
            *x += y * factor;
        }
    }

    pub fn divide(&mut self, num: usize) {
        self.scale(1.0 / num as f64)
    }

    pub fn scale(&mut self, factor: f64) {
        for x in self.high.iter_mut() {
            *x *= factor;
        }
        for x in self.low.iter_mut() {
            *x *= factor;
        }
    }

    pub fn total(&self) -> f64 {
        self.high.iter().sum::<f64>() + self.low.iter().sum::<f64>()
    }

    pub fn normalize(&mut self, value: f64) {
        let current = self.total();
        if current <= 0.0 {
            let v = value / (2.0 * self.high.len() as f64);
            for x in self.high.iter_mut() {
                *x = v;
            }
            for x in self.low.iter_mut() {
                *x = v;
            }
        } else {
            self.scale(value / current);
        }
    }

    pub fn dimensions(&self) -> usize {
        self.high.len()
    }

    pub fn high_low_sum(&self, index: usize) -> f64 {
        self.high[index] + self.low[index]
    }

    pub fn max_contribution(&self, base_dimension: usize) -> usize {
        let mut val = 0.0;
        let mut index = 0;
        for i in 0..base_dimension {
            val += self.high_low_sum(i);
        }
        for j in 1..(self.dimensions() / base_dimension) {
            let mut sum = 0.0;
            for i in 0..base_dimension {
                sum += self.high_low_sum(j * base_dimension + i);
            }
            if sum > val {
                val = sum;
                index = j;
            }
        }
        index
    }
}


use std::fmt::Debug;

#[repr(C)]
pub struct IntervalStoreManager<T> {
    capacity: usize,
    last_in_use: usize,
    free_indices_start: Vec<T>,
    free_indices_end: Vec<T>,
}

impl<T: Copy + std::convert::TryFrom<usize>> IntervalStoreManager<T>
where
    T: std::fmt::Display + std::cmp::PartialEq,
    usize: From<T>,
{
    pub fn new(size: usize) -> Self
    where
        <T as TryFrom<usize>>::Error: Debug,
    {
        IntervalStoreManager {
            capacity: size,
            last_in_use: 1,
            free_indices_start: vec![0.try_into().unwrap()],
            free_indices_end: vec![(size - 1).try_into().unwrap()],
        }
    }

    pub fn get_capacity(&self) -> usize {
        self.capacity
    }

    pub fn change_capacity(&mut self, new_capacity: usize)
    where
        <T as TryFrom<usize>>::Error: Debug,
    {
        if new_capacity > self.capacity {
            let start: T = self.capacity.try_into().unwrap();
            let end: T = (new_capacity - 1).try_into().unwrap();
            if self.free_indices_start.len() == self.last_in_use {
                self.free_indices_start.resize(self.last_in_use + 1, start);
                self.free_indices_end.resize(self.last_in_use + 1, end);
            } else {
                self.free_indices_start[self.last_in_use] = start;
                self.free_indices_end[self.last_in_use] = end;
            }
            self.last_in_use += 1;
            self.capacity = new_capacity;
        }
    }

    pub fn is_empty(&self) -> bool {
        self.last_in_use == 0
    }

    pub fn get(&mut self) -> usize
    where
        <T as TryFrom<usize>>::Error: Debug,
    {
        if self.is_empty() {
            println!(" no more indices left");
            panic!();
        }
        let answer = self.free_indices_start[self.last_in_use - 1];
        let new_value: usize = answer.into();
        if answer == self.free_indices_end[self.last_in_use - 1] {
            self.last_in_use -= 1;
        } else {
            self.free_indices_start[self.last_in_use - 1] = (new_value + 1).try_into().unwrap();
        }
        new_value
    }

    pub fn release(&mut self, index: usize)
    where
        <T as TryFrom<usize>>::Error: Debug,
    {
        let val: T = TryFrom::try_from(index).unwrap();
        if self.last_in_use != 0 {
            let start: usize = self.free_indices_start[self.last_in_use - 1].into();
            let end: usize = self.free_indices_end[self.last_in_use - 1].into();
            if start == index + 1 {
                self.free_indices_start[self.last_in_use - 1] = val;
                return;
            } else if end + 1 == index {
                self.free_indices_end[self.last_in_use - 1] = val;
                return;
            }
        }
        if self.last_in_use < self.free_indices_start.len() {
            self.free_indices_start[self.last_in_use] = val;
            self.free_indices_end[self.last_in_use] = val;
        } else {
            self.free_indices_start.resize(self.last_in_use + 1, val);
            self.free_indices_end.resize(self.last_in_use + 1, val);
        }
        self.last_in_use += 1;
    }

    pub fn used(&self) -> usize {
        let mut answer = 0;
        for i in 0..self.last_in_use {
            let start: usize = self.free_indices_start[i].into();
            let end: usize = self.free_indices_end[i].into();
            answer += end - start + 1;
        }
        self.capacity - answer
    }

    pub fn get_size(&self) -> usize {
        self.free_indices_start.len() * 2 * std::mem::size_of::<T>()
            + std::mem::size_of::<IntervalStoreManager<T>>()
    }
}


pub mod cluster;
pub mod conditionalfieldsummarizer;
pub mod directionaldensity;
pub mod divector;
pub mod intervalstoremanager;
pub mod multidimdatawithkey;
pub mod samplesummary;
pub mod deviation;
pub mod anomalydescriptor;
pub mod rangevector;


extern crate rand;

extern crate rand_chacha;
use std::f32::consts::PI;

use rand::SeedableRng;
use rand_chacha::ChaCha20Rng;
use rand_core::RngCore;

use crate::rand::Rng;

pub struct MultiDimDataWithKey {
    pub data: Vec<Vec<f32>>,
    pub change_indices: Vec<usize>,
    pub labels: Vec<usize>,
    pub changes: Vec<Vec<f32>>,
}

impl MultiDimDataWithKey {
    pub fn multi_cosine(
        num: usize,
        period: &[usize],
        amplitude: &[f32],
        noise: f32,
        seed: u64,
        base_dimension: usize,
    ) -> Self {
        assert!(
            period.len() == base_dimension,
            " need a period for each dimension "
        );
        assert!(
            amplitude.len() == base_dimension,
            " need an amplitude for each dimension"
        );
        let mut rng = ChaCha20Rng::seed_from_u64(seed);
        let mut noiserng = ChaCha20Rng::seed_from_u64(seed + 1);
        let mut phase: Vec<usize> = Vec::new();

        for i in 0..base_dimension {
            phase.push(rng.next_u64() as usize % period[i]);
        }

        let mut data: Vec<Vec<f32>> = Vec::new();
        let mut change_indices: Vec<usize> = Vec::new();
        let mut changes: Vec<Vec<f32>> = Vec::new();

        for i in 0..num {
            let mut elem = vec![0.0; base_dimension];
            let flag = noiserng.gen::<f32>() < 0.01;
            let mut new_change = vec![0.0; base_dimension];
            let mut used: bool = false;
            for j in 0..base_dimension {
                elem[j] = amplitude[j]
                    * (2.0 * PI * (i + phase[j]) as f32 / period[j] as f32).cos()
                    + noise * noiserng.gen::<f32>();
                if flag && noiserng.gen::<f64>() < 0.3 {
                    let factor: f32 = 5.0 * (1.0 + noiserng.gen::<f32>());
                    let mut change: f32 = factor * noise;
                    if noiserng.gen::<f32>() < 0.5 {
                        change = -change;
                    }
                    elem[j] += change;
                    new_change[j] = change;
                    used = true;
                }
            }
            data.push(elem);
            if used {
                change_indices.push(i);
                changes.push(new_change);
            }
        }
        MultiDimDataWithKey {
            data,
            change_indices,
            labels: Vec::new(),
            changes,
        }
    }

    pub fn mixture(
        num: usize,
        mean: &[Vec<f32>],
        scale: &[Vec<f32>],
        weight: &[f32],
        seed: u64,
    ) -> Self {
        let mut rng = ChaCha20Rng::seed_from_u64(seed);
        assert!(num > 0, " number of elements cannot be 0");
        assert!(mean.len() > 0, " cannot be null");
        let base_dimension = mean[0].len();
        assert!(
            mean.len() == scale.len(),
            " need scales and means to be 1-1"
        );
        assert!(
            weight.len() == mean.len(),
            " need weights and means to be 1-1"
        );
        for i in 0..mean.len() {
            assert!(
                mean[i].len() == base_dimension,
                " must have the same dimensions"
            );
            assert!(
                scale[i].len() == base_dimension,
                "sclaes must have the same dimension as the mean"
            );
            assert!(weight[i] >= 0.0, " weights cannot be negative");
        }
        let sum: f32 = weight.iter().sum();

        let mut data = Vec::new();
        let mut labels = Vec::new();
        for _j in 0..num {
            let mut i = 0;
            let mut wt: f32 = sum * rng.gen::<f32>();
            while wt > weight[i] {
                wt -= weight[i];
                i += 1;
            }
            data.push(new_vec(&mean[i], &scale[i], &mut rng));
            labels.push(i);
        }

        MultiDimDataWithKey {
            data,
            labels,
            change_indices: vec![],
            changes: vec![],
        }
    }
}

fn next_element(mean: f32, scale: f32, rng: &mut ChaCha20Rng) -> f32 {
    let mut r: f32 = f64::sqrt(-2.0f64 * f64::ln(rng.gen::<f64>())) as f32;
    // the following is to discard inf being returned from ln()
    while r.is_infinite() {
        r = f64::sqrt(-2.0f64 * f64::ln(rng.gen::<f64>())) as f32;
    }

    let switch: f32 = rng.gen();
    if 0.5 < switch {
        mean + scale * r * f32::cos(2.0 * PI * rng.gen::<f32>())
    } else {
        mean + scale * r * f32::sin(2.0 * PI * rng.gen::<f32>())
    }
}

pub fn new_vec(mean: &[f32], scale: &[f32], rng: &mut ChaCha20Rng) -> Vec<f32> {
    let dimensions = mean.len();
    let mut answer = Vec::new();
    for i in 0..dimensions {
        answer.push(next_element(mean[i], scale[i], rng));
    }
    answer
}


/**
 * A RangeVector is used when we want to track a quantity and its upper and
 * lower bounds
 */
#[repr(C)]
#[derive(Clone)]
pub struct RangeVector {
    pub values: Vec<f32>,
    pub upper: Vec<f32>,
    pub lower: Vec<f32>
}

impl RangeVector {
    pub fn new(dimensions: usize) -> Self {
        RangeVector {
            values: vec![0.0; dimensions],
            upper: vec![0.0; dimensions],
            lower: vec![0.0; dimensions]
        }
    }

    pub fn from(values : Vec<f32>) -> Self {
        RangeVector{
            values : values.clone(),
            upper : values.clone(),
            lower : values.clone()
        }
    }

    pub fn create(values: &[f32], upper: &[f32], lower:&[f32]) -> Self {
        assert!(values.len() == upper.len() && upper.len() == lower.len(), " incorrect lengths");
        for i in 0..values.len() {
            assert!(values[i] <= upper[i], " incorrect upper bound at {}", i);
            assert!(lower[i] <= values [i], "incorrect lower bounds at {}",i);
        }
        RangeVector{
            values :Vec::from(values),
            upper : Vec::from(upper),
            lower : Vec::from(lower)
        }
    }

    pub fn shift(&mut self, i:usize, shift: f32) {
        self.values[i] += shift;
        self.upper[i] += shift;
        self.lower[i] += shift;
        // managing precision explicitly
        if self.upper[i] < self.values[i] {
            self.upper[i] = self.values[i];
        }
        if self.lower[i] > self.values[i] {
            self.lower[i] = self.values[i];
        }
    }

    pub fn scale(&mut self, i:usize, scale: f32) {
        self.values[i] *= scale;
        self.upper[i] *= scale;
        self.lower[i] *= scale;
        // managing precision explicitly
        if self.upper[i] < self.values[i] {
            self.upper[i] = self.values[i];
        }
        if self.lower[i] > self.values[i] {
            self.lower[i] = self.values[i];
        }
    }
}


use std::cmp::min;
use std::ops::Index;
use std::slice::SliceIndex;
use crate::types::Result;
use crate::errors;
use crate::util::check_argument;
use rand::{Rng, SeedableRng};
use rand_chacha::ChaCha20Rng;
use rayon::range;

use crate::common::cluster::{Center, multi_cluster_as_weighted_obj, multi_cluster_as_weighted_ref, single_centroid_cluster_weighted_vec_with_distance_over_slices};

///
/// The goal of the summarization below is as follows: on being provided a collection of sampled weighted points
/// represented by a slice &[(Vec<f32>,f32)] where each of the Vec<f32> has the same length/dimension
/// and the f32 in the pair is the corresponding weight.
/// The algorithm uses the philosophy of RCFs, in repeatedly using randomization. It proceeds as follows:
/// 1. It uses an initial sampling which serves as a basis of efficiency as well as denoising, borrowing from
/// https://en.wikipedia.org/wiki/CURE_algorithm, in that algorithm's robustness to outliers.
/// 2. It uses a sampling mechanism to initialize some clusters based on https://en.wikipedia.org/wiki/Data_stream_clustering
/// where the radom sampling achieves half of the the same effects as hierarchical compression.
///3.  It repeatedly merges the most overlapping clusters, failing that, eliminates the least weighted cluster to achieve
/// the same effect as hieararchical compression.
///
/// The algorithm takes a distance function as an input, and tends to produce spherical (measured in the input
/// distance function) clusters. These types of algorithms are unlikely to be useful for large number of output clusters.
/// The output is the SampleSummary, which provides basic statistics of mean, median and deviation
/// in addition it performs a grouping/clustering, assuming that the maximum number of clusters are not large
/// the routine below bounds the number to be max_number_per_dimension times the dimension of Vec<f32>
/// and a smaller number can also be provided in the summarize() function
///
///

const MAX_NUMBER_PER_DIMENSION: usize = 5;

const PHASE2_THRESHOLD: usize = 2;

const LENGTH_BOUND: usize = 1000;

const UPPER_FRACTION : f64 = 0.9;

const LOWER_FRACTION : f64 = 0.1;

#[repr(C)]
pub struct SampleSummary {
    pub summary_points: Vec<Vec<f32>>,

    // a measure of comparison among the typical points;
    pub relative_weight: Vec<f32>,

    // number of samples, often the number of summary, but can handle weighted points
    // (possibly indicating confidence or othe measure) in the future
    pub total_weight: f32,

    // the global mean, median
    pub mean: Vec<f32>,
    pub median: Vec<f32>,

    // percentiles and bounds
    pub upper: Vec<f32>,
    pub lower : Vec<f32>,

    // This is the global deviation,
    pub deviation: Vec<f32>,
}

impl SampleSummary {
    pub fn new(
        total_weight: f32,
        summary_points: Vec<Vec<f32>>,
        relative_weight: Vec<f32>,
        median: Vec<f32>,
        mean: Vec<f32>,
        upper: Vec<f32>,
        lower: Vec<f32>,
        deviation: Vec<f32>,
    ) -> Self {
        SampleSummary {
            total_weight,
            summary_points: summary_points.clone(),
            relative_weight: relative_weight.clone(),
            median: median.clone(),
            mean: mean.clone(),
            upper: upper.clone(),
            lower: lower.clone(),
            deviation: deviation.clone(),
        }
    }

    pub fn add_typical(&mut self, summary_points: Vec<Vec<f32>>, relative_weight: Vec<f32>) {
        self.summary_points = summary_points.clone();
        self.relative_weight = relative_weight.clone();
    }

    pub fn pick(weighted_points : &[(f32,f32)], weight: f64, start: usize, initial_weight : f64) -> (usize,f64) {
        let mut running = initial_weight;
        let mut index = start;
        while index + 1 < weighted_points.len() && weighted_points[index].1 as f64 + running < weight {
            running += weighted_points[index].1 as f64;
            index += 1;
        }
        (index, running)
    }

    pub fn from_points<Q>(dimensions: usize,points: &[(Q, f32)], lower_fraction: f64, upper_fraction:f64) -> Result<Self>
        where Q: Index<usize, Output = f32>
    {
        check_argument(points.len() > 0, "cannot be empty list")?;
        check_argument(lower_fraction < 0.5, " has to be less than half")?;
        check_argument(upper_fraction > 0.5, "has to be larger than half")?;
        check_argument(dimensions > 0, " cannot have 0 dimensions")?;
        let total_weight: f64 = points.iter().map(|x| x.1 as f64).sum();
        check_argument(total_weight > 0.0, "weights cannot be all zero")?;
        check_argument(total_weight.is_finite(), " cannot have infinite weights")?;
        let mut mean = vec![0.0f32; dimensions];
        let mut deviation = vec![0.0f32; dimensions];
        let mut sum_values_sq = vec![0.0f64; dimensions];
        let mut sum_values = vec![0.0f64; dimensions];
        for i in 0..points.len() {
            check_argument(points[i].1 >= 0.0, "point weights have to be non-negative")?;
            for j in 0..dimensions {
                check_argument(
                    points[i].0[j].is_finite() && !points[i].0[j].is_nan(),
                    " cannot have NaN or infinite values"
                )?;
                sum_values[j] += points[i].1 as f64 * points[i].0[j] as f64;
                sum_values_sq[j] +=
                    points[i].1 as f64 * points[i].0[j] as f64 * points[i].0[j] as f64;
            }
        }
        for j in 0..dimensions {
            mean[j] = (sum_values[j] / total_weight) as f32;
            let t: f64 = sum_values_sq[j] / total_weight
                - sum_values[j] * sum_values[j] / (total_weight * total_weight);
            deviation[j] = f64::sqrt(if t > 0.0 { t } else { 0.0 }) as f32;
        }
        let mut median = vec![0.0f32; dimensions];
        let mut upper_vec = vec![0.0f32;dimensions];
        let mut lower_vec = vec![0.0f32;dimensions];
        let num = total_weight/2.0;
        let lower = total_weight * lower_fraction;
        let upper = total_weight * upper_fraction;
        for j in 0..dimensions {
            let mut y: Vec<(f32,f32)> = points.iter().map(|x| (x.0[j],x.1)).collect();
            y.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap());
            let first = Self::pick(&y,lower,0,0.0);
            lower_vec[j] = y[first.0].0;
            let second = Self::pick(&y,num,first.0,first.1);
            median[j] = y[second.0].0;
            let third = Self::pick(&y,upper,second.0,second.1);
            upper_vec[j] = y[third.0].0;
        }

        Ok(SampleSummary {
            summary_points: Vec::new(),
            relative_weight: Vec::new(),
            total_weight: total_weight as f32,
            mean,
            upper: upper_vec,
            lower: lower_vec,
            median,
            deviation,
        })
    }

    pub fn from_references<Q>(dimensions: usize, points: &[(&Q, f32)], lower_fraction: f64, upper_fraction:f64) -> Result<Self>
    where Q:?Sized + Index<usize, Output = f32>
    {
        check_argument(points.len() > 0, "cannot be empty list")?;
        check_argument(lower_fraction < 0.5, " has to be less than half")?;
        check_argument(upper_fraction > 0.5, "has to be larger than half")?;
        check_argument(dimensions > 0, " cannot have 0 dimensions")?;
        let total_weight: f64 = points.iter().map(|x| x.1 as f64).sum();
        check_argument(total_weight > 0.0, "weights cannot be all zero")?;
        check_argument(total_weight.is_finite(), " cannot have infinite weights")?;
        let mut mean = vec![0.0f32; dimensions];
        let mut deviation = vec![0.0f32; dimensions];
        let mut sum_values_sq = vec![0.0f64; dimensions];
        let mut sum_values = vec![0.0f64; dimensions];
        for i in 0..points.len() {
            check_argument(points[i].1 >= 0.0, "point weights have to be non-negative")?;
            for j in 0..dimensions {
                check_argument(
                    points[i].0[j].is_finite() && !points[i].0[j].is_nan(),
                    " cannot have NaN or infinite values"
                )?;
                sum_values[j] += points[i].1 as f64 * points[i].0[j] as f64;
                sum_values_sq[j] +=
                    points[i].1 as f64 * points[i].0[j] as f64 * points[i].0[j] as f64;
            }
        }
        for j in 0..dimensions {
            mean[j] = (sum_values[j] / total_weight) as f32;
            let t: f64 = sum_values_sq[j] / total_weight
                - sum_values[j] * sum_values[j] / (total_weight * total_weight);
            deviation[j] = f64::sqrt(if t > 0.0 { t } else { 0.0 }) as f32;
        }
        let mut median = vec![0.0f32; dimensions];
        let mut upper_vec = vec![0.0f32;dimensions];
        let mut lower_vec = vec![0.0f32;dimensions];
        let num = total_weight/2.0;
        let lower = total_weight * lower_fraction;
        let upper = total_weight * upper_fraction;
        for j in 0..dimensions {
            let mut y: Vec<(f32,f32)> = points.iter().map(|x| (x.0[j],x.1)).collect();
            y.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap());
            let first = Self::pick(&y,lower,0,0.0);
            lower_vec[j] = y[first.0].0;
            let second = Self::pick(&y,num,first.0,first.1);
            median[j] = y[second.0].0;
            let third = Self::pick(&y,upper,second.0,second.1);
            upper_vec[j] = y[third.0].0;
        }

        Ok(SampleSummary {
            summary_points: Vec::new(),
            relative_weight: Vec::new(),
            total_weight: total_weight as f32,
            mean,
            upper: upper_vec,
            lower: lower_vec,
            median,
            deviation,
        })
    }
}

pub fn summarize(
    points: &[(Vec<f32>, f32)],
    distance: fn(&[f32], &[f32]) -> f64,
    max_number: usize,
    parallel_enabled: bool,
) -> Result<SampleSummary> {
    let dimensions = points[0].0.len();
    let mut summary = SampleSummary::from_points(dimensions,&points,LOWER_FRACTION,UPPER_FRACTION)?;

    if max_number > 0 {
        let max_allowed = min(dimensions * MAX_NUMBER_PER_DIMENSION, max_number);

        let mut list: Vec<Center> = single_centroid_cluster_weighted_vec_with_distance_over_slices(
            &points,
            distance,
            max_allowed,
            parallel_enabled,
        )?;
        list.sort_by(|o1, o2| o2.weight().partial_cmp(&o1.weight()).unwrap()); // decreasing order
        let mut summary_points: Vec<Vec<f32>> = Vec::new();
        let mut relative_weight: Vec<f32> = Vec::new();
        let center_sum: f64 = list.iter().map(|x| x.weight()).sum();
        for i in 0..list.len() {
            summary_points.push(list[i].representative().clone());
            relative_weight.push((list[i].weight() / center_sum) as f32);
        }
        summary.add_typical(summary_points, relative_weight);
    }
    return Ok(summary);
}

pub fn multi_summarize_ref(
    points: &[(&[f32], f32)],
    distance: fn(&[f32], &[f32]) -> f64,
    number_of_representatives: usize,
    shrinkage : f32,
    max_number: usize,
    parallel_enabled: bool,
) -> Result<SampleSummary> {
    let dimensions = points[0].0.len();
    let mut summary = SampleSummary::from_references(dimensions,points,LOWER_FRACTION,UPPER_FRACTION)?;

    if max_number > 0 {
        let max_allowed = min(dimensions * MAX_NUMBER_PER_DIMENSION, max_number);

        let mut list= multi_cluster_as_weighted_ref(
            &points,
            distance,
            number_of_representatives,
            shrinkage,
            false,
            max_allowed,
            parallel_enabled,
        )?;
        list.sort_by(|o1, o2| o2.weight().partial_cmp(&o1.weight()).unwrap()); // decreasing order
        let mut summary_points: Vec<Vec<f32>> = Vec::new();
        let mut relative_weight: Vec<f32> = Vec::new();
        let center_sum: f64 = list.iter().map(|x| x.weight()).sum();
        for i in 0..list.len() {
            summary_points.push(Vec::from(list[i].representatives()[0].0));
            relative_weight.push((list[i].weight() / center_sum) as f32);
        }
        summary.add_typical(summary_points, relative_weight);
    }

    return Ok(summary);
}


#[repr(C)]
pub struct BoundingBox {
    range_sum: f64,
    min_values: Vec<f32>,
    max_values: Vec<f32>,
}

impl BoundingBox {
    pub fn new(first_values: &[f32], second_values: &[f32]) -> Self {
        assert!(first_values.len() == second_values.len());
        let minv: Vec<f32> = first_values
            .iter()
            .zip(second_values)
            .map(|(x, y)| if *x < *y { *x } else { *y })
            .collect();
        let maxv: Vec<f32> = first_values
            .iter()
            .zip(second_values)
            .map(|(x, y)| if *x > *y { *x } else { *y })
            .collect();

        let sum = minv.iter().zip(&maxv).map(|(x, y)| (y - x) as f64).sum();
        BoundingBox {
            min_values: minv,
            max_values: maxv,
            range_sum: sum,
        }
    }

    pub fn check_contains_and_add_point(&mut self, values: &[f32]) -> bool {
        self.add_two_arrays(values, values)
    }

    pub fn add_box(&mut self, x: &BoundingBox) {
        self.add_two_arrays(x.get_min_values(), x.get_max_values());
    }

    fn add_two_arrays(&mut self, minvalues: &[f32], maxvalues: &[f32]) -> bool {
        let old_sum = self.range_sum;

        for (x, y) in self.min_values.iter_mut().zip(minvalues) {
            *x = if *x < *y { *x } else { *y };
        }
        for (x, y) in self.max_values.iter_mut().zip(maxvalues) {
            *x = if *x < *y { *y } else { *x };
        }

        self.range_sum = self
            .min_values
            .iter()
            .zip(self.get_max_values())
            .map(|(x, y)| (y - x) as f64)
            .sum();
        if old_sum > self.range_sum {
            panic!();
        }
        old_sum == self.range_sum
    }

    pub fn get_range_sum(&self) -> f64 {
        self.range_sum
    }

    pub fn copy(&self) -> BoundingBox {
        BoundingBox::new(&self.min_values, &self.max_values)
    }

    pub fn get_min_values(&self) -> &[f32] {
        &self.min_values
    }

    pub fn get_max_values(&self) -> &[f32] {
        &self.max_values
    }

    pub fn probability_of_cut(&self, point: &[f32]) -> f64 {
        let minsum: f32 = self
            .min_values
            .iter()
            .zip(point)
            .map(|(&x, &y)| if x - y > 0.0 { x - y } else { 0.0 })
            .sum();
        let maxsum: f32 = point
            .iter()
            .zip(self.get_max_values())
            .map(|(&x, &y)| if x - y > 0.0 { x - y } else { 0.0 })
            .sum();
        let sum = maxsum + minsum;

        if sum == 0.0 {
            return 0.0;
        } else if self.range_sum == 0.0 {
            return 1.0;
        }
        (sum as f64) / (self.range_sum + sum as f64)
    }

    pub fn probability_of_cut_with_missing_coordinates(
        &self,
        point: &[f32],
        missing_coordinates: &[bool],
    ) -> f64 {
        let minsum: f32 = self
            .min_values
            .iter()
            .zip(point)
            .zip(missing_coordinates)
            .map(|((&x, &y), &z)| if !z && x - y > 0.0 { x - y } else { 0.0 })
            .sum();
        let maxsum: f32 = point
            .iter()
            .zip(self.get_max_values())
            .zip(missing_coordinates)
            .map(|((&x, &y), &z)| if !z && x - y > 0.0 { x - y } else { 0.0 })
            .sum();
        let sum = maxsum + minsum;

        if sum == 0.0 {
            return 0.0;
        } else if self.range_sum == 0.0 {
            return 1.0;
        }
        (sum as f64) / (self.range_sum + sum as f64)
    }
}


use crate::samplerplustree::boundingbox::BoundingBox;

/**
*  this is a class that helps manage the cut information; the nodes do not store information in
*  this format
*/

pub struct Cut {
    pub dimension: usize,
    pub value: f32,
}

impl Cut {
    pub fn new(dimension: usize, value: f32) -> Self {
        Cut { dimension, value }
    }

    pub fn random_cut_and_separation(
        bounding_box: &BoundingBox,
        factor: f64,
        point: &[f32],
    ) -> (Cut, bool) {
        let min_values = bounding_box.get_min_values();
        let max_values = bounding_box.get_max_values();
        let mut range: f64 = min_values
            .iter()
            .zip(max_values)
            .zip(point)
            .map(|((x, y), z)| {
                if z < x {
                    (x - z) as f64
                } else if y < z {
                    (z - y) as f64
                } else {
                    0.0
                }
            })
            .sum();
        if range == 0.0 {
            return (Cut::new(usize::MAX, 0.0), false);
        }
        range += bounding_box.get_range_sum();
        range *= factor;

        let mut dim: usize = 0;
        let mut new_cut: f32 = f32::MAX;

        while dim < point.len() {
            let minv = if point[dim] < min_values[dim] {
                point[dim]
            } else {
                min_values[dim]
            };
            let maxv = if point[dim] > max_values[dim] {
                point[dim]
            } else {
                max_values[dim]
            };

            let gap: f32 = maxv - minv;
            if gap > range as f32 || (gap == range as f32 && gap > 0.0) {
                new_cut = minv + range as f32; // precision lost here
                if new_cut <= minv || new_cut >= maxv {
                    new_cut = minv;
                }
                break;
            }
            range = range - gap as f64;
            dim += 1;
        }

        let minvalue = min_values[dim];
        let maxvalue = max_values[dim];

        let separation: bool = ((point[dim] <= new_cut) && (new_cut < minvalue))
            || ((maxvalue <= new_cut) && (new_cut < point[dim]));
        (Cut::new(dim.try_into().unwrap(), new_cut), separation)
    }
}


pub mod boundingbox;
mod cut;
pub mod nodestore;
pub mod nodeview;
mod randomcuttree;
pub mod sampler;
pub mod samplerplustree;


use std::{collections::HashMap, fmt::Debug, mem};

use crate::{
    common::{divector::DiVector, intervalstoremanager::IntervalStoreManager},
    pointstore::PointStore,
    samplerplustree::{boundingbox::BoundingBox, cut::Cut},
    types::Location,
};

///
/// capacity is the number of leaves in the tree
/// this is the (per tree) samplesize in RCF
/// in the encoding below, the leaves are point_index + capacity
/// the value capacity - 1 stands for null
/// the values 0..(capacity-2) corresponds to the internal nodes; note that a regular binary tree
/// where each node has 0 or 2 children, has (capacity - 1) internal nodes
///
/// the nodestore does not need to save the parent information; it is saved if the bounding box cache is
/// more than 0.
///
/// Note that the mass of each node (in use) is at least 1. Subtracting 1 from each node implicitly
/// makes the values between [0..(capacity-1)] which is very convenient for 2^8 and 2^16.
///

#[repr(C)]
pub struct VectorNodeStore<C, P, N>
where
    C: Location,
    usize: From<C>,
    P: Location,
    usize: From<P>,
    N: Location,
    usize: From<N>,
{
    capacity: usize,
    dimensions: usize,
    using_transforms: bool,
    project_to_tree: fn(Vec<f32>) -> Vec<f32>,
    bounding_box_cache_fraction: f64,
    parent_index: Vec<N>,
    mass: Vec<N>,
    pub left_index: Vec<P>,
    pub right_index: Vec<P>,
    pub cut_dimension: Vec<C>,
    pub cut_value: Vec<f32>,
    bounding_box_data: Vec<f32>,
    range_sum_data: Vec<f64>,
    hash_mass_leaves: HashMap<usize, usize>,
    pub internal_node_manager: IntervalStoreManager<usize>,
}

const SWITCH_THRESHOLD: f64 = 0.5;

pub trait NodeStore {
    fn get_mass(&self, index: usize) -> usize;
    fn get_box(&self, index: usize, point_store: &dyn PointStore) -> BoundingBox;
    fn get_probability_of_cut(
        &self,
        index: usize,
        point: &[f32],
        point_store: &dyn PointStore,
    ) -> f64;
    fn get_probability_of_cut_with_missing_coordinates(
        &self,
        index: usize,
        point: &[f32],
        missing_coordinates: &[bool],
        point_store: &dyn PointStore,
    ) -> f64;
    fn modify_in_place_probability_of_cut_di_vector(
        &self,
        index: usize,
        point: &[f32],
        point_store: &dyn PointStore,
        di_vector: &mut DiVector,
    );
    fn modify_in_place_probability_of_cut_di_vector_with_missing_coordinates(
        &self,
        index: usize,
        point: &[f32],
        missing_coordinates: &[bool],
        point_store: &dyn PointStore,
        di_vector: &mut DiVector,
    );
    fn grow_node_box_pair(
        &self,
        first: &mut BoundingBox,
        second: &mut BoundingBox,
        point_store: &dyn PointStore,
        node: usize,
        sibling: usize,
    );
    fn grow_node_box(
        &self,
        bounding_box: &mut BoundingBox,
        point_store: &dyn PointStore,
        node: usize,
        sibling: usize,
    );
    fn get_sibling(&self, node: usize, parent: usize) -> usize;
    fn get_leaf_point_index(&self, index: usize) -> usize;
    fn get_cut_dimension(&self, index: usize) -> usize;
    fn get_left_index(&self, index: usize) -> usize;
    fn get_right_index(&self, index: usize) -> usize;
    fn get_cut_value(&self, index: usize) -> f32;
    fn is_leaf(&self, index: usize) -> bool;
    fn is_left_of(&self, index: usize, point: &[f32]) -> bool;
    fn use_path_for_box(&self) -> bool;
    fn get_distribution(&self, index: usize) -> (usize, f32, usize, usize);
    fn get_cut_and_children(&self, index: usize) -> (usize, f32, usize, usize);
    fn set_path(&self, answer: &mut Vec<(usize, usize)>, root: usize, point: &[f32]);
    fn null_node(&self) -> usize;
}

impl<C, P, N> VectorNodeStore<C, P, N>
where
    C: Location,
    usize: From<C>,
    P: Location,
    usize: From<P>,
    N: Location,
    usize: From<N>,
    <C as TryFrom<usize>>::Error: Debug,
    <P as TryFrom<usize>>::Error: Debug,
    <N as TryFrom<usize>>::Error: Debug,
{
    pub fn new(
        capacity: usize,
        dimensions: usize,
        using_transforms: bool,
        project_to_tree: fn(Vec<f32>) -> Vec<f32>,
        bounding_box_cache_fraction: f64,
    ) -> Self {
        if capacity - 1 > N::MAX.into() {
            println!(
                " invalid parameter, increase size of N to represent {}",
                capacity
            );
            panic!();
        }
        let cache_limit: usize = (bounding_box_cache_fraction * capacity as f64) as usize;
        let null_node = Self::null_value(capacity);
        VectorNodeStore {
            capacity,
            dimensions,
            using_transforms,
            project_to_tree,
            bounding_box_cache_fraction,
            left_index: vec![null_node.try_into().unwrap(); capacity - 1],
            right_index: vec![null_node.try_into().unwrap(); capacity - 1],
            mass: vec![0.try_into().unwrap(); capacity - 1],
            parent_index: if bounding_box_cache_fraction > 0.0 {
                vec![null_node.try_into().unwrap(); capacity - 1]
            } else {
                Vec::new()
            },
            cut_value: vec![0.0; capacity - 1],
            cut_dimension: vec![C::MAX; capacity - 1],
            bounding_box_data: vec![0.0; dimensions * 2 * cache_limit],
            range_sum_data: vec![0.0; cache_limit],
            hash_mass_leaves: HashMap::new(),
            internal_node_manager: IntervalStoreManager::<usize>::new(capacity - 1),
        }
    }

    /// 0 is indicative of null given unsigned representation
    /// otherwise index X uses slot X-1

    fn translate(&self, index: usize) -> usize {
        if index != self.null_node() && self.range_sum_data.len() <= index {
            usize::MAX
        } else {
            index
        }
    }

    fn copy_box_to_data(&mut self, index: usize, bounding_box: &BoundingBox) {
        let idx: usize = self.translate(index);
        if idx != usize::MAX {
            let base: usize = 2 * idx * self.dimensions;
            let mid = base + self.dimensions;
            let minarray = &mut self.bounding_box_data[base..mid];
            for (x, y) in minarray.iter_mut().zip((*bounding_box).get_min_values()) {
                *x = *y;
            }
            let maxarray = &mut self.bounding_box_data[mid..mid + self.dimensions];
            for (x, y) in maxarray.iter_mut().zip((*bounding_box).get_max_values()) {
                *x = *y;
            }
            self.range_sum_data[idx] = (*bounding_box).get_range_sum();
        }
    }

    fn check_contains_and_add_point(&mut self, index: usize, point: &[f32]) -> bool {
        let idx: usize = self.translate(index);
        if idx != usize::MAX {
            let base = 2 * idx * self.dimensions;
            let mid = base + self.dimensions;
            let minarray = &mut self.bounding_box_data[base..mid];
            for (x, y) in minarray.iter_mut().zip(point) {
                *x = if (*x) > (*y) { *y } else { *x };
            }

            let maxarray = &mut self.bounding_box_data[mid..mid + self.dimensions];
            for (x, y) in maxarray.iter_mut().zip(point) {
                *x = if *x < *y { *y } else { *x };
            }

            let newminarray = &self.bounding_box_data[base..mid];
            let newmaxarray = &self.bounding_box_data[mid..mid + self.dimensions];
            let newsum: f64 = newminarray
                .iter()
                .zip(newmaxarray)
                .map(|(x, y)| (y - x) as f64)
                .sum();
            let answer = self.range_sum_data[idx] == newsum;
            self.range_sum_data[idx] = newsum;
            return answer;
        }
        false
    }

    fn check_strictly_contains(&mut self, index: usize, point: &[f32]) -> bool {
        let idx: usize = self.translate(index);
        if idx != usize::MAX {
            let base = 2 * idx * self.dimensions;
            let mid = base + self.dimensions;
            let minarray = &self.bounding_box_data[base..mid];
            let maxarray = &self.bounding_box_data[mid..mid + self.dimensions];
            let not_inside = minarray
                .iter()
                .zip(point)
                .zip(maxarray)
                .any(|((x, y), z)| x >= y || y >= z);
            return !not_inside;
        }
        false
    }

    fn get_box_from_data(&self, idx: usize) -> BoundingBox {
        let dimensions = self.dimensions;
        let base = 2 * idx * dimensions;
        return BoundingBox::new(
            &self.bounding_box_data[base..base + dimensions],
            &self.bounding_box_data[base + dimensions..base + 2 * dimensions],
        );
    }

    pub fn reconstruct_box(&self, index: usize, point_store: &dyn PointStore) -> BoundingBox {
        let mut mutated_bounding_box = self.get_box(self.left_index[index].into(), point_store);
        self.grow_node_box(
            &mut mutated_bounding_box,
            point_store,
            index,
            self.right_index[index].into(),
        );
        mutated_bounding_box
    }

    pub fn check_contains_and_rebuild_box(
        &mut self,
        index: usize,
        point: &[f32],
        point_store: &dyn PointStore,
    ) -> bool {
        let idx = self.translate(index);
        if idx != usize::MAX {
            if !self.check_strictly_contains(index, point) {
                let mutated_bounding_box = self.reconstruct_box(index, point_store);
                self.copy_box_to_data(index, &mutated_bounding_box);
                return false;
            }
            true
        } else {
            false
        }
    }

    pub fn add_node(
        &mut self,
        parent_index: usize,
        point: &[f32],
        child: usize,
        point_index: usize,
        cut: Cut,
        saved_box: &BoundingBox,
    ) -> usize {
        let index: usize = self.internal_node_manager.get().into();
        self.cut_value[index] = cut.value;
        self.cut_dimension[index] = cut.dimension.try_into().unwrap();
        if point[cut.dimension] <= cut.value {
            self.left_index[index] = self.leaf_index(point_index).try_into().unwrap();
            self.right_index[index] = child.try_into().unwrap();
        } else {
            self.left_index[index] = child.try_into().unwrap();
            self.right_index[index] = self.leaf_index(point_index).try_into().unwrap();
        }

        self.mass[index] = (self.get_mass(child)).try_into().unwrap();
        // Not adding 1 to the above (new leaf) since all mass is represented as mass- 1
        if self.bounding_box_cache_fraction > 0.0 {
            self.copy_box_to_data(index, saved_box);
            self.check_contains_and_add_point(index, point);

            self.parent_index[index] = parent_index.try_into().unwrap();
            if !self.is_leaf(child) {
                self.parent_index[child] = index.try_into().unwrap();
            }
        }

        if parent_index != self.null_node() {
            self.replace_node(parent_index, child, index);
        }
        index
    }

    pub fn leaf_index(&self, point_index: usize) -> usize {
        point_index + self.capacity
    }

    pub fn set_root(&mut self, index: usize)
    where
        <N as TryFrom<usize>>::Error: Debug,
    {
        if !self.is_leaf(index) && self.bounding_box_cache_fraction > 0.0 {
            self.parent_index[index] = 0.try_into().unwrap();
        }
    }

    // capacity is the number of leaves
    pub fn increase_leaf_mass(&mut self, index: usize) {
        if index >= self.capacity {
            let y = index - self.capacity;
            if let Some(a) = self.hash_mass_leaves.remove(&y) {
                self.hash_mass_leaves.insert(y, a + 1);
            } else {
                self.hash_mass_leaves.insert(y, 1);
            }
        } else {
            panic!(" incorrect call with a non-leaf index");
        }
    }

    pub fn decrease_leaf_mass(&mut self, index: usize) -> usize {
        let y = index - self.capacity;
        return if let Some(a) = self.hash_mass_leaves.remove(&y) {
            if a > 1 {
                self.hash_mass_leaves.insert(y, a - 1);
                a
            } else {
                1 //default
            }
        } else {
            0
        };
    }

    pub fn manage_ancestors_add(
        &mut self,
        path: &mut Vec<(usize, usize)>,
        point: &[f32],
        _point_store: &dyn PointStore,
        box_resolved: bool,
    ) {
        let mut resolved = box_resolved;
        while path.len() != 0 {
            let index = path.pop().unwrap().0;
            let val: usize = self.mass[index].into();
            self.mass[index] = (val + 1).try_into().unwrap();
            if self.bounding_box_cache_fraction > 0.0 && !resolved {
                resolved = self.check_contains_and_add_point(index.into(), point);
            }
        }
    }

    pub fn manage_ancestors_delete(
        &mut self,
        path: &mut Vec<(usize, usize)>,
        point: &[f32],
        point_store: &dyn PointStore,
        box_resolved: bool,
    ) {
        let mut resolved = box_resolved;
        while path.len() != 0 {
            let index = path.pop().unwrap().0;
            let val: usize = self.mass[index].into();
            self.mass[index] = (val - 1).try_into().unwrap();
            if self.bounding_box_cache_fraction > 0.0 && !resolved {
                resolved = self.check_contains_and_rebuild_box(index, point, point_store);
            }
        }
    }

    pub fn delete_internal_node(&mut self, index: usize) {
        let null_node = self.null_node();

        self.left_index[index] = null_node.try_into().unwrap();
        self.right_index[index] = null_node.try_into().unwrap();
        self.mass[index] = 0.try_into().unwrap();
        if self.bounding_box_cache_fraction > 0.0 {
            self.parent_index[index] = null_node.try_into().unwrap(); // null
        }
        self.cut_dimension[index] = C::MAX;
        self.cut_value[index] = 0.0;
        self.internal_node_manager.release(index);
    }

    pub fn get_point_index(&self, index: usize) -> usize {
        assert!(self.is_leaf(index));
        index - self.capacity
    }

    pub fn get_cut_value(&self, index: usize) -> f32 {
        self.cut_value[index]
    }

    pub fn get_cut_dimension(&self, index: usize) -> usize {
        self.cut_dimension[index].into()
    }

    pub fn check_left(
        &self,
        index: usize,
        dim: usize,
        value: f32,
        point_store: &dyn PointStore,
    ) -> bool {
        if self.is_leaf(index) {
            let point = (self.project_to_tree)(point_store.get_copy(self.get_point_index(index)));
            return point[dim] < value;
        }
        self.check_left(self.get_left_index(index), dim, value, point_store)
            && self.check_left(self.get_right_index(index), dim, value, point_store)
    }

    pub fn check_right(
        &self,
        index: usize,
        dim: usize,
        value: f32,
        point_store: &dyn PointStore,
    ) -> bool {
        if self.is_leaf(index) {
            let point = (self.project_to_tree)(point_store.get_copy(self.get_point_index(index)));
            return point[dim] >= value;
        }
        self.check_right(self.get_left_index(index), dim, value, point_store)
            && self.check_right(self.get_right_index(index), dim, value, point_store)
    }

    pub fn replace_node(&mut self, grand_parent: usize, parent: usize, node: usize) {
        if parent == self.left_index[grand_parent].into() {
            self.left_index[grand_parent] = node.try_into().unwrap();
        } else {
            self.right_index[grand_parent] = node.try_into().unwrap();
        }
        if !self.is_leaf(node) && self.bounding_box_cache_fraction > 0.0 {
            self.parent_index[node] = grand_parent.try_into().unwrap();
        }
    }

    pub fn get_size(&self, _dimensions: usize) -> usize {
        (self.internal_node_manager.get_size() + self.left_index.len() + self.right_index.len())
            * std::mem::size_of::<P>()
            + (self.parent_index.len() + self.mass.len()) * std::mem::size_of::<N>()
            + (self.cut_dimension.len()) * std::mem::size_of::<C>()
            + (self.cut_value.len()) * mem::size_of::<f32>()
            + (self.bounding_box_data.len() + 2 * self.range_sum_data.len()) * mem::size_of::<f32>()
            + std::mem::size_of::<VectorNodeStore<C, P, N>>()
    }

    fn null_value(capacity: usize) -> usize {
        capacity - 1
    }

    fn is_internal(&self, index: usize) -> bool {
        index != self.null_node() && index < self.capacity
    }
}

impl<C, P, N> NodeStore for VectorNodeStore<C, P, N>
where
    C: Location,
    usize: From<C>,
    P: Location,
    usize: From<P>,
    N: Location,
    usize: From<N>,
    <C as TryFrom<usize>>::Error: Debug,
    <P as TryFrom<usize>>::Error: Debug,
    <N as TryFrom<usize>>::Error: Debug,
{
    fn get_mass(&self, index: usize) -> usize {
        if self.is_leaf(index) {
            let y = index - self.capacity;
            return if let Some(a) = self.hash_mass_leaves.get(&y) {
                (*a).into()
            } else {
                1
            };
        }
        let base: usize = self.mass[index].into();
        base + 1
    }

    fn get_box(&self, index: usize, point_store: &dyn PointStore) -> BoundingBox {
        if self.is_leaf(index) {
            return if self.using_transforms {
                let point =
                    &(self.project_to_tree)(point_store.get_copy(self.get_point_index(index)));
                BoundingBox::new(point, point)
            } else {
                let point = point_store
                    .get_reference_and_offset(self.get_point_index(index))
                    .0;
                BoundingBox::new(point, point)
            };
        } else {
            let idx: usize = self.translate(index);
            if idx != usize::MAX {
                return self.get_box_from_data(idx);
            }
            let mut mutated_bounding_box = self.get_box(self.left_index[index].into(), point_store);
            self.grow_node_box(
                &mut mutated_bounding_box,
                point_store,
                index,
                self.right_index[index].into(),
            );
            return mutated_bounding_box;
        }
    }

    fn get_probability_of_cut(
        &self,
        index: usize,
        point: &[f32],
        point_store: &dyn PointStore,
    ) -> f64 {
        let node_idx: usize = self.translate(index);
        if node_idx != usize::MAX {
            let base = 2 * node_idx * self.dimensions;
            let mid = base + self.dimensions;
            let minarray = &self.bounding_box_data[base..mid];
            let maxarray = &self.bounding_box_data[mid..mid + self.dimensions];
            let minsum: f32 = minarray
                .iter()
                .zip(point)
                .map(|(&x, &y)| if x - y > 0.0 { x - y } else { 0.0 })
                .sum();
            let maxsum: f32 = point
                .iter()
                .zip(maxarray)
                .map(|(&x, &y)| if x - y > 0.0 { x - y } else { 0.0 })
                .sum();
            let sum = maxsum + minsum;

            if sum == 0.0 {
                return 0.0;
            }
            sum as f64 / (self.range_sum_data[node_idx] + sum as f64)
        } else {
            let bounding_box = self.get_box(index, point_store);
            bounding_box.probability_of_cut(point)
        }
    }
    fn get_probability_of_cut_with_missing_coordinates(
        &self,
        index: usize,
        point: &[f32],
        missing_coordinates: &[bool],
        point_store: &dyn PointStore,
    ) -> f64 {
        let node_idx: usize = self.translate(index);
        if node_idx != usize::MAX {
            let base = 2 * node_idx * self.dimensions;
            let mid = base + self.dimensions;
            let minarray = &self.bounding_box_data[base..mid];
            let maxarray = &self.bounding_box_data[mid..mid + self.dimensions];
            let minsum: f32 = minarray
                .iter()
                .zip(point)
                .zip(missing_coordinates)
                .map(|((&x, &y), &b)| if !b && x - y > 0.0 { x - y } else { 0.0 })
                .sum();
            let maxsum: f32 = point
                .iter()
                .zip(maxarray)
                .zip(missing_coordinates)
                .map(|((&x, &y), &b)| if !b && x - y > 0.0 { x - y } else { 0.0 })
                .sum();
            let sum = maxsum + minsum;

            if sum == 0.0 {
                return 0.0;
            }
            sum as f64 / (self.range_sum_data[node_idx] + sum as f64)
        } else {
            let bounding_box = self.get_box(index, point_store);
            bounding_box.probability_of_cut_with_missing_coordinates(point, missing_coordinates)
        }
    }
    fn modify_in_place_probability_of_cut_di_vector(
        &self,
        index: usize,
        point: &[f32],
        point_store: &dyn PointStore,
        di_vector: &mut DiVector,
    ) {
        assert!(di_vector.high.len() == point.len());
        let node_idx: usize = self.translate(index);
        if node_idx != usize::MAX {
            let base = 2 * node_idx * self.dimensions;
            let mid = base + self.dimensions;
            let minsum: f64 = di_vector
                .low
                .iter_mut()
                .zip(&self.bounding_box_data[base..mid])
                .zip(point)
                .map(|((x, &y), &z)| {
                    if y - z > 0.0 {
                        *x = (y - z) as f64;
                        *x
                    } else {
                        *x = 0.0;
                        *x
                    }
                })
                .sum();
            let maxsum: f64 = di_vector
                .high
                .iter_mut()
                .zip(point)
                .zip(&self.bounding_box_data[mid..mid + self.dimensions])
                .map(|((x, &y), &z)| {
                    if y - z > 0.0 {
                        *x = (y - z) as f64;
                        *x
                    } else {
                        *x = 0.0;
                        *x
                    }
                })
                .sum();
            let sum = maxsum + minsum;
            if sum > 0.0 {
                di_vector.scale(1.0 / (self.range_sum_data[node_idx] + sum));
            }
        } else {
            let bounding_box = self.get_box(index, point_store);
            di_vector.assign_as_probability_of_cut(&bounding_box, point);
        };
    }

    fn modify_in_place_probability_of_cut_di_vector_with_missing_coordinates(
        &self,
        index: usize,
        point: &[f32],
        missing_coordinates: &[bool],
        point_store: &dyn PointStore,
        di_vector: &mut DiVector,
    ) {
        assert!(di_vector.high.len() == point.len());
        let node_idx: usize = self.translate(index);
        if node_idx != usize::MAX {
            let base = 2 * node_idx * self.dimensions;
            let mid = base + self.dimensions;
            let minsum: f64 = di_vector
                .low
                .iter_mut()
                .zip(&self.bounding_box_data[base..mid])
                .zip(point)
                .zip(missing_coordinates)
                .map(|(((x, &y), &z), &b)| {
                    if !b && y - z > 0.0 {
                        *x = (y - z) as f64;
                        *x
                    } else {
                        *x = 0.0;
                        *x
                    }
                })
                .sum();
            let maxsum: f64 = di_vector
                .high
                .iter_mut()
                .zip(point)
                .zip(&self.bounding_box_data[mid..mid + self.dimensions])
                .zip(missing_coordinates)
                .map(|(((x, &y), &z), &b)| {
                    if !b && y - z > 0.0 {
                        *x = (y - z) as f64;
                        *x
                    } else {
                        *x = 0.0;
                        *x
                    }
                })
                .sum();
            let sum = maxsum + minsum;
            if sum > 0.0 {
                di_vector.scale(1.0 / (self.range_sum_data[node_idx] + sum));
            }
        } else {
            let bounding_box = self.get_box(index, point_store);
            di_vector.assign_as_probability_of_cut_with_missing_coordinates(
                &bounding_box,
                point,
                missing_coordinates,
            );
        };
    }

    fn grow_node_box_pair(
        &self,
        first: &mut BoundingBox,
        second: &mut BoundingBox,
        point_store: &dyn PointStore,
        _node: usize,
        sibling: usize,
    ) {
        if self.is_leaf(sibling) {
            if self.using_transforms {
                let point =
                    &(self.project_to_tree)(point_store.get_copy(self.get_point_index(sibling)));
                (*first).check_contains_and_add_point(point);
                (*second).check_contains_and_add_point(point);
            } else {
                let point = point_store
                    .get_reference_and_offset(self.get_point_index(sibling))
                    .0;
                (*first).check_contains_and_add_point(point);
                (*second).check_contains_and_add_point(point);
            }
        } else {
            let idx: usize = self.translate(sibling);
            if idx != usize::MAX {
                let dimensions = self.dimensions;
                let base = 2 * idx * dimensions;
                (*first)
                    .check_contains_and_add_point(&self.bounding_box_data[base..base + dimensions]);
                (*second)
                    .check_contains_and_add_point(&self.bounding_box_data[base..base + dimensions]);
                (*first).check_contains_and_add_point(
                    &self.bounding_box_data[base + dimensions..base + 2 * dimensions],
                );
                (*second).check_contains_and_add_point(
                    &self.bounding_box_data[base + dimensions..base + 2 * dimensions],
                );
            } else {
                self.grow_node_box_pair(
                    first,
                    second,
                    point_store,
                    sibling,
                    self.get_left_index(sibling),
                );
                self.grow_node_box_pair(
                    first,
                    second,
                    point_store,
                    sibling,
                    self.get_right_index(sibling),
                );
            }
        }
    }

    fn grow_node_box(
        &self,
        bounding_box: &mut BoundingBox,
        point_store: &dyn PointStore,
        _node: usize,
        sibling: usize,
    ) {
        if self.is_leaf(sibling) {
            if self.using_transforms {
                let point =
                    &(self.project_to_tree)(point_store.get_copy(self.get_point_index(sibling)));
                (*bounding_box).check_contains_and_add_point(point);
            } else {
                let point = point_store
                    .get_reference_and_offset(self.get_point_index(sibling))
                    .0;
                (*bounding_box).check_contains_and_add_point(point);
            }
        } else {
            let idx: usize = self.translate(sibling.into());
            if idx != usize::MAX {
                let dimensions = self.dimensions;
                let base = 2 * idx * dimensions;
                (*bounding_box)
                    .check_contains_and_add_point(&self.bounding_box_data[base..base + dimensions]);
                (*bounding_box).check_contains_and_add_point(
                    &self.bounding_box_data[base + dimensions..base + 2 * dimensions],
                );
            } else {
                self.grow_node_box(
                    bounding_box,
                    point_store,
                    sibling,
                    self.get_left_index(sibling),
                );
                self.grow_node_box(
                    bounding_box,
                    point_store,
                    sibling,
                    self.get_right_index(sibling),
                );
            }
        }
    }

    fn get_sibling(&self, node: usize, parent: usize) -> usize {
        let mut sibling = self.left_index[parent].into();
        if node == sibling {
            sibling = self.right_index[parent].into();
        }
        sibling
    }

    fn get_leaf_point_index(&self, index: usize) -> usize {
        if !self.is_leaf(index) {
            println!(" not at a leaf");
            panic!();
        }
        self.get_point_index(index)
    }

    fn get_cut_dimension(&self, index: usize) -> usize {
        self.cut_dimension[index].into()
    }

    fn get_left_index(&self, index: usize) -> usize {
        self.left_index[index].try_into().unwrap()
    }

    fn get_right_index(&self, index: usize) -> usize {
        self.right_index[index].try_into().unwrap()
    }

    fn get_cut_value(&self, index: usize) -> f32 {
        self.cut_value[index]
    }

    fn is_leaf(&self, index: usize) -> bool {
        index != self.null_node() && index >= self.capacity
    }

    fn is_left_of(&self, index: usize, point: &[f32]) -> bool {
        let dim_idx: usize = self.cut_dimension[index].try_into().unwrap();
        point[dim_idx] <= self.cut_value[index]
    }

    fn use_path_for_box(&self) -> bool {
        self.bounding_box_cache_fraction < SWITCH_THRESHOLD
    }

    fn get_distribution(&self, index: usize) -> (usize, f32, usize, usize) {
        (
            self.cut_dimension[index].into(),
            self.cut_value[index],
            self.get_mass(self.get_left_index(index)),
            self.get_mass(self.get_right_index(index)),
        )
    }

    fn get_cut_and_children(&self, index: usize) -> (usize, f32, usize, usize) {
        if self.is_internal(index) {
            (
                self.cut_dimension[index].into(),
                self.cut_value[index],
                self.left_index[index].into(),
                self.right_index[index].into(),
            )
        } else {
            (usize::MAX, f32::MAX, usize::MAX, usize::MAX)
        }
    }

    fn set_path(&self, answer: &mut Vec<(usize, usize)>, root: usize, point: &[f32]) {
        let mut node = root;
        answer.push((root, self.null_node()));
        while !self.is_leaf(node) {
            if self.is_left_of(node, point) {
                answer.push((self.left_index[node].into(), self.right_index[node].into()));
                node = self.left_index[node].into();
            } else {
                answer.push((self.right_index[node].into(), self.left_index[node].into()));
                node = self.right_index[node].into();
            }
        }
    }

    fn null_node(&self) -> usize {
        Self::null_value(self.capacity)
    }
}