use rand::{Rng, SeedableRng};
use rand_chacha::ChaCha20Rng;
use rand_core::RngCore;
use crate::common::cluster::{multi_cluster_as_object_with_weight_array, multi_cluster_as_weighted_obj, MultiCenter, persist};
use crate::util::check_argument;
use crate::trcf::basicthresholder::BasicThresholder;
use crate::common::intervalstoremanager;
use crate::common::intervalstoremanager::IntervalStoreManager;
use crate::types::Result;

pub const DEFAULT_MAX_CLUSTERS :usize = 10;
pub const SCORE_MAX : f32 = 10.0;
pub const DEFAULT_IGNORE_SMALL_CLUSTER_REPRESENTATIVE : f32 = 0.005;
//ignore clusters that are 10 or more times away from the closest
pub const CLUSTER_COMPARISON_THRESHOLD : f64 = 10.0;

#[repr(C)]
pub struct GenericAnomalyDescriptor<T> {
    pub representative_list: Vec<(T, f32)>,
    pub score: f64,
    pub threshold: f32,
    pub grade: f32,
}

#[repr(C)]
pub struct GlobalLocalAnomalyDetector<T:Clone + Sync> {
    capacity: usize,
    current_size: usize,
    random_seed: u64,
    heap: Vec<(f64,usize)>,
    object_list : Vec<(T,f32)>,
    time_decay: f64,
    most_recent_time_decay_update: u64,
    accumulated_decay: f64,
    interval_manager: IntervalStoreManager<usize>,
    basic_thresholder: BasicThresholder,
    last_cluster: u64,
    do_not_recluster_within : u64,
    entries_seen: u64,
    sequence_number: u64,
    last_mean: f32,
    evicted : Option<(T,f32)>,
    clusters: Vec<MultiCenter<T>>,
    max_allowed: usize,
    shrinkage: f32,
    is_compact: bool,
    number_of_representatives: usize,
    ignore_below: f32,
    initial_accept_fraction: f64,
    //global_distance : fn(&T,&T) -> f64
}

impl<T:Clone + Sync> GlobalLocalAnomalyDetector<T> {
    pub fn new(capacity: usize, random_seed: u64, time_decay: f64, number_of_representatives: usize, shrinkage: f32, is_compact: bool) -> Self{
        let mut basic_thresholder = BasicThresholder::new_adjustible(time_decay as f32,false);
        basic_thresholder.set_absolute_threshold(1.2);
        if !is_compact {
            basic_thresholder.set_z_factor(2.5);
        }
        GlobalLocalAnomalyDetector{
            capacity,
            current_size: 0,
            random_seed,
            heap: vec![],
            object_list: vec![],
            time_decay,
            most_recent_time_decay_update: 0,
            accumulated_decay: 0.0,
            interval_manager: IntervalStoreManager::new(capacity),
            basic_thresholder,
            last_cluster: 0,
            do_not_recluster_within: (capacity / 2) as u64,
            entries_seen: 0,
            sequence_number: 0,
            last_mean: -1.0, // forcing the first clustering
            evicted: Option::None,
            clusters: Vec::new(),
            max_allowed: 10,
            shrinkage,
            is_compact,
            number_of_representatives,
            ignore_below: DEFAULT_IGNORE_SMALL_CLUSTER_REPRESENTATIVE,
            initial_accept_fraction: 0.125,
            //global_distance: ()
        }
    }

    fn initial_accept_probability(&self, fill_fraction: f64) -> f64 {
        return if fill_fraction < self.initial_accept_fraction {
            1.0
        } else if self.initial_accept_fraction >= 1.0 {
            0.0
        } else {
            1.0 - (fill_fraction - self.initial_accept_fraction)
                / (1.0 - self.initial_accept_fraction)
        };
    }

    fn fill_fraction(&self) -> f64 {
        if self.current_size == self.capacity {
            return 1.0;
        };
        (self.current_size as f64 / self.capacity as f64)
    }

    fn compute_weight(&self, random_number: f64, weight: f32) -> f64 {
        f64::ln(-f64::ln(random_number) / weight as f64) -
            ((self.entries_seen - self.most_recent_time_decay_update) as f64
                * self.time_decay - self.accumulated_decay)
    }

    fn swap_down(&mut self, start_index: usize) {
        let mut current: usize = start_index;
        while 2 * current + 1 < self.current_size {
            let mut max_index: usize = 2 * current + 1;
            if 2 * current + 2 < self.current_size
                && self.heap[2 * current + 2].0 > self.heap[max_index].0
            {
                max_index = 2 * current + 2;
            }
            if self.heap[max_index].0 > self.heap[current].0 {
                self.swap_weights(current, max_index);
                current = max_index;
            } else {
                break;
            }
        }
    }

    fn swap_weights(&mut self, a: usize, b: usize) {
        let tmp = self.heap[a];
        self.heap[a] = self.heap[b];
        self.heap[b] = tmp;
    }

    fn evict_max(&mut self) -> (f64, usize) {
        let evicted_point = self.heap[0];
        self.current_size -= 1;
        let current: usize = self.current_size.into();
        self.heap[0] = self.heap[current];
        self.heap[0] = self.heap[current];
        self.swap_down(0);
        evicted_point
    }

    fn sample(&mut self, object: &T, weight: f32) -> bool {
        self.sequence_number += 1;
        self.entries_seen += 1;
        let mut initial = false;
        let mut rng = ChaCha20Rng::seed_from_u64(self.random_seed);
        self.random_seed = rng.next_u64();
        let random_number: f64 = rng.gen();
        let heap_weight = self.compute_weight(random_number, weight);
        if self.current_size < self.capacity {
            let other_random: f64 = rng.gen();
            initial = other_random < self.initial_accept_probability(self.fill_fraction());
        }
        if initial || (heap_weight < self.heap[0].0) {
            if !initial {
                let old_index = self.evict_max().1;
                self.evicted = Some(self.object_list[old_index].clone());
                self.interval_manager.release(old_index);
            }
            let index = self.interval_manager.get();
            if index < self.object_list.len() {
                self.object_list[index] = (object.clone(), weight);
            } else {
                self.object_list.push((object.clone(), weight));
            }
            if (self.heap.len() == self.current_size){
                self.heap.push((heap_weight, index));
            } else {
                self.heap[self.current_size] = (heap_weight, index);
            }
            let mut current = self.current_size;
            self.current_size += 1;

            while current > 0 {
                let tmp = (current - 1) / 2;
                if self.heap[tmp].0 < self.heap[current].0 {
                    self.swap_weights(current, tmp);
                    current = tmp;
                } else {
                    break;
                }
            }
            return true;
        };
        false
    }

    pub fn set_z_factor(&mut self, z_factor : f32){
        self.basic_thresholder.set_z_factor(z_factor);
    }

    pub fn score(&self, current: &T, local_distance: fn(&T, &T) -> f64, consider_occlusion: bool) -> Result<Vec<(T, f32)>> {
        if (self.clusters.len() == 0) {
            return Ok(Vec::new());
        } else {
            let mut candidate_list: Vec<(usize, (f64, &T), f64)> = Vec::new();
            for j in 0..self.clusters.len() {
                let rad = self.clusters[j].average_radius();
                let close = self.clusters[j].distance_to_point_and_ref(current, self.ignore_below, local_distance)?;
                candidate_list.push((j, close, rad));
            }
            candidate_list.sort_by(|a, b| a.1.0.partial_cmp(&b.1.0).unwrap());

            if (candidate_list[0].1.0 == 0.0) {
                return Ok(vec![(candidate_list[0].1.1.clone(), 0.0)]);
            }
            let mut index = 0;
            while (index < candidate_list.len()) {
                let head = candidate_list[index];
                if (consider_occlusion) {
                    for j in index + 1..candidate_list.len() {
                        let occlude = (local_distance)(head.1.1, candidate_list[j].1.1);
                        check_argument(occlude>=0.0, "distances cannot be negative")?;
                        if candidate_list[j].2 > f64::sqrt(occlude * occlude + head.2 * head.2) {
                            candidate_list.remove(j);
                        }
                    }
                }
                index += 1;
            }
            let mut answer = Vec::new();
            let distance_threshold = candidate_list[0].1.0 * CLUSTER_COMPARISON_THRESHOLD;
            for head in &candidate_list {
                if head.1.0 < distance_threshold {
                    let temp_measure = if head.2 > 0.0 && head.1.0 < SCORE_MAX as f64 * head.2 {
                        (head.1.0 / head.2) as f32
                    } else {
                        SCORE_MAX
                    };
                    answer.push((head.1.1.clone(), temp_measure));
                }
            }
            Ok(answer)
        }
    }

    pub fn process(&mut self, object: &T, weight: f32, global_distance: fn(&T, &T) -> f64, local_distance: fn(&T, &T) -> f64, consider_occlusion: bool)
                   -> Result<GenericAnomalyDescriptor<T>> {
        check_argument(weight >= 0.0, "weight cannot be negative")?;
        // recompute clusters first; this enables easier merges and deserialization
        if (self.sequence_number > self.last_cluster + self.do_not_recluster_within) {
            let current_mean = self.basic_thresholder.primary_mean() as f32;
            if (f32::abs(current_mean - self.last_mean) > 0.1 || current_mean > 1.7f32
                || self.sequence_number > self.last_cluster + 20 * self.do_not_recluster_within) {
                self.last_cluster = self.sequence_number;
                self.last_mean = current_mean;
                let temp = multi_cluster_as_weighted_obj(&self.object_list,
                                                         global_distance, 5, 0.1, self.is_compact, self.max_allowed, false)?;
                self.clusters = persist(&temp);
            }
        }
        let mut score_list = self.score(object, local_distance, consider_occlusion)?;
        let threshold = self.basic_thresholder.threshold();
        let mut grade: f32 = 0.0;
        let score: f32 = if score_list.len() == 0 { 0.0 } else {
            score_list.iter().map(|a| a.1).min_by(|a,b| a.partial_cmp(b).unwrap()).unwrap()
        };

        if score_list.len() > 0 {
            if score < SCORE_MAX {
                // an exponential attribution
                let sum: f64 = score_list.iter().map(|a|
                    if a.1 == SCORE_MAX { 0.0f64 } else {
                        f64::exp(-( a.1 * a.1) as f64)
                    }
                ).sum();
                for mut item in &mut score_list {
                    let t = if item.1 == f32::MAX { 0.0 } else {
                        f64::min(1.0, f64::exp(-(item.1 * item.1) as f64) / sum)
                    };
                    item.1 = t as f32;
                }
            } else {
                let y = score_list.len();
                for mut item in &mut score_list {
                    item.1 = 1.0 / (y as f32);
                }
            }
            grade = self.basic_thresholder.anomaly_grade(score, false);
            let other = self.basic_thresholder.z_factor();
            self.basic_thresholder.update_both(score, f32::min(score, other));
        }
        self.sample(object, weight);

        return Ok(GenericAnomalyDescriptor {
            representative_list: score_list,
            score: score as f64,
            threshold,
            grade
        })
    }

    pub fn clusters(&self) -> Vec<MultiCenter<T>> {
        self.clusters.clone()
    }
}


pub mod common;
pub mod errors;
mod pointstore;
pub mod rcf;
mod samplerplustree;
mod types;
mod util;
pub mod visitor;
pub mod trcf;
pub mod glad;

extern crate rand;
extern crate rand_chacha;

use num::abs;

pub fn l1distance(a: &[f32], b: &[f32]) -> f64 {
    a.iter()
        .zip(b)
        .map(|(&x, &y)| abs(x as f64 - y as f64))
        .sum()
}

pub fn linfinitydistance(a: &[f32], b: &[f32]) -> f64 {
    let mut dist = 0.0;
    for i in 0..a.len() {
        let t = abs(a[i] as f64 - b[i] as f64);
        if dist < t {
            dist = t;
        };
    }
    dist
}

pub fn l2distance(a: &[f32], b: &[f32]) -> f64 {
    f64::sqrt(
        a.iter()
            .zip(b)
            .map(|(&x, &y)| (abs(x as f64 - y as f64) * abs(x as f64 - y as f64)))
            .sum(),
    )
}


extern crate num;
use std::{collections::HashMap, convert::TryFrom, fmt::Debug};

use crate::{common::intervalstoremanager::IntervalStoreManager, types::Location};

pub trait PointStore {
    fn get_shingled_point(&self, point: &[f32]) -> Vec<f32>;
    fn get_size(&self) -> usize;
    fn get_missing_indices(&self, look_ahead: usize, values: &[usize]) -> Vec<usize>;
    fn get_next_indices(&self, look_ahead: usize) -> Vec<usize>;
    fn get_copy(&self, index: usize) -> Vec<f32>;
    fn is_equal(&self, point: &[f32], index: usize) -> bool;
    fn get_reference_and_offset(&self, index: usize) -> (&[f32], usize);

    fn add(&mut self, point: &[f32]) -> usize;
    fn inc(&mut self, index: usize);
    fn dec(&mut self, index: usize);
    fn adjust_count(&mut self, result: &[(usize, usize)]);
    fn compact(&mut self);
}

#[repr(C)]
pub struct VectorizedPointStore<L>
where
    L: Location,
{
    internal_shingling: bool,
    internal_rotation: bool,
    last_known_shingle: Vec<f32>,
    dimensions: usize,
    shingle_size: usize,
    capacity: usize,
    store: Vec<f32>,
    reference_count: Vec<u8>,
    location: Vec<L>,
    next_sequence_index: usize,
    start_free_region: usize,
    index_manager: IntervalStoreManager<usize>,
    hash_reference_counts: HashMap<usize, usize>,
    entries_seen: i32,
}

impl<L> VectorizedPointStore<L>
where
    L: Location,
    usize: From<L>,
    <L as TryFrom<usize>>::Error: Debug,
{
    pub fn new(
        dimensions: usize,
        shingle_size: usize,
        capacity: usize,
        initial_capacity: usize,
        internal_shingling: bool,
        internal_rotation: bool,
    ) -> Self {
        VectorizedPointStore {
            internal_shingling,
            internal_rotation,
            dimensions,
            shingle_size,
            capacity,
            store: vec![0.0; initial_capacity * dimensions],
            location: vec![L::MAX; initial_capacity],
            reference_count: vec![0; initial_capacity],
            start_free_region: 0,
            index_manager: IntervalStoreManager::<usize>::new(initial_capacity),
            last_known_shingle: vec![0.0; dimensions],
            hash_reference_counts: HashMap::new(),
            entries_seen: 0,
            next_sequence_index: 0,
        }
    }

    fn ready_to_copy(&self, point: &[f32]) -> bool {
        let mut answer: bool = self.shingle_size > 1;
        let base = self.dimensions / self.shingle_size;
        let mut index: usize = self.start_free_region;
        let extra = self.dimensions - base;
        if answer && index > extra {
            index -= extra;
            for i in 0..(extra) {
                answer = answer && (self.store[index + i] == point[i]);
            }
        } else {
            answer = false;
        }
        answer
    }
}

impl<L> PointStore for VectorizedPointStore<L>
where
    L: Location,
    usize: From<L>,
    <L as TryFrom<usize>>::Error: Debug,
{
    fn get_shingled_point(&self, point: &[f32]) -> Vec<f32> {
        let mut new_point = vec![0.0; self.dimensions];
        let base = self.dimensions / self.shingle_size;
        if point.len() == base && self.shingle_size > 1 {
            if !self.internal_shingling {
                println!(
                    "The point must be '{}' floats long, got {} ",
                    self.dimensions,
                    point.len()
                );
                panic!();
            }
            if !self.internal_rotation {
                for i in 0..(self.dimensions - base) {
                    new_point[i] = self.last_known_shingle[i + base];
                }
                for i in 0..base {
                    new_point[self.dimensions - base + i] = point[i];
                }
            } else {
                for i in 0..(self.dimensions) {
                    new_point[i] = self.last_known_shingle[i];
                }
                let offset = (self.next_sequence_index * base) % self.dimensions;
                for i in 0..base {
                    new_point[offset + i] = point[i];
                }
            }
            return new_point;
        } else if point.len() != self.dimensions {
            println!(
                "The point must be '{}' floats long, got {}",
                self.dimensions,
                point.len()
            );
            panic!();
        }
        for i in 0..self.dimensions {
            new_point[i] = point[i];
        }
        new_point
    }

    fn get_size(&self) -> usize {
        self.store.len() * std::mem::size_of::<f32>()
            + self.location.len() * std::mem::size_of::<L>()
            + self.reference_count.len() * std::mem::size_of::<u8>()
            + self.index_manager.get_size()
            + std::mem::size_of::<VectorizedPointStore<L>>()
    }

    fn get_missing_indices(&self, look_ahead: usize, values: &[usize]) -> Vec<usize> {
        if !self.internal_shingling {
            assert!(
                look_ahead == 0,
                "look ahead is meaningless for external shingling"
            );
            return Vec::from(values);
        }
        let mut answer = Vec::new();
        let base = self.dimensions / self.shingle_size;
        for i in 0..values.len() {
            assert!(values[i] < base);
            if self.internal_rotation {
                answer.push(
                    ((self.next_sequence_index + look_ahead) * base + values[i]) % self.dimensions,
                );
            } else {
                answer.push(self.dimensions - base + values[i]);
            }
        }
        answer
    }

    fn get_next_indices(&self, look_ahead: usize) -> Vec<usize> {
        let base = self.dimensions / self.shingle_size;
        let vec: Vec<usize> = (0..base).collect();
        self.get_missing_indices(look_ahead, &vec)
    }

    fn get_copy(&self, index: usize) -> Vec<f32> {
        let mut new_point = vec![0.0; self.dimensions];
        let (reference, offset) = self.get_reference_and_offset(index);
        if self.internal_rotation {
            for i in 0..self.dimensions {
                new_point[(i + offset) % self.dimensions] = reference[i];
            }
        } else {
            for i in 0..self.dimensions {
                new_point[i] = reference[i];
            }
        }
        new_point
    }

    fn is_equal(&self, point: &[f32], index: usize) -> bool {
        let (reference, offset) = self.get_reference_and_offset(index);
        if self.internal_rotation {
            for i in 0..self.dimensions {
                if point[(i + offset) % self.dimensions] != reference[i] {
                    return false;
                }
            }
            return true;
        } else {
            return point.eq(reference);
        }
    }

    fn get_reference_and_offset(&self, index: usize) -> (&[f32], usize) {
        let base = self.dimensions / self.shingle_size;
        if self.reference_count[index] == 0 {
            println!(" Index '{}' not in use", index);
            panic!();
        }
        let locn: usize = self.location[index].try_into().unwrap(); // because of u32
        let adj_locn = locn * base;
        let offset = if !self.internal_rotation {
            0
        } else {
            adj_locn % self.dimensions
        };
        (&self.store[adj_locn..(adj_locn + self.dimensions)], offset)
    }

    fn add(&mut self, point: &[f32]) -> usize {
        let base = self.dimensions / self.shingle_size;
        self.next_sequence_index += 1;
        if self.internal_shingling {
            if point.len() != base {
                println!("The point must be '{}' floats long", self.dimensions);
                panic!();
            }
            for i in 0..(self.dimensions - base) {
                self.last_known_shingle[i] = self.last_known_shingle[i + base];
            }
            for i in 0..base {
                self.last_known_shingle[self.dimensions - base + i] = point[i];
            }
            if self.next_sequence_index < self.shingle_size {
                return usize::MAX;
            }
        } else if point.len() != self.dimensions {
            println!("The point must be '{}' floats long", self.dimensions);
            panic!();
        }

        if self.dimensions + self.start_free_region > self.store.len() {
            self.compact();
            if self.dimensions + self.start_free_region > self.store.len() {
                let mut new_size = self.store.len() + self.store.len() / 5;
                if new_size > self.capacity * self.dimensions {
                    new_size = self.capacity * self.dimensions;
                }
                self.store.resize(new_size, 0.0);
            }
        }

        if self.index_manager.is_empty() {
            assert!(self.reference_count.len() == self.location.len());
            let mut new_size = self.location.len() + self.location.len() / 5;
            if new_size > self.capacity {
                new_size = self.capacity;
            }
            self.location.resize(new_size, L::MAX);
            self.reference_count.resize(new_size, 0);
            self.index_manager.change_capacity(new_size);
        }
        let position: usize = self.index_manager.get();
        assert!(self.reference_count[position] == 0);
        self.reference_count[position] = 1;
        let new_point: &[f32] = if self.internal_shingling {
            &self.last_known_shingle
        } else {
            &point
        };

        if self.ready_to_copy(&new_point) {
            let base = self.dimensions / self.shingle_size;
            let mut index: usize = self.start_free_region;
            let extra = self.dimensions - base;
            let idx_value: usize = (index - extra) / base;
            self.location[position] = idx_value.try_into().unwrap();
            for i in 0..base {
                self.store[index] = new_point[extra + i];
                index += 1;
            }
            self.start_free_region += base;
        } else {
            let mut index: usize = self.start_free_region;
            let idx_value: usize = index / base;
            self.location[position] = idx_value.try_into().unwrap();
            for i in 0..self.dimensions {
                self.store[index] = new_point[i];
                index += 1;
            }
            self.start_free_region += self.dimensions;
        }
        position
    }

    fn inc(&mut self, index: usize) {
        if self.reference_count[index] == 0 {
            println!(" Index '{}' not in use", index);
            panic!();
        } else if self.reference_count[index] == u8::MAX {
            if let Some(a) = self.hash_reference_counts.remove(&index) {
                self.hash_reference_counts.insert(index, a + 1);
            } else {
                self.hash_reference_counts.insert(index, 1);
            }
        } else {
            self.reference_count[index] += 1;
        }
    }

    fn dec(&mut self, index: usize) {
        if self.reference_count[index] == 0 {
            println!(" Index '{}' not in use", index);
            panic!();
        } else if let Some(a) = self.hash_reference_counts.remove(&index) {
            if a > 1 {
                self.hash_reference_counts.insert(index, a - 1);
            }
        } else {
            self.reference_count[index] -= 1;
        }

        if self.reference_count[index] == 0 {
            self.index_manager.release(index);
            self.location[index] = L::MAX;
        }
    }

    fn adjust_count(&mut self, result: &[(usize, usize)]) {
        for (insert, delete) in result {
            if *insert != usize::MAX {
                self.inc(*insert);
                if *delete != usize::MAX {
                    self.dec(*delete);
                }
            }
        }
    }

    fn compact(&mut self) {
        let base = self.dimensions / self.shingle_size;
        let mut reverse_reference: Vec<(usize, usize)> = Vec::new();
        for i in 0..self.location.len() {
            if self.location[i] != L::MAX {
                reverse_reference.push((self.location[i].try_into().unwrap(), i));
            }
        }
        reverse_reference.sort();
        let mut fresh_start: usize = 0;
        let mut j_static: usize = 0;
        let mut j_dynamic: usize;
        let end: usize = reverse_reference.len();
        while j_static < end {
            let mut block_start: usize = reverse_reference[j_static].0;
            block_start = block_start * base;
            let mut block_end: usize = block_start + self.dimensions;
            let initial = if self.internal_rotation {
                (self.dimensions - fresh_start + block_start) % self.dimensions
            } else {
                0
            };

            let mut k = j_static + 1;
            j_dynamic = j_static + 1;
            while k < end {
                let new_locn: usize = reverse_reference[k].0;
                let new_elem: usize = base * new_locn;
                if block_end >= new_elem {
                    k += 1;
                    j_dynamic += 1;
                    if block_end < new_elem + self.dimensions {
                        block_end = new_elem + self.dimensions;
                    }
                } else {
                    k = end;
                }
            }

            // aligning the boundaries
            for _i in 0..initial {
                self.store[fresh_start] = 0.0;
                fresh_start += 1;
            }

            for i in block_start..block_end {
                self.store[fresh_start] = self.store[i];
                assert!(
                    !self.internal_rotation || fresh_start % self.dimensions == i % self.dimensions
                );
                if j_static < end {
                    let locn: usize = reverse_reference[j_static].0;
                    if i == base * locn {
                        let new_idx: usize = reverse_reference[j_static].1;
                        self.location[new_idx] = (fresh_start / base).try_into().unwrap();
                        j_static += 1;
                    }
                }
                fresh_start += 1;
            }

            if j_static != j_dynamic {
                println!(
                    "There is discrepancy in indices between '{}' versus '{}'",
                    j_static, j_dynamic
                );
                panic!();
            }
        }
        self.start_free_region = fresh_start.try_into().unwrap();
    }
}


extern crate num;
extern crate rand;
extern crate rand_chacha;

use core::fmt::Debug;

use rand::SeedableRng;
use rand_chacha::ChaCha20Rng;
use rand_core::RngCore;
use rayon::prelude::*;

use crate::{
    common::{
        conditionalfieldsummarizer::FieldSummarizer, directionaldensity::InterpolationMeasure,
        divector::DiVector, samplesummary::SampleSummary,
    },
    l1distance,
    pointstore::{PointStore, VectorizedPointStore},
    samplerplustree::{
        nodestore::VectorNodeStore, nodeview::UpdatableNodeView, samplerplustree::SamplerPlusTree,
    },
    types::{Location, Result},
    util::{add_nbr, add_to, check_argument, divide, nbr_finish},
    visitor::{
        attributionvisitor::AttributionVisitor,
        imputevisitor::ImputeVisitor,
        interpolationvisitor::InterpolationVisitor,
        scalarscorevisitor::ScalarScoreVisitor,
        visitor::{Visitor, VisitorInfo},
    },
};
use crate::common::rangevector::RangeVector;

pub(crate) fn score_seen(x: usize, y: usize) -> f64 {
    1.0 / (x as f64 + f64::log2(1.0 + y as f64))
}
pub(crate) fn score_unseen(x: usize, _y: usize) -> f64 {
    1.0 / (x as f64 + 1.0)
}
pub(crate) fn normalizer(x: f64, y: usize) -> f64 {
    x * f64::log2(1.0 + y as f64)
}
pub(crate) fn damp(x: usize, y: usize) -> f64 {
    1.0 - (x as f64) / (2.0 * y as f64)
}

pub(crate) fn score_seen_displacement(_x: usize, y: usize) -> f64 {
    1.0 / (1.0 + y as f64)
}

// the following would be used for density estimation as well; note that for density estimation
// we are only focused about similarity of points and thus (previously) seen  and
// (previously) unseen points have little distinction
// that distinction can be crucial for some applications of anomaly detection

pub(crate) fn score_unseen_displacement(_x: usize, y: usize) -> f64 {
    y as f64
}

// the normalization is now multiplication by 1/treesize; this makes the
// max score to be 1.0 whereas for the standard score the average is close to 1

pub(crate) fn displacement_normalizer(x: f64, y: usize) -> f64 {
    x * 1.0 / (1.0 + y as f64)
}

pub(crate) fn identity(x: f64, _y: usize) -> f64 {
    x
}

pub trait RCF {
    fn update(&mut self, point: &[f32], timestamp: u64) -> Result<()>;

    fn dimensions(&self) -> usize;
    fn shingle_size(&self) -> usize;
    fn is_internal_shingling_enabled(&self) -> bool;
    fn entries_seen(&self) -> u64;
    fn size(&self) -> usize;
    fn point_store_size(&self) -> usize;
    fn shingled_point(&self,point:&[f32]) -> Vec<f32>;

    fn score(&self, point: &[f32]) -> Result<f64> {
        self.score_visitor_traversal(point, &VisitorInfo::default())
    }

    fn displacement_score(&self, point: &[f32]) -> Result<f64> {
        self.score_visitor_traversal(point, &VisitorInfo::displacement())
    }

    fn generic_score(
        &self,
        point: &[f32],
        ignore_mass: usize,
        score_seen: fn(usize, usize) -> f64,
        score_unseen: fn(usize, usize) -> f64,
        damp: fn(usize, usize) -> f64,
        normalizer: fn(f64, usize) -> f64,
    ) -> Result<f64> {
        self.score_visitor_traversal(
            point,
            &VisitorInfo::use_score(ignore_mass, score_seen, score_unseen, damp, normalizer),
        )
    }

    fn score_visitor_traversal(&self, point: &[f32], visitor_info: &VisitorInfo) -> Result<f64>;

    fn attribution(&self, point: &[f32]) -> Result<DiVector> {
        self.attribution_visitor_traversal(point, &VisitorInfo::default())
    }

    fn generic_attribution(
        &self,
        point: &[f32],
        ignore_mass: usize,
        score_seen: fn(usize, usize) -> f64,
        score_unseen: fn(usize, usize) -> f64,
        damp: fn(usize, usize) -> f64,
        normalizer: fn(f64, usize) -> f64,
    ) -> Result<DiVector> {
        self.attribution_visitor_traversal(
            point,
            &VisitorInfo::use_score(ignore_mass, score_seen, score_unseen, damp, normalizer),
        )
    }

    fn attribution_visitor_traversal(
        &self,
        point: &[f32],
        visitor_info: &VisitorInfo,
    ) -> Result<DiVector>;

    fn density(&self, point: &[f32]) -> Result<f64> {
        self.interpolation_visitor_traversal(point, &VisitorInfo::density())
            .map(|meas| meas.density())
    }

    fn directional_density(&self, point: &[f32]) -> Result<DiVector> {
        self.interpolation_visitor_traversal(point, &VisitorInfo::density())
            .map(|meas| meas.directional_density())
    }

    fn density_interpolant(&self, point: &[f32]) -> Result<InterpolationMeasure> {
        self.interpolation_visitor_traversal(point, &VisitorInfo::density())
    }

    fn interpolation_visitor_traversal(
        &self,
        point: &[f32],
        visitor_info: &VisitorInfo,
    ) -> Result<InterpolationMeasure>;

    /// the answer format is (score, point, distance from original)
    fn near_neighbor_list(
        &self,
        point: &[f32],
        percentile: usize,
    ) -> Result<Vec<(f64, Vec<f32>, f64)>> {
        self.near_neighbor_traversal(point, percentile, &VisitorInfo::default())
    }

    fn near_neighbor_traversal(
        &self,
        point: &[f32],
        percentile: usize,
        visitor_info: &VisitorInfo,
    ) -> Result<Vec<(f64, Vec<f32>, f64)>>;

    fn impute_missing_values(&self, positions: &[usize], point: &[f32]) -> Result<Vec<f32>> {
        assert!(positions.len() > 0, "nothing to impute");
        self.conditional_field(positions, point, 1.0, true, 0)
            .map(|summary| summary.median)
    }

    fn extrapolate(&self, look_ahead: usize) -> Result<RangeVector>;

    fn conditional_field(
        &self,
        positions: &[usize],
        point: &[f32],
        centrality: f64,
        project: bool,
        max_number: usize,
    ) -> Result<SampleSummary> {
        self.generic_conditional_field_visitor(
            positions,
            point,
            centrality,
            project,
            max_number,
            &VisitorInfo::default(),
        )
    }

    fn generic_conditional_field_visitor(
        &self,
        positions: &[usize],
        point: &[f32],
        centrality: f64,
        project: bool,
        max_number: usize,
        visitor_info: &VisitorInfo,
    ) -> Result<SampleSummary>;

    // to be extended to match Java version
}

pub struct RCFStruct<C, L, P, N>
where
    C: Location,
    usize: From<C>,
    L: Location,
    usize: From<L>,
    P: Location,
    usize: From<P>,
    N: Location,
    usize: From<N>,
{
    dimensions: usize,
    capacity: usize,
    number_of_trees: usize,
    sampler_plus_trees: Vec<SamplerPlusTree<C, P, N>>,
    time_decay: f64,
    shingle_size: usize,
    entries_seen: u64,
    internal_shingling: bool,
    internal_rotation: bool,
    store_attributes: bool,
    initial_accept_fraction: f64,
    bounding_box_cache_fraction: f64,
    parallel_enabled: bool,
    random_seed: u64,
    output_after: usize,
    point_store: VectorizedPointStore<L>,
}

pub type RCFTiny = RCFStruct<u8, u16, u16, u8>; // sampleSize <= 256 for these and shingleSize * { max { base_dimensions, (number_of_trees + 1) } <= 256
pub type RCFSmall = RCFStruct<u8, usize, u16, u8>; // sampleSize <= 256 and (number_of_trees + 1) <= 256 and dimensions = shingle_size*base_dimensions <= 256
pub type RCFMedium = RCFStruct<u16, usize, usize, u16>; // sampleSize, dimensions <= u16::MAX
pub type RCFLarge = RCFStruct<usize, usize, usize, usize>; // as large as the machine would allow

impl<C, L, P, N> RCFStruct<C, L, P, N>
where
    C: Location,
    usize: From<C>,
    L: Location,
    usize: From<L>,
    P: Location,
    usize: From<P>,
    N: Location,
    usize: From<N>,
    <C as TryFrom<usize>>::Error: Debug,
    <L as TryFrom<usize>>::Error: Debug,
    <P as TryFrom<usize>>::Error: Debug,
    <N as TryFrom<usize>>::Error: Debug,
{
    pub fn new(
        dimensions: usize,
        shingle_size: usize,
        capacity: usize,
        number_of_trees: usize,
        random_seed: u64,
        store_attributes: bool,
        parallel_enabled: bool,
        internal_shingling: bool,
        internal_rotation: bool,
        time_decay: f64,
        initial_accept_fraction: f64,
        bounding_box_cache_fraction: f64,
        output_after: usize,
    ) -> Self {
        let mut point_store_capacity: usize = (capacity * number_of_trees + 1).try_into().unwrap();
        if point_store_capacity < 2 * capacity {
            point_store_capacity = 2 * capacity;
        }
        let initial_capacity = 2 * capacity;
        if shingle_size != 1 && dimensions % shingle_size != 0 {
            println!("Shingle size must divide dimensions.");
            panic!();
        }
        assert!(
            !internal_rotation || internal_shingling,
            " internal shingling required for rotations"
        );
        let mut rng = ChaCha20Rng::seed_from_u64(random_seed);
        let _new_random_seed = rng.next_u64();
        let mut models: Vec<SamplerPlusTree<C, P, N>> = Vec::new();
        let using_transforms = internal_rotation; // other conditions may be added eventually
        for _i in 0..number_of_trees {
            models.push(SamplerPlusTree::<C, P, N>::new(
                dimensions,
                capacity,
                using_transforms,
                rng.next_u64(),
                store_attributes,
                time_decay,
                initial_accept_fraction,
                bounding_box_cache_fraction,
            ));
        }
        RCFStruct {
            random_seed,
            dimensions,
            capacity,
            sampler_plus_trees: models,
            number_of_trees,
            store_attributes,
            shingle_size,
            entries_seen: 0,
            time_decay,
            initial_accept_fraction,
            bounding_box_cache_fraction,
            parallel_enabled,
            point_store: VectorizedPointStore::<L>::new(
                dimensions.into(),
                shingle_size.into(),
                point_store_capacity,
                initial_capacity,
                internal_shingling,
                internal_rotation,
            ),
            internal_shingling,
            internal_rotation,
            output_after
        }
    }

    pub fn generic_conditional_field_point_list_and_distances(
        &self,
        positions: &[usize],
        point: &[f32],
        centrality: f64,
        visitor_info: &VisitorInfo,
    ) -> Vec<(f64, usize, f64)> {
        let new_point = self.point_store.get_shingled_point(point);
        let mut list: Vec<(f64, usize, f64)> = if self.parallel_enabled {
            self.sampler_plus_trees
                .par_iter()
                .map(|m| {
                    m.conditional_field(
                        &positions,
                        centrality,
                        &new_point,
                        &self.point_store,
                        visitor_info,
                    )
                })
                .collect()
        } else {
            self.sampler_plus_trees
                .iter()
                .map(|m| {
                    m.conditional_field(
                        &positions,
                        centrality,
                        &new_point,
                        &self.point_store,
                        visitor_info,
                    )
                })
                .collect()
        };
        list.sort_by(|&o1, &o2| o1.2.partial_cmp(&o2.2).unwrap());
        list
    }

    pub fn simple_traversal<NodeView, V, R, S>(
        &self,
        point: &[f32],
        parameters: &[usize],
        visitor_info: &VisitorInfo,
        visitor_factory: fn(usize, &[usize], &VisitorInfo) -> V,
        default: &R,
        initial: &S,
        collect_to: fn(&R, &mut S),
        finish: fn(&mut S, usize),
    ) -> Result<S>
    where
        NodeView: UpdatableNodeView<VectorNodeStore<C, P, N>, VectorizedPointStore<L>>,
        V: Visitor<NodeView, R>,
        R: Clone + std::marker::Send + std::marker::Sync,
        S: Clone,
    {
        check_argument(
            point.len() == self.dimensions || point.len() * self.shingle_size == self.dimensions,
            "invalid input length",
        )?;

        let mut answer = initial.clone();
        let new_point = self.point_store.get_shingled_point(point);

        if self.parallel_enabled {
            let list: Vec<R> = self
                .sampler_plus_trees
                .par_iter()
                .map(|m| {
                    //m.generic_visitor_traversal(
                    m.simple_traversal(
                        &new_point,
                        &self.point_store,
                        parameters,
                        &visitor_info,
                        visitor_factory,
                        default,
                    )
                })
                .collect();
            // given the overhead of parallelism, it seems appropriate to collect()
            // the below transformation is single threaded and the same function can be used
            // as is used in the single threaded case
            list.iter().for_each(|m| (collect_to)(m, &mut answer));
        } else {
            self.sampler_plus_trees
                .iter()
                .map(|m| {
                    m.simple_traversal(
                        &new_point,
                        &self.point_store,
                        parameters,
                        &visitor_info,
                        visitor_factory,
                        default,
                    )
                })
                .for_each(|m| collect_to(&m, &mut answer));
        }
        (finish)(&mut answer, self.sampler_plus_trees.len());
        Ok(answer)
    }
}

pub fn create_rcf(
    dimensions: usize,
    shingle_size: usize,
    capacity: usize,
    number_of_trees: usize,
    random_seed: u64,
    store_attributes: bool,
    parallel_enabled: bool,
    internal_shingling: bool,
    internal_rotation: bool,
    time_decay: f64,
    initial_accept_fraction: f64,
    bounding_box_cache_fraction: f64
) -> Box<dyn RCF> {
    if (dimensions < u8::MAX as usize) && (capacity - 1 <= u8::MAX as usize) {
        if capacity * (1 + number_of_trees) * shingle_size <= u16::MAX as usize {
            Box::new(RCFTiny::new(
                dimensions,
                shingle_size,
                capacity,
                number_of_trees,
                random_seed,
                store_attributes,
                parallel_enabled,
                internal_shingling,
                internal_rotation,
                time_decay,
                initial_accept_fraction,
                bounding_box_cache_fraction,
                capacity/4
            ))
        } else {
            Box::new(RCFSmall::new(
                dimensions,
                shingle_size,
                capacity,
                number_of_trees,
                random_seed,
                store_attributes,
                parallel_enabled,
                internal_shingling,
                internal_rotation,
                time_decay,
                initial_accept_fraction,
                bounding_box_cache_fraction,
                capacity/4
            ))
        }
    } else if (dimensions < u16::MAX as usize) && (capacity - 1 <= u16::MAX as usize) {
        Box::new(RCFMedium::new(
            dimensions,
            shingle_size,
            capacity,
            number_of_trees,
            random_seed,
            store_attributes,
            parallel_enabled,
            internal_shingling,
            internal_rotation,
            time_decay,
            initial_accept_fraction,
            bounding_box_cache_fraction,
            capacity/4
        ))
    } else {
        Box::new(RCFLarge::new(
            dimensions,
            shingle_size,
            capacity,
            number_of_trees,
            random_seed,
            store_attributes,
            parallel_enabled,
            internal_shingling,
            internal_rotation,
            time_decay,
            initial_accept_fraction,
            bounding_box_cache_fraction,
            capacity/4
        ))
    }
}

impl<C, L, P, N> RCF for RCFStruct<C, L, P, N>
where
    C: Location,
    usize: From<C>,
    L: Location,
    usize: From<L>,
    P: Location,
    usize: From<P>,
    N: Location,
    usize: From<N>,
    <C as TryFrom<usize>>::Error: Debug,
    <L as TryFrom<usize>>::Error: Debug,
    <P as TryFrom<usize>>::Error: Debug,
    <N as TryFrom<usize>>::Error: Debug,
{
    fn shingled_point(&self,point:&[f32]) -> Vec<f32> {
        assert!(self.is_internal_shingling_enabled(), " incorrect function call");
        self.point_store.get_shingled_point(point)
    }

    fn update(&mut self, point: &[f32], _timestamp: u64) -> Result<()> {
        let point_index = self.point_store.add(&point);
        if point_index != usize::MAX {
            let result: Vec<(usize, usize)> = if self.parallel_enabled {
                self.sampler_plus_trees
                    .par_iter_mut()
                    .map(|m| m.update(point_index, usize::MAX, &self.point_store))
                    .collect()
            } else {
                self.sampler_plus_trees
                    .iter_mut()
                    .map(|m| m.update(point_index, usize::MAX, &self.point_store))
                    .collect()
            };
            self.point_store.adjust_count(&result);
            self.point_store.dec(point_index);
            self.entries_seen += 1;
        }
        Ok(())
    }

    fn dimensions(&self) -> usize {
        self.dimensions
    }

    fn shingle_size(&self) -> usize {
        self.shingle_size
    }

    fn is_internal_shingling_enabled(&self) -> bool {
        self.internal_shingling
    }

    fn entries_seen(&self) -> u64 {
        self.entries_seen
    }

    fn score_visitor_traversal(&self, point: &[f32], visitor_info: &VisitorInfo) -> Result<f64> {
        // parameter unused for score traversal
        if self.output_after > self.entries_seen as usize {
            return Ok(0.0);
        }
        self.simple_traversal(
            point,
            &Vec::new(),
            visitor_info,
            ScalarScoreVisitor::default,
            &0.0,
            &0.0,
            add_to,
            divide,
        )
    }

    fn attribution_visitor_traversal(
        &self,
        point: &[f32],
        visitor_info: &VisitorInfo,
    ) -> Result<DiVector> {
        if self.output_after > self.entries_seen as usize {
            return Ok(DiVector::empty(self.dimensions));
        }
        // tells the visitor what dimension to expect for each tree
        let parameters = &vec![self.dimensions];
        self.simple_traversal(
            point,
            parameters,
            visitor_info,
            AttributionVisitor::create_visitor,
            &DiVector::empty(self.dimensions),
            &DiVector::empty(self.dimensions),
            DiVector::add_to,
            DiVector::divide,
        )
    }

    fn interpolation_visitor_traversal(
        &self,
        point: &[f32],
        visitor_info: &VisitorInfo,
    ) -> Result<InterpolationMeasure> {
        // tells the visitor what dimension to expect for each tree
        let parameters = &vec![self.dimensions];
        self.simple_traversal(
            point,
            parameters,
            visitor_info,
            InterpolationVisitor::create_visitor,
            &InterpolationMeasure::empty(self.dimensions, 0.0),
            &InterpolationMeasure::empty(self.dimensions, 0.0),
            InterpolationMeasure::add_to,
            InterpolationMeasure::divide,
        )
    }

    fn near_neighbor_traversal(
        &self,
        point: &[f32],
        percentile: usize,
        visitor_info: &VisitorInfo,
    ) -> Result<Vec<(f64, Vec<f32>, f64)>> {
        let x = (0.0f64, usize::MAX, f64::MAX);
        let parameters = &vec![percentile];
        let list = self.simple_traversal(
            point,
            parameters,
            visitor_info,
            ImputeVisitor::create_nbr_visitor,
            &x,
            &Vec::new(),
            add_nbr,
            nbr_finish,
        )?;
        let mut answer = Vec::new();
        for e in list.iter() {
            answer.push((e.0, self.point_store.get_copy(e.1), e.2));
        }
        Ok(answer)
    }

    fn generic_conditional_field_visitor(
        &self,
        positions: &[usize],
        point: &[f32],
        centrality: f64,
        project: bool,
        max_number: usize,
        visitor_info: &VisitorInfo,
    ) -> Result<SampleSummary> {
        check_argument(
            point.len() == self.dimensions || point.len() * self.shingle_size == self.dimensions,
            "invalid input length",
        )?;
        let new_positions = if point.len() == self.dimensions {
            Vec::from(positions)
        } else {
            // internal shingling
            self.point_store.get_missing_indices(0, positions)
        };

        let raw_list = self.generic_conditional_field_point_list_and_distances(
            &new_positions,
            point,
            centrality,
            visitor_info,
        );
        let field_summarizer = FieldSummarizer::new(centrality, project, max_number, l1distance);
        Ok(field_summarizer.summarize_list(&self.point_store, &raw_list, &new_positions))
    }

    fn extrapolate(&self, look_ahead: usize) -> Result<RangeVector> {
        check_argument(
            self.internal_shingling,
            "look ahead is not meaningful without internal shingling mechanism",
        )?;
        check_argument(
            self.shingle_size > 1,
            "need shingle size > 1 for extrapolation",
        )?;
        let mut values = Vec::new();
        let mut upper = Vec::new();
        let mut lower = Vec::new();
        let base = self.dimensions / self.shingle_size;
        let mut fictitious_point = self.point_store.get_shingled_point(&vec![0.0f32; base]);
        for i in 0..look_ahead {
            let missing = self.point_store.get_next_indices(i);
            assert!(missing.len() == base, "incorrect imputation");
            let iterate = self.conditional_field(&missing, &fictitious_point, 1.0, true, 0).unwrap();
            for j in 0..base {
                values.push(iterate.median[j]);
                lower.push(iterate.lower[j]);
                upper.push(iterate.upper[j]);
                fictitious_point[missing[j]] = values[j];
            }
        }
        Ok(RangeVector::create(&values,&upper,&lower))
    }

    fn size(&self) -> usize {
        let mut sum: usize = 0;
        for model in &self.sampler_plus_trees {
            sum += model.get_size();
        }
        sum + self.point_store.get_size() + std::mem::size_of::<RCFStruct<C, L, P, N>>()
    }

    fn point_store_size(&self) -> usize {
        self.point_store.get_size()
    }
}


use crate::errors;

/// A trait that defines a maximum value constant.
pub trait Max {
    const MAX: Self;
}

impl Max for u8 {
    const MAX: u8 = u8::MAX;
}

impl Max for u16 {
    const MAX: u16 = u16::MAX;
}

impl Max for usize {
    const MAX: usize = usize::MAX;
}

pub type PointIndex = usize;

/// The Location trait is used as a shorthand for the various traits needed by store (e.g., point
/// store, node store) locations. These are the values vended by stores to reference a stored
/// value.
pub trait Location:
    Copy + Max + std::cmp::PartialEq + TryFrom<usize> + std::marker::Send + Sync
{
}

impl Location for u8 {}
impl Location for u16 {}
impl Location for usize {}

pub type Result<R> = std::result::Result<R, errors::RCFError>;


use crate::{errors::RCFError, types::Result};

pub(crate) fn add_to(a: &f64, b: &mut f64) {
    *b += *a;
}
pub(crate) fn divide(a: &mut f64, b: usize) {
    *a /= b as f64;
}

pub(crate) fn add_nbr(a: &(f64, usize, f64), b: &mut Vec<(f64, usize, f64)>) {
    b.push(*a)
}

pub(crate) fn nbr_finish(_a: &mut Vec<(f64, usize, f64)>, _b: usize) {}

/// If the test condition is false, return an InvalidArgument error with
/// the given error message. Otherwise return Ok.
pub(crate) fn check_argument(test: bool, msg: &'static str) -> Result<()> {
    if test {
        Ok(())
    } else {
        Err(RCFError::InvalidArgument { msg: msg })
    }
}


use crate::common::divector::DiVector;

/**
 * This class maintains a simple discounted statistics. Setters are avoided
 * except for discount rate which is useful as initialization from raw scores
 */
#[repr(C)]
#[derive(Clone)]
pub struct AnomalyDescriptor {

    // the current input point; can have missing values
    pub current_input: Vec<f32>,

    // current timestamp
    pub current_timestamp: usize,

    // potential missing values in the current input (ideally None)
    pub missing_values: Option<Vec<i32>>,

    // potentially transformed point used by RCF, can have different dimensions than input
    pub rcf_point: Option<Vec<f32>>,

    pub score: f32,

    pub internal_timestamp: usize,

    pub threshold: f32,

    pub anomaly_grade: f32,

    pub data_confidence: f32,

    pub forecast_reasonable: bool,

    // present only if grade > 0
    pub attribution: Option<DiVector>,

    pub expected_rcf_point: Option<Vec<f32>>,

    pub relative_index: Option<i32>,

    // useful for time augmented forests
    pub expected_timestamp: Option<usize>,

    pub start_of_anomaly: Option<bool>,

    pub in_high_score_region: Option<bool>,

    pub relevant_attribution: Option<Vec<f32>>,

    pub time_attribution: Option<f32>,

    // the values being replaced; may correspond to past
    pub past_values: Option<Vec<f32>>,

    pub past_timestamp: Option<usize>,

    pub expected_values_list: Option<Vec<Vec<f32>>>,

    pub likelihood_of_values: Option<Vec<f32>>

}

impl AnomalyDescriptor {
    pub fn new(point: &[f32], timestamp: usize) -> Self {
        AnomalyDescriptor {
            current_input: Vec::from(point),
            current_timestamp: timestamp,
            missing_values: None,
            rcf_point: None,
            score: 0.0,
            internal_timestamp: 0,
            threshold: 0.0,
            anomaly_grade: 0.0,
            data_confidence: 0.0,
            forecast_reasonable: false,
            attribution: None,
            expected_rcf_point: None,
            relative_index: None,
            expected_timestamp: None,
            start_of_anomaly: None,
            in_high_score_region: None,
            relevant_attribution: None,
            time_attribution: None,
            past_values: None,
            past_timestamp: None,
            expected_values_list: None,
            likelihood_of_values: None
        }
    }

    pub fn new_with_missing_values(point: Vec<f32>, timestamp: usize, missing_values: Vec<i32>) -> Self {
        assert!(missing_values.len() <= point.len(), "incorrect input");
        for i in &missing_values {
            assert!( *i >=0 && (*i as usize) < point.len(), "incorrect input")
        }
        AnomalyDescriptor {
            current_input: point.clone(),
            current_timestamp: timestamp,
            missing_values: Some(missing_values.clone()),
            rcf_point: None,
            score: 0.0,
            internal_timestamp: 0,
            threshold: 0.0,
            anomaly_grade: 0.0,
            data_confidence: 0.0,
            forecast_reasonable: false,
            attribution: None,
            expected_rcf_point: None,
            relative_index: None,
            expected_timestamp: None,
            start_of_anomaly: None,
            in_high_score_region: None,
            relevant_attribution: None,
            time_attribution: None,
            past_values: None,
            past_timestamp: None,
            expected_values_list: None,
            likelihood_of_values: None
        }
    }
}


use std::cmp::{max, min};
use std::f32::NAN;
use std::ops::{Deref, Index};
use std::slice;
use rand::{Rng, SeedableRng};
use rand_chacha::ChaCha20Rng;
use rand_core::RngCore;
use rayon::iter::{IntoParallelRefMutIterator, ParallelIterator};
use crate::util::check_argument;
use crate::types::Result;

const PHASE2_THRESHOLD: usize = 2;
const SEPARATION_RATIO_FOR_MERGE: f64 = 0.8;
const WEIGHT_THRESHOLD: f64 = 1.25;
const LENGTH_BOUND: usize = 5000;
/**
* In the following, the goal is to cluster objects of type T, given a distance function over a pair
* of references. The clustering need not create any new object of type T, but would find representative
* points that express the cluster. The dictionary of points is expressed as pairs of reference and
* corresponding weights of objects. However for vectors/slives over f32 and single representative scenario
* the clustering does allow computation of an approximation median to create new "central" points.
* Q is the struct that corresponds to a representative of a cluster. Note that
* a cluster can have multiple representatives (see for example https://en.wikipedia.org/wiki/CURE_algorithm)
* thus the entire information is a vector of tuples corresponding to (representative, weight of representative)
*
**/

pub trait IntermediateCluster<Z, Q, T: ?Sized> {
    // aclsuter should provide a measure of the point set convered by the cluster
    fn weight(&self) -> f64;
    // A cluster is an extended object of a type different from the base type of a
    // point indicated by T (and only the reference is used in the distance function
    // every clustering algorithm implicitly/explicitly defines this function
    // the return value is (distance, the integer identifier of the cluster representative)
    // if the cluster is represented by a single value then that identifier would be 0
    fn distance_to_point<'a>(&self, dictionary: &'a [Z], get_point: fn (usize, &'a [Z]) -> &'a T, point: &T, distance: fn(&T, &T) -> f64) -> (f64,usize);
    // Likewise, the distance function needs to be extended (implicitly/explicitly)
    // to a distance function between clusters
    // the return value is the distance and the pair of identifiers in the corresponding clusters which
    // define that closest distance
    fn distance_to_cluster<'a>(&self, dictionary: &'a [Z], get_point: fn (usize, &'a [Z]) -> &'a T, other: &dyn IntermediateCluster<Z, Q, T>, distance: fn(&T, &T) -> f64) -> (f64, usize, usize);
    // a function that assigns a point indexed by usize from a list of samples to
    // the cluster; note the weight used in the function need not be the entire weight of the
    // sampled point (for example in case of soft assignments)
    fn add_point(&mut self, index: usize, weight: f32, dist: f64, representative: usize);
    // given a set of previous assignments, recomputes the optimal set of representatives
    // this is the classic optimization step for k-Means; but the analogue exists for every
    // clustering; note that it is possible that recompute does nothing
    fn recompute<'a>(&mut self, points: &'a [Z], get_point: fn (usize,&'a [Z]) -> &'a T, distance: fn(&T, &T) -> f64) -> f64;
    // resets  the statistics of a clusters; preparing for a sequence of add_point followed
    // by recompute
    fn reset(&mut self);
    // a function that allows a cluster to absorb another cluster in an agglomerative \
    // clustering algorithm
    fn extent_measure(&self) -> f64;
    // a function that indicates cluster quality
    fn average_radius(&self) -> f64;
    // a function that absorbs another cluster
    fn absorb<'a>(&mut self, dictionary: &'a [Z], get_point: fn (usize,&'a [Z]) -> &'a T, another: &dyn IntermediateCluster<Z, Q, T>, distance: fn(&T, &T) -> f64);
    // a function to return a list of representatives corresponding to pairs (Q,weight)
    fn representatives(&self) -> Vec<(Q, f32)>;
    // a function that helps scale (by multiplication) the cluster weight
    fn scale_weight(&mut self, factor: f64);
}

fn pick<T>(points: &[(T, f32)], wt: f32) -> usize {
    let mut position = 0;
    let mut running = wt;
    for i in 0..points.len() {
        position = i;
        if running - points[i].1 <= 0.0 {
            break;
        } else {
            running -= points[i].1;
        }
    }
    position
}

fn median<'a, Z,Q:?Sized>(dimensions:usize,points: &'a [Z], get_point: fn(usize,&'a [Z]) -> &'a Q,list: &mut [(usize, f32)]) -> Vec<f32>
where Q: Index<usize, Output = f32>
{
    let mut answer = vec![0.0f32;dimensions];
    let total: f64 = list.iter().map(|a| a.1 as f64).sum();
    for i in 0..dimensions {
        list.sort_by(|a, b| get_point(a.0,&points)[i].partial_cmp(&get_point(b.0,points)[i]).unwrap());
        let position = pick(list, (total / 2.0) as f32);
        answer[i] = get_point(list[position].0,&points)[i];
    }
    answer
}

#[repr(C)]
pub struct Center {
    representative: Vec<f32>,
    weight: f64,
    points: Vec<(usize, f32)>,
    sum_of_radii: f64,
}

impl Center {
    pub fn new(representative: usize, point:&[f32], weight: f32, _params:usize) -> Self {
        Center {
            representative: Vec::from(point),
            weight: weight as f64,
            points: Vec::new(),
            sum_of_radii: 0.0,
        }
    }

    pub fn new_as_vec(representative: usize, point:&Vec<f32>, weight: f32,_params:usize) -> Self {
        Center {
            representative: point.clone(),
            weight: weight as f64,
            points: Vec::new(),
            sum_of_radii: 0.0,
        }
    }

    pub fn average_radius(&self) -> f64 {
        if self.weight == 0.0 {
            0.0
        } else {
            self.sum_of_radii / self.weight
        }
    }

    pub fn distance(&self, point: &[f32], dist: fn(&[f32],&[f32]) -> f64) -> f64 {
        (dist)(&self.representative,point)
    }

    pub fn representative(&self) -> Vec<f32> {
        self.representative.clone()
    }

    pub fn weight(&self) -> f64 {
        self.weight
    }

    fn re_optimize<'a, Z, Q:?Sized>(&mut self, points:&'a [Z],
                      get_point: fn(usize,&'a [Z]) ->&'a Q,
                      picker: fn(usize,&'a [Z],fn(usize,&'a [Z]) -> &'a Q, a: &mut [(usize,f32)]) -> Vec<f32>)
    {
        if self.weight == 0.0 {
            let dimensions = self.representative.len();
            // the following computes an approximate median
            if self.points.len() < 500 {
                self.representative = picker(dimensions, points, get_point, &mut self.points);
            } else {
                let mut samples = Vec::new();
                let mut rng = ChaCha20Rng::seed_from_u64(0);
                for i in 0..self.points.len() {
                    if rng.gen::<f64>() < (200.0 * self.points[i].1 as f64) / self.weight {
                        samples.push((self.points[i].0, 1.0));
                    }
                }
                self.representative = picker(dimensions, points, get_point, &mut samples);
            };
        }
    }

    fn recompute_rad<'a,Z>(&mut self, points: &'a [Z], get_point: fn(usize,&'a [Z]) -> &'a [f32], dist: fn(&[f32],&[f32]) -> f64)
        -> f64
    {
        let old_value = self.sum_of_radii;
        self.sum_of_radii = 0.0;
        for j in 0..self.points.len() {
            self.sum_of_radii += self.points[j].1 as f64
                * dist(&self.representative, get_point(self.points[j].0,&points)) as f64;
        }
        old_value - self.sum_of_radii
    }

    fn recompute_rad_vec<'a,Z>(&mut self, points: &'a [Z], get_point: fn(usize,&'a [Z]) -> &'a Vec<f32>, dist: fn(&Vec<f32>,&Vec<f32>) -> f64)
                           -> f64
    {
        let old_value = self.sum_of_radii;
        self.sum_of_radii = 0.0;
        for j in 0..self.points.len() {
            self.sum_of_radii += self.points[j].1 as f64
                * dist(&self.representative, get_point(self.points[j].0,&points)) as f64;
        }
        old_value - self.sum_of_radii
    }

    fn add_point(&mut self, index: usize, weight: f32, dist: f64) {
        self.points.push((index, weight));
        self.weight += weight as f64;
        self.sum_of_radii += weight as f64 * dist;
    }

    fn reset(&mut self) {
        self.points.clear();
        self.weight = 0.0;
        self.sum_of_radii = 0.0;
    }

    fn absorb_list(&mut self, other_weight: f64, other_list: &Vec<(Vec<f32>,f32)>, closest: (f64,usize)){
        let t = f64::exp(2.0 * (self.weight - other_weight) / (self.weight + other_weight));
        let factor = t / (1.0 + t);
        let dimensions = self.representative.len();
        for i in 0..dimensions {
            self.representative[i] = (factor * (self.representative[i] as f64)
                + (1.0 - factor) * (other_list[closest.1].0[i] as f64)) as f32;
        }

        self.sum_of_radii += (self.weight * (1.0 - factor) + factor * other_weight) * closest.0;
    }
}

impl<Z> IntermediateCluster<Z,Vec<f32>, [f32]> for Center {
    fn weight(&self) -> f64 {
        self.weight()
    }

    fn scale_weight(&mut self, factor: f64){
        assert!(!factor.is_nan() && factor>0.0," has to be positive");
        self.weight = (self.weight as f64 * factor);
    }

    fn distance_to_point<'a>(&self, _points:&'a [Z],_get_point: fn(usize,&'a [Z]) ->&'a [f32],point: &[f32], distance: fn(&[f32], &[f32]) -> f64) -> (f64,usize) {
        ((distance)(&self.representative, point),0)
    }

    fn distance_to_cluster<'a>(
        &self,
        _points:&'a [Z],
        _get_point: fn(usize,&'a [Z]) ->&'a [f32],
        other: &dyn IntermediateCluster<Z,Vec<f32>, [f32]>,
        distance: fn(&[f32], &[f32]) -> f64,
    ) -> (f64,usize,usize) {
        let tuple = other.distance_to_point(_points,_get_point,&self.representative, distance);
        (tuple.0,0,tuple.1)
    }

    fn add_point(&mut self, index: usize, weight: f32, dist: f64, representative:usize) {
        assert!(representative==0,"can have only one representative");
        assert!(!weight.is_nan() && weight >= 0.0f32, "non-negative weight");
        self.add_point(index,weight,dist);
    }

    fn recompute<'a>(&mut self, points:&'a [Z],get_point: fn(usize,&'a [Z]) ->&'a [f32], distance: fn(&[f32], &[f32]) -> f64) -> f64 {
        self.re_optimize(&points,get_point, median);
        self.recompute_rad(&points,get_point, distance)
    }

    fn reset(&mut self) {
        self.reset();
    }

    fn extent_measure(&self) -> f64 {
        self.average_radius()
    }

    fn average_radius(&self) -> f64 {
        self.average_radius()
    }

    fn absorb<'a>(
        &mut self,
        points:&'a [Z],
        get_point: fn(usize,&'a [Z]) ->&'a [f32],
        another: &dyn IntermediateCluster<Z, Vec<f32>, [f32]>,
        distance: fn(&[f32], &[f32]) -> f64,
    ) {
        let closest = another.distance_to_point(points,get_point, &self.representative,distance);
        self.absorb_list(another.weight(),&another.representatives(),closest);
    }

    fn representatives(&self) -> Vec<(Vec<f32>, f32)> {
        vec![(self.representative.clone(), self.weight as f32); 1]
    }
}

impl<Z> IntermediateCluster<Z,Vec<f32>, Vec<f32>> for Center {
    fn weight(&self) -> f64 {
        self.weight()
    }

    fn scale_weight(&mut self, factor: f64){
        assert!(!factor.is_nan() && factor>0.0," has to be positive");
        self.weight = (self.weight as f64 * factor);
    }

    fn distance_to_point<'a>(&self, _points:&'a [Z],_get_point: fn(usize,&'a [Z]) ->&'a Vec<f32>,point: &Vec<f32>, distance: fn(&Vec<f32>, &Vec<f32>) -> f64) -> (f64,usize) {
        ((distance)(&self.representative, point),0)
    }

    fn distance_to_cluster<'a>(
        &self,
        _points:&'a [Z],
        _get_point: fn(usize,&'a [Z]) ->&'a Vec<f32>,
        other: &dyn IntermediateCluster<Z,Vec<f32>, Vec<f32>>,
        distance: fn(&Vec<f32>, &Vec<f32>) -> f64,
    ) -> (f64,usize,usize) {
        let tuple = other.distance_to_point(_points,_get_point,&self.representative, distance);
        (tuple.0,0,tuple.1)
    }

    fn add_point(&mut self, index: usize, weight: f32, dist: f64, representative:usize) {
        assert!(representative==0,"can have only one representative");
        assert!(!weight.is_nan() && weight >= 0.0f32, "non-negative weight");
        self.add_point(index,weight,dist);
    }

    fn recompute<'a>(&mut self, points:&'a [Z],get_point: fn(usize,&'a [Z]) ->&'a Vec<f32>, distance: fn(&Vec<f32>, &Vec<f32>) -> f64) -> f64 {
        self.re_optimize(&points,get_point, median);
        self.recompute_rad_vec(&points,get_point, distance)
    }

    fn reset(&mut self) {
        self.reset();
    }

    fn extent_measure(&self) -> f64 {
        self.average_radius()
    }

    fn average_radius(&self) -> f64 {
        self.average_radius()
    }

    fn absorb<'a>(
        &mut self,
        points:&'a [Z],
        get_point: fn(usize,&'a [Z]) ->&'a Vec<f32>,
        another: &dyn IntermediateCluster<Z, Vec<f32>, Vec<f32>>,
        distance: fn(&Vec<f32>, &Vec<f32>) -> f64,
    ) {
        let closest = another.distance_to_point(points,get_point, &self.representative,distance);
        self.absorb_list(another.weight(),&another.representatives(),closest);
    }

    fn representatives(&self) -> Vec<(Vec<f32>, f32)> {
        vec![(self.representative.clone(), self.weight as f32); 1]
    }
}

fn process_point<'a,Z,U,Q,T :?Sized>(dictionary: &'a [Z], get_point: fn(usize,&'a [Z])->&'a T, index: usize, centers: &mut [U], weight : f32, distance: fn(&T, &T) -> f64) -> Result<()>
    where
        U: IntermediateCluster<Z,Q, T> + Send,
        T: std::marker::Sync,
{
    let mut dist = vec![(0.0, 1); centers.len()];
    let mut min_distance = (f64::MAX, 1);
    for j in 0..centers.len() {
        dist[j] = centers[j].distance_to_point(dictionary, get_point,get_point(index,dictionary), distance);
        check_argument(dist[j].0>=0.0," distances cannot be negative")?;
        if min_distance.0 > dist[j].0 {
            min_distance = dist[j];
        }
    };
    //check_argument(min_distance.0>=0.0," distances cannot be negative")?;
    if min_distance.0 == 0.0 {
        for j in 0..centers.len() {
            if dist[j].0 == 0.0 {
                centers[j].add_point(index, weight, 0.0, dist[j].1);
            }
        }
    } else {
        let mut sum = 0.0;
        for j in 0..centers.len() {
            if dist[j].0 <= WEIGHT_THRESHOLD * min_distance.0 {
                sum += min_distance.0 / dist[j].0;
            }
        }
        for j in 0..centers.len() {
            if dist[j].0 <= WEIGHT_THRESHOLD * min_distance.0 {
                centers[j].add_point(
                    index,
                    (weight as f64 * min_distance.0 / (sum * dist[j].0)) as f32,
                    dist[j].0, dist[j].1
                );
            }
        }
    }
    Ok(())
}

fn assign_and_recompute<'a, Z, Q, U, T: ?Sized>(
    dictionary: &'a [Z],
    weights: &'a [f32],
    get_point: fn(usize,&'a [Z]) -> &'a T,
    get_weight: fn(usize,&'a [Z],&'a [f32]) -> f32,
    samples: &[(usize,f32)],
    centers: &mut [U],
    distance: fn(&T, &T) -> f64,
    parallel_enabled: bool,
) -> Result<f64>
    where
        U: IntermediateCluster<Z,Q, T> + Send,
        T: std::marker::Sync,
        Z: std::marker::Sync,
{
    for j in 0..centers.len() {
        centers[j].reset();
    }

    if (samples.len() == 0){
        for i in 0..dictionary.len() {
            process_point(dictionary,get_point,i,centers,get_weight(i,dictionary,weights),distance)?;
        }
    } else {
        for i in 0..samples.len() {
            process_point(dictionary,get_point,i,centers,samples[i].1,distance)?;
        }
    }

    let gain: f64 = if parallel_enabled {
        centers
            .par_iter_mut()
            .map(|x| x.recompute(dictionary, get_point,distance))
            .sum()
    } else {
        centers
            .iter_mut()
            .map(|x| x.recompute(dictionary, get_point,distance))
            .sum()
    };
    Ok(gain)
}

fn down_sample<'a,Z>(points: &'a [Z], weights:&'a [f32], get_weight: fn(usize,&'a [Z],&'a [f32]) -> f32, seed:u64, approximate_bound: usize) ->  Vec<(usize, f32)> {
    let mut total_weight: f64 = 0.0;
    for j in 0..points.len() {
        total_weight += get_weight(j, &points, &weights) as f64;
    };

    let mut rng = ChaCha20Rng::seed_from_u64(seed as u64);
    let mut sampled_points = Vec::new();
    let mut remainder = 0.0f64;
    for j in 0..points.len() {
        let point_weight = get_weight(j,&points,&weights);
        if point_weight > (0.005 * total_weight) as f32 {
            sampled_points.push((j, point_weight));
        } else {
            remainder += point_weight as f64;
        }
    }
    for j in 0..points.len() {
        let point_weight = get_weight(j,&points,&weights);
        if point_weight <= (0.005 * total_weight) as f32
            && rng.gen::<f64>() < approximate_bound as f64 / (points.len() as f64)
        {
            let t = point_weight as f64
                * (points.len() as f64 / approximate_bound as f64)
                * (remainder / total_weight);
            sampled_points.push((j, t as f32));
        }
    }
    sampled_points
}

fn pick_from<'a,Z>(points: &'a [Z], weights: &'a [f32], get_weight: fn(usize,&'a [Z],&'a [f32]) -> f32, wt: f32) -> (usize,f32) {
    let mut position = 0;
    let mut weight = get_weight(position,points,weights);
    let mut running = wt;
    for i in 0..points.len() {
        position = i;
        weight = get_weight(position,points,weights);
        if running - weight <= 0.0 {
            break;
        } else {
            running -= weight;
        }
    }
    (position,weight)
}

pub fn general_iterative_clustering<'a, U, V, Q, Z, T: ?Sized>(
    max_allowed: usize,
    dictionary: &'a [Z],
    weights: &'a [f32],
    get_point: fn(usize,&'a [Z]) -> &'a T,
    get_weight: fn(usize,&'a [Z],&'a [f32]) -> f32,
    approximate_bound: usize,
    seed: u64,
    parallel_enabled: bool,
    create: fn(usize, &'a T, f32, V) -> U,
    create_params: V,
    distance: fn(&T, &T) -> f64,
    phase_2_reassign: bool,
    enable_phase_3: bool,
    overlap_parameter: f64,
) -> Result<Vec<U>>
where
    U: IntermediateCluster<Z,Q, T> + Send,
    T: std::marker::Sync,
    Z: std::marker::Sync,
    V: Copy,
{
    check_argument(max_allowed < 51, " for large number of clusters, other methods may be better, consider recursively removing clusters")?;
    check_argument(max_allowed > 0, " number of clusters has to be greater or equal to 1")?;
    let mut rng = ChaCha20Rng::seed_from_u64(seed);

    let mut centers: Vec<U> = Vec::new();

    let mut samples : Vec<(usize,f32)> = if dictionary.len() > approximate_bound {
        down_sample(dictionary,weights,get_weight,rng.next_u64(),approximate_bound)
    } else {
        Vec::new()
    };
    //
    // we now peform an initialization; the sampling corresponds a denoising
    // note that if we are look at 2k random points, we are likely hitting every group of points
    // with weight 1/k whp
    let sampled_sum: f32 = if dictionary.len() > approximate_bound {
        samples.iter().map(|x| x.1).sum()
    } else {
        (0..dictionary.len()).into_iter().map(|x| get_weight(x,dictionary,weights)).sum()
    };

    for _k in 0..10 * max_allowed {
        let wt = (rng.gen::<f64>() * sampled_sum as f64) as f32;
        let mut min_dist = f64::MAX;
        let (index,weight) = if dictionary.len() > approximate_bound {
            let i = pick(&samples, wt);
            (i,samples[i].1)
        } else {
            pick_from(dictionary,weights,get_weight,wt)
        };
        for i in 0..centers.len() {
            let t = centers[i].distance_to_point(dictionary,get_point,get_point(index,&dictionary), distance);
            if t.0 < min_dist {
                min_dist = t.0;
            };
        }
        if min_dist > 0.0 {
            centers.push(create(index,get_point(index,dictionary), weight,create_params));
        }
    }

    assign_and_recompute(&dictionary,weights, get_point, get_weight,&samples,&mut centers, distance, parallel_enabled)?;

    // sort in increasing order of weight
    centers.sort_by(|o1, o2| o1.weight().partial_cmp(&o2.weight()).unwrap());
    while centers.len() > 0 && centers[0].weight() == 0.0 {
        centers.remove(0);
    }

    let mut phase_3_distance = 0.0f64;
    let mut keep_reducing_centers = centers.len() > max(max_allowed, 1);

    while keep_reducing_centers {
        let mut measure = 0.0f64;
        let mut measure_dist = f64::MAX;
        let mut lower = 0;
        let mut first = lower;
        let mut second = lower + 1;
        let mut found_merge = false;
        while lower < centers.len() - 1 && !found_merge {
            let mut min_dist = f64::MAX;
            let mut min_nbr = usize::MAX;
            for j in lower + 1..centers.len() {
                let dist = centers[lower].distance_to_cluster(&dictionary,get_point,&centers[j], distance);
                if min_dist > dist.0 {
                    min_nbr = j;
                    min_dist = dist.0;
                }
                let numerator = centers[lower].extent_measure()
                    + centers[j].extent_measure()
                    + phase_3_distance;
                if numerator >= overlap_parameter * dist.0 {
                    if measure * dist.0 < numerator {
                        first = lower;
                        second = j;
                        if dist.0 == 0.0f64 {
                            found_merge = true;
                        } else {
                            measure = numerator / dist.0;
                        }
                        measure_dist = dist.0;
                    }
                }
            }
            if lower == 0 && !found_merge {
                measure_dist = min_dist;
                second = min_nbr;
            }
            lower += 1;
        }

        let inital = centers.len();
        if inital > max_allowed || found_merge || (enable_phase_3 && measure > overlap_parameter) {
            let (small, large) = centers.split_at_mut(second);
            large.first_mut().unwrap().absorb(&dictionary,get_point, &small[first], distance);
            centers.swap_remove(first);
            if phase_2_reassign && centers.len() <= PHASE2_THRESHOLD * max_allowed + 1{
                assign_and_recompute(&dictionary, weights,get_point,get_weight, &samples, &mut centers, distance, parallel_enabled);
            }

            centers.sort_by(|o1, o2| o1.weight().partial_cmp(&o2.weight()).unwrap());
            while centers.len() > 0 && centers[0].weight() == 0.0 {
                centers.remove(0);
            }

            if inital > max_allowed && centers.len() <= max_allowed {
                // phase 3 kicks in; but this will execute at most once
                // note that measureDist can be 0 as well
                phase_3_distance = measure_dist;
            }
        } else {
            keep_reducing_centers = false;
        }
    }

    centers.sort_by(|o1, o2| o2.weight().partial_cmp(&o1.weight()).unwrap()); // decreasing order
    let center_sum: f64 = centers.iter().map(|x| x.weight() as f64).sum();
    for i in 0..centers.len() {
        centers[i].scale_weight(1.0/center_sum);
    }
    Ok(centers)
}

fn pick_slice_to_slice<'a>(index: usize, entry:&'a [&[f32]]) -> &'a [f32]{
    &entry[index]
}

fn pick_first_slice_to_slice<'a>(index: usize, entry:&'a [(&[f32],f32)]) -> &'a [f32]{
    &entry[index].0
}

fn pick_first_to_slice<'a>(index: usize, entry:&'a [(Vec<f32>,f32)]) -> &'a [f32]{
    &entry[index].0
}

fn pick_to_slice<'a>(index: usize, entry:&'a [Vec<f32>]) -> &'a [f32]{
    &entry[index]
}

fn pick_tuple_weight<T>(index:usize, entry:&[(T,f32)], weights: &[f32]) -> f32{
    entry[index].1
}

fn pick_weight<T>(index:usize, entry:&[T], weights: &[f32]) -> f32{
    weights[index]
}

fn one<'a,Z>(_i:usize,_points : &'a[Z], _weight : &'a [f32]) -> f32{
    1.0
}

pub fn single_centroid_cluster_weighted_vec_with_distance_over_slices(
    dictionary: &[(Vec<f32>, f32)],
    distance: fn(&[f32], &[f32]) -> f64,
    max_allowed: usize,
    parallel_enabled: bool,
) -> Result<Vec<Center>> {
    general_iterative_clustering(
        max_allowed,
        dictionary,
        empty_weights,
        pick_first_to_slice,
        pick_tuple_weight,
        LENGTH_BOUND,
        max_allowed as u64,
        parallel_enabled,
        Center::new,
        0,
        distance,
        false,
        true,
        SEPARATION_RATIO_FOR_MERGE,
    )
}

const empty_weights:&Vec<f32> = &Vec::new();

pub fn single_centroid_unweighted_cluster_vec_as_slice(
    dictionary: &[Vec<f32>],
    distance: fn(&[f32], &[f32]) -> f64,
    max_allowed: usize,
    parallel_enabled: bool,
) -> Result<Vec<Center>> {
    general_iterative_clustering(
        max_allowed,
        dictionary,
        empty_weights,
        pick_to_slice,
        one,
        LENGTH_BOUND,
        max_allowed as u64,
        parallel_enabled,
        Center::new,
        0,
        distance,
        false,
        true,
        SEPARATION_RATIO_FOR_MERGE,
    )
}

pub fn single_centroid_unweighted_cluster_slice(
    dictionary: &[&[f32]],
    distance: fn(&[f32], &[f32]) -> f64,
    max_allowed: usize,
    parallel_enabled: bool,
) -> Result<Vec<Center>> {
    general_iterative_clustering(
        max_allowed,
        dictionary,
        empty_weights,
        pick_slice_to_slice,
        one,
        LENGTH_BOUND,
        max_allowed as u64,
        parallel_enabled,
        Center::new,
        0,
        distance,
        false,
        true,
        SEPARATION_RATIO_FOR_MERGE,
    )
}

pub fn single_centroid_cluster_slice_with_weight_arrays(
    dictionary: &[&[f32]],
    weights : &[f32],
    distance: fn(&[f32], &[f32]) -> f64,
    max_allowed: usize,
    parallel_enabled: bool,
) -> Result<Vec<Center>> {
    general_iterative_clustering(
        max_allowed,
        dictionary,
        &weights,
        pick_slice_to_slice,
        pick_weight,
        LENGTH_BOUND,
        max_allowed as u64,
        parallel_enabled,
        Center::new,
        0,
        distance,
        false,
        true,
        SEPARATION_RATIO_FOR_MERGE,
    )
}

pub fn single_centroid_cluster_weighted_vec(
    dictionary: &[(Vec<f32>, f32)],
    distance: fn(&Vec<f32>, &Vec<f32>) -> f64,
    max_allowed: usize,
    parallel_enabled: bool,
) -> Result<Vec<Center>> {
    general_iterative_clustering(
        max_allowed,
        dictionary,
        empty_weights,
        pick_first_to_ref,
        pick_tuple_weight,
        LENGTH_BOUND,
        max_allowed as u64,
        parallel_enabled,
        Center::new_as_vec,
        0,
        distance,
        false,
        true,
        SEPARATION_RATIO_FOR_MERGE,
    )
}

fn pick_ref<'a,T :?Sized>(index: usize, entry:&[&'a T]) -> &'a T{
    entry[index]
}

fn pick_ref_tuple_first<'a,T :?Sized>(index: usize, entry:&[(&'a T,f32)]) -> &'a T{
    entry[index].0
}

fn pick_first_to_ref<'a,T>(index: usize, entry:&'a [(T,f32)]) -> &'a T{
    &entry[index].0
}

fn pick_to_ref<'a,T>(index: usize, entry:&'a [T]) -> &'a T{
    &entry[index]
}

pub fn single_centroid_cluster_vec(
    dictionary: &[Vec<f32>],
    distance: fn(&Vec<f32>, &Vec<f32>) -> f64,
    max_allowed: usize,
    parallel_enabled: bool,
) -> Result<Vec<Center>> {
    general_iterative_clustering(
        max_allowed,
        dictionary,
        empty_weights,
        pick_to_ref,
        one,
        LENGTH_BOUND,
        max_allowed as u64,
        parallel_enabled,
        Center::new_as_vec,
        0,
        distance,
        false,
        true,
        SEPARATION_RATIO_FOR_MERGE,
    )
}

#[repr(C)]
pub struct MultiCenterRef<'b, T :?Sized> {
    representatives: Vec<(&'b T,f32)>,
    number_of_representatives: usize,
    is_compact: bool,
    shrinkage: f32,
    weight: f64,
    sum_of_radii: f64,
}

impl<'b,T :?Sized> MultiCenterRef<'b,T>{

    pub fn representatives(& self) -> Vec<(&'b T,f32)>{
        self.representatives.clone()
    }

    pub fn new(representative: usize, point: &'b T, weight: f32, params : (usize,f32,bool)) -> Self {
        let (number_of_representatives, shrinkage,is_compact) = params;
        assert!(number_of_representatives>0,"has to be positive");
        assert!(shrinkage>=0.0 && shrinkage<= 1.0," has to between [0,1]");
        MultiCenterRef {
            representatives: vec![(point, weight as f32);1],
            number_of_representatives,
            shrinkage,
            is_compact,
            weight: weight as f64,
            sum_of_radii: 0.0,
        }
    }

    pub fn average_radius(&self) -> f64 {
        if self.weight == 0.0 {
            0.0
        } else {
            self.sum_of_radii / self.weight
        }
    }

    pub fn weight(&self) -> f64 {
        self.weight
    }
}

impl<'b, Z, T:?Sized> IntermediateCluster<Z, &'b T,T> for MultiCenterRef<'b,T> {
    fn weight(&self) -> f64 {
        self.weight()
    }

    fn distance_to_point<'a>(&self, _points:&'a [Z],_get_point: fn(usize,&'a [Z]) ->&'a T,point: &T, distance: fn(&T, &T) -> f64) -> (f64,usize) {
        let original = ((distance)(point, self.representatives[0].0), 0);
        let mut closest = original;
        for i in 1..self.representatives.len() {
            let t = ((distance)(point, self.representatives[i].0), i);
            if closest.0 > t.0 {
                closest = t;
            }
        }
        ((closest.0 * (1.0 - self.shrinkage as f64) + self.shrinkage as f64 * original.0), closest.1)
    }

    fn distance_to_cluster<'a>(
        &self,
        _points:&'a [Z],
        _get_point: fn(usize,&'a [Z]) ->&'a T,
        other: &dyn IntermediateCluster<Z,&'b T, T>,
        distance: fn(&T, &T) -> f64,
    ) -> (f64,usize,usize) {
        let list = other.representatives();
        let original = ((distance)(list[0].0, self.representatives[0].0), 0, 0);
        let mut closest = original;
        for i in 1..self.representatives.len() {
            for j in 1..list.len() {
                let t = ((distance)(list[j].0, self.representatives[i].0), i, j);
                if closest.0 > t.0 {
                    closest = t;
                }
            }
        }
        ((closest.0 * (1.0 - self.shrinkage as f64) + self.shrinkage as f64 * original.0), closest.1, closest.2)
    }

    fn add_point(&mut self, index: usize, weight: f32, dist: f64, representative: usize) {
        self.representatives[representative].1 += weight;
        self.sum_of_radii += weight as f64 * dist;
        self.weight += weight as f64;
    }

    fn recompute<'a>(&mut self, points:&'a [Z],get_point: fn(usize,&'a [Z]) ->&'a T, distance: fn(&T, &T) -> f64) -> f64 {
        self.representatives.sort_by(|a,b| a.1.partial_cmp(&b.1).unwrap());
        0.0
    }

    fn reset(&mut self) {
        self.sum_of_radii = 0.0;
        self.weight = 0.0;
        for i in 0..self.representatives.len() {
            self.representatives[i].1 = 0.0;
        }
    }

    fn extent_measure(&self) -> f64 {
        0.5 * self.average_radius() / self.number_of_representatives as f64
    }

    fn average_radius(&self) -> f64 {
        self.average_radius()
    }

    fn absorb<'a>(
        &mut self,
        points:&'a [Z],
        get_point: fn(usize,&'a [Z]) ->&'a T,
        another: &dyn IntermediateCluster<Z, &'b T, T>,
        distance: fn(&T, &T) -> f64,
    ) {
        self.sum_of_radii += if self.is_compact {
            another.average_radius()*another.weight()
        } else {
            another.extent_measure()*another.weight()
        };
        self.weight += another.weight();
        let mut representatives = Vec::new();
        representatives.append(&mut self.representatives);
        representatives.append(&mut another.representatives());
        self.representatives = Vec::with_capacity(self.number_of_representatives);

        let mut max_index: usize = 0;
        let mut weight = representatives[0].1;
        for i in 1..representatives.len() {
            if representatives[i].1 > weight {
                weight = representatives[i].1;
                max_index = i;
            }
        }
        self.representatives.push(representatives[max_index]);
        representatives.swap_remove(max_index);

        /**
         * create a list of representatives based on the farthest point method, which
         * correspond to a well scattered set. See
         * https://en.wikipedia.org/wiki/CURE_algorithm
         */
        while (representatives.len() > 0 && self.representatives.len() < self.number_of_representatives) {
            let mut farthest_weighted_distance = 0.0;
            let mut farthest_index: usize = usize::MAX;
            for j in 0..representatives.len() {
                if representatives[j].1 as f64 > (weight as f64) / (2.0 * self.number_of_representatives as f64) {
                    let mut new_weighted_distance = (distance)(self.representatives[0].0,
                                                               representatives[j].0) * representatives[j].1 as f64;
                    assert!(new_weighted_distance >= 0.0, " weights or distances cannot be negative");
                    for i in 1..self.representatives.len() {
                        let t = (distance)(self.representatives[i].0,
                                           representatives[j].0) * representatives[j].1 as f64;
                        assert!(t >= 0.0, " weights or distances cannot be negative");
                        if (t < new_weighted_distance) {
                            new_weighted_distance = t;
                        }
                    }
                    if new_weighted_distance > farthest_weighted_distance {
                        farthest_weighted_distance = new_weighted_distance;
                        farthest_index = j;
                    }
                }
            }
            if farthest_weighted_distance == 0.0 {
                break;
            }
            self.representatives.push(representatives[farthest_index]);
            representatives.swap_remove(farthest_index);
        }

        // absorb the remainder into existing representatives
        for j in 0..representatives.len() {
            let dist = (distance)(representatives[0].0, self.representatives[0].0);
            assert!(dist >= 0.0, "distance cannot be negative");
            let mut min_dist = dist;
            let mut min_index: usize = 0;
            for i in 1..self.representatives.len() {
                let new_dist = (distance)(self.representatives[i].0, representatives[j].0);
                assert!(new_dist >= 0.0, "distance cannot be negative");
                if (new_dist < min_dist) {
                    min_dist = new_dist;
                    min_index = i;
                }
            }
            self.representatives[min_index].1 += representatives[j].1;
            self.sum_of_radii += representatives[j].1 as f64 * min_dist;
        }
        self.representatives.sort_by(|a,b| b.1.partial_cmp(&a.1).unwrap());
    }

    fn representatives(&self) -> Vec<(&'b T, f32)> {
        self.representatives()
    }

    fn scale_weight(&mut self, factor: f64) {
        for i in 0..self.representatives.len() {
            self.representatives[i].1 = (self.representatives[i].1 as f64 * factor) as f32;
        }
    }

}

pub fn multi_cluster_obj<'a,T:Sync>(
    dictionary: &'a [T],
    distance: fn(&T, &T) -> f64,
    number_of_representatives: usize,
    shrinkage: f32,
    is_compact: bool,
    max_allowed: usize,
    parallel_enabled: bool,
) -> Result<Vec<MultiCenterRef<'a, T>>> {
    general_iterative_clustering(
        max_allowed,
        dictionary,
        empty_weights,
        pick_to_ref,
        one,
        LENGTH_BOUND,
        max_allowed as u64,
        parallel_enabled,
        MultiCenterRef::new,
        (number_of_representatives,shrinkage,is_compact),
        distance,
        false,
        true,
        SEPARATION_RATIO_FOR_MERGE,
    )
}

pub fn multi_cluster_as_ref<'a,T:Sync>(
    dictionary: &'a [&T],
    distance: fn(&T, &T) -> f64,
    number_of_representatives: usize,
    shrinkage: f32,
    is_compact: bool,
    max_allowed: usize,
    parallel_enabled: bool,
) -> Result<Vec<MultiCenterRef<'a, T>>> {
    general_iterative_clustering(
        max_allowed,
        dictionary,
        empty_weights,
        pick_ref,
        one,
        LENGTH_BOUND,
        max_allowed as u64,
        parallel_enabled,
        MultiCenterRef::new,
        (number_of_representatives,shrinkage,is_compact),
        distance,
        false,
        true,
        SEPARATION_RATIO_FOR_MERGE,
    )
}

pub fn multi_cluster_as_weighted_obj<'a,T:Sync>(
    dictionary: &'a [(T,f32)],
    distance: fn(&T, &T) -> f64,
    number_of_representatives: usize,
    shrinkage: f32,
    is_compact: bool,
    max_allowed: usize,
    parallel_enabled: bool,
) -> Result<Vec<MultiCenterRef<'a, T>>> {
    general_iterative_clustering(
        max_allowed,
        dictionary,
        empty_weights,
        pick_first_to_ref,
        pick_tuple_weight,
        LENGTH_BOUND,
        max_allowed as u64,
        parallel_enabled,
        MultiCenterRef::new,
        (number_of_representatives,shrinkage,is_compact),
        distance,
        false,
        true,
        SEPARATION_RATIO_FOR_MERGE,
    )
}

pub fn multi_cluster_as_weighted_ref<'a,T:Sync + ?Sized>(
    dictionary: &'a [(&T,f32)],
    distance: fn(&T, &T) -> f64,
    number_of_representatives: usize,
    shrinkage: f32,
    is_compact: bool,
    max_allowed: usize,
    parallel_enabled: bool,
) -> Result<Vec<MultiCenterRef<'a, T>>> {

    general_iterative_clustering(
        max_allowed,
        dictionary,
        empty_weights,
        pick_ref_tuple_first,
        pick_tuple_weight,
        LENGTH_BOUND,
        max_allowed as u64,
        parallel_enabled,
        MultiCenterRef::new,
        (number_of_representatives,shrinkage,is_compact),
        distance,
        false,
        true,
        SEPARATION_RATIO_FOR_MERGE,
    )
}

pub fn multi_cluster_as_object_with_weight_array<'a,T :Sync>(
    dictionary: &'a [T],
    weights: &'a [f32],
    distance: fn(&T, &T) -> f64,
    number_of_representatives: usize,
    shrinkage: f32,
    is_compact: bool,
    max_allowed: usize,
    parallel_enabled: bool,
) -> Result<Vec<MultiCenterRef<'a, T>>> {
    general_iterative_clustering(
        max_allowed,
        dictionary,
        weights,
        pick_to_ref,
        pick_weight,
        LENGTH_BOUND,
        max_allowed as u64,
        parallel_enabled,
        MultiCenterRef::new,
        (number_of_representatives,shrinkage,is_compact),
        distance,
        false,
        true,
        SEPARATION_RATIO_FOR_MERGE,
    )

}

#[repr(C)]
#[derive(Clone)]
pub struct MultiCenter<T : Clone> {
    representatives: Vec<(T,f32)>,
    shrinkage: f32,
    weight: f64,
    sum_of_radii: f64,
}

impl<T : Clone> MultiCenter<T>{

    pub fn create<'a>( refc : &MultiCenterRef<'a,T> ) -> Self {
        let mut rep_list = Vec::new();
        for j in refc.representatives() {
            rep_list.push((j.0.clone(),j.1));
        }
        MultiCenter {
            representatives: rep_list,
            weight: refc.weight,
            shrinkage : refc.shrinkage,
            sum_of_radii: refc.sum_of_radii,
        }
    }

    pub fn representatives(& self) -> Vec<(T,f32)>{
        self.representatives.clone()
    }

    pub fn representative(& self, number: usize) -> T {
        self.representatives[number].0.clone()
    }

    pub fn average_radius(&self) -> f64 {
        if self.weight == 0.0 {
            0.0
        } else {
            self.sum_of_radii / self.weight
        }
    }

    pub fn weight(&self) -> f64 {
        self.weight
    }

    pub fn distance_to_point(&self, point: &T, ignore: f32, distance: fn(&T, &T) -> f64) -> Result<(f64,usize)> {
        let original = ((distance)(point, &self.representatives[0].0), 0);
        check_argument(original.0>=0.0,"distances cannot be negative")?;
        let mut closest = original;
        for i in 1..self.representatives.len() {
            if self.representatives[i].1 > ignore {
                let t = ((distance)(point, &self.representatives[i].0), i);
                check_argument(t.0 >= 0.0, "distances cannot be negative")?;
                if closest.0 > t.0 {
                    closest = t;
                }
            }
        }
        Ok(((closest.0 * (1.0 - self.shrinkage as f64) + self.shrinkage as f64 * original.0), closest.1))
    }

    pub fn distance_to_point_and_ref<'a>(&'a self, point: &T, ignore: f32, distance: fn(&T, &T) -> f64) -> Result<(f64,&'a T)> {
        let original = ((distance)(point, &self.representatives[0].0), 0);
        check_argument(original.0>=0.0,"distances cannot be negative")?;
        let mut closest = original;
        for i in 1..self.representatives.len() {
            if self.representatives[i].1 > ignore {
                let t = ((distance)(point, &self.representatives[i].0), i);
                check_argument(t.0 >= 0.0, "distances cannot be negative")?;
                if closest.0 > t.0 {
                    closest = t;
                }
            }
        }
        Ok(((closest.0 * (1.0 - self.shrinkage as f64) + self.shrinkage as f64 * original.0), &self.representatives[closest.1].0))
    }
}

pub fn persist<'a,T:Clone>(list:&Vec<MultiCenterRef<'a,T>>) -> Vec<MultiCenter<T>> {
    let mut answer = Vec::new();
    for item in list {
       answer.push(MultiCenter::create(item));
    }
    answer
}