Spaces:
Build error
Build error
| use std::collections::HashMap; | |
| use std::path::Path; | |
| use segment::data_types::named_vectors::NamedVectors; | |
| use segment::data_types::vectors::{only_default_vector, DenseVector, VectorRef}; | |
| use segment::entry::entry_point::SegmentEntry; | |
| use segment::index::sparse_index::sparse_index_config::{SparseIndexConfig, SparseIndexType}; | |
| use segment::segment::Segment; | |
| use segment::segment_constructor::build_segment; | |
| use segment::segment_constructor::simple_segment_constructor::build_simple_segment; | |
| use segment::types::{ | |
| Distance, Indexes, SegmentConfig, SparseVectorDataConfig, VectorDataConfig, VectorStorageType, | |
| }; | |
| use serde_json::json; | |
| use sparse::common::sparse_vector::SparseVector; | |
| pub fn empty_segment(path: &Path) -> Segment { | |
| build_simple_segment(path, 4, Distance::Dot).unwrap() | |
| } | |
| pub const PAYLOAD_KEY: &str = "color"; | |
| pub fn build_segment_1(path: &Path) -> Segment { | |
| let mut segment1 = empty_segment(path); | |
| let vec1 = vec![1.0, 0.0, 1.0, 1.0]; | |
| let vec2 = vec![1.0, 0.0, 1.0, 0.0]; | |
| let vec3 = vec![1.0, 1.0, 1.0, 1.0]; | |
| let vec4 = vec![1.0, 1.0, 0.0, 1.0]; | |
| let vec5 = vec![1.0, 0.0, 0.0, 0.0]; | |
| segment1 | |
| .upsert_point(1, 1.into(), only_default_vector(&vec1)) | |
| .unwrap(); | |
| segment1 | |
| .upsert_point(2, 2.into(), only_default_vector(&vec2)) | |
| .unwrap(); | |
| segment1 | |
| .upsert_point(3, 3.into(), only_default_vector(&vec3)) | |
| .unwrap(); | |
| segment1 | |
| .upsert_point(4, 4.into(), only_default_vector(&vec4)) | |
| .unwrap(); | |
| segment1 | |
| .upsert_point(5, 5.into(), only_default_vector(&vec5)) | |
| .unwrap(); | |
| let payload_key = PAYLOAD_KEY; | |
| let payload_option1 = json!({ payload_key: vec!["red".to_owned()] }).into(); | |
| let payload_option2 = json!({ payload_key: vec!["red".to_owned(), "blue".to_owned()] }).into(); | |
| let payload_option3 = json!({ payload_key: vec!["blue".to_owned()] }).into(); | |
| segment1 | |
| .set_payload(6, 1.into(), &payload_option1, &None) | |
| .unwrap(); | |
| segment1 | |
| .set_payload(6, 2.into(), &payload_option1, &None) | |
| .unwrap(); | |
| segment1 | |
| .set_payload(6, 3.into(), &payload_option3, &None) | |
| .unwrap(); | |
| segment1 | |
| .set_payload(6, 4.into(), &payload_option2, &None) | |
| .unwrap(); | |
| segment1 | |
| .set_payload(6, 5.into(), &payload_option2, &None) | |
| .unwrap(); | |
| segment1 | |
| } | |
| pub fn build_segment_2(path: &Path) -> Segment { | |
| let mut segment2 = empty_segment(path); | |
| let vec1 = vec![-1.0, 0.0, 1.0, 1.0]; | |
| let vec2 = vec![-1.0, 0.0, 1.0, 0.0]; | |
| let vec3 = vec![-1.0, 1.0, 1.0, 1.0]; | |
| let vec4 = vec![-1.0, 1.0, 0.0, 1.0]; | |
| let vec5 = vec![-1.0, 0.0, 0.0, 0.0]; | |
| segment2 | |
| .upsert_point(11, 11.into(), only_default_vector(&vec1)) | |
| .unwrap(); | |
| segment2 | |
| .upsert_point(12, 12.into(), only_default_vector(&vec2)) | |
| .unwrap(); | |
| segment2 | |
| .upsert_point(13, 13.into(), only_default_vector(&vec3)) | |
| .unwrap(); | |
| segment2 | |
| .upsert_point(14, 14.into(), only_default_vector(&vec4)) | |
| .unwrap(); | |
| segment2 | |
| .upsert_point(15, 15.into(), only_default_vector(&vec5)) | |
| .unwrap(); | |
| let payload_key = PAYLOAD_KEY; | |
| let payload_option1 = json!({ payload_key: vec!["red".to_owned()] }).into(); | |
| let payload_option2 = json!({ payload_key: vec!["red".to_owned(), "blue".to_owned()] }).into(); | |
| let payload_option3 = json!({ payload_key: vec!["blue".to_owned()] }).into(); | |
| segment2 | |
| .set_payload(16, 11.into(), &payload_option1, &None) | |
| .unwrap(); | |
| segment2 | |
| .set_payload(16, 12.into(), &payload_option1, &None) | |
| .unwrap(); | |
| segment2 | |
| .set_payload(16, 13.into(), &payload_option3, &None) | |
| .unwrap(); | |
| segment2 | |
| .set_payload(16, 14.into(), &payload_option2, &None) | |
| .unwrap(); | |
| segment2 | |
| .set_payload(16, 15.into(), &payload_option2, &None) | |
| .unwrap(); | |
| segment2 | |
| } | |
| pub fn build_segment_3(path: &Path) -> Segment { | |
| let mut segment3 = build_segment( | |
| path, | |
| &SegmentConfig { | |
| vector_data: HashMap::from([ | |
| ( | |
| "vector1".to_owned(), | |
| VectorDataConfig { | |
| size: 4, | |
| distance: Distance::Dot, | |
| storage_type: VectorStorageType::Memory, | |
| index: Indexes::Plain {}, | |
| quantization_config: None, | |
| multivector_config: None, | |
| datatype: None, | |
| }, | |
| ), | |
| ( | |
| "vector2".to_owned(), | |
| VectorDataConfig { | |
| size: 1, | |
| distance: Distance::Dot, | |
| storage_type: VectorStorageType::Memory, | |
| index: Indexes::Plain {}, | |
| quantization_config: None, | |
| multivector_config: None, | |
| datatype: None, | |
| }, | |
| ), | |
| ( | |
| "vector3".to_owned(), | |
| VectorDataConfig { | |
| size: 4, | |
| distance: Distance::Euclid, | |
| storage_type: VectorStorageType::Memory, | |
| index: Indexes::Plain {}, | |
| quantization_config: None, | |
| multivector_config: None, | |
| datatype: None, | |
| }, | |
| ), | |
| ]), | |
| sparse_vector_data: Default::default(), | |
| payload_storage_type: Default::default(), | |
| }, | |
| true, | |
| ) | |
| .unwrap(); | |
| let collect_points_data = |vectors: &[DenseVector]| { | |
| NamedVectors::from_pairs([ | |
| ("vector1".to_owned(), vectors[0].clone()), | |
| ("vector2".to_owned(), vectors[1].clone()), | |
| ("vector3".to_owned(), vectors[2].clone()), | |
| ]) | |
| }; | |
| let vec1 = [ | |
| vec![1.0, 0.0, 1.0, 1.0], | |
| vec![0.0], | |
| vec![-1.0, 0.0, 1.0, 1.0], | |
| ]; | |
| let vec2 = [ | |
| vec![1.0, 0.0, 1.0, 0.0], | |
| vec![1.0], | |
| vec![-1.0, 0.0, 1.0, 1.0], | |
| ]; | |
| let vec3 = [ | |
| vec![1.0, 1.0, 1.0, 1.0], | |
| vec![2.0], | |
| vec![-1.0, 0.0, 1.0, 1.0], | |
| ]; | |
| let vec4 = [ | |
| vec![1.0, 1.0, 0.0, 1.0], | |
| vec![3.0], | |
| vec![-1.0, 0.0, 1.0, 1.0], | |
| ]; | |
| let vec5 = [ | |
| vec![1.0, 0.0, 0.0, 0.0], | |
| vec![4.0], | |
| vec![-1.0, 0.0, 1.0, 1.0], | |
| ]; | |
| segment3 | |
| .upsert_point(1, 1.into(), collect_points_data(&vec1)) | |
| .unwrap(); | |
| segment3 | |
| .upsert_point(2, 2.into(), collect_points_data(&vec2)) | |
| .unwrap(); | |
| segment3 | |
| .upsert_point(3, 3.into(), collect_points_data(&vec3)) | |
| .unwrap(); | |
| segment3 | |
| .upsert_point(4, 4.into(), collect_points_data(&vec4)) | |
| .unwrap(); | |
| segment3 | |
| .upsert_point(5, 5.into(), collect_points_data(&vec5)) | |
| .unwrap(); | |
| let payload_key = PAYLOAD_KEY; | |
| let payload_option1 = json!({ payload_key: vec!["red".to_owned()] }).into(); | |
| let payload_option2 = json!({ payload_key: vec!["red".to_owned(), "blue".to_owned()] }).into(); | |
| let payload_option3 = json!({ payload_key: vec!["blue".to_owned()] }).into(); | |
| segment3 | |
| .set_payload(6, 1.into(), &payload_option1, &None) | |
| .unwrap(); | |
| segment3 | |
| .set_payload(6, 2.into(), &payload_option1, &None) | |
| .unwrap(); | |
| segment3 | |
| .set_payload(6, 3.into(), &payload_option3, &None) | |
| .unwrap(); | |
| segment3 | |
| .set_payload(6, 4.into(), &payload_option2, &None) | |
| .unwrap(); | |
| segment3 | |
| .set_payload(6, 5.into(), &payload_option2, &None) | |
| .unwrap(); | |
| segment3 | |
| } | |
| pub fn build_segment_sparse_1(path: &Path) -> Segment { | |
| let mut segment1 = build_segment( | |
| path, | |
| &SegmentConfig { | |
| vector_data: Default::default(), | |
| sparse_vector_data: HashMap::from([( | |
| "sparse".to_owned(), | |
| SparseVectorDataConfig { | |
| index: SparseIndexConfig::new(None, SparseIndexType::MutableRam, None), | |
| }, | |
| )]), | |
| payload_storage_type: Default::default(), | |
| }, | |
| true, | |
| ) | |
| .unwrap(); | |
| let vec1 = SparseVector::new(vec![0, 1, 2, 3], vec![1.0, 0.0, 1.0, 1.0]).unwrap(); | |
| let vec2 = SparseVector::new(vec![0, 1, 2, 3], vec![1.0, 0.0, 1.0, 0.0]).unwrap(); | |
| let vec3 = SparseVector::new(vec![0, 1, 2, 3], vec![1.0, 1.0, 1.0, 1.0]).unwrap(); | |
| let vec4 = SparseVector::new(vec![0, 1, 2, 3], vec![1.0, 1.0, 0.0, 1.0]).unwrap(); | |
| let vec5 = SparseVector::new(vec![0, 1, 2, 3], vec![1.0, 0.0, 0.0, 0.0]).unwrap(); | |
| segment1 | |
| .upsert_point( | |
| 1, | |
| 1.into(), | |
| NamedVectors::from_ref("sparse", VectorRef::Sparse(&vec1)), | |
| ) | |
| .unwrap(); | |
| segment1 | |
| .upsert_point( | |
| 2, | |
| 2.into(), | |
| NamedVectors::from_ref("sparse", VectorRef::Sparse(&vec2)), | |
| ) | |
| .unwrap(); | |
| segment1 | |
| .upsert_point( | |
| 3, | |
| 3.into(), | |
| NamedVectors::from_ref("sparse", VectorRef::Sparse(&vec3)), | |
| ) | |
| .unwrap(); | |
| segment1 | |
| .upsert_point( | |
| 4, | |
| 4.into(), | |
| NamedVectors::from_ref("sparse", VectorRef::Sparse(&vec4)), | |
| ) | |
| .unwrap(); | |
| segment1 | |
| .upsert_point( | |
| 5, | |
| 5.into(), | |
| NamedVectors::from_ref("sparse", VectorRef::Sparse(&vec5)), | |
| ) | |
| .unwrap(); | |
| let payload_key = PAYLOAD_KEY; | |
| let payload_option1 = json!({ payload_key: vec!["red".to_owned()] }).into(); | |
| let payload_option2 = json!({ payload_key: vec!["red".to_owned(), "blue".to_owned()] }).into(); | |
| let payload_option3 = json!({ payload_key: vec!["blue".to_owned()] }).into(); | |
| segment1 | |
| .set_payload(6, 1.into(), &payload_option1, &None) | |
| .unwrap(); | |
| segment1 | |
| .set_payload(6, 2.into(), &payload_option1, &None) | |
| .unwrap(); | |
| segment1 | |
| .set_payload(6, 3.into(), &payload_option3, &None) | |
| .unwrap(); | |
| segment1 | |
| .set_payload(6, 4.into(), &payload_option2, &None) | |
| .unwrap(); | |
| segment1 | |
| .set_payload(6, 5.into(), &payload_option2, &None) | |
| .unwrap(); | |
| segment1 | |
| } | |
| pub fn build_segment_sparse_2(path: &Path) -> Segment { | |
| let mut segment2 = build_segment( | |
| path, | |
| &SegmentConfig { | |
| vector_data: Default::default(), | |
| sparse_vector_data: HashMap::from([( | |
| "sparse".to_owned(), | |
| SparseVectorDataConfig { | |
| index: SparseIndexConfig::new(None, SparseIndexType::MutableRam, None), | |
| }, | |
| )]), | |
| payload_storage_type: Default::default(), | |
| }, | |
| true, | |
| ) | |
| .unwrap(); | |
| let vec1 = SparseVector::new(vec![0, 1, 2, 3], vec![-1.0, 0.0, 1.0, 1.0]).unwrap(); | |
| let vec2 = SparseVector::new(vec![0, 1, 2, 3], vec![-1.0, 0.0, 1.0, 0.0]).unwrap(); | |
| let vec3 = SparseVector::new(vec![0, 1, 2, 3], vec![-1.0, 1.0, 1.0, 1.0]).unwrap(); | |
| let vec4 = SparseVector::new(vec![0, 1, 2, 3], vec![-1.0, 1.0, 0.0, 1.0]).unwrap(); | |
| let vec5 = SparseVector::new(vec![0, 1, 2, 3], vec![-1.0, 0.0, 0.0, 0.0]).unwrap(); | |
| segment2 | |
| .upsert_point( | |
| 11, | |
| 11.into(), | |
| NamedVectors::from_ref("sparse", VectorRef::Sparse(&vec1)), | |
| ) | |
| .unwrap(); | |
| segment2 | |
| .upsert_point( | |
| 12, | |
| 12.into(), | |
| NamedVectors::from_ref("sparse", VectorRef::Sparse(&vec2)), | |
| ) | |
| .unwrap(); | |
| segment2 | |
| .upsert_point( | |
| 13, | |
| 13.into(), | |
| NamedVectors::from_ref("sparse", VectorRef::Sparse(&vec3)), | |
| ) | |
| .unwrap(); | |
| segment2 | |
| .upsert_point( | |
| 14, | |
| 14.into(), | |
| NamedVectors::from_ref("sparse", VectorRef::Sparse(&vec4)), | |
| ) | |
| .unwrap(); | |
| segment2 | |
| .upsert_point( | |
| 15, | |
| 15.into(), | |
| NamedVectors::from_ref("sparse", VectorRef::Sparse(&vec5)), | |
| ) | |
| .unwrap(); | |
| let payload_key = PAYLOAD_KEY; | |
| let payload_option1 = json!({ payload_key: vec!["red".to_owned()] }).into(); | |
| let payload_option2 = json!({ payload_key: vec!["red".to_owned(), "blue".to_owned()] }).into(); | |
| let payload_option3 = json!({ payload_key: vec!["blue".to_owned()] }).into(); | |
| segment2 | |
| .set_payload(16, 11.into(), &payload_option1, &None) | |
| .unwrap(); | |
| segment2 | |
| .set_payload(16, 12.into(), &payload_option1, &None) | |
| .unwrap(); | |
| segment2 | |
| .set_payload(16, 13.into(), &payload_option3, &None) | |
| .unwrap(); | |
| segment2 | |
| .set_payload(16, 14.into(), &payload_option2, &None) | |
| .unwrap(); | |
| segment2 | |
| .set_payload(16, 15.into(), &payload_option2, &None) | |
| .unwrap(); | |
| segment2 | |
| } | |