Change:
vec.truncate(len); black_box(vec.first()); <ins> let vec = black_box(vec); </ins> vec.clear(); vec.extend_from_slice(&template); }); } <del> fn bench_vec_dedup_new(b: &mut Bencher, sz: usize) { </del> <ins> // Measures performance of Vec::dedup on random data. fn bench_vec_dedup_random(b: &mut Bencher, sz: usize) { </ins> let mut template = vec![0u32; sz]; b.bytes = std::mem::size_of_val(template.as_slice()) as u64; random_sorted_fill(0x43, &mut template); let mut vec = template.clone(); b.iter(|| { <ins> let vec = black_box(&mut vec); </ins> vec.dedup(); black_box(vec.first()); <ins> let vec = black_box(vec); vec.clear(); vec.extend_from_slice(&template); }); } // Measures performance of Vec::dedup when there is no items removed fn bench_vec_dedup_none(b: &mut Bencher, sz: usize) { let mut template = vec![0u32; sz]; b.bytes = std::mem::size_of_val(template.as_slice()) as u64; template.chunks_exact_mut(2).for_each(|w| { w[0] = black_box(0); w[1] = black_box(5); }); let mut vec = template.clone(); b.iter(|| { let vec = black_box(&mut vec); vec.dedup(); black_box(vec.first()); // Unlike other benches of `dedup` // this doesn't reinitialize vec // because we measure how efficient dedup is // when no memory written }); } // Measures performance of Vec::dedup when there is all items removed fn bench_vec_dedup_all(b: &mut Bencher, sz: usize) { let mut template = vec![0u32; sz]; b.bytes = std::mem::size_of_val(template.as_slice()) as u64; template.iter_mut().for_each(|w| { *w = black_box(0); }); let mut vec = template.clone(); b.iter(|| { let vec = black_box(&mut vec); vec.dedup(); black_box(vec.first()); let vec = black_box(vec); </ins> vec.clear(); vec.extend_from_slice(&template); }); } #[bench] <del> fn bench_dedup_old_100(b: &mut Bencher) { bench_vec_dedup_old(b, 100); </del> <ins> fn bench_dedup_slice_truncate_100(b: &mut Bencher) { bench_dedup_slice_truncate(b, 100); </ins> } #[bench] <del> fn bench_dedup_new_100(b: &mut Bencher) { bench_vec_dedup_new(b, 100); </del> <ins> fn bench_dedup_random_100(b: &mut Bencher) { bench_vec_dedup_random(b, 100); </ins> } #[bench] <del> fn bench_dedup_old_1000(b: &mut Bencher) { bench_vec_dedup_old(b, 1000); </del> <ins> fn bench_dedup_none_100(b: &mut Bencher) { bench_vec_dedup_none(b, 100); </ins> } <ins> #[bench] fn bench_dedup_all_100(b: &mut Bencher) { bench_vec_dedup_all(b, 100); } #[bench] fn bench_dedup_slice_truncate_1000(b: &mut Bencher) { bench_dedup_slice_truncate(b, 1000); } #[bench] fn bench_dedup_random_1000(b: &mut Bencher) { bench_vec_dedup_random(b, 1000); } #[bench] fn bench_dedup_none_1000(b: &mut Bencher) { bench_vec_dedup_none(b, 1000); } </ins> #[bench] <del> fn bench_dedup_new_1000(b: &mut Bencher) { bench_vec_dedup_new(b, 1000); </del> <ins> fn bench_dedup_all_1000(b: &mut Bencher) { bench_vec_dedup_all(b, 1000); </ins> } #[bench] <del> fn bench_dedup_old_10000(b: &mut Bencher) { bench_vec_dedup_old(b, 10000); </del> <ins> fn bench_dedup_slice_truncate_10000(b: &mut Bencher) { bench_dedup_slice_truncate(b, 10000); </ins> } #[bench] <del> fn bench_dedup_new_10000(b: &mut Bencher) { bench_vec_dedup_new(b, 10000); </del> <ins> fn bench_dedup_random_10000(b: &mut Bencher) { bench_vec_dedup_random(b, 10000); </ins> } #[bench] <del> fn bench_dedup_old_100000(b: &mut Bencher) { bench_vec_dedup_old(b, 100000); </del> <ins> fn bench_dedup_none_10000(b: &mut Bencher) { bench_vec_dedup_none(b, 10000); </ins> } <ins> #[bench] fn bench_dedup_all_10000(b: &mut Bencher) { bench_vec_dedup_all(b, 10000); } #[bench] fn bench_dedup_slice_truncate_100000(b: &mut Bencher) { bench_dedup_slice_truncate(b, 100000); } #[bench] fn bench_dedup_random_100000(b: &mut Bencher) { bench_vec_dedup_random(b, 100000); } #[bench] fn bench_dedup_none_100000(b: &mut Bencher) { bench_vec_dedup_none(b, 100000); } </ins> #[bench] <del> fn bench_dedup_new_100000(b: &mut Bencher) { bench_vec_dedup_new(b, 100000); </del> <ins> fn bench_dedup_all_100000(b: &mut Bencher) { bench_vec_dedup_all(b, 100000); </ins> } #[bench]

Discussion:
It seems like the implementation makes two passes over the data, making many more copies than are strictly necessary. Doing this allows the implementation to panic without causing a leak. I wouldn't call this bad, but it was a surprise to see that it was doing the extra work. Mimicking the way handles this and temporarily shortening the vector allows us to cut down on the copies (and also conveniently removes a panicking branch). I wonder if it makes sense to document the trade-off that makes here? Imo, it'd make it easier to use the library correctly in the way that
I wonder if it would be even faster to split it into two loops since once it will always be hitting that branch in the loop to do ? Maybe it could help reduce branch misprediction? I feel like this could be applied to the same thing right now. I wonder why are we not using but instead use ? And do we really need to , from what I see it does not do that right now? cc who may be interested in this.
The leaking behavior is unnecessary. A drop guard handle the tail.
is being called in in the current implementation. If you''re asking why I'm using , it's because it's faster than , and we don't need to use in this implementation. I'm pretty sure you're right My mistake, I wound myself up trying to justify the call. I think that means this would be a direct upgrade to the current implementation? (Note: My implementation seems to be broken in a couple ways, it's just for reference)
I tried to flesh out the implementation a little, and make sure it's working properly. The behaviour of the still needs to be tested, and it'd be good to do a more thorough benchmark if this was being considered for addition. I also implemented the nested loop version. I can't tell if it helped with performance yet, since my computer gives the benchmarks too much noise.
Do you think separating the branch into a different loop is a good idea in case they are not in the same bucket?
I've used a second (and third) loop in the implementation to avoid the branch, did I misunderstand your recommendation?
It still does seemed to be in the loop , I think making that in the inner loop is the same as the one currently since that branch will always be there.

I didn't benchmark and I am not sure if it will be faster, just a guess avoiding that branch may end up reducing branch mispredictions.
:thinking: I honestly don't know how I'd implement that - it seems like we need to branch somewhere to verify the the copies are legal. I'd welcome a PR showing what you mean.
I mean we are branching it in another loop since once the dedup and is different, it will always copy stuff when it hit the other branch, so instead of having 2 branches when the bucket is different, we could keep it to the other loop to remove the branch. Not sure if it's faster though.
proposed additional optimizations for this issue. Details here: