Repository: commonwarexyz/monorepo
Pull request #603: [codec] Add more test coverage
Author: login: BrendanChou
Created: 2025-03-13T20:25:27Z | Merged: 2025-03-15T21:27:44Z

- Generally cleans up code
- Adds more tests
- Removes unnecessary functions for 32-bit varints
  - Instead just use the 64-bit versions and cast before/after decoding for not much loss in performance
  - Eventually we can just encode/decode using arbitrary integer primitives using the same code. I made an issue [here](https://github.com/commonwarexyz/monorepo/issues/606)


Diff:
diff --git a/codec/src/buffer.rs b/codec/src/buffer.rs
index abcaf4cb1e..e9f2edbcbb 100644
--- a/codec/src/buffer.rs
+++ b/codec/src/buffer.rs
@@ -29,9 +29,7 @@ impl ReadBuffer {
     /// Gets a byte from the buffer
     #[inline]
     pub fn get_u8(&mut self) -> Result<u8, Error> {
-        if !self.has_remaining() {
-            return Err(Error::EndOfBuffer);
-        }
+        self.at_least(1)?;
         Ok(self.inner.get_u8())
     }
 
diff --git a/codec/src/codec.rs b/codec/src/codec.rs
index ee39542fc7..9f6e84692e 100644
--- a/codec/src/codec.rs
+++ b/codec/src/codec.rs
@@ -42,24 +42,33 @@ pub trait Codec: Sized {
 /// Trait for types that have a fixed-length encoding
 pub trait SizedCodec: Codec {
     /// The encoded length of this value.
-    const LEN_CODEC: usize;
+    const LEN_ENCODED: usize;
 
     /// Returns the encoded length of this value.
+    ///
+    /// Should not be overridden by implementations.
     fn len_encoded(&self) -> usize {
-        Self::LEN_CODEC
+        Self::LEN_ENCODED
     }
 
     /// Encodes a value to fixed-size bytes.
     fn encode_fixed<const N: usize>(&self) -> [u8; N] {
+        // Ideally this is a compile-time check, but we can't do that in the current Rust version
+        // without adding a new generic parameter to the trait.
+        assert_eq!(
+            N,
+            Self::LEN_ENCODED,
+            "Can't encode {} bytes into {} bytes",
+            Self::LEN_ENCODED,
+            N
+        );
+
         self.encode().try_into().unwrap()
     }
 }
 
 /// Trait for codec read operations
 pub trait Reader {
-    /// Reads a value of type T
-    fn read<T: Codec>(&mut self) -> Result<T, Error>;
-
     /// Reads a u8 value
     fn read_u8(&mut self) -> Result<u8, Error>;
 
@@ -126,9 +135,6 @@ pub trait Reader {
 
 /// Trait for codec write operations
 pub trait Writer {
-    /// Writes a value of type T
-    fn write<T: Codec>(&mut self, value: &T);
-
     /// Writes a u8 value
     fn write_u8(&mut self, value: u8);
 
@@ -186,10 +192,6 @@ pub trait Writer {
 
 // Implement Reader for ReadBuffer
 impl Reader for ReadBuffer {
-    fn read<T: Codec>(&mut self) -> Result<T, Error> {
-        T::read(self)
-    }
-
     fn read_u8(&mut self) -> Result<u8, Error> {
         self.get_u8()
     }
@@ -278,7 +280,7 @@ impl Reader for ReadBuffer {
         let has_value = self.read_bool()?;
 
         if has_value {
-            Ok(Some(self.read()?))
+            Ok(Some(T::read(self)?))
         } else {
             Ok(None)
         }
@@ -288,7 +290,7 @@ impl Reader for ReadBuffer {
         let len = self.read_varint()? as usize;
         let mut items = Vec::with_capacity(len);
         for _ in 0..len {
-            items.push(self.read()?);
+            items.push(T::read(self)?);
         }
         Ok(items)
     }
@@ -302,7 +304,7 @@ impl Reader for ReadBuffer {
 
         let mut items = Vec::with_capacity(len);
         for _ in 0..len {
-            items.push(self.read()?);
+            items.push(T::read(self)?);
         }
         Ok(items)
     }
@@ -310,10 +312,6 @@ impl Reader for ReadBuffer {
 
 // Implement Writer for WriteBuffer
 impl Writer for WriteBuffer {
-    fn write<T: Codec>(&mut self, value: &T) {
-        value.write(self);
-    }
-
     fn write_u8(&mut self, value: u8) {
         self.put_u8(value)
     }
@@ -383,7 +381,7 @@ impl Writer for WriteBuffer {
         match value {
             Some(v) => {
                 self.write_bool(true);
-                self.write(v);
+                v.write(self);
             }
             None => {
                 self.write_bool(false);
@@ -394,15 +392,15 @@ impl Writer for WriteBuffer {
     fn write_vec<T: Codec>(&mut self, values: &[T]) {
         self.write_varint(values.len() as u64);
         for value in values {
-            self.write(value);
+            value.write(self);
         }
     }
 }
+
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::buffer::{ReadBuffer, WriteBuffer};
-    use crate::error::Error;
+    use crate::{varint::varint_size, Codec, Error, ReadBuffer, WriteBuffer};
     use bytes::Bytes;
 
     #[test]
@@ -424,14 +422,13 @@ mod tests {
     }
 
     #[test]
-    fn test_invalid_varint() {
-        let encoded = Bytes::from_static(&[
-            0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
-        ]);
-        assert!(matches!(
-            ReadBuffer::new(encoded).read_varint(),
-            Err(Error::InvalidVarint)
-        ));
+    fn test_varint() {
+        let value = u64::MAX / 2;
+        let mut writer = WriteBuffer::new(varint_size(value));
+        writer.write_varint(value);
+        let mut reader = ReadBuffer::new(writer.freeze());
+        let result = reader.read_varint().unwrap();
+        assert_eq!(result, value);
     }
 
     #[test]
@@ -483,4 +480,18 @@ mod tests {
             Err(Error::LengthExceeded(3, 2))
         ));
     }
+
+    #[test]
+    fn test_encode_fixed() {
+        let value = 42u32;
+        let encoded: [u8; 4] = value.encode_fixed();
+        let decoded = u32::decode(Bytes::copy_from_slice(&encoded)).unwrap();
+        assert_eq!(value, decoded);
+    }
+
+    #[test]
+    #[should_panic(expected = "Can't encode 4 bytes into 5 bytes")]
+    fn test_encode_fixed_panic() {
+        let _: [u8; 5] = 42u32.encode_fixed();
+    }
 }
diff --git a/codec/src/lib.rs b/codec/src/lib.rs
index dcf3625858..3a27148ba8 100644
--- a/codec/src/lib.rs
+++ b/codec/src/lib.rs
@@ -33,16 +33,16 @@
 //! impl Codec for Point {
 //!     fn write(&self, writer: &mut impl Writer) {
 //!         // Basic types can be written by inferring the type
-//!         writer.write(&self.xy);
-//!         writer.write(&self.z);
-//!         writer.write(&self.metadata);
+//!         self.xy.write(writer);
+//!         self.z.write(writer);
+//!         self.metadata.write(writer);
 //!     }
 //!
 //!     fn read(reader: &mut impl Reader) -> Result<Self, Error> {
 //!         // Basic types can be inferred by the return type
-//!         let xy = reader.read()?;
-//!         let z = reader.read()?;
-//!         let metadata = reader.read()?;
+//!         let xy = <(u64, u64)>::read(reader)?;
+//!         let z = <Option<u32>>::read(reader)?;
+//!         let metadata = <[u8; 11]>::read(reader)?;
 //!         Ok(Self { xy, z, metadata })
 //!     }
 //!
diff --git a/codec/src/types.rs b/codec/src/types.rs
index 06f5868f6d..3cffc20296 100644
--- a/codec/src/types.rs
+++ b/codec/src/types.rs
@@ -7,10 +7,8 @@ use crate::{
 use bytes::Bytes;
 use paste::paste;
 
-// ===== Primitive implementations =====
-
 macro_rules! impl_primitive {
-    ($type:ty, $read_method:ident, $write_method:ident) => {
+    ($type:ty, $read_method:ident, $write_method:ident, $size:expr) => {
         impl Codec for $type {
             #[inline]
             fn write(&self, writer: &mut impl Writer) {
@@ -29,24 +27,24 @@ macro_rules! impl_primitive {
         }
 
         impl SizedCodec for $type {
-            const LEN_CODEC: usize = std::mem::size_of::<$type>();
+            const LEN_ENCODED: usize = $size;
         }
     };
 }
 
-impl_primitive!(u8, read_u8, write_u8);
-impl_primitive!(u16, read_u16, write_u16);
-impl_primitive!(u32, read_u32, write_u32);
-impl_primitive!(u64, read_u64, write_u64);
-impl_primitive!(u128, read_u128, write_u128);
-impl_primitive!(i8, read_i8, write_i8);
-impl_primitive!(i16, read_i16, write_i16);
-impl_primitive!(i32, read_i32, write_i32);
-impl_primitive!(i64, read_i64, write_i64);
-impl_primitive!(i128, read_i128, write_i128);
-impl_primitive!(f32, read_f32, write_f32);
-impl_primitive!(f64, read_f64, write_f64);
-impl_primitive!(bool, read_bool, write_bool);
+impl_primitive!(u8, read_u8, write_u8, 1);
+impl_primitive!(u16, read_u16, write_u16, 2);
+impl_primitive!(u32, read_u32, write_u32, 4);
+impl_primitive!(u64, read_u64, write_u64, 8);
+impl_primitive!(u128, read_u128, write_u128, 16);
+impl_primitive!(i8, read_i8, write_i8, 1);
+impl_primitive!(i16, read_i16, write_i16, 2);
+impl_primitive!(i32, read_i32, write_i32, 4);
+impl_primitive!(i64, read_i64, write_i64, 8);
+impl_primitive!(i128, read_i128, write_i128, 16);
+impl_primitive!(f32, read_f32, write_f32, 4);
+impl_primitive!(f64, read_f64, write_f64, 8);
+impl_primitive!(bool, read_bool, write_bool, 1);
 
 // Bytes implementation
 impl Codec for Bytes {
@@ -85,7 +83,7 @@ impl<const N: usize> Codec for [u8; N] {
 }
 
 impl<const N: usize> SizedCodec for [u8; N] {
-    const LEN_CODEC: usize = N;
+    const LEN_ENCODED: usize = N;
 }
 
 // Option implementation
@@ -115,7 +113,7 @@ macro_rules! impl_codec_for_tuple {
         paste! {
             impl<$( [<T $index>]: Codec ),*> Codec for ( $( [<T $index>], )* ) {
                 fn write(&self, writer: &mut impl Writer) {
-                    $( writer.write(&self.$index); )*
+                    $( self.$index.write(writer); )*
                 }
 
                 fn len_encoded(&self) -> usize {
@@ -123,13 +121,9 @@ macro_rules! impl_codec_for_tuple {
                 }
 
                 fn read(reader: &mut impl Reader) -> Result<Self, Error> {
-                    Ok(( $( reader.read::<[<T $index>]>()? , )* ))
+                    Ok(( $( [<T $index>]::read(reader)?, )* ))
                 }
             }
-
-            impl<$( [<T $index>]: SizedCodec ),*> SizedCodec for ( $( [<T $index>], )* ) {
-                const LEN_CODEC: usize = 0 $( + [<T $index>]::LEN_CODEC )*;
-            }
         }
     };
 }
@@ -170,54 +164,96 @@ impl<T: Codec> Codec for Vec<T> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::{
-        buffer::{ReadBuffer, WriteBuffer},
-        codec::{Codec, SizedCodec},
-    };
+    use crate::codec::{Codec, SizedCodec};
     use bytes::Bytes;
 
+    // Float tests
+    macro_rules! impl_num_test {
+        ($type:ty, $size:expr) => {
+            paste! {
+                #[test]
+                fn [<test_ $type>]() {
+                    let expected_len = std::mem::size_of::<$type>();
+                    let values: [$type; 5] =
+                        [0 as $type, 1 as $type, 42 as $type, <$type>::MAX, <$type>::MIN];
+                    for value in values.iter() {
+                        let encoded = value.encode();
+                        assert_eq!(encoded.len(), expected_len);
+                        let decoded = <$type>::decode(encoded).unwrap();
+                        assert_eq!(*value, decoded);
+                        assert_eq!(Codec::len_encoded(value), expected_len);
+                        assert_eq!(SizedCodec::len_encoded(value), expected_len);
+
+                        let fixed: [u8; $size] = value.encode_fixed();
+                        assert_eq!(fixed.len(), expected_len);
+                        let decoded = <$type>::decode(Bytes::copy_from_slice(&fixed)).unwrap();
+                        assert_eq!(*value, decoded);
+                    }
+                }
+            }
+        };
+    }
+    impl_num_test!(u8, 1);
+    impl_num_test!(u16, 2);
+    impl_num_test!(u32, 4);
+    impl_num_test!(u64, 8);
+    impl_num_test!(u128, 16);
+    impl_num_test!(i8, 1);
+    impl_num_test!(i16, 2);
+    impl_num_test!(i32, 4);
+    impl_num_test!(i64, 8);
+    impl_num_test!(i128, 16);
+    impl_num_test!(f32, 4);
+    impl_num_test!(f64, 8);
+
     #[test]
-    fn test_primitives_round_trip() {
-        let u32_values = [0, 1, 42, u32::MAX];
-        for value in u32_values {
-            let encoded = value.encode();
-            let decoded = u32::decode(encoded).unwrap();
-            assert_eq!(value, decoded);
-        }
+    fn test_endianness() {
+        // u16
+        let encoded = 0x0102u16.encode();
+        assert_eq!(encoded, Bytes::from_static(&[0x01, 0x02]));
 
-        let f32_values = [0.0, 1.0, -1.0, 0.5];
-        for value in f32_values {
-            let encoded = value.encode();
-            let decoded = f32::decode(encoded).unwrap();
-            assert_eq!(value, decoded);
-        }
+        // u32
+        let encoded = 0x01020304u32.encode();
+        assert_eq!(encoded, Bytes::from_static(&[0x01, 0x02, 0x03, 0x04]));
 
-        let bool_values = [true, false];
-        for value in bool_values {
+        // f32
+        let encoded = 1.0f32.encode();
+        assert_eq!(encoded, Bytes::from_static(&[0x3F, 0x80, 0x00, 0x00])); // Big-endian IEEE 754
+    }
+
+    #[test]
+    fn test_bool() {
+        let values = [true, false];
+        for value in values.iter() {
             let encoded = value.encode();
+            assert_eq!(encoded.len(), 1);
             let decoded = bool::decode(encoded).unwrap();
-            assert_eq!(value, decoded);
+            assert_eq!(*value, decoded);
+            assert_eq!(Codec::len_encoded(value), 1);
+            assert_eq!(SizedCodec::len_encoded(value), 1);
         }
     }
 
     #[test]
-    fn test_bytes_round_trip() {
-        let bytes_values = [
+    fn test_bytes() {
+        let values = [
             Bytes::new(),
             Bytes::from_static(&[1, 2, 3]),
             Bytes::from(vec![0; 300]),
         ];
-        for value in bytes_values {
-            let mut writer = WriteBuffer::new(value.len() + 4);
-            writer.write_bytes(&value);
-            let mut reader = ReadBuffer::new(writer.freeze());
-            let decoded = reader.read_bytes().unwrap();
+        for value in values {
+            let encoded = value.encode();
+            assert_eq!(
+                encoded.len(),
+                varint::varint_size(value.len() as u64) + value.len()
+            );
+            let decoded = Bytes::decode(encoded).unwrap();
             assert_eq!(value, decoded);
         }
     }
 
     #[test]
-    fn test_arrays_round_trip() {
+    fn test_array() {
         let values = [1u8, 2, 3];
         let encoded = values.encode();
         let decoded = <[u8; 3]>::decode(encoded).unwrap();
@@ -225,7 +261,7 @@ mod tests {
     }
 
     #[test]
-    fn test_option_round_trip() {
+    fn test_option() {
         let option_values = [Some(42u32), None];
         for value in option_values {
             let encoded = value.encode();
@@ -235,7 +271,17 @@ mod tests {
     }
 
     #[test]
-    fn test_tuples_round_trip() {
+    fn test_option_length() {
+        let some = Some(42u32);
+        assert_eq!(Codec::len_encoded(&some), 1 + 4);
+        assert_eq!(some.encode().len(), 1 + 4);
+        let none: Option<u32> = None;
+        assert_eq!(Codec::len_encoded(&none), 1);
+        assert_eq!(none.encode().len(), 1);
+    }
+
+    #[test]
+    fn test_tuple() {
         let tuple_values = [(1u16, None), (1u16, Some(2u32))];
         for value in tuple_values {
             let encoded = value.encode();
@@ -245,78 +291,13 @@ mod tests {
     }
 
     #[test]
-    fn test_vec_round_trip() {
+    fn test_vec() {
         let vec_values = [vec![], vec![1u8], vec![1u8, 2u8, 3u8]];
         for value in vec_values {
-            let mut writer = WriteBuffer::new(value.len() + 4);
-            writer.write_vec(&value);
-            let mut reader = ReadBuffer::new(writer.freeze());
-            let decoded = reader.read_vec::<u8>().unwrap();
+            let encoded = value.encode();
+            assert_eq!(encoded.len(), value.len() * std::mem::size_of::<u8>() + 1);
+            let decoded = Vec::<u8>::decode(encoded).unwrap();
             assert_eq!(value, decoded);
         }
     }
-
-    #[test]
-    fn test_u16_endianness() {
-        let value = 0x0102u16;
-        let encoded = value.encode();
-        assert_eq!(encoded, Bytes::from_static(&[0x01, 0x02]));
-    }
-
-    #[test]
-    fn test_u32_endianness() {
-        let value = 0x01020304u32;
-        let encoded = value.encode();
-        assert_eq!(encoded, Bytes::from_static(&[0x01, 0x02, 0x03, 0x04]));
-    }
-
-    #[test]
-    fn test_f32_encoding() {
-        let value = 1.0f32;
-        let encoded = value.encode();
-        assert_eq!(encoded, Bytes::from_static(&[0x3F, 0x80, 0x00, 0x00])); // Big-endian IEEE 754
-    }
-
-    #[test]
-    fn test_primitives_length() {
-        assert_eq!(Codec::len_encoded(&42u32), 4);
-        assert_eq!(SizedCodec::len_encoded(&42u32), 4);
-        assert_eq!(42u32.encode().len(), 4);
-        assert_eq!(Codec::len_encoded(&1.0f32), 4);
-        assert_eq!(SizedCodec::len_encoded(&1.0f32), 4);
-        assert_eq!(1.0f32.encode().len(), 4);
-        assert_eq!(Codec::len_encoded(&true), 1);
-        assert_eq!(SizedCodec::len_encoded(&true), 1);
-        assert_eq!(true.encode().len(), 1);
-    }
-
-    #[test]
-    fn test_bytes_length() {
-        let bytes = Bytes::from_static(&[1, 2, 3]);
-        let expected = 1 + 3; // Varint length + data
-        let mut writer = WriteBuffer::new(expected);
-        writer.write_bytes(&bytes);
-        assert_eq!(bytes.len_encoded(), expected);
-        assert_eq!(writer.freeze().len(), expected);
-    }
-
-    #[test]
-    fn test_option_length() {
-        let some = Some(42u32);
-        assert_eq!(Codec::len_encoded(&some), 1 + 4);
-        assert_eq!(some.encode().len(), 1 + 4);
-        let none: Option<u32> = None;
-        assert_eq!(Codec::len_encoded(&none), 1);
-        assert_eq!(none.encode().len(), 1);
-    }
-
-    #[test]
-    fn test_vec_length() {
-        let vec = vec![1u8, 2u8, 3u8];
-        let expected = 1 + 3; // Varint length + data
-        let mut writer = WriteBuffer::new(expected);
-        writer.write_vec(&vec);
-        assert_eq!(vec.len_encoded(), expected);
-        assert_eq!(writer.freeze().len(), expected);
-    }
 }
diff --git a/codec/src/varint.rs b/codec/src/varint.rs
index d5032a9584..be00db7e9d 100644
--- a/codec/src/varint.rs
+++ b/codec/src/varint.rs
@@ -6,12 +6,6 @@
 use crate::error::Error;
 use bytes::{Buf, BufMut};
 
-/// Maximum number of bytes needed to encode a 64-bit integer as a varint
-pub const MAX_VARINT_LEN_U64: usize = 10;
-
-/// Maximum number of bytes needed to encode a 32-bit integer as a varint
-pub const MAX_VARINT_LEN_U32: usize = 5;
-
 /// Encodes a unsigned 64-bit integer as a varint
 pub fn encode_varint(value: u64, buf: &mut impl BufMut) {
     if value < 0x80 {
@@ -33,32 +27,29 @@ pub fn decode_varint(buf: &mut impl Buf) -> Result<u64, Error> {
     let mut result = 0u64;
     let mut shift = 0;
 
+    // Loop over all the bytes.
     loop {
+        // Read the next byte.
         if !buf.has_remaining() {
             return Err(Error::EndOfBuffer);
         }
-
         let byte = buf.get_u8();
-        if shift > 63 && byte > 1 {
+
+        // If we have read more than 9 bytes, the next byte must be 0 or 1.
+        if shift >= (9 * 7) && byte > 1 {
             return Err(Error::InvalidVarint);
         }
 
+        // Write the 7 bits of data to the result.
         result |= ((byte & 0x7F) as u64) << shift;
 
+        // If the continuation bit is not set, return.
         if byte & 0x80 == 0 {
             return Ok(result);
         }
 
+        // Each byte has 7 bits of data.
         shift += 7;
-
-        if shift > 63 {
-            // We've read 9 bytes but still have the continuation bit set,
-            // which would push our value beyond the range of u64
-            if buf.has_remaining() && buf.get_u8() & 0x80 != 0 {
-                return Err(Error::InvalidVarint);
-            }
-            return Ok(result);
-        }
     }
 }
 
@@ -78,41 +69,30 @@ pub fn varint_size(value: u64) -> usize {
     }
 }
 
-/// Encodes a unsigned 32-bit integer as a varint (more efficient than u64 version)
-pub fn encode_varint_u32(value: u32, buf: &mut impl BufMut) {
-    if value < 0x80 {
-        // Fast path for small values
-        buf.put_u8(value as u8);
-        return;
-    }
+/// Converts a signed integer to an unsigned integer using ZigZag encoding
+fn to_u64(value: i64) -> u64 {
+    ((value << 1) ^ (value >> 63)) as u64
+}
 
-    let mut val = value;
-    while val >= 0x80 {
-        buf.put_u8((val as u8) | 0x80);
-        val >>= 7;
-    }
-    buf.put_u8(val as u8);
+/// Converts an unsigned integer to a signed integer using ZigZag encoding
+fn to_i64(value: u64) -> i64 {
+    ((value >> 1) as i64) ^ (-((value & 1) as i64))
 }
 
 /// Encodes a signed 64-bit integer as a varint using ZigZag encoding
 pub fn encode_varint_i64(value: i64, buf: &mut impl BufMut) {
-    // Convert to ZigZag encoding
-    let zigzag = ((value << 1) ^ (value >> 63)) as u64;
-    encode_varint(zigzag, buf);
+    encode_varint(to_u64(value), buf);
 }
 
 /// Decodes a signed 64-bit integer from a varint using ZigZag encoding
 pub fn decode_varint_i64(buf: &mut impl Buf) -> Result<i64, Error> {
     let zigzag = decode_varint(buf)?;
-    // Convert from ZigZag encoding
-    Ok(((zigzag >> 1) as i64) ^ (-((zigzag & 1) as i64)))
+    Ok(to_i64(zigzag))
 }
 
-/// Encodes a signed 32-bit integer as a varint using ZigZag encoding
-pub fn encode_varint_i32(value: i32, buf: &mut impl BufMut) {
-    // Convert to ZigZag encoding
-    let zigzag = ((value << 1) ^ (value >> 31)) as u32;
-    encode_varint_u32(zigzag, buf);
+/// Calculates the number of bytes needed to encode a signed integer as a varint
+pub fn varint_i64_size(value: i64) -> usize {
+    varint_size(to_u64(value))
 }
 
 #[cfg(test)]
@@ -139,7 +119,7 @@ mod tests {
             0xFFFFFFFF,
             0x1FFFFFFFFFF,
             0xFFFFFFFFFFFFFF,
-            0xFFFFFFFFFFFFFFFF,
+            u64::MAX,
         ];
 
         for &value in &test_cases {
@@ -151,13 +131,8 @@ mod tests {
             let mut read_buf = &buf[..];
             let decoded = decode_varint(&mut read_buf).unwrap();
 
-            assert_eq!(decoded, value, "Failed for value: {}", value);
-            assert_eq!(
-                read_buf.len(),
-                0,
-                "Not all bytes consumed for value: {}",
-                value
-            );
+            assert_eq!(decoded, value);
+            assert_eq!(read_buf.len(), 0);
         }
     }
 
@@ -177,24 +152,21 @@ mod tests {
             -129,
             0x7FFFFFFF,
             -0x7FFFFFFF,
-            0x7FFFFFFFFFFFFFFF,
-            -0x7FFFFFFFFFFFFFFF,
+            i64::MIN,
+            i64::MAX,
         ];
 
         for &value in &test_cases {
             let mut buf = Vec::new();
             encode_varint_i64(value, &mut buf);
 
+            assert_eq!(buf.len(), varint_i64_size(value));
+
             let mut read_buf = &buf[..];
             let decoded = decode_varint_i64(&mut read_buf).unwrap();
 
-            assert_eq!(decoded, value, "Failed for value: {}", value);
-            assert_eq!(
-                read_buf.len(),
-                0,
-                "Not all bytes consumed for value: {}",
-                value
-            );
+            assert_eq!(decoded, value);
+            assert_eq!(read_buf.len(), 0,);
         }
     }
 
@@ -226,7 +198,14 @@ mod tests {
 
     #[test]
     fn test_varint_insufficient_buffer() {
-        let mut buf = Bytes::from_static(&[0x80]); // Incomplete varint
+        let mut buf = Bytes::from_static(&[0x80]);
         assert!(matches!(decode_varint(&mut buf), Err(Error::EndOfBuffer)));
     }
+
+    #[test]
+    fn test_varint_invalid() {
+        let mut buf =
+            Bytes::from_static(&[0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x02]);
+        assert!(matches!(decode_varint(&mut buf), Err(Error::InvalidVarint)));
+    }
 }


Reviews:
user: login: BrendanChou
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-13T20:27:47Z
user: login: patrick-ogrady
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T20:45:42Z
user: login: patrick-ogrady
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T21:53:26Z
user: login: patrick-ogrady
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T21:55:03Z
user: login: BrendanChou
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-14T16:25:11Z
user: login: copilot-pull-request-reviewer[bot]
body: ## Pull Request Overview

This PR enhances test coverage and refactors the codec API by leveraging const generics for fixed-size types while removing outdated functions for 32-bit varints. Key changes include:
- Refactoring the SizedCodec trait to use a const generic parameter.
- Updating primitive implementations and related tests to utilize the new generic API.
- Adding and improving tests for varint encoding/decoding, tuples, and option types.

### Reviewed Changes

Copilot reviewed 4 out of 4 changed files in this pull request and generated no comments.

| File                 | Description                                                     |
| -------------------- | --------------------------------------------------------------- |
| codec/src/codec.rs   | Refactored SizedCodec to use const generics and updated tests.  |
| codec/src/types.rs   | Adjusted primitive and tuple implementations to support const sizes and added additional tests. |
| codec/src/varint.rs  | Removed unused constants and refined the varint encoding/decoding logic. |
| codec/src/buffer.rs  | Simplified buffer read logic by consolidating error checking.   |




state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T19:49:11Z
user: login: BrendanChou
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-14T22:23:20Z
user: login: BrendanChou
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-15T00:34:46Z
user: login: BrendanChou
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-15T00:36:22Z
user: login: patrick-ogrady
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-15T21:27:38Z

Review comments:
diff_hunk: @@ -40,17 +40,16 @@ pub trait Codec: Sized {
 }
 
 /// Trait for types that have a fixed-length encoding
-pub trait SizedCodec: Codec {
-    /// The encoded length of this value.
-    const LEN_CODEC: usize;
-
+pub trait SizedCodec<const N: usize>: Codec {
     /// Returns the encoded length of this value.
+    ///
+    /// Should not be overridden by implementations.
     fn len_encoded(&self) -> usize {
-        Self::LEN_CODEC
+        N
     }
 
     /// Encodes a value to fixed-size bytes.
-    fn encode_fixed<const N: usize>(&self) -> [u8; N] {
+    fn encode_fixed(&self) -> [u8; N] {
path: codec/src/codec.rs
user: login: BrendanChou
body: This is safer now, you can't do stuff like `const buf: [u8; 3] = (1234 as u64).encode_fixed();` as it won't compile anymore due to that `3` not matching the length of `u64`

Wasn't able to figure out compiling tuples as fixed due to this change anymore but I think that's a fine tradeoff
created_at: 2025-03-13T20:27:47Z
author_association: COLLABORATOR
diff_hunk: @@ -40,17 +40,16 @@ pub trait Codec: Sized {
 }
 
 /// Trait for types that have a fixed-length encoding
-pub trait SizedCodec: Codec {
-    /// The encoded length of this value.
-    const LEN_CODEC: usize;
-
+pub trait SizedCodec<const N: usize>: Codec {
     /// Returns the encoded length of this value.
+    ///
+    /// Should not be overridden by implementations.
     fn len_encoded(&self) -> usize {
-        Self::LEN_CODEC
+        N
     }
 
     /// Encodes a value to fixed-size bytes.
-    fn encode_fixed<const N: usize>(&self) -> [u8; N] {
+    fn encode_fixed(&self) -> [u8; N] {
path: codec/src/codec.rs
user: login: patrick-ogrady
body: Agreed. Live and die by the compiler as much as possible.
created_at: 2025-03-13T20:45:42Z
author_association: CONTRIBUTOR
path: codec/src/varint.rs
user: login: patrick-ogrady
body: Now that we have our own varint, should we clean this up?

https://github.com/commonwarexyz/monorepo/blob/f51acf8732116fc8f6f01773dceb6fcaaf4856f8/utils/src/lib.rs#L70-L80
created_at: 2025-03-13T21:53:26Z
author_association: CONTRIBUTOR
diff_hunk: @@ -78,41 +75,30 @@ pub fn varint_size(value: u64) -> usize {
     }
 }
 
-/// Encodes a unsigned 32-bit integer as a varint (more efficient than u64 version)
-pub fn encode_varint_u32(value: u32, buf: &mut impl BufMut) {
-    if value < 0x80 {
-        // Fast path for small values
-        buf.put_u8(value as u8);
-        return;
-    }
+/// Converts a signed integer to an unsigned integer using ZigZag encoding
+fn to_u64(value: i64) -> u64 {
+    ((value << 1) ^ (value >> 63)) as u64
+}
 
-    let mut val = value;
-    while val >= 0x80 {
-        buf.put_u8((val as u8) | 0x80);
-        val >>= 7;
-    }
-    buf.put_u8(val as u8);
+/// Converts an unsigned integer to a signed integer using ZigZag encoding
+fn to_i64(value: u64) -> i64 {
+    ((value >> 1) as i64) ^ (-((value & 1) as i64))
 }
 
 /// Encodes a signed 64-bit integer as a varint using ZigZag encoding
-pub fn encode_varint_i64(value: i64, buf: &mut impl BufMut) {
-    // Convert to ZigZag encoding
-    let zigzag = ((value << 1) ^ (value >> 63)) as u64;
-    encode_varint(zigzag, buf);
+pub fn encode_svarint(value: i64, buf: &mut impl BufMut) {
path: codec/src/varint.rs
user: login: patrick-ogrady
body: tbh, I like `_i64` better than `svarint`
created_at: 2025-03-13T21:55:02Z
author_association: CONTRIBUTOR
path: codec/src/varint.rs
user: login: BrendanChou
body: https://github.com/commonwarexyz/monorepo/issues/608
created_at: 2025-03-14T16:25:11Z
author_association: COLLABORATOR
diff_hunk: @@ -40,17 +40,16 @@ pub trait Codec: Sized {
 }
 
 /// Trait for types that have a fixed-length encoding
-pub trait SizedCodec: Codec {
-    /// The encoded length of this value.
-    const LEN_CODEC: usize;
-
+pub trait SizedCodec<const N: usize>: Codec {
     /// Returns the encoded length of this value.
+    ///
+    /// Should not be overridden by implementations.
     fn len_encoded(&self) -> usize {
-        Self::LEN_CODEC
+        N
     }
 
     /// Encodes a value to fixed-size bytes.
-    fn encode_fixed<const N: usize>(&self) -> [u8; N] {
+    fn encode_fixed(&self) -> [u8; N] {
path: codec/src/codec.rs
user: login: BrendanChou
body: I actually found an issue with this change so I will revert it.

This is called using a "generic parameter" rather than an "associated constant". And if `Array` trait has to implement this trait, it would cause `Array` to have to have a new generic parameter `N`. So `Array` becomes `Array<N>` which doesn't seem that bad, so far.

But then `Scheme` must then become: `Scheme<PRIVATE_LEN, PUBLIC_LEN, SIG_LEN>` with three different associated constants, and a `Config` that includes `Scheme` would also have to add three more associated constants, and so on. It starts to get really messy :confused:
created_at: 2025-03-14T22:23:20Z
author_association: COLLABORATOR
diff_hunk: @@ -40,25 +40,35 @@ pub trait Codec: Sized {
 }
 
 /// Trait for types that have a fixed-length encoding
-pub trait SizedCodec<const N: usize>: Codec {
+pub trait SizedCodec: Codec {
+    /// The encoded length of this value.
+    const LEN_ENCODED: usize;
+
     /// Returns the encoded length of this value.
     ///
     /// Should not be overridden by implementations.
     fn len_encoded(&self) -> usize {
-        N
+        Self::LEN_ENCODED
     }
 
     /// Encodes a value to fixed-size bytes.
-    fn encode_fixed(&self) -> [u8; N] {
+    fn encode_fixed<const N: usize>(&self) -> [u8; N] {
+        // Ideally this is a compile-time check, but we can't do that in the current Rust version
+        // without adding a new generic parameter to the trait.
+        assert_eq!(
+            N,
+            Self::LEN_ENCODED,
+            "Can't encode {} bytes into {} bytes",
+            Self::LEN_ENCODED,
+            N
+        );
+
         self.encode().try_into().unwrap()
     }
 }
 
 /// Trait for codec read operations
 pub trait Reader {
-    /// Reads a value of type T
-    fn read<T: Codec>(&mut self) -> Result<T, Error>;
-
path: codec/src/codec.rs
user: login: BrendanChou
body: Allowed a client to easily accidentally write infinite recursion code that ends up with a stack overflow:
```
impl Codec for X {
    fn read(reader: &mut impl Reader) -> Result<Self, Error> {
        reader.read(self)
    }
}
```

instead, force use of `<type>::read(reader);`. If you tried `self.read(reader)` inside of the `self.read(...)` function, the linter would catch the recursion issue
created_at: 2025-03-15T00:34:46Z
author_association: COLLABORATOR
diff_hunk: @@ -125,9 +135,6 @@ pub trait Reader {
 
 /// Trait for codec write operations
 pub trait Writer {
-    /// Writes a value of type T
-    fn write<T: Codec>(&mut self, value: &T);
-
path: codec/src/codec.rs
user: login: BrendanChou
body: Allowed a client to easily accidentally write infinite recursion code that ends up with a stack overflow:
```
impl Codec for X {
    fn write(&self, writer: &mut impl Writer);
        writer.write(self);
    }
}
```

instead, force use of `Z.write(writer);`. If you tried `self.write(writer)` inside of the `self.write(...)` function, the linter would catch the recursion issue
created_at: 2025-03-15T00:36:22Z
author_association: COLLABORATOR

PR comments:
user: login: codecov[bot]
created_at: 2025-03-15T21:27:52Z
body: ## [Codecov](https://app.codecov.io/gh/commonwarexyz/monorepo/pull/603?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=commonwarexyz) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 88.94%. Comparing base [(`8040660`)](https://app.codecov.io/gh/commonwarexyz/monorepo/commit/804066098ca141f61d81085b024ae35ff4aebcb2?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=commonwarexyz) to head [(`6cde465`)](https://app.codecov.io/gh/commonwarexyz/monorepo/commit/6cde465c1eaed35f6be0171e833030ae4d5c1b1a?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=commonwarexyz).
> Report is 1 commits behind head on main.

```diff
@@            Coverage Diff             @@
##             main     #603      +/-   ##
==========================================
+ Coverage   88.56%   88.94%   +0.38%     
==========================================
  Files         137      137              
  Lines       35316    35287      -29     
==========================================
+ Hits        31277    31387     +110     
+ Misses       4039     3900     -139     
```

| [Files with missing lines](https://app.codecov.io/gh/commonwarexyz/monorepo/pull/603?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=commonwarexyz) | Coverage Δ | |
|---|---|---|
| [codec/src/buffer.rs](https://app.codecov.io/gh/commonwarexyz/monorepo/pull/603?src=pr&el=tree&filepath=codec%2Fsrc%2Fbuffer.rs&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=commonwarexyz#diff-Y29kZWMvc3JjL2J1ZmZlci5ycw==) | `84.48% <100.00%> (+30.50%)` | :arrow_up: |
| [codec/src/codec.rs](https://app.codecov.io/gh/commonwarexyz/monorepo/pull/603?src=pr&el=tree&filepath=codec%2Fsrc%2Fcodec.rs&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=commonwarexyz#diff-Y29kZWMvc3JjL2NvZGVjLnJz) | `97.55% <100.00%> (+21.60%)` | :arrow_up: |
| [codec/src/types.rs](https://app.codecov.io/gh/commonwarexyz/monorepo/pull/603?src=pr&el=tree&filepath=codec%2Fsrc%2Ftypes.rs&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=commonwarexyz#diff-Y29kZWMvc3JjL3R5cGVzLnJz) | `100.00% <100.00%> (+7.22%)` | :arrow_up: |
| [codec/src/varint.rs](https://app.codecov.io/gh/commonwarexyz/monorepo/pull/603?src=pr&el=tree&filepath=codec%2Fsrc%2Fvarint.rs&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=commonwarexyz#diff-Y29kZWMvc3JjL3ZhcmludC5ycw==) | `100.00% <100.00%> (+13.85%)` | :arrow_up: |

... and [1 file with indirect coverage changes](https://app.codecov.io/gh/commonwarexyz/monorepo/pull/603/indirect-changes?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=commonwarexyz)

------

[Continue to review full report in Codecov by Sentry](https://app.codecov.io/gh/commonwarexyz/monorepo/pull/603?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=commonwarexyz).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=commonwarexyz)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://app.codecov.io/gh/commonwarexyz/monorepo/pull/603?dropdown=coverage&src=pr&el=footer&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=commonwarexyz). Last update [8040660...6cde465](https://app.codecov.io/gh/commonwarexyz/monorepo/pull/603?dropdown=coverage&src=pr&el=lastupdated&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=commonwarexyz). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=commonwarexyz).

<details><summary>🚀 New features to boost your workflow: </summary>

- ❄ [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE