Repository: ParkMyCar/compact_str
Pull request #207 (2022-08-14 22:52:37)

Implement `from_utf16le`, `from_utf16be`, `from_utf16ne`

Kijewski: Thank you for your review! :) I made the requested changes. I also saw the comment in `from_utf16()` about <https://github.com/rust-lang/rust/issues/48994>, so I don't use `collect::<Result<_, _>>()`.

Yes, with `from_utf16ne()` we would give the user a footgun to create platform incompatible code. The input should contain a BOM, or the user should use some other means to tell the endianness.

Patch:
diff --git a/compact_str/src/lib.rs b/compact_str/src/lib.rs
index f1b62e6a..f564ff73 100644
--- a/compact_str/src/lib.rs
+++ b/compact_str/src/lib.rs
@@ -8,7 +8,6 @@ use core::borrow::{
     BorrowMut,
 };
 use core::cmp::Ordering;
-use core::fmt;
 use core::hash::{
     Hash,
     Hasher,
@@ -26,6 +25,10 @@ use core::str::{
     FromStr,
     Utf8Error,
 };
+use core::{
+    fmt,
+    slice,
+};
 use std::borrow::Cow;
 use std::ffi::OsStr;
 use std::iter::FusedIterator;
@@ -1170,12 +1173,90 @@ impl CompactString {
         let mut result = Self::with_capacity(v.len());
         let mut iter = v.iter();
         while let Some(s) = next_char(&mut iter, &mut buf) {
-            // SAFETY: next_char() only returns valid string
+            // SAFETY: next_char() only returns valid strings
             let s = unsafe { std::str::from_utf8_unchecked(s) };
             result.push_str(s);
         }
         result
     }
+
+    fn from_utf16x(
+        v: &[u8],
+        from_int: impl Fn(u16) -> u16,
+        from_bytes: impl Fn([u8; 2]) -> u16,
+    ) -> Result<Self, Utf16Error> {
+        if v.len() % 2 != 0 {
+            // Input had an odd number of bytes.
+            return Err(Utf16Error(()));
+        }
+
+        // Note: we don't use collect::<Result<_, _>>() because that fails to pre-allocate a buffer,
+        // even though the size of our iterator, `buf`, is known ahead of time.
+        //
+        // rustlang issue #48994 is tracking the fix
+        let mut result = CompactString::with_capacity(v.len());
+
+        // SAFETY: `u8` and `u16` are `Copy`, so if the alignment fits, we can transmute a
+        //         `[u8; 2*N]` to `[u16; N]`. `slice::align_to()` checks if the alignment is right.
+        match unsafe { v.align_to::<u16>() } {
+            (&[], v, _) => {
+                // Input is correcty aligned.
+                for c in std::char::decode_utf16(v.iter().copied().map(from_int)) {
+                    result.push(c.map_err(|_| Utf16Error(()))?);
+                }
+            }
+            _ => {
+                // Input's alignment is off.
+                // SAFETY: we can always reinterpret a `[u8; 2*N]` slice as `[[u8; 2]; N]`
+                let v = unsafe { slice::from_raw_parts(v.as_ptr().cast(), v.len() / 2) };
+                for c in std::char::decode_utf16(v.iter().copied().map(from_bytes)) {
+                    result.push(c.map_err(|_| Utf16Error(()))?);
+                }
+            }
+        }
+
+        Ok(result)
+    }
+
+    /// Decode a slice of bytes as UTF-16 encoded string, in little endian.
+    ///
+    /// # Errors
+    ///
+    /// If the slice has an odd number of bytes, or if it did not contain valid UTF-16 characters,
+    /// a [`Utf16Error`] is returned.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use compact_str::CompactString;
+    /// const DANCING_MEN: &[u8] = b"\x3d\xd8\x6f\xdc\x0d\x20\x42\x26\x0f\xfe";
+    /// let dancing_men = CompactString::from_utf16le(DANCING_MEN).unwrap();
+    /// assert_eq!(dancing_men, "👯‍♂️");
+    /// ```
+    #[inline]
+    pub fn from_utf16le(v: impl AsRef<[u8]>) -> Result<Self, Utf16Error> {
+        CompactString::from_utf16x(v.as_ref(), u16::from_le, u16::from_le_bytes)
+    }
+
+    /// Decode a slice of bytes as UTF-16 encoded string, in big endian.
+    ///
+    /// # Errors
+    ///
+    /// If the slice has an odd number of bytes, or if it did not contain valid UTF-16 characters,
+    /// a [`Utf16Error`] is returned.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use compact_str::CompactString;
+    /// const DANCING_WOMEN: &[u8] = b"\xd8\x3d\xdc\x6f\x20\x0d\x26\x40\xfe\x0f";
+    /// let dancing_women = CompactString::from_utf16be(DANCING_WOMEN).unwrap();
+    /// assert_eq!(dancing_women, "👯‍♀️");
+    /// ```
+    #[inline]
+    pub fn from_utf16be(v: impl AsRef<[u8]>) -> Result<Self, Utf16Error> {
+        CompactString::from_utf16x(v.as_ref(), u16::from_be, u16::from_be_bytes)
+    }
 }
 
 impl Default for CompactString {
diff --git a/fuzz/src/actions.rs b/fuzz/src/actions.rs
index e8dc7f6b..f07a1817 100644
--- a/fuzz/src/actions.rs
+++ b/fuzz/src/actions.rs
@@ -44,6 +44,8 @@ pub enum Action<'a> {
     Retain(u8, char),
     /// Interpret random bytes as UTF-8 characters
     FromUtf8Lossy(&'a [u8]),
+    /// Interpret a random `&[u16]` slice as UTF-16 characters
+    FromUtf16x(Vec<u16>),
 }
 
 impl Action<'_> {
@@ -278,6 +280,61 @@ impl Action<'_> {
                 assert_eq!(compact, control);
                 assert_eq!(compact.len(), control.len());
             }
+            FromUtf16x(buf) => {
+                #[allow(clippy::type_complexity)]
+                const FUNCS: &[(
+                    fn(&[u8]) -> Result<CompactString, compact_str::Utf16Error>,
+                    fn(u16) -> u16,
+                    fn([u8; 2]) -> u16,
+                )] = &[
+                    (
+                        |v| CompactString::from_utf16le(v),
+                        u16::from_le,
+                        u16::from_le_bytes,
+                    ),
+                    (
+                        |v| CompactString::from_utf16be(v),
+                        u16::from_be,
+                        u16::from_be_bytes,
+                    ),
+                ];
+
+                for (new_compact_string, from_int, from_bytes) in FUNCS {
+                    let buf = &*buf;
+                    let bytes: &[u8] =
+                        unsafe { core::slice::from_raw_parts(buf.as_ptr().cast(), buf.len() * 2) };
+
+                    let compact = new_compact_string(bytes);
+                    let control = String::from_utf16(
+                        &buf.iter().copied().map(from_int).collect::<Vec<u16>>(),
+                    );
+                    assert_eq!(compact.is_ok(), control.is_ok());
+
+                    if let (Ok(compact), Ok(control)) = (compact, control) {
+                        assert_eq!(compact.len(), control.len());
+                        assert_eq!(compact, control);
+                    }
+
+                    if bytes.len() >= 2 {
+                        // Test if `CompactString::from_utf16x()` works with misaligned slices.
+
+                        let bytes: &[u8] = &bytes[1..bytes.len() - 1];
+                        let buf: Vec<u16> = bytes
+                            .chunks_exact(2)
+                            .map(|v| from_bytes([v[0], v[1]]))
+                            .collect();
+
+                        let compact = new_compact_string(bytes);
+                        let control = String::from_utf16(&buf);
+                        assert_eq!(compact.is_ok(), control.is_ok());
+
+                        if let (Ok(compact), Ok(control)) = (compact, control) {
+                            assert_eq!(compact.len(), control.len());
+                            assert_eq!(compact, control);
+                        }
+                    }
+                }
+            }
         }
     }
 }


Test patch:
diff --git a/compact_str/src/tests.rs b/compact_str/src/tests.rs
index 988604d5..30de6b71 100644
--- a/compact_str/src/tests.rs
+++ b/compact_str/src/tests.rs
@@ -1,3 +1,4 @@
+use core::slice;
 use std::borrow::Cow;
 use std::num;
 use std::str::FromStr;
@@ -1287,3 +1288,80 @@ fn proptest_from_utf8_lossy(#[strategy(rand_bytes())] bytes: Vec<u8>) {
     assert_eq!(compact, control);
     assert_eq!(compact.len(), control.len());
 }
+
+#[proptest]
+#[cfg_attr(miri, ignore)]
+fn proptest_from_utf16(#[strategy(rand_u16s())] buf: Vec<u16>) {
+    const FUNCS: &[(
+        fn(&[u8]) -> Result<CompactString, crate::Utf16Error>,
+        fn(u16) -> u16,
+        fn([u8; 2]) -> u16,
+    )] = &[
+        (
+            |v| CompactString::from_utf16le(v),
+            u16::from_le,
+            u16::from_le_bytes,
+        ),
+        (
+            |v| CompactString::from_utf16be(v),
+            u16::from_be,
+            u16::from_be_bytes,
+        ),
+    ];
+
+    for (new_compact_string, from_int, from_bytes) in FUNCS {
+        let buf = &*buf;
+        let bytes: &[u8] = unsafe { slice::from_raw_parts(buf.as_ptr().cast(), buf.len() * 2) };
+
+        let compact = new_compact_string(bytes);
+        let control = String::from_utf16(&buf.iter().copied().map(from_int).collect::<Vec<u16>>());
+        assert_eq!(compact.is_ok(), control.is_ok());
+
+        if let (Ok(compact), Ok(control)) = (compact, control) {
+            assert_eq!(compact.len(), control.len());
+            assert_eq!(compact, control);
+        }
+
+        if bytes.len() >= 2 {
+            // Test if `CompactString::from_utf16x()` works with misaligned slices.
+
+            let bytes: &[u8] = &bytes[1..bytes.len() - 1];
+            let buf: Vec<u16> = bytes
+                .chunks_exact(2)
+                .map(|v| from_bytes([v[0], v[1]]))
+                .collect();
+
+            let compact = new_compact_string(bytes);
+            let control = String::from_utf16(&buf);
+            assert_eq!(compact.is_ok(), control.is_ok());
+
+            if let (Ok(compact), Ok(control)) = (compact, control) {
+                assert_eq!(compact.len(), control.len());
+                assert_eq!(compact, control);
+            }
+        }
+    }
+}
+
+#[test]
+fn test_from_utf16x() {
+    let dancing_men = b"\x3d\xd8\x6f\xdc\x0d\x20\x42\x26\x0f\xfe";
+    assert_eq!(CompactString::from_utf16le(dancing_men).unwrap(), "👯‍♂️");
+
+    let dancing_men = b"0\x3d\xd8\x6f\xdc\x0d\x20\x42\x26\x0f\xfe";
+    assert!(CompactString::from_utf16le(dancing_men).is_err());
+    assert_eq!(
+        CompactString::from_utf16le(&dancing_men[1..]).unwrap(),
+        "👯‍♂️",
+    );
+
+    let dancing_women = b"\xd8\x3d\xdc\x6f\x20\x0d\x26\x40\xfe\x0f";
+    assert_eq!(CompactString::from_utf16be(dancing_women).unwrap(), "👯‍♀️");
+
+    let dancing_women = b"0\xd8\x3d\xdc\x6f\x20\x0d\x26\x40\xfe\x0f";
+    assert!(CompactString::from_utf16be(dancing_women).is_err());
+    assert_eq!(
+        CompactString::from_utf16be(&dancing_women[1..]).unwrap(),
+        "👯‍♀️",
+    );
+}