Repository: pyfisch/cbor
Pull request #172 (2020-01-06 15:27:20)

Add support for CBOR tags
Using a thread local value as a way to get the tag value over the fence from the CBOR format to a CBOR aware serializer/deserializer.

Replaces https://github.com/pyfisch/cbor/pull/164

Here is a more complex real world use case from @vmx : https://github.com/vmx/rust-ipld-dag-cbor


Patch:
diff --git a/Cargo.toml b/Cargo.toml
index 40f725a..6659dda 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -30,3 +30,4 @@ default = ["std"]
 alloc = ["serde/alloc"]
 std = ["serde/std" ]
 unsealed_read_write = []
+tags = []
diff --git a/examples/tags.rs b/examples/tags.rs
new file mode 100644
index 0000000..9281b9b
--- /dev/null
+++ b/examples/tags.rs
@@ -0,0 +1,84 @@
+use serde::de::{Deserialize, Deserializer};
+use serde::ser::{Serialize, Serializer};
+use serde_cbor::tags::Tagged;
+use serde_cbor::Value;
+use serde_derive::{Deserialize, Serialize};
+use std::error::Error;
+
+/// https://tools.ietf.org/html/rfc7049#section-2.4.1
+#[derive(Debug, PartialEq)]
+struct Date(String);
+
+impl Serialize for Date {
+    fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
+        Tagged::new(Some(0), &self.0).serialize(s)
+    }
+}
+
+impl<'de> Deserialize<'de> for Date {
+    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+        let tagged = Tagged::<String>::deserialize(deserializer)?;
+        match tagged.tag {
+            Some(0) | None => Ok(Date(tagged.value)),
+            Some(_) => Err(serde::de::Error::custom("unexpected tag")),
+        }
+    }
+}
+
+/// https://tools.ietf.org/html/rfc7049#section-2.4.4.3
+#[derive(Debug, PartialEq)]
+struct Uri(String);
+
+impl Serialize for Uri {
+    fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
+        Tagged::new(Some(32), &self.0).serialize(s)
+    }
+}
+impl<'de> Deserialize<'de> for Uri {
+    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+        let tagged = Tagged::<String>::deserialize(deserializer)?;
+        match tagged.tag {
+            // allow deserialization even if there is no tag. Allows roundtrip via other formats such as json
+            Some(0) | None => Ok(Uri(tagged.value)),
+            Some(_) => Err(serde::de::Error::custom("unexpected tag")),
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+struct Bookmark {
+    title: String,
+    link: Uri,
+    created: Date,
+}
+
+fn main() -> Result<(), Box<dyn Error>> {
+    let bookmark = Bookmark {
+        title: "The Example Domain".into(),
+        link: Uri("http://example.org/".into()),
+        created: Date("2003-12-13T18:30:02Z".into()),
+    };
+
+    // serialize the struct to bytes
+    let bytes1 = serde_cbor::to_vec(&bookmark)?;
+    // deserialize to a serde_cbor::Value
+    let value1: Value = serde_cbor::from_slice(&bytes1)?;
+    println!("{:?}", value1);
+    // serialize the value to bytes
+    let bytes2 = serde_cbor::to_vec(&value1)?;
+    // deserialize to a serde_cbor::Value
+    let value2: Value = serde_cbor::from_slice(&bytes2)?;
+    println!("{:?}", value2);
+    // deserialize to a Bookmark
+    let result: Bookmark = serde_cbor::from_slice(&bytes2)?;
+
+    // check that the roundtrip was successful
+    assert_eq!(value1, value2);
+    assert_eq!(bookmark, result);
+
+    // check that going via a format that does not support tags does work
+    // let json = serde_json::to_vec(&bookmark)?;
+    // let result: Bookmark = serde_json::from_slice(&json)?;
+    // assert_eq!(bookmark, result);
+    Ok(())
+}
diff --git a/src/de.rs b/src/de.rs
index af0aeb7..f1fc925 100644
--- a/src/de.rs
+++ b/src/de.rs
@@ -20,6 +20,7 @@ use crate::read::Offset;
 #[cfg(any(feature = "std", feature = "alloc"))]
 pub use crate::read::SliceRead;
 pub use crate::read::{MutSliceRead, Read, SliceReadFixed};
+use crate::tags::set_tag;
 /// Decodes a value from CBOR data in a slice.
 ///
 /// # Examples
@@ -399,6 +400,18 @@ where
         }
     }
 
+    fn handle_tagged_value<V>(&mut self, tag: u64, visitor: V) -> Result<V::Value>
+    where
+        V: de::Visitor<'de>,
+    {
+        self.recursion_checked(|d| {
+            set_tag(Some(tag));
+            let r = visitor.visit_newtype_struct(d);
+            set_tag(None);
+            r
+        })
+    }
+
     fn recursion_checked<F, T>(&mut self, f: F) -> Result<T>
     where
         F: FnOnce(&mut Deserializer<R>) -> Result<T>,
@@ -702,22 +715,25 @@ where
             0xbf => self.parse_indefinite_map(visitor),
 
             // Major type 6: optional semantic tagging of other major types
-            0xc0..=0xd7 => self.recursion_checked(|de| de.parse_value(visitor)),
+            0xc0..=0xd7 => {
+                let tag = u64::from(byte) - 0xc0;
+                self.handle_tagged_value(tag, visitor)
+            }
             0xd8 => {
-                self.parse_u8()?;
-                self.recursion_checked(|de| de.parse_value(visitor))
+                let tag = self.parse_u8()?;
+                self.handle_tagged_value(tag.into(), visitor)
             }
             0xd9 => {
-                self.parse_u16()?;
-                self.recursion_checked(|de| de.parse_value(visitor))
+                let tag = self.parse_u16()?;
+                self.handle_tagged_value(tag.into(), visitor)
             }
             0xda => {
-                self.parse_u32()?;
-                self.recursion_checked(|de| de.parse_value(visitor))
+                let tag = self.parse_u32()?;
+                self.handle_tagged_value(tag.into(), visitor)
             }
             0xdb => {
-                self.parse_u64()?;
-                self.recursion_checked(|de| de.parse_value(visitor))
+                let tag = self.parse_u64()?;
+                self.handle_tagged_value(tag, visitor)
             }
             0xdc..=0xdf => Err(self.error(ErrorCode::UnassignedCode)),
 
diff --git a/src/lib.rs b/src/lib.rs
index 9e3add9..1736ce8 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -271,6 +271,7 @@ pub mod de;
 pub mod error;
 mod read;
 pub mod ser;
+pub mod tags;
 mod write;
 
 #[cfg(feature = "std")]
diff --git a/src/ser.rs b/src/ser.rs
index 9274032..7016dc3 100644
--- a/src/ser.rs
+++ b/src/ser.rs
@@ -13,6 +13,8 @@ use serde::ser::{self, Serialize};
 #[cfg(feature = "std")]
 use std::io;
 
+use crate::tags::{get_tag, CBOR_NEWTYPE_NAME};
+
 /// Serializes a value to a vector.
 #[cfg(any(feature = "std", feature = "alloc"))]
 pub fn to_vec<T>(value: &T) -> Result<Vec<u8>>
@@ -399,10 +401,15 @@ where
     }
 
     #[inline]
-    fn serialize_newtype_struct<T>(self, _name: &'static str, value: &T) -> Result<()>
+    fn serialize_newtype_struct<T>(self, name: &'static str, value: &T) -> Result<()>
     where
         T: ?Sized + ser::Serialize,
     {
+        if name == CBOR_NEWTYPE_NAME {
+            for tag in get_tag().into_iter() {
+                self.write_u64(6, tag)?;
+            }
+        }
         value.serialize(self)
     }
 
diff --git a/src/tags.rs b/src/tags.rs
new file mode 100644
index 0000000..50e039e
--- /dev/null
+++ b/src/tags.rs
@@ -0,0 +1,221 @@
+//! Support for cbor tags
+use core::fmt;
+use core::marker::PhantomData;
+use serde::de::{
+    Deserialize, Deserializer, EnumAccess, IntoDeserializer, MapAccess, SeqAccess, Visitor,
+};
+use serde::forward_to_deserialize_any;
+use serde::ser::{Serialize, Serializer};
+
+/// signals that a newtype is from a CBOR tag
+pub(crate) const CBOR_NEWTYPE_NAME: &str = "\0cbor_tag";
+
+/// A value that is optionally tagged with a cbor tag
+///
+/// this only serves as an intermediate helper for tag serialization or deserialization
+pub struct Tagged<T> {
+    /// cbor tag
+    pub tag: Option<u64>,
+    /// value
+    pub value: T,
+}
+
+impl<T> Tagged<T> {
+    /// Create a new tagged value
+    pub fn new(tag: Option<u64>, value: T) -> Self {
+        Self { tag, value }
+    }
+}
+
+impl<T: Serialize> Serialize for Tagged<T> {
+    fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
+        set_tag(self.tag);
+        let r = s.serialize_newtype_struct(CBOR_NEWTYPE_NAME, &self.value);
+        set_tag(None);
+        r
+    }
+}
+
+fn untagged<T>(value: T) -> Tagged<T> {
+    Tagged::new(None, value)
+}
+
+macro_rules! delegate {
+    ($name: ident, $type: ty) => {
+        fn $name<E: serde::de::Error>(self, v: $type) -> Result<Self::Value, E>
+        {
+            T::deserialize(v.into_deserializer()).map(untagged)
+        }
+    };
+}
+
+struct EnumDeserializer<A>(A);
+
+impl<'de, A> Deserializer<'de> for EnumDeserializer<A>
+where
+    A: EnumAccess<'de>,
+{
+    type Error = A::Error;
+
+    fn deserialize_any<V: Visitor<'de>>(self, visitor: V) -> Result<V::Value, Self::Error> {
+        visitor.visit_enum(self.0)
+    }
+
+    forward_to_deserialize_any! {
+        bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string
+        bytes byte_buf option unit unit_struct newtype_struct seq tuple
+        tuple_struct map struct enum identifier ignored_any
+    }
+}
+
+struct NoneDeserializer<E>(PhantomData<E>);
+
+impl<'de, E> Deserializer<'de> for NoneDeserializer<E>
+where
+    E: serde::de::Error,
+{
+    type Error = E;
+
+    fn deserialize_any<V: Visitor<'de>>(self, visitor: V) -> Result<V::Value, Self::Error> {
+        visitor.visit_none()
+    }
+
+    forward_to_deserialize_any! {
+        bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string
+        bytes byte_buf option unit unit_struct newtype_struct seq tuple
+        tuple_struct map struct enum identifier ignored_any
+    }
+}
+
+struct BytesDeserializer<'a, E>(&'a [u8], PhantomData<E>);
+
+impl<'de, 'a, E> Deserializer<'de> for BytesDeserializer<'a, E>
+where
+    E: serde::de::Error,
+{
+    type Error = E;
+
+    fn deserialize_any<V: Visitor<'de>>(self, visitor: V) -> Result<V::Value, Self::Error> {
+        visitor.visit_bytes(self.0)
+    }
+
+    forward_to_deserialize_any! {
+        bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string
+        bytes byte_buf option unit unit_struct newtype_struct seq tuple
+        tuple_struct map struct enum identifier ignored_any
+    }
+}
+
+/// A visitor that intercepts *just* visit_newtype_struct and passes through everything else.
+struct MaybeTaggedVisitor<T>(PhantomData<T>);
+
+impl<'de, T: Deserialize<'de>> Visitor<'de> for MaybeTaggedVisitor<T> {
+    type Value = Tagged<T>;
+
+    fn expecting(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt.write_str("a cbor tag newtype")
+    }
+
+    delegate!(visit_bool, bool);
+
+    delegate!(visit_i8, i8);
+    delegate!(visit_i16, i16);
+    delegate!(visit_i32, i32);
+    delegate!(visit_i64, i64);
+
+    delegate!(visit_u8, u8);
+    delegate!(visit_u16, u16);
+    delegate!(visit_u32, u32);
+    delegate!(visit_u64, u64);
+
+    delegate!(visit_f32, f32);
+    delegate!(visit_f64, f64);
+
+    delegate!(visit_char, char);
+    delegate!(visit_str, &str);
+    delegate!(visit_borrowed_str, &'de str);
+
+    #[cfg(feature = "std")]
+    delegate!(visit_byte_buf, Vec<u8>);
+
+    #[cfg(feature = "std")]
+    delegate!(visit_string, String);
+
+    fn visit_bytes<E: serde::de::Error>(self, value: &[u8]) -> Result<Self::Value, E> {
+        T::deserialize(BytesDeserializer(value, PhantomData)).map(untagged)
+    }
+
+    fn visit_borrowed_bytes<E: serde::de::Error>(self, value: &'de [u8]) -> Result<Self::Value, E> {
+        T::deserialize(serde::de::value::BorrowedBytesDeserializer::new(value)).map(untagged)
+    }
+
+    fn visit_unit<E: serde::de::Error>(self) -> Result<Self::Value, E> {
+        T::deserialize(().into_deserializer()).map(untagged)
+    }
+
+    fn visit_none<E: serde::de::Error>(self) -> Result<Self::Value, E> {
+        T::deserialize(NoneDeserializer(PhantomData)).map(untagged)
+    }
+
+    fn visit_some<D: Deserializer<'de>>(self, deserializer: D) -> Result<Self::Value, D::Error> {
+        T::deserialize(deserializer).map(untagged)
+    }
+
+    fn visit_seq<A: SeqAccess<'de>>(self, seq: A) -> Result<Self::Value, A::Error> {
+        T::deserialize(serde::de::value::SeqAccessDeserializer::new(seq)).map(untagged)
+    }
+
+    fn visit_map<V: MapAccess<'de>>(self, map: V) -> Result<Self::Value, V::Error> {
+        T::deserialize(serde::de::value::MapAccessDeserializer::new(map)).map(untagged)
+    }
+
+    fn visit_enum<A: EnumAccess<'de>>(self, data: A) -> Result<Self::Value, A::Error> {
+        T::deserialize(EnumDeserializer(data)).map(untagged)
+    }
+
+    fn visit_newtype_struct<D: serde::Deserializer<'de>>(
+        self,
+        deserializer: D,
+    ) -> Result<Self::Value, D::Error> {
+        let t = get_tag();
+        T::deserialize(deserializer).map(|v| Tagged::new(t, v))
+    }
+}
+
+impl<'de, T: serde::de::Deserialize<'de>> serde::de::Deserialize<'de> for Tagged<T> {
+    fn deserialize<D: serde::de::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+        deserializer.deserialize_any(MaybeTaggedVisitor::<T>(PhantomData))
+    }
+}
+
+/// function to get the current cbor tag
+///
+/// The only place where it makes sense to call this function is within visit_newtype_struct of a serde visitor.
+/// This is a low level API. In most cases it is preferable to use Tagged
+pub fn current_cbor_tag() -> Option<u64> {
+    get_tag()
+}
+
+#[cfg(feature = "tags")]
+pub(crate) fn set_tag(value: Option<u64>) {
+    CBOR_TAG.with(|f| *f.borrow_mut() = value);
+}
+
+#[cfg(feature = "tags")]
+pub(crate) fn get_tag() -> Option<u64> {
+    CBOR_TAG.with(|f| *f.borrow())
+}
+
+#[cfg(not(feature = "tags"))]
+pub(crate) fn set_tag(_value: Option<u64>) {}
+
+#[cfg(not(feature = "tags"))]
+pub(crate) fn get_tag() -> Option<u64> {
+    None
+}
+
+#[cfg(feature = "tags")]
+use std::cell::RefCell;
+
+#[cfg(feature = "tags")]
+thread_local!(static CBOR_TAG: RefCell<Option<u64>> = RefCell::new(None));
diff --git a/src/value/de.rs b/src/value/de.rs
index 2905f3b..f5bdbb7 100644
--- a/src/value/de.rs
+++ b/src/value/de.rs
@@ -1,9 +1,8 @@
 use std::collections::BTreeMap;
 use std::fmt;
 
-use serde::de;
-
 use crate::value::Value;
+use serde::de;
 
 impl<'de> de::Deserialize<'de> for Value {
     #[inline]
@@ -134,6 +133,18 @@ impl<'de> de::Deserialize<'de> for Value {
             {
                 Ok(Value::Float(v))
             }
+
+            fn visit_newtype_struct<D>(self, deserializer: D) -> Result<Self::Value, D::Error>
+            where
+                D: serde::Deserializer<'de>,
+            {
+                let tag = crate::tags::get_tag();
+                let inner = deserializer.deserialize_any(self);
+                match tag {
+                    Some(tag) => inner.map(|v| Value::Tag(tag, Box::new(v))),
+                    None => inner,
+                }
+            }
         }
 
         deserializer.deserialize_any(ValueVisitor)
diff --git a/src/value/mod.rs b/src/value/mod.rs
index 3e83c46..7bd2255 100644
--- a/src/value/mod.rs
+++ b/src/value/mod.rs
@@ -51,6 +51,8 @@ pub enum Value {
     /// to establish canonical order may be slow and therefore insertion
     /// and retrieval of values will be slow too.
     Map(BTreeMap<Value, Value>),
+    /// Represents a tagged value
+    Tag(u64, Box<Value>),
     // The hidden variant allows the enum to be extended
     // with variants for tags and simple values.
     #[doc(hidden)]
@@ -142,6 +144,7 @@ impl Value {
                     1
                 }
             }
+            Tag(_, _) => 6,
             Float(_) => 7,
             Bytes(_) => 2,
             Text(_) => 3,
diff --git a/src/value/ser.rs b/src/value/ser.rs
index e51dea8..347aae9 100644
--- a/src/value/ser.rs
+++ b/src/value/ser.rs
@@ -11,6 +11,7 @@ use std::collections::BTreeMap;
 use crate::error::Error;
 use serde::{self, Serialize};
 
+use crate::tags::Tagged;
 use crate::value::Value;
 
 impl serde::Serialize for Value {
@@ -25,6 +26,7 @@ impl serde::Serialize for Value {
             Value::Text(ref v) => serializer.serialize_str(&v),
             Value::Array(ref v) => v.serialize(serializer),
             Value::Map(ref v) => v.serialize(serializer),
+            Value::Tag(tag, ref v) => Tagged::new(Some(tag), v).serialize(serializer),
             Value::Float(v) => serializer.serialize_f64(v),
             Value::Bool(v) => serializer.serialize_bool(v),
             Value::Null => serializer.serialize_unit(),


Test patch:
diff --git a/tests/de.rs b/tests/de.rs
index 49f6e8b..01d7914 100644
--- a/tests/de.rs
+++ b/tests/de.rs
@@ -224,7 +224,15 @@ mod std_tests {
     fn test_self_describing() {
         let value: error::Result<Value> =
             de::from_slice(&[0xd9, 0xd9, 0xf7, 0x66, 0x66, 0x6f, 0x6f, 0x62, 0x61, 0x72]);
-        assert_eq!(value.unwrap(), Value::Text("foobar".to_owned()));
+        let expected = Value::Text("foobar".to_owned());
+        let strip_tags = |x: Value| {
+            if let Value::Tag(_, inner) = x {
+                *inner
+            } else {
+                x
+            }
+        };
+        assert_eq!(strip_tags(value.unwrap()), expected);
     }
 
     #[test]
diff --git a/tests/tags.rs b/tests/tags.rs
new file mode 100644
index 0000000..92534c0
--- /dev/null
+++ b/tests/tags.rs
@@ -0,0 +1,48 @@
+#[cfg(feature = "tags")]
+mod tagtests {
+    use serde_cbor::value::Value;
+    use serde_cbor::{from_slice, to_vec};
+
+    fn decode_hex(s: &str) -> std::result::Result<Vec<u8>, std::num::ParseIntError> {
+        (0..s.len())
+            .step_by(2)
+            .map(|i| u8::from_str_radix(&s[i..i + 2], 16))
+            .collect()
+    }
+
+    // get bytes from http://cbor.me/ trees
+    fn parse_cbor_me(example: &str) -> std::result::Result<Vec<u8>, std::num::ParseIntError> {
+        let hex = example
+            .split("\n")
+            .flat_map(|line| line.split("#").take(1))
+            .collect::<Vec<&str>>()
+            .join("")
+            .replace(" ", "");
+        decode_hex(&hex)
+    }
+
+    #[test]
+    fn tagged_cbor_roundtrip() {
+        let data = r#"
+C1                   # tag(1)
+   82                # array(2)
+      C2             # tag(2)
+         63          # text(3)
+            666F6F   # "foo"
+      C3             # tag(3)
+         A1          # map(1)
+            C4       # tag(4)
+               61    # text(1)
+                  61 # "a"
+            C5       # tag(5)
+               61    # text(1)
+                  62 # "b"
+            "#;
+        let bytes1 = parse_cbor_me(&data).unwrap();
+        let value1: Value = from_slice(&bytes1).unwrap();
+        let bytes2 = to_vec(&value1).unwrap();
+        let value2: Value = from_slice(&bytes2).unwrap();
+        assert_eq!(bytes1, bytes2);
+        assert_eq!(value1, value2);
+    }
+}