Repository: shuttle-hq/synth (language: rust)

Issue:
Feature: Allow fields to be omitted from output, hiding them
It is sometimes useful to generate a more complex structure and reference parts of it in the same namespace. For example, a database table might have day/month/year as 3 separate columns, and for the sake of consistency a single datetime has to be generated (to deal with cases like different months having different days etc.)

One approach to implement this is by declaring a field as `hidden` and omitting it from the output entirely. 

Hypothetical example:

```json
{
  "type": "object",
  "_date": {
    "type": "string",
    "hidden": true,
    "date_time": {
      "format": "%Y-%m-%d",
      "subtype": "naive_date",
      "begin": "1930-01-01",
      "end": "2010-01-01"
    }
  },
  "day": {
    "type": "number",
    "format": {
      "format": "{date@day}",
      "arguments": {
        "date": "@namespace.content.date"
      }
    }
  }
}
```


Discussion / hints:
Hey @vlushn thanks for the request! Clearly, this is something we should support.

Initially I thought it'd be more straightforward to have all these "hidden" fields be contained in a 'special' JSON schema file which would never get serialized but could be referenced to from other files. But I think your solution is much better because you can control at what level in a nested structure you want fields to be consistent.

Are you thinking `"hidden"` would only be a valid attribute for fields of `"type": "object"`? Or would `"hidden"` be a valid attribute on any JSON schema node type? I think the latter would probably be better.

An alternative, which is similar could be to have an attribute `"variables"` on nodes of `"type": "object"` in which we can store all the "variables" we want generated but hidden from serialization. For instance:

```json
{
  "type": "object",
  "variables": {
    "date": {
      "type": "string",
      "date_time": {
        "format": "%Y-%m-%d",
        "subtype": "naive_date",
        "begin": "1930-01-01",
        "end": "2010-01-01"
      }
    }
  },
  "day": {
    "type": "number",
    "format": {
      "format": "{date@day}",
      "arguments": {
        "date": "@namespace.variables.date"
      }
    }
  }
}
```

of course, in that case, this would only apply for nodes of type `object`.

Fix patch:
diff --git a/core/src/graph/mod.rs b/core/src/graph/mod.rs
index 19822709..d20e37cf 100644
--- a/core/src/graph/mod.rs
+++ b/core/src/graph/mod.rs
@@ -94,23 +94,6 @@ macro_rules! derive_from {
 			Self::$variant(value)
 		    }
 		}
-
-		impl TryInto<$ty> for $id {
-		    type Error = Error;
-		    fn try_into(self) -> Result<$ty, Self::Error> {
-			match self {
-			    Self::$variant(value) => Ok(value),
-			    otherwise => Err(
-				failed_crate!(
-				    target: Release,
-				    "invalid type: expected '{}', found '{}'",
-				    stringify!($variant),
-				    otherwise.type_()
-				)
-			    )
-			}
-		    }
-		}
 	    )?
 	)*
     };
@@ -174,6 +157,38 @@ derive_from! {
     }
 }
 
+impl TryFrom<Value> for String {
+    type Error = Error;
+
+    fn try_from(value: Value) -> Result<String, Self::Error> {
+        match value {
+            Value::String(str) => Ok(str),
+            Value::Number(num) => Ok(num.to_string()),
+            Value::DateTime(date) => Ok(date.format_to_string()),
+            otherwise => Err(failed_crate!(
+                target: Release,
+                "invalid type: expected 'String', found '{}'",
+                otherwise.type_()
+            )),
+        }
+    }
+}
+
+impl TryFrom<Value> for Number {
+    type Error = Error;
+
+    fn try_from(value: Value) -> Result<Number, Self::Error> {
+        match value {
+            Value::Number(num) => Ok(num),
+            otherwise => Err(failed_crate!(
+                target: Release,
+                "invalid type: expected 'Number', found '{}'",
+                otherwise.type_()
+            )),
+        }
+    }
+}
+
 impl Display for Value {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}", serde_json::to_string(&self).unwrap())
@@ -501,7 +516,8 @@ derive_generator!(
         OneOf(OneOfNode),
         Series(SeriesNode),
         Unique(UniqueNode),
-        Link(Box<LinkNode>)
+        Link(Box<LinkNode>),
+        Hidden(Box<Graph>),
     }
 );
 
diff --git a/core/src/graph/object.rs b/core/src/graph/object.rs
index ae4c23d2..4f34bba4 100644
--- a/core/src/graph/object.rs
+++ b/core/src/graph/object.rs
@@ -7,6 +7,8 @@ pub struct KeyValueOrNothing {
     inner: Concatenate<JustToken<String>, Graph>,
     p: f64,
     active: bool,
+    pub hidden: bool,
+    pub key: String,
 }
 
 impl Generator for KeyValueOrNothing {
@@ -33,20 +35,22 @@ impl Generator for KeyValueOrNothing {
 }
 
 impl KeyValueOrNothing {
-    pub fn new_with(key: &str, content: Graph, freq: f64) -> Self {
+    pub fn new_with(key: &str, content: Graph, freq: f64, is_hidden: bool) -> Self {
         Self {
             inner: content.with_key(key.to_string().yield_token()),
             p: freq,
             active: false,
+            hidden: is_hidden,
+            key: key.to_string(),
         }
     }
 
-    pub fn always(key: &str, content: Graph) -> Self {
-        Self::new_with(key, content, 1.0)
+    pub fn always(key: &str, content: Graph, is_hidden: bool) -> Self {
+        Self::new_with(key, content, 1.0, is_hidden)
     }
 
-    pub fn sometimes(key: &str, content: Graph) -> Self {
-        Self::new_with(key, content, 0.5)
+    pub fn sometimes(key: &str, content: Graph, is_hidden: bool) -> Self {
+        Self::new_with(key, content, 0.5, is_hidden)
     }
 }
 
@@ -64,8 +68,15 @@ impl Generator for ObjectNode {
     type Return = Result<Value, Error>;
 
     fn next<R: Rng>(&mut self, rng: &mut R) -> GeneratorState<Self::Yield, Self::Return> {
+        let hidden_fields: std::collections::HashSet<String> = self.0.inner.inner.inners.iter().filter(|&p| p.hidden)
+            .map(|p| p.key.clone()).collect();
+
         self.0.next(rng).map_complete(|kv| {
             kv.into_iter()
+                .filter(|p| match p {
+                    Some((k, _)) => !hidden_fields.contains(k),
+                    _ => true
+                })
                 .filter_map(|m_kv| m_kv.map(|(k, vr)| vr.map(|v| (k, v))))
                 .collect::<Result<BTreeMap<_, _>, Error>>()
                 .map(|hm| hm.into())
diff --git a/core/src/graph/string/format.rs b/core/src/graph/string/format.rs
index 48782a96..4312ae3d 100644
--- a/core/src/graph/string/format.rs
+++ b/core/src/graph/string/format.rs
@@ -86,7 +86,11 @@ impl dynfmt::FormatArgs for FormatArgs<String> {
 #[cfg(test)]
 pub mod tests {
     use super::*;
-    use crate::graph::{Graph, RandFaker, RandomString, StringNode};
+    use crate::graph::{
+        ChronoValue, Graph, NumberNode, RandFaker, RandomDateTime, RandomI64, RandomString,
+        StringNode,
+    };
+    use chrono::naive::NaiveDate;
 
     fn faker_graph(name: &str) -> Graph {
         Graph::String(StringNode::from(RandomString::from(
@@ -135,4 +139,56 @@ pub mod tests {
             .collect::<Result<Vec<_>, _>>()
             .unwrap();
     }
+
+    #[test]
+    fn format_with_number_args() {
+        let mut rng = rand::thread_rng();
+
+        let args = FormatArgs {
+            named: vec![(
+                "id".to_string(),
+                Graph::Number(NumberNode::from(RandomI64::constant(42))),
+            )]
+            .into_iter()
+            .collect(),
+            ..Default::default()
+        };
+        let formatted = Format::new("{id}_suffix".to_string(), args);
+        let gen = formatted
+            .repeat(1024)
+            .complete(&mut rng)
+            .into_iter()
+            .collect::<Result<Vec<_>, _>>()
+            .unwrap();
+
+        assert_eq!(gen[0], "42_suffix");
+    }
+
+    #[test]
+    fn format_with_date_args() {
+        let mut rng = rand::thread_rng();
+
+        let args = FormatArgs {
+            named: vec![(
+                "date".to_string(),
+                Graph::String(StringNode::from(RandomDateTime::new(
+                    ChronoValue::NaiveDate(NaiveDate::from_ymd(2021, 10, 4))
+                        ..ChronoValue::NaiveDate(NaiveDate::from_ymd(2021, 10, 4)),
+                    "%Y-%m-%d",
+                ))),
+            )]
+            .into_iter()
+            .collect(),
+            ..Default::default()
+        };
+        let formatted = Format::new("{date}.png".to_string(), args);
+        let gen = formatted
+            .repeat(1024)
+            .complete(&mut rng)
+            .into_iter()
+            .collect::<Result<Vec<_>, _>>()
+            .unwrap();
+
+        assert_eq!(gen[0], "2021-10-04.png");
+    }
 }
diff --git a/core/src/schema/content/hidden.rs b/core/src/schema/content/hidden.rs
new file mode 100644
index 00000000..5474ac78
--- /dev/null
+++ b/core/src/schema/content/hidden.rs
@@ -0,0 +1,20 @@
+#![allow(clippy::derivable_impls)]
+
+use crate::compile::Compile;
+use crate::{Compiler, Content, Graph};
+use anyhow::Result;
+use serde::{Deserialize, Serialize};
+
+
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
+#[serde(deny_unknown_fields)]
+pub struct HiddenContent {
+    pub content: Box<Content>,
+}
+
+impl Compile for HiddenContent {
+    fn compile<'a, C: Compiler<'a>>(&'a self, compiler: C) -> Result<Graph> {
+        let graph = self.content.compile(compiler)?;
+        Ok(Graph::Hidden(Box::new(graph)))
+    }
+}
diff --git a/core/src/schema/content/mod.rs b/core/src/schema/content/mod.rs
index dd9190f2..a5b47003 100644
--- a/core/src/schema/content/mod.rs
+++ b/core/src/schema/content/mod.rs
@@ -49,6 +49,9 @@ pub use series::SeriesContent;
 pub mod unique;
 pub use unique::{UniqueAlgorithm, UniqueContent};
 
+pub mod hidden;
+pub use hidden::{HiddenContent};
+
 use prelude::*;
 
 use super::{FieldRef, Namespace};
@@ -97,6 +100,8 @@ pub struct ContentLabels {
     optional: bool,
     #[serde(default)]
     unique: bool,
+    #[serde(default)]
+    hidden: bool,
 }
 
 impl ContentLabels {
@@ -114,6 +119,10 @@ impl ContentLabels {
             output = output.into_nullable();
         }
 
+        if self.hidden {
+            output = output.into_hidden();
+        }
+
         Ok(output)
     }
 }
@@ -235,6 +244,7 @@ content! {
         OneOf(OneOfContent),
         Series(SeriesContent),
         Unique(UniqueContent),
+        Hidden(HiddenContent),
     }
 }
 
@@ -266,6 +276,18 @@ impl Content {
         }
     }
 
+    pub fn into_hidden(self) -> Self {
+        if !self.is_hidden() {
+            Content::Hidden(HiddenContent{ content: Box::new(self) })
+        } else {
+            self
+        }
+    }
+
+    pub fn is_hidden(&self) -> bool {
+        matches!(self, Self::Hidden(_))
+    }
+
     pub fn is_unique(&self) -> bool {
         matches!(self, Self::Unique(_))
     }
@@ -297,6 +319,7 @@ impl Content {
     pub fn accepts(&self, value: &Value) -> Result<()> {
         match self {
             Self::Unique(unique_content) => unique_content.content.accepts(value),
+            Self::Hidden(_) => Ok(()),
             Self::SameAs(_) => Ok(()),
             Self::OneOf(one_of_content) => {
                 let res: Vec<_> = one_of_content
@@ -381,6 +404,7 @@ impl Content {
             Content::OneOf(_) => "one_of",
             Content::Series(_) => "series",
             Content::Unique(_) => "unique",
+            Content::Hidden(_) => "hidden",
         }
     }
 }
@@ -495,6 +519,7 @@ impl Compile for Content {
             Self::OneOf(one_of_content) => one_of_content.compile(compiler),
             Self::Series(series_content) => series_content.compile(compiler),
             Self::Unique(unique_content) => unique_content.compile(compiler),
+            Self::Hidden(hidden_content) => hidden_content.compile(compiler),
             Self::Null(_) => Ok(Graph::null()),
         }
     }
@@ -550,6 +575,11 @@ pub mod tests {
         pub static ref USER_SCHEMA: Content = schema!({
             "type": "object",
             "skip_when_null": true,
+            "_uuid": {
+                "type": "string",
+                "uuid": {},
+                "hidden": true
+            },
             "user_id": {
                 "type": "number",
                 "subtype": "u64",
diff --git a/core/src/schema/content/object.rs b/core/src/schema/content/object.rs
index 0e2b0de1..6b940920 100644
--- a/core/src/schema/content/object.rs
+++ b/core/src/schema/content/object.rs
@@ -78,6 +78,8 @@ impl ObjectContent {
                 if let Some(value) = obj.get(k) {
                     content.accepts(value)?;
                 }
+            } else if matches!(v, Content::Hidden(_)) {
+                // ok, hidden is not expected to appear
             } else {
                 let json_value = obj
                     .get(k)
@@ -160,11 +162,13 @@ impl Compile for ObjectContent {
                     let nullable = field.as_nullable().unwrap();
                     compiler
                         .build(name, nullable)
-                        .map(|graph| KeyValueOrNothing::sometimes(name, graph))
+                        .map(|graph| KeyValueOrNothing::sometimes(name, graph, false))
                 } else {
                     compiler
                         .build(name, field)
-                        .map(|graph| KeyValueOrNothing::always(name, graph))
+                        .map(|graph| {
+                            KeyValueOrNothing::always(name, graph, matches!(field, Content::Hidden(_)))
+                        })
                 }
             })
             .collect::<Result<ObjectNode>>()?;
diff --git a/core/src/schema/namespace.rs b/core/src/schema/namespace.rs
index 0efa9f77..d28ac5ae 100644
--- a/core/src/schema/namespace.rs
+++ b/core/src/schema/namespace.rs
@@ -171,7 +171,7 @@ impl Compile for Namespace {
             .map(|(name, content)| {
                 compiler
                     .build(name.as_ref(), content)
-                    .map(|graph| KeyValueOrNothing::always(name.as_ref(), graph))
+                    .map(|graph| KeyValueOrNothing::always(name.as_ref(), graph, false))
             })
             .collect::<Result<_>>()?;
         Ok(Graph::Object(object_node))
diff --git a/gen/src/error.rs b/gen/src/error.rs
index 7cc07259..f4394ed0 100644
--- a/gen/src/error.rs
+++ b/gen/src/error.rs
@@ -1,4 +1,4 @@
-use std::fmt::Display;
+use std::fmt::{Display, Debug};
 
 use crate::value::{IntoToken, Special, Token};
 
@@ -11,10 +11,10 @@ pub enum Error {
 }
 
 impl Error {
-    pub fn type_<T1: ToString, T2: ToString>(expected: T1, got: T2) -> Self {
+    pub fn type_<T1: ToString, T2: Debug>(expected: T1, got: T2) -> Self {
         Self::Type {
             expected: expected.to_string(),
-            got: got.to_string(),
+            got: format!("{:?}", got),
         }
     }
 
diff --git a/gen/src/generator/mod.rs b/gen/src/generator/mod.rs
index c02eeb85..db37b3c5 100644
--- a/gen/src/generator/mod.rs
+++ b/gen/src/generator/mod.rs
@@ -483,7 +483,7 @@ where
     EG: Generator,
 {
     begin: BG,
-    inner: G,
+    pub inner: G,
     end: EG,
     state: BraceState,
     complete: Option<G::Return>,
@@ -700,7 +700,7 @@ pub struct Chain<G>
 where
     G: Generator,
 {
-    inners: Vec<G>,
+    pub inners: Vec<G>,
     idx: usize,
     completed: Vec<G::Return>,
 }
diff --git a/gen/src/value.rs b/gen/src/value.rs
index f75e6c1d..d0970550 100644
--- a/gen/src/value.rs
+++ b/gen/src/value.rs
@@ -36,6 +36,16 @@ macro_rules! generate_enum {
 	}
 
 	impl std::fmt::Display for $id {
+	    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		match self {
+		    $(
+			Self::$variant(inner) => write!(f, "{:?}", inner),
+		    )*
+		}
+	    }
+	}
+
+	impl std::fmt::Debug for $id {
 	    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
 		match self {
 		    $(
@@ -144,7 +154,7 @@ macro_rules! generate_special_enum {
 			    Token::$id($id::$variant$(($($var,)*))?) => Ok(($($($var,)*)*)),
 			    otherwise => {
 				let err = format!(
-				    "unexpected: wanted {}, got {}",
+				    "unexpected: wanted {}, got {:?}",
 				    stringify!($variant),
 				    otherwise
 				);
@@ -216,7 +226,7 @@ macro_rules! data_model_variant_impl_ext {
 generate_enum!(
     /// Token variant for serde primitive numerical types.
     #[allow(missing_docs)]
-    #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
+    #[derive(Clone, Copy, Hash, PartialEq, Eq)]
     pub enum Number {
         I8(i8),
         I16(i16),
@@ -290,7 +300,7 @@ impl serde::Serialize for Number {
 generate_enum!(
     /// Token variant for all serde primitive (non-composite) types.
     #[allow(missing_docs)]
-    #[derive(Clone, Debug, PartialEq, Eq, Hash)]
+    #[derive(Clone, PartialEq, Eq, Hash)]
     pub enum Primitive {
         Bool(bool),
         String(String),
@@ -342,7 +352,7 @@ generate_special_enum!(
 
 generate_enum!(
     /// A custom tokenization for the serde data model.
-    #[derive(Clone, Debug, PartialEq, Eq, Hash)]
+    #[derive(Clone, PartialEq, Eq, Hash)]
     pub enum Token {
         /// A token encoding serde primitive types.
         Primitive(Primitive),
@@ -525,11 +535,11 @@ macro_rules! data_model_variant {
 	$id:ident<G>($($arg:ident: $ty:ty$(,)?)*) -> $inner:ty = $cl:expr
     } => {
 	#[doc = $name]
-	pub struct $id<G>
+    pub struct $id<G>
 	where
 	    G: Generator<Yield = Token>
 	{
-	    inner: $inner
+	    pub inner: $inner
 	}
 
 	impl<G> $id<G>


Test patch: