Repository: shuttle-hq/synth (language: rust)

Issue:
Using a `u32` constant as an array length causes an error while a `u32` range does not
**Describe the bug**
I've noticed an issue with `"length"` types of `"array"` generators - `u32` constants cause an error while 'range' does not. I think we should either disallow all 32-bit types to be used as a `"length"`, or we should start allowing 32-bit constants in addition to ranges.

**To Reproduce**
Steps to reproduce the behavior:
1. Schema:
```
{
  "type": "array",
  "length": {
    "type": "number",
    "subtype": "u32",
    "constant": 1
  },
  "content": {
      "type": "string",
      "pattern": "hello world"
  }
}
```
2. See error:
```
Error: Unable to open the namespace "temp"

Caused by:
    0: at file temp/a.json
    1: Failed to parse collection
    2: cannot use number::U32::Constant as an array length at line 11 column 1
```
3. And yet, all works expected (no error, `["hello world"]` is generated) when the schema is modified such that:
```
...
  "length": {
    "type": "number",
    "subtype": "u32",
    "range": {
        "low": 1,
        "high": 2,
        "step": 1
    }
  },
...
```
4. Interestingly, using the `"range"` workaround as above still gives an error with `"subtype": "i32"`:
```
Error: At namespace "temp"

Caused by:
    BadRequest: could not convert from value '1': Type { expected: "U32", got: "i32(1)" }
```

**Expected behavior**
Either allow all variations of 32-bit integer types as array lengths (constants, ranges, etc.) or none at all (i.e. limit to just 64-bit integers).


Discussion / hints:

Fix patch:
diff --git a/Cargo.lock b/Cargo.lock
index 83897983..5950e84f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1317,6 +1317,19 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "globset"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10463d9ff00a2a068db14231982f5132edebad0d7660cd956a1c30292dbcbfbd"
+dependencies = [
+ "aho-corasick",
+ "bstr",
+ "fnv",
+ "log",
+ "regex",
+]
+
 [[package]]
 name = "gloo-timers"
 version = "0.2.1"
@@ -1586,6 +1599,24 @@ dependencies = [
  "unicode-normalization",
 ]
 
+[[package]]
+name = "ignore"
+version = "0.4.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "713f1b139373f96a2e0ce3ac931cd01ee973c3c5dd7c40c0c2efe96ad2b6751d"
+dependencies = [
+ "crossbeam-utils",
+ "globset",
+ "lazy_static",
+ "log",
+ "memchr",
+ "regex",
+ "same-file",
+ "thread_local",
+ "walkdir",
+ "winapi-util",
+]
+
 [[package]]
 name = "indexmap"
 version = "1.7.0"
@@ -2693,6 +2724,15 @@ version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
 
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
 [[package]]
 name = "schannel"
 version = "0.1.19"
@@ -3260,6 +3300,7 @@ dependencies = [
  "synth-gen",
  "sysinfo",
  "tempfile",
+ "test_macros",
  "uriparse",
  "uuid",
 ]
@@ -3380,6 +3421,17 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "test_macros"
+version = "0.1.0"
+dependencies = [
+ "ignore",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "tokenstream2-tmpl",
+]
+
 [[package]]
 name = "textwrap"
 version = "0.11.0"
@@ -3409,6 +3461,15 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "thread_local"
+version = "1.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180"
+dependencies = [
+ "once_cell",
+]
+
 [[package]]
 name = "tide"
 version = "0.16.0"
@@ -3496,6 +3557,17 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
 
+[[package]]
+name = "tokenstream2-tmpl"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0cf48a946556c35fd2c2332204b5502d50376c1f11e36c707d3cd3a2aad40a6e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "tokio"
 version = "1.11.0"
@@ -3776,6 +3848,17 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca"
 
+[[package]]
+name = "walkdir"
+version = "2.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56"
+dependencies = [
+ "same-file",
+ "winapi",
+ "winapi-util",
+]
+
 [[package]]
 name = "want"
 version = "0.3.0"
diff --git a/Cargo.toml b/Cargo.toml
index 60ca881e..0480748b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -3,7 +3,8 @@ members = [
   "gen",
   "core",
   "synth",
-  "dist/playground"
+  "dist/playground",
+  "test_macros"
 ]
 
 # blocked by diesel error for now.
diff --git a/README.md b/README.md
index 65fb345a..e124ed46 100644
--- a/README.md
+++ b/README.md
@@ -82,7 +82,7 @@ $ mkdir my_app
 
 Next let's create a `users` collection using Synth's configuration language, and put it into `my_app/users.json`:
 
-```json
+```json synth
 {
     "type": "array",
     "length": {
diff --git a/core/src/graph/string/faker.rs b/core/src/graph/string/faker.rs
index b2b0d81f..a114aba4 100644
--- a/core/src/graph/string/faker.rs
+++ b/core/src/graph/string/faker.rs
@@ -32,7 +32,7 @@ impl Default for Locale {
 #[derive(Clone, Default, Deserialize, Debug, Serialize, PartialEq, Eq, Hash)]
 pub struct FakerArgs {
     #[serde(default)]
-    locales: Vec<Locale>,
+    pub(crate) locales: Vec<Locale>,
 }
 
 type FakerFunction = for<'r> fn(&'r mut dyn RngCore, &FakerArgs) -> String;
diff --git a/core/src/schema/content/array.rs b/core/src/schema/content/array.rs
index 5f78f7cc..e43b9a07 100644
--- a/core/src/schema/content/array.rs
+++ b/core/src/schema/content/array.rs
@@ -70,6 +70,8 @@ impl<'de> Deserialize<'de> for ArrayContent {
                 let content = content.ok_or_else(|| de::Error::missing_field("content"))?;
 
                 match length {
+                    // Default for positive constants
+                    Content::Number(NumberContent::U32(number_content::U32::Constant(_))) => { }
                     Content::Number(NumberContent::U64(number_content::U64::Range(r))) => {
                         if r.high.is_none() {
                             return Err(A::Error::custom(
@@ -218,6 +220,27 @@ mod tests {
                 "type": "object"
             }
         },
+        default_constant: {
+            "type": "array",
+            "length": {
+                "type": "number",
+                "constant": 8
+            },
+            "content": {
+                "type": "object"
+            }
+        },
+        u32_constant: {
+            "type": "array",
+            "length": {
+                "type": "number",
+                "subtype": "u32",
+                "constant": 8
+            },
+            "content": {
+                "type": "object"
+            }
+        },
         u64_constant: {
             "type": "array",
             "length": {
@@ -317,17 +340,6 @@ mod tests {
                 "type": "object"
             }
         },
-        u32_constant: {
-            "type": "array",
-            "length": {
-                "type": "number",
-                "subtype": "u32",
-                "constant": 8
-            },
-            "content": {
-                "type": "object"
-            }
-        },
         i64_range: {
             "type": "array",
             "length": {
diff --git a/core/src/schema/content/string.rs b/core/src/schema/content/string.rs
index 7fc05d6f..862887d8 100644
--- a/core/src/schema/content/string.rs
+++ b/core/src/schema/content/string.rs
@@ -2,6 +2,7 @@ use std::hash::{Hash, Hasher};
 
 use super::prelude::*;
 use super::Categorical;
+use crate::graph::string::FakerArgs;
 use crate::graph::string::{Constant, Serialized, Sliced};
 
 #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Hash)]
@@ -207,7 +208,7 @@ impl<'de> Deserialize<'de> for FakerContent {
                 A: serde::de::MapAccess<'de>,
             {
                 let mut generator = None;
-                let mut args = None;
+                let mut locales = None;
                 while let Some(key) = map.next_key()? {
                     match key {
                         Field::Generator => {
@@ -217,19 +218,19 @@ impl<'de> Deserialize<'de> for FakerContent {
                             generator = Some(map.next_value()?);
                         }
                         Field::Locales => {
-                            if args.is_some() {
+                            if locales.is_some() {
                                 return Err(A::Error::duplicate_field("locales"));
                             }
-                            args = Some(map.next_value()?);
+                            locales = Some(map.next_value()?);
                         }
                         Field::Unknown => {}
                     }
                 }
                 let generator = generator.ok_or_else(|| A::Error::missing_field("generator"))?;
-                let args = args.unwrap_or_default();
+                let locales = locales.unwrap_or_default();
                 Ok(FakerContent {
                     generator,
-                    args,
+                    args: FakerArgs { locales },
                     locales: Vec::new(),
                 })
             }
diff --git a/docs/blog/2021-03-08-mern.md b/docs/blog/2021-03-08-mern.md
index 19e5e44f..204d0876 100644
--- a/docs/blog/2021-03-08-mern.md
+++ b/docs/blog/2021-03-08-mern.md
@@ -268,7 +268,7 @@ First let's change the `rating` field. Our app can only accept numbers between 0
 
 The `time` field has been correctly detected as an array of values. First of all, let's say a movie can be shown up to 5 times a day, so we'll change the `high` field at `time.length.range` to 6 (`high` is exclusive). At this stage, the values are just random strings, so let's instead use the `String::DateTime` content type to generate hours of the day.
 
-```json synth
+```json synth[expect = "unknown variant `date_time`"]
 {
     "type": "array",
     "length": {
@@ -326,7 +326,7 @@ So let's use the `String::Faker` content type to generate some fake movie names!
 
 So, making all the changes above, we can use our beautiful finished schema to generate data for our app:
 
-```json synth
+```json synth[expect = "unknown variant `date_time`"]
 {
     "type": "array",
     "length": {
@@ -388,7 +388,7 @@ So, making all the changes above, we can use our beautiful finished schema to ge
 }
 ```
 
-```json
+```bash
 $ synth generate cinema/ --size 5
 {
   "movies": [
diff --git a/docs/blog/2021-03-09-postgres-data-gen.md b/docs/blog/2021-03-09-postgres-data-gen.md
index c0bb4f65..427daee7 100644
--- a/docs/blog/2021-03-09-postgres-data-gen.md
+++ b/docs/blog/2021-03-09-postgres-data-gen.md
@@ -138,7 +138,7 @@ Now comes the fun part! Using Synth's configuration language we can specify how
 
 To tell Synth that `companies` is a table (or collection in the Synth lingo) we'll create a new file `app/companies.json`.
 
-```json
+```json synth[companies.json]
 {
     "type": "array",
     "length": {
@@ -184,7 +184,7 @@ $ synth generate my_app/ --size 2
 Now we can do the same thing for the `contacts` table by create a file `my_app/contacts.json`. Here we have the added complexity of a foreign key constraints to the company table, but we can solve it easily using Synth's [`same_as`](/docs/content/same-as) generator.
 
 
-```json
+```json synth[contacts.json]
 {
     "type": "array",
     "length": {
@@ -207,7 +207,7 @@ Now we can do the same thing for the `contacts` table by create a file `my_app/c
             "type": "string",
             "faker": {
                 "generator": "phone_number",
-                "locales": ["en_GB"]
+                "locales": ["FR_FR"]
             }
         },
         "email": {
@@ -218,7 +218,7 @@ Now we can do the same thing for the `contacts` table by create a file `my_app/c
         }
     }
 }
-``` 
+```
 
 There is quite a bit going on here - to get an in-depth understanding of the synth configuration refer I'd recommend reading the comprehensive docs. There are tons of cool features which this schema can't really explore!
 
@@ -253,4 +253,4 @@ We explored 3 different ways to generate data.
 - [**Synth**](https://github.com/getsynth/synth): Synth has a small learning curve, but to create realistic testing data at scale it reduces most of the manual labour.
 
 
-In the next post we'll explore how to subset your existing database for testing purposes. And don't worry if you have sensitive / personal data - we'll cover that too. 
\ No newline at end of file
+In the next post we'll explore how to subset your existing database for testing purposes. And don't worry if you have sensitive / personal data - we'll cover that too.
diff --git a/docs/blog/2021-08-31-seeding-databases-tutorial.md b/docs/blog/2021-08-31-seeding-databases-tutorial.md
index 7287abe0..8bf665f9 100644
--- a/docs/blog/2021-08-31-seeding-databases-tutorial.md
+++ b/docs/blog/2021-08-31-seeding-databases-tutorial.md
@@ -572,7 +572,7 @@ OK, so we now know how to generate the `id` and the `email` properties of
 our [`User` model](#prisma-is-awesome). But we do not yet know how to put them
 together in one object. For that we need the [`object`][synth-object] type:
 
-```json synth
+```json synth[User.json]
 {
     "type": "object",
     "id": {
@@ -597,7 +597,7 @@ the `createdAt` and `nickname` fields.
 
 Here is the finished result for our `User.json` collection:
 
-```json synth
+```json synth[expect = "unknown variant `date_time`"]
 {
     "type": "array",
     "length": 3,
@@ -695,7 +695,7 @@ the `Post.json` collection.
 
 Here is the end result:
 
-```json
+```json synth[expect = "unknown variant `date_time`"]
 {
   "type": "array",
   "length": 5,
@@ -732,7 +732,7 @@ Here is the end result:
 It all looks pretty similar to the `User.json` collection, except for one
 important difference at the line
 
-```json
+```json synth
     "authorId": "@User.content.id"
 ```
 
diff --git a/docs/blog/2021-09-27-crash.md b/docs/blog/2021-09-27-crash.md
index 4c4cf2c6..2e5a4b6c 100644
--- a/docs/blog/2021-09-27-crash.md
+++ b/docs/blog/2021-09-27-crash.md
@@ -42,7 +42,7 @@ I then called out `synth generate` until finding a problem. First, I attempted t
 
 \#1 The first thing I tried is using a negative length:
 
-```json synth
+```json synth[expect = "cannot have a negative array length"]
 {
     "type": "array",
     "length": -1,
@@ -71,7 +71,7 @@ The line number is wrong here, the length should be in line six in the `content`
 
 \#3 It's not that long that we can use literal numbers for number constants here (for example given the `length`). The old way would use a [`number`](/docs/content/number) generator. A recent improvement let us generate arbitrary numbers, however this is likely not a good idea for a `length` field:
 
-```json synth
+```json synth[expect = "missing high value for array length range"]
 {
     "type": "array",
     "length": {
@@ -88,7 +88,7 @@ This might be done very quickly, but far more likely it will work for a long tim
 
 \#4 Let's now add a `string`:
 
-```json synth
+```json synth[expect = "`string` generator is missing a subtype"]
 {
     "type": "array",
     "length": {
@@ -117,7 +117,7 @@ Caused by:
 
 \#5 Ok, let's make that a [`format`](/docs/content/string#format) then. However, I forgot that the `format`must contains a map with the keys "format" and "arguments", putting them into the `s` map directly:
 
-```json synth
+```json synth[expect = "`arguments` is expected to be a field of `format`"]
 {
     "type": "array",
     "length": {
@@ -148,7 +148,7 @@ Caused by:
 
 \#6 Ok, then let's try to use a faker. Unfortunately, I haven't really read the docs, so I'll just try the first thing that comes to mind:
 
-```json synth
+```json synth[expect = "`faker` is expected to have a `generator` field. Try '"faker": {"generator": "name"}'"]
 {
     "type": "array",
     "length": {
@@ -180,7 +180,7 @@ One could say that the error is not exactly misleading, but not exactly helpful
 
 \#7 Trying to mess up with `date_time`, I mistakenly specify a date format for a `naive_time` value. 
 
-```json synth
+```json synth[expect = "unknown variant `date_time`, expected one of `pattern`, `faker`, `categorical`, `serialized`, `uuid`, `truncated`, `sliced`, `format`, `constant`"]
 {
    "type": "array",
    "length": 1,
@@ -228,7 +228,7 @@ For the record, the error is on line 11.
 
 \#9 Now we let `date_time` have some rest and go on to [`categorical`](/docs/content/string#categorical). Having just one variant with a weight of `0` will actually trigger an `unreachable` error:
 
-```json
+```json synth
 {
     "type": "array",
     "length": 1,
@@ -325,4 +325,4 @@ Caused by:
 
 So I give up. I've found 1 way to crash our product with an unintended error, reproduced some known limitations and outlined a number of error messages we can improve on. I fell far short of my original goal, which either means I'm really bad at finding errors, or our code is incredibly reliable. Given the track record of software written in Rust, I'd like to think it's the latter, but I'll leave judgement to you.
 
-Anyway, this was a fun exercise and I looked at many more things that turned out to just work well, so that's a good thing™. With all the test automation we have today, it's easy to forget that the manual approach also has its upsides. So feel free and try to break your (or our) code!
\ No newline at end of file
+Anyway, this was a fun exercise and I looked at many more things that turned out to just work well, so that's a good thing™. With all the test automation we have today, it's easy to forget that the manual approach also has its upsides. So feel free and try to break your (or our) code!
diff --git a/docs/docs/content/array.md b/docs/docs/content/array.md
index 9387d2d1..69e1c9f0 100644
--- a/docs/docs/content/array.md
+++ b/docs/docs/content/array.md
@@ -27,4 +27,4 @@ The example below generates arrays of credit card numbers with `3` to `10` eleme
     }
   }
 }
-```
\ No newline at end of file
+```
diff --git a/docs/docs/content/datasource.md b/docs/docs/content/datasource.md
index 7478ccac..7be3f176 100644
--- a/docs/docs/content/datasource.md
+++ b/docs/docs/content/datasource.md
@@ -15,7 +15,7 @@ exhausted when set to `true`.
 When pulling from a JSON file, the JSON is expected to be an array with every item being the value for a single Synth
 generator. The following is a valid JSON datasource:
 
-```json
+```json[addresses.json]
 [
   "21 Mary Street",
   "5 Diascia Avenue",
diff --git a/docs/docs/content/null.md b/docs/docs/content/null.md
index 9fa46347..741356e2 100644
--- a/docs/docs/content/null.md
+++ b/docs/docs/content/null.md
@@ -6,4 +6,4 @@ The `null` generator type simply evaluates to the JSON [`null`](https://www.json
 {
    "type": "null"
 }
-```
\ No newline at end of file
+```
diff --git a/docs/docs/content/same-as.md b/docs/docs/content/same-as.md
index 2c230706..57d833f8 100644
--- a/docs/docs/content/same-as.md
+++ b/docs/docs/content/same-as.md
@@ -6,9 +6,12 @@ Synth's `same_as` generator type establishes a relation between two generators.
 a different level in say, an [`object`](object), at a different level in the same object. It is often
 used to specify foreign key relationships in complex datasets.
 
+A full path is needed at all times. So to refer to the `name` in a file named `example.json`, we need to use
+`example.content.name`.
+
 #### Example
 
-```json synth
+```json synth[example.json]
 {
   "type": "object",
   "name": {
@@ -19,17 +22,17 @@ used to specify foreign key relationships in complex datasets.
   },
   "same_name": {
     "type": "same_as",
-    "ref": "name"
+    "ref": "example.content.name"
   }
 }
 ```
 
 The `"ref"` field must point to another existing field. Complex objects can be traversed by concatenating levels
-with a period `.`.
+with a period `.` as seen in the following `home.json` file.
 
 #### Example
 
-```json synth
+```json synth[home.json]
 {
   "type": "object",
   "address": {
@@ -49,14 +52,14 @@ with a period `.`.
   },
   "same_zip_code": {
     "type": "same_as",
-    "ref": "address.zip_code"
+    "ref": "home.content.address.zip_code"
   }
 }
 ```
 
 The `same_as` generator can also be simply declared by the value of the `"ref"` field prefixed with `@`:
 
-```json synth
+```json synth[home2.json]
 {
   "type": "object",
   "address": {
@@ -74,6 +77,6 @@ The `same_as` generator can also be simply declared by the value of the `"ref"`
       }
     }
   },
-  "same_zip_code": "@address.zip_code"
+  "same_zip_code": "@home2.content.address.zip_code"
 }
-```
\ No newline at end of file
+```
diff --git a/docs/docs/content/unique.md b/docs/docs/content/unique.md
index 7598ce04..1a6c3740 100644
--- a/docs/docs/content/unique.md
+++ b/docs/docs/content/unique.md
@@ -36,7 +36,7 @@ By default, the unique generator will give up if it sees the same value more tha
 
 Below is an example of a generator which will fail since the inner generator cannot generate 20 distinct values.
 
-```json synth
+```json synth[expect = "Could not generate enough unique values from generator"]
 {
     "type": "array",
     "length": {
diff --git a/docs/docs/examples/bank.md b/docs/docs/examples/bank.md
index 949a52dc..a0748b70 100644
--- a/docs/docs/examples/bank.md
+++ b/docs/docs/examples/bank.md
@@ -224,7 +224,7 @@ As `synth` evolves, inference will get better - but for now, we need to tweak th
 To modify the schema, open the workspace in your favourite editor. Let's take a look at `bank_db/transactions.json`
 first.
 
-```json synth
+```json synth[users.json]
 {
   "type": "array",
   "length": {
@@ -278,7 +278,7 @@ first.
       "type": "string",
       "pattern": "[a-zA-Z0-9]*"
     },
-    description": {
+    "description": {
       "type": "string",
       "pattern": "[a-zA-Z0-9]*"
     }
@@ -345,7 +345,7 @@ it should be a [date_time](/content/date-time).
 The `user_id` field should point to a valid entry in the `users` collection, so let's use
 the [same_as](../content/same-as) content type to express this foreign key relationship.
 
-```json
+```json synth
 {
   "type": "same_as",
   "ref": "users.content.id"
@@ -373,7 +373,7 @@ generator, but don't want to put thousands of descriptive lines in our declarati
 external JSON [datasource](../content/datasource) to pull descriptions from. First copy the following JSON to
 a `descriptions.json` file in the workspace.
 
-```json
+```json[descriptions.json]
 [
   "Walmart Credit Purchase",
   "Amazon - Kindle reading tablet",
diff --git a/docs/docs/getting_started/core-concepts.md b/docs/docs/getting_started/core-concepts.md
index d7396831..9337ee97 100644
--- a/docs/docs/getting_started/core-concepts.md
+++ b/docs/docs/getting_started/core-concepts.md
@@ -65,7 +65,7 @@ refer to. A field address takes the
 form `<collection name>.<level_0>.<level_1>...`. For example, say we have a
 collection `users.json` containing the following schema:
 
-```json
+```json synth[users.json]
 {
   "type": "array",
   "length": {
@@ -104,16 +104,16 @@ A reference to the `username` field would have the
 address `users.content.username`. If we want to add a reference to this field
 from another collection we would simply add:
 
-```json
+```json synth
 {
-  ...
+  "type": "array",
+  "length": 1,
   "content": {
-    ...
+    "type": "object",
     "username": {
       "type": "same_as",
       "ref": "users.content.username"
-    },
-    ...
+    }
   }
 }
 ```
diff --git a/docs/docs/getting_started/schema.md b/docs/docs/getting_started/schema.md
index 72cea2bc..262cabdc 100644
--- a/docs/docs/getting_started/schema.md
+++ b/docs/docs/getting_started/schema.md
@@ -106,7 +106,7 @@ Below is a tree representation of the schema Schema tree:
 
 The corresponding namespace can be broken into 2 collections. The first, `transactions`:
 
-```json
+```json synth[transactions.json]
 {
   "type": "array",
   "length": {
@@ -151,7 +151,7 @@ The corresponding namespace can be broken into 2 collections. The first, `transa
     },
     "user_id": {
       "type": "same_as",
-      "ref": "users.Schema.user_id"
+      "ref": "users.content.user_id"
     }
   }
 }
@@ -159,7 +159,7 @@ The corresponding namespace can be broken into 2 collections. The first, `transa
 
 And the second, the `users` collection:
 
-```json synth
+```json synth[users.json]
 {
     "type": "array",
     "length": {
@@ -183,7 +183,7 @@ And the second, the `users` collection:
         "user_email": {
             "type": "string",
             "faker": {
-                "generator": "email"
+                "generator": "safe_email"
             }
         }
     }
diff --git a/docs/docs/integrations/postgres.md b/docs/docs/integrations/postgres.md
index 93db1a54..4168c280 100644
--- a/docs/docs/integrations/postgres.md
+++ b/docs/docs/integrations/postgres.md
@@ -94,7 +94,7 @@ create table doctors
 ```
 
 And the corresponding `synth` collection:
-```json
+```json synth[expect = "unknown field: hospitals"]
 {
   "type": "array",
   "length": {
@@ -139,7 +139,7 @@ And the corresponding `synth` collection:
       "pattern": "[a-zA-Z0-9]{0, 255}"
     }
   }
-
+}
 ```
 ### Example Import Command
 
diff --git a/docs/docs/tutorials/creating-logs-with-synth.md b/docs/docs/tutorials/creating-logs-with-synth.md
index bdbb9fea..a2fc1b9d 100644
--- a/docs/docs/tutorials/creating-logs-with-synth.md
+++ b/docs/docs/tutorials/creating-logs-with-synth.md
@@ -345,7 +345,7 @@ In this case the we're assigning weights to the variants of the categorical.
 And finally `bytes`. For `bytes` we'll use a [number range](/docs/content/number#range) generator 
 from 1 b to 1 MiB:
 
-```json synth
+```json synth[logs.json]
 {
     "type": "array",
     "length": 1,
@@ -426,7 +426,7 @@ creating another file in our namespace called `formatted.json`. We'll then
 use a combination of the `format` and [`same_as`](/docs/content/same-as) generators to compose 
 together fields from our original collection `logs.json`.
 
-```json
+```json synth[formatted.json]
 {
     "type": "array",
     "length": 1,
diff --git a/synth/.gitignore b/synth/.gitignore
new file mode 100644
index 00000000..3fec32c8
--- /dev/null
+++ b/synth/.gitignore
@@ -0,0 +1,1 @@
+tmp/
diff --git a/synth/Cargo.toml b/synth/Cargo.toml
index b79313e3..d01173fa 100644
--- a/synth/Cargo.toml
+++ b/synth/Cargo.toml
@@ -9,7 +9,7 @@ authors = [
 edition = "2021"
 homepage = "https://getsynth.com"
 license = "Apache-2.0"
-exclude = ["testing_harness/*"]
+exclude = ["testing_harness/*", "tmp"]
 
 [[bench]]
 name = "bench"
@@ -26,6 +26,7 @@ git2 = "0.13.20"
 lazy_static = "1.4.0"
 tempfile = "3.1.0"
 iai = "0.1"
+test_macros = { path = "../test_macros" }
 
 [dependencies]
 lazy_static = "1.4.0"
diff --git a/www/README.md b/www/README.md
index 8a4279e4..93d5a23e 100644
--- a/www/README.md
+++ b/www/README.md
@@ -43,4 +43,7 @@ in the root `www` folder will deploy to a test deployment, and:
 $ vercel --prod
 ```
 
-will deploy to production and override the current `getsynth.com` page.
\ No newline at end of file
+will deploy to production and override the current `getsynth.com` page.
+
+## Testing
+All Synth code block examples should be marked as 'json synth' to automatically run them on the CI.


Test patch:
diff --git a/synth/tests/docs.rs b/synth/tests/docs.rs
new file mode 100644
index 00000000..44b45555
--- /dev/null
+++ b/synth/tests/docs.rs
@@ -0,0 +1,227 @@
+use lazy_static::lazy_static;
+use std::{
+    collections::HashSet,
+    env,
+    fs::{self, File},
+    io::{BufRead, BufReader, Write},
+    path::{Path, PathBuf},
+};
+use test_macros::tmpl_ignore;
+
+use anyhow::Result;
+
+// Skipping fmt is needed until this fix is released
+// https://github.com/rust-lang/rustfmt/pull/5142
+#[rustfmt::skip]
+mod helpers;
+
+use helpers::generate;
+use regex::Regex;
+
+#[tmpl_ignore("./", exclude_dir = true, filter_extension = "md")]
+#[async_std::test]
+async fn PATH_IDENT() -> Result<()> {
+    let path = Path::new("../").join(PATH);
+    let tmp = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tmp");
+    let tmp = tmp.as_path();
+
+    fs::create_dir_all(&tmp)?;
+    env::set_current_dir(tmp)?;
+    let path = Path::new("../").join(path);
+
+    let ns = get_ns_dir(tmp, &path);
+    fs::create_dir_all(&ns)?;
+
+    let mut expects = HashSet::new();
+
+    extract_code_blocks(&path)?
+        .into_iter()
+        .filter(is_json_block)
+        .try_for_each(|block| -> Result<()> {
+            let expect = write_code_block(&path, block, tmp)?;
+
+            if let Some(expect) = expect {
+                expects.insert(expect);
+            }
+
+            Ok(())
+        })?;
+
+    let actual = generate(&get_ns(&path).unwrap()).await;
+
+    // Did we expect any errors for this document
+    if expects.is_empty() {
+        assert!(
+            actual.is_ok(),
+            "should not have error: {:?}\n",
+            actual.unwrap_err()
+        );
+    } else {
+        assert!(
+            actual.is_err(),
+            "should be one of the following errors: {:#?}\n",
+            expects
+        );
+
+        let err = actual.unwrap_err();
+        let err = format!("{:?}", err);
+
+        assert!(
+            expects.iter().any(|expect| err.contains(expect)),
+            "{}\nshould contain one of the following errors: {:#?}",
+            err,
+            expects
+        );
+    }
+
+    fs::remove_dir_all(ns)?;
+
+    Ok(())
+}
+
+struct Line {
+    content: String,
+    index: usize,
+}
+
+fn extract_code_blocks(path: &Path) -> Result<Vec<Vec<Line>>> {
+    let file = File::open(path)?;
+    let reader = BufReader::new(file);
+    let mut blocks = Vec::new();
+    let mut block: Option<Vec<Line>> = None;
+    let mut count = 0;
+
+    for line in reader.lines() {
+        let line = line?;
+        count += 1;
+
+        // We are currently in a code block
+        if let Some(block) = block.as_mut() {
+            block.push(Line {
+                content: line.clone(),
+                index: count,
+            });
+        }
+
+        if line.starts_with("```") {
+            // We are currently in a block so this must be an end marker
+            if let Some(block_some) = block {
+                blocks.push(block_some);
+                block = None;
+            } else {
+                block = Some(vec![Line {
+                    content: line,
+                    index: count,
+                }]);
+            }
+        }
+    }
+
+    Ok(blocks)
+}
+
+#[allow(clippy::ptr_arg)]
+fn is_json_block(block: &Vec<Line>) -> bool {
+    block.len() > 1 && block[0].content.starts_with("```json")
+}
+
+fn get_ns_dir(tmp: &Path, ns: &Path) -> PathBuf {
+    let ns = get_ns(ns).unwrap();
+    tmp.join(ns)
+}
+
+fn get_ns(path: &Path) -> Option<String> {
+    let ns = path.with_extension("");
+
+    Some(ns.strip_prefix("../../").unwrap().display().to_string())
+}
+
+lazy_static! {
+    /// Regex to extract whether a code block has synth schema, a custom filename, or expected
+    /// errors
+    /// The following lines should be matched with their respective capture groups
+    /// ```json synth                                       (synth= synth)
+    /// ```json synth[custom.json]                          (synth= synth)(file=custom.json)
+    /// ```json synth[expect = "error"]                     (synth= synth)(expect=error)
+    /// ```json[data.json]                                  (file=data.json)
+     static ref BLOCK_IDENTIFIER: Regex =Regex::new(
+        r#"^```json(?P<synth> synth)?(?:\[(?P<file>.*\.json)\])?(?:\[expect\s=\s"(?P<expect>.*)"\])?$"#,
+    ).unwrap();
+
+    /// Regex to capture any comments at the end of a line
+    static ref COMMENT: Regex = Regex::new("(?P<comment>//.*$)").unwrap();
+}
+
+fn write_code_block(ns: &Path, block: Vec<Line>, tmp: &Path) -> Result<Option<String>> {
+    let ns = get_ns_dir(tmp, ns);
+
+    let (is_synth, file, expect) = BLOCK_IDENTIFIER
+        .captures(&block[0].content)
+        .map(|cap| {
+            (
+                cap.name("synth").is_some(),
+                cap.name("file").map(|file| file.as_str().to_string()),
+                cap.name("expect").map(|expect| expect.as_str().to_string()),
+            )
+        })
+        .unwrap();
+
+    let file = match (is_synth, file) {
+        (false, None) => {
+            println!(
+                "{}:{} has a JSON only code block that will be skipped",
+                ns.display(),
+                block[0].index
+            );
+            return Ok(None);
+        }
+        (false, Some(file)) => {
+            println!(
+                "{}:{} has a JSON data file that will be copied",
+                ns.display(),
+                block[0].index
+            );
+            tmp.join(file)
+        }
+        (true, Some(file)) => ns.join(file),
+        (true, None) => ns.join(format!("{}.json", block[0].index)),
+    };
+
+    let mut file = File::create(file)?;
+    let is_array_block = block[2].content.contains("\"type\": \"array\",");
+
+    if is_synth && !is_array_block {
+        write!(
+            file,
+            r#"{{
+            "type": "array",
+            "length": 1,
+            "content": "#
+        )?;
+    }
+
+    for line in &block[1..block.len() - 1] {
+        // Strip comments since they will crash serde
+        let comment = COMMENT
+            .captures(&line.content)
+            .map(|cap| {
+                cap.name("comment")
+                    .map(|comment| comment.as_str().to_string())
+            })
+            .unwrap_or(None);
+
+        let mut safe_line = line.content.clone();
+
+        if let Some(comment) = comment {
+            safe_line = safe_line.trim_end_matches(&comment).to_string();
+        }
+
+        writeln!(file, "{}", safe_line)?;
+    }
+
+    if is_synth && !is_array_block {
+        writeln!(file, "}}")?;
+    }
+
+    Ok(expect)
+}
diff --git a/test_macros/Cargo.toml b/test_macros/Cargo.toml
new file mode 100644
index 00000000..3dd9790c
--- /dev/null
+++ b/test_macros/Cargo.toml
@@ -0,0 +1,15 @@
+[package]
+name = "test_macros"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+ignore = "0.4"
+proc-macro2 = "1.0"
+quote = "1.0"
+syn = "1.0"
+tokenstream2-tmpl = "0.1"
+
diff --git a/test_macros/src/lib.rs b/test_macros/src/lib.rs
new file mode 100644
index 00000000..6a95b7ec
--- /dev/null
+++ b/test_macros/src/lib.rs
@@ -0,0 +1,257 @@
+extern crate proc_macro;
+use std::{collections::HashMap, ffi::OsStr};
+
+use ignore::{DirEntry, WalkBuilder};
+use proc_macro2::{Ident, Span, TokenStream};
+use quote::ToTokens;
+use syn::{
+    parse::{Parse, ParseStream, Result},
+    parse_macro_input, LitBool, LitStr, Token,
+};
+use tokenstream2_tmpl::{interpolate, Interpolate};
+
+mod kw {
+    syn::custom_keyword!(exclude_dir);
+    syn::custom_keyword!(filter_extension);
+}
+
+fn is_extension(extension: &str, dir_entry: &DirEntry) -> bool {
+    is_dir(dir_entry) || dir_entry.path().extension() == Some(OsStr::new(extension))
+}
+
+fn is_dir(dir_entry: &DirEntry) -> bool {
+    dir_entry.path().is_dir()
+}
+
+#[cfg_attr(test, derive(Debug, PartialEq))]
+struct IgnoreInput {
+    path: LitStr,
+    exclude_dir: bool,
+    filter_extension: Option<String>,
+}
+
+impl IgnoreInput {
+    fn get_files(&self) -> Vec<File> {
+        let filter_extension = self.filter_extension.clone();
+
+        WalkBuilder::new(self.path.value())
+            .filter_entry(move |entry| {
+                if let Some(extension) = &filter_extension {
+                    is_extension(extension, entry)
+                } else {
+                    true
+                }
+            })
+            .build()
+            .filter_map(std::result::Result::ok)
+            .filter(|entry| !self.exclude_dir || !is_dir(entry))
+            .map(|entry| {
+                let path = entry.path().display().to_string();
+                let path = path.trim_start_matches("./");
+
+                File {
+                    path: path.to_string(),
+                    path_ident: path
+                        .replace('.', "_dot_")
+                        .replace('/', "_")
+                        .replace('\\', "_")
+                        .replace('-', "_")
+                        .to_lowercase(),
+                }
+            })
+            .collect()
+    }
+}
+
+impl Parse for IgnoreInput {
+    fn parse(input: ParseStream) -> Result<Self> {
+        let path = input.parse()?;
+        let mut exclude_dir = false;
+        let mut filter_extension = None;
+
+        while !input.is_empty() {
+            input.parse::<Token!(,)>()?;
+            let lookahead = input.lookahead1();
+
+            if lookahead.peek(kw::exclude_dir) {
+                input.parse::<kw::exclude_dir>()?;
+                input.parse::<Token!(=)>()?;
+
+                exclude_dir = input.parse::<LitBool>()?.value;
+            } else if lookahead.peek(kw::filter_extension) {
+                input.parse::<kw::filter_extension>()?;
+                input.parse::<Token!(=)>()?;
+
+                filter_extension = Some(input.parse::<LitStr>()?.value());
+            } else {
+                return Err(lookahead.error());
+            }
+        }
+
+        Ok(Self {
+            path,
+            exclude_dir,
+            filter_extension,
+        })
+    }
+}
+
+impl Interpolate for IgnoreInput {
+    fn interpolate(&self, stream: TokenStream) -> TokenStream {
+        let mut output = TokenStream::new();
+
+        let files = self
+            .get_files()
+            .into_iter()
+            .map(|file| file.interpolate(stream.clone()));
+
+        output.extend(files);
+
+        output
+    }
+}
+
+#[cfg_attr(test, derive(Debug, PartialEq))]
+struct File {
+    path_ident: String,
+    path: String,
+}
+
+impl Interpolate for File {
+    fn interpolate(&self, stream: TokenStream) -> TokenStream {
+        let mut replacements: HashMap<_, &dyn ToTokens> = HashMap::new();
+
+        let path = LitStr::new(&self.path, Span::call_site());
+        let path_ident = Ident::new(&self.path_ident, Span::call_site());
+
+        replacements.insert("PATH", &path);
+        replacements.insert("PATH_IDENT", &path_ident);
+
+        interpolate(stream, &replacements)
+    }
+}
+
+/// Use `ignore` to find files or directories at compile time, then apply the supplied template to
+/// each file or directory found
+/// The template can include the following two placeholders:
+/// - PATH_IDENT: a rust safe identifier for the path
+/// - PATH: a string literal to the path
+///
+/// # Arguments
+/// * String path to search in
+/// * `exclude_dir` - Should directories be excluded from results
+/// * `filter_extension` - Filter results to only those with the given file extension
+///
+/// # Examples
+/// The following creates a function for each toml file
+/// ```
+/// use test_macros::tmpl_ignore;
+///
+/// #[tmpl_ignore("./", exclude_dir = true, filter_extension = "toml")]
+/// fn PATH_IDENT() -> String {
+///     PATH.to_string()
+/// }
+///
+/// fn main() {
+///     assert_eq!(cargo_dot_toml(), "Cargo.toml");
+/// }
+/// ```
+#[proc_macro_attribute]
+pub fn tmpl_ignore(
+    tokens: proc_macro::TokenStream,
+    template: proc_macro::TokenStream,
+) -> proc_macro::TokenStream {
+    let input = parse_macro_input!(tokens as IgnoreInput);
+
+    input.interpolate(template.into()).into()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{File, IgnoreInput};
+    use syn::parse_str;
+
+    type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
+
+    #[test]
+    fn parse() -> Result<()> {
+        let actual: IgnoreInput = parse_str("\"foo\"")?;
+        let expected = IgnoreInput {
+            path: parse_str("\"foo\"")?,
+            exclude_dir: false,
+            filter_extension: None,
+        };
+
+        assert_eq!(actual, expected);
+
+        Ok(())
+    }
+
+    #[test]
+    fn parse_exclude_dir() -> Result<()> {
+        let actual: IgnoreInput = parse_str("\"foo\", exclude_dir = true")?;
+        let expected = IgnoreInput {
+            path: parse_str("\"foo\"")?,
+            exclude_dir: true,
+            filter_extension: None,
+        };
+
+        assert_eq!(actual, expected);
+
+        Ok(())
+    }
+
+    #[test]
+    fn parse_filter_extension() -> Result<()> {
+        let actual: IgnoreInput = parse_str("\"foo\", filter_extension = \"json\"")?;
+        let expected = IgnoreInput {
+            path: parse_str("\"foo\"")?,
+            exclude_dir: false,
+            filter_extension: Some("json".to_string()),
+        };
+
+        assert_eq!(actual, expected);
+
+        Ok(())
+    }
+
+    #[test]
+    fn parse_all_options() -> Result<()> {
+        let actual: IgnoreInput =
+            parse_str("\"foo\", filter_extension = \"json\", exclude_dir = true")?;
+        let expected = IgnoreInput {
+            path: parse_str("\"foo\"")?,
+            exclude_dir: true,
+            filter_extension: Some("json".to_string()),
+        };
+
+        assert_eq!(actual, expected);
+
+        Ok(())
+    }
+
+    #[test]
+    #[should_panic(expected = "expected string literal")]
+    fn parse_missing_path() {
+        parse_str::<IgnoreInput>("filter_extension = \"json\", exclude_dir = true").unwrap();
+    }
+
+    #[test]
+    fn get_files() -> Result<()> {
+        let options = IgnoreInput {
+            path: parse_str("\"./\"")?,
+            exclude_dir: true,
+            filter_extension: Some("toml".to_string()),
+        };
+
+        let actual = options.get_files();
+        let expected = vec![File {
+            path: "Cargo.toml".to_string(),
+            path_ident: "cargo_dot_toml".to_string(),
+        }];
+
+        assert_eq!(actual, expected);
+
+        Ok(())
+    }
+}