Repository: stoplightio/spectral
Pull request #101 (2019-03-15 11:42:29)

feat(oas): check examples are valid against their schema
If an example is provided, make sure it is valid against the schema for that property.

SL-2065

### Please check if the PR fulfills these requirements
- [x] Tests for the changes have been added (for bug fixes / features)
- [ ] Docs have been added / updated (for bug fixes / features)

### What kind of change does this PR introduce?

Feature

### What is the current behavior?

Example is pretty much ignored.

### If this is a feature change, what is the new behavior?

Now examples with nonsense values, formats that are not matched, etc, will product validation errors. 

### Does this PR introduce a breaking change?

This will create a breaking change if you have invalid examples, so you will want to fix the examples.



Patch:
diff --git a/README.md b/README.md
index 6afc3cd7..22825dd8 100644
--- a/README.md
+++ b/README.md
@@ -31,7 +31,7 @@ There are two key concepts in Spectral: **Rules** and **Functions**.
 
 Think of a set of **rules** and **functions** as a flexible and customizable style guide for your JSON objects.
 
-### Creating a custom rule:
+### Creating a custom rule
 
 Spectral has a built-in set of functions which you can reference in your rules. This example uses the `RuleFunction.PATTERN` to create a rule that checks that all property values are in snake case.
 
@@ -147,7 +147,7 @@ console.log(JSON.stringify(results, null, 4));
 // }
 ```
 
-### Linting an OAS 2 document:
+### Linting an OAS 2 document
 
 Spectral also includes a number of ready made rules and functions for OpenAPI Specification (OAS) 2 and 3 documents. This example uses the OAS 2 rules to lint a document.
 
diff --git a/src/functions/index.ts b/src/functions/index.ts
index 8e3942d4..8106b6cd 100644
--- a/src/functions/index.ts
+++ b/src/functions/index.ts
@@ -3,6 +3,7 @@ export const functions = {
   length: require('./length').length,
   pattern: require('./pattern').pattern,
   schema: require('./schema').schema,
+  schemaPath: require('./schema-path').schemaPath,
   truthy: require('./truthy').truthy,
   xor: require('./xor').xor,
 };
diff --git a/src/functions/schema-path.ts b/src/functions/schema-path.ts
new file mode 100644
index 00000000..b979a0ea
--- /dev/null
+++ b/src/functions/schema-path.ts
@@ -0,0 +1,36 @@
+/*
+ * Schema Path is a variant of the normal Schema rule with a slight twist.
+ * Instead of passing the rule a schema to validate, there is already a schema
+ * somewhere inside the target, and we are just selecting it, and applying it to
+ * a given field.
+ *
+ * The primary use case for this was validating OpenAPI examples
+ * against their schema, but this could be used for other things.
+ */
+import { IFunction, ISchemaPathOptions } from '../types';
+import { schema } from './schema';
+
+import jp = require('jsonpath');
+
+export const schemaPath: IFunction<ISchemaPathOptions> = (targetVal, opts, paths, otherValues) => {
+  if (!targetVal) return [];
+  let value = targetVal;
+
+  // We want to run this on a specific field, for example: example
+  if (opts.field && typeof targetVal === 'object') {
+    value = targetVal[opts.field];
+  }
+
+  if (!value) return [];
+
+  // Hunt in the original targetVal for the schema we are going to apply
+  let schemaObj;
+  try {
+    schemaObj = jp.value(targetVal, opts.schemaPath);
+  } catch (error) {
+    console.error(error);
+  }
+  if (!schemaObj) return [];
+
+  return schema(value, { schema: schemaObj }, paths, otherValues);
+};
diff --git a/src/rulesets/oas2/index.ts b/src/rulesets/oas2/index.ts
index 58efbff0..563dcb5c 100644
--- a/src/rulesets/oas2/index.ts
+++ b/src/rulesets/oas2/index.ts
@@ -89,7 +89,6 @@ export const oas2Rules = () => {
       },
     },
     'operation-security-defined': {
-      enabled: true,
       summary: 'Operation `security` values must match a scheme defined in the `securityDefinitions` object.',
       type: RuleType.VALIDATION,
       given: '$',
@@ -101,5 +100,17 @@ export const oas2Rules = () => {
       },
       tags: ['operation'],
     },
+    'valid-example': {
+      summary: 'Examples must be valid against their defined schema.',
+      type: RuleType.VALIDATION,
+      given: '$..[?(@.example)]',
+      then: {
+        function: RuleFunction.SCHEMAPATH,
+        functionOptions: {
+          field: 'example',
+          schemaPath: '$',
+        },
+      },
+    },
   });
 };
diff --git a/src/rulesets/oas3/index.ts b/src/rulesets/oas3/index.ts
index 1bb2e963..936d9339 100644
--- a/src/rulesets/oas3/index.ts
+++ b/src/rulesets/oas3/index.ts
@@ -54,7 +54,6 @@ export const oas3Rules = () => {
       },
     },
     'operation-security-defined': {
-      enabled: true,
       summary: 'Operation `security` values must match a scheme defined in the `components.securitySchemes` object.',
       type: RuleType.VALIDATION,
       given: '$',
@@ -91,5 +90,17 @@ export const oas3Rules = () => {
         },
       },
     },
+    'valid-example': {
+      summary: 'Examples must be valid against their defined schema.',
+      type: RuleType.VALIDATION,
+      given: '$..[?(@.example)]',
+      then: {
+        function: RuleFunction.SCHEMAPATH,
+        functionOptions: {
+          field: 'example',
+          schemaPath: '$',
+        },
+      },
+    },
   });
 };
diff --git a/src/types/enums.ts b/src/types/enums.ts
index afe5f390..2364c41a 100644
--- a/src/types/enums.ts
+++ b/src/types/enums.ts
@@ -8,6 +8,7 @@ export enum RuleFunction {
   LENGTH = 'length',
   PATTERN = 'pattern',
   SCHEMA = 'schema',
+  SCHEMAPATH = 'schemaPath',
   TRUTHY = 'truthy',
   XOR = 'xor',
 }
diff --git a/src/types/rule.ts b/src/types/rule.ts
index 9c60d2b8..f0abd1d8 100644
--- a/src/types/rule.ts
+++ b/src/types/rule.ts
@@ -82,3 +82,10 @@ export interface ISchemaOptions {
   schema: object;
 }
 export type SchemaRule = IRule<RuleFunction.SCHEMA, ISchemaOptions>;
+
+export interface ISchemaPathOptions {
+  schemaPath: string;
+  // the `path.to.prop` to field, or special `@key` value to target keys for matched `given` object
+  field?: string;
+}
+export type SchemaPathRule = IRule<RuleFunction.SCHEMAPATH, ISchemaPathOptions>;


Test patch:
diff --git a/src/functions/__tests__/schema-path.test.ts b/src/functions/__tests__/schema-path.test.ts
new file mode 100644
index 00000000..109ff80a
--- /dev/null
+++ b/src/functions/__tests__/schema-path.test.ts
@@ -0,0 +1,103 @@
+import { schemaPath } from '../schema-path';
+
+function runSchemaPath(target: any, field: string, schemaPathStr: string) {
+  return schemaPath(target, { field, schemaPath: schemaPathStr }, { given: ['$'] }, { given: null, original: null });
+}
+
+describe('schema', () => {
+  // Check the example field matches the conents of schema
+  const fieldToCheck = 'example';
+  const path = '$.schema';
+
+  test('will pass when example is valid', () => {
+    const target = {
+      schema: {
+        type: 'string',
+      },
+      example: 'turtle',
+    };
+
+    expect(runSchemaPath(target, fieldToCheck, path)).toHaveLength(0);
+  });
+
+  test('will pass when example is valid', () => {
+    const target = {
+      schema: {
+        type: 'object',
+        properties: {
+          url: {
+            type: 'string',
+          },
+        },
+        required: ['url'],
+      },
+      example: {
+        url: 'images/38.png',
+      },
+    };
+
+    expect(runSchemaPath(target, fieldToCheck, path)).toHaveLength(0);
+  });
+
+  test('will error with totally invalid input', () => {
+    const target = {
+      schema: {
+        type: 'object',
+        properties: {
+          url: {
+            type: 'string',
+          },
+        },
+        required: ['url'],
+      },
+      example: {
+        notUrl: 'images/38.png',
+      },
+    };
+    expect(runSchemaPath(target, fieldToCheck, path)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "message": "should have required property 'url'",
+    "path": Array [
+      "$",
+    ],
+  },
+]
+`);
+  });
+
+  test('will error formats', () => {
+    const target = {
+      schema: {
+        type: 'string',
+        format: 'url',
+      },
+      example: 'turtle',
+    };
+
+    expect(runSchemaPath(target, fieldToCheck, path)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "message": "should match format \\"url\\"",
+    "path": Array [
+      "$",
+    ],
+  },
+]
+`);
+  });
+
+  describe('when schema path is not there', () => {
+    const invalidFieldToCheck = 'nonsense';
+
+    test('will pass when field is not there', () => {
+      const target = {
+        schema: {
+          type: 'string',
+        },
+        notNonsense: 'turtle',
+      };
+      expect(runSchemaPath(target, invalidFieldToCheck, path)).toHaveLength(0);
+    });
+  });
+});
diff --git a/src/functions/__tests__/schema.test.ts b/src/functions/__tests__/schema.test.ts
new file mode 100644
index 00000000..efdc5bae
--- /dev/null
+++ b/src/functions/__tests__/schema.test.ts
@@ -0,0 +1,43 @@
+import { schema } from '../schema';
+
+function runSchema(target: any, schemaObj: object) {
+  return schema(target, { schema: schemaObj }, { given: ['$'] }, { given: null, original: null });
+}
+
+describe('schema', () => {
+  const testSchema = {
+    type: 'array',
+    items: {
+      type: 'string',
+    },
+    maxItems: 1,
+  };
+
+  test('will error with totally invalid input', () => {
+    const input = { foo: 'bar' };
+    expect(runSchema(input, testSchema)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "message": "should be array",
+    "path": Array [
+      "$",
+    ],
+  },
+]
+`);
+  });
+
+  test('will error with subtly invalid input', () => {
+    const input = ['1', '2'];
+    expect(runSchema(input, testSchema)).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "message": "should NOT have more than 1 items",
+    "path": Array [
+      "$",
+    ],
+  },
+]
+`);
+  });
+});
diff --git a/src/rulesets/oas2/__tests__/valid-example.ts b/src/rulesets/oas2/__tests__/valid-example.ts
new file mode 100644
index 00000000..84f61e10
--- /dev/null
+++ b/src/rulesets/oas2/__tests__/valid-example.ts
@@ -0,0 +1,141 @@
+import { Spectral } from '../../../spectral';
+import { oas2Rules } from '../index';
+
+const ruleset = { rules: oas2Rules() };
+
+describe('valid-example', () => {
+  const s = new Spectral();
+  s.addRules({
+    'valid-example': Object.assign(ruleset.rules['valid-example'], {
+      enabled: true,
+    }),
+  });
+
+  test('will pass when simple example is valid', () => {
+    const results = s.run({
+      xoxo: {
+        type: 'string',
+        example: 'doggie',
+      },
+    });
+    expect(results.results).toHaveLength(0);
+  });
+
+  test('will fail when simple example is invalid', () => {
+    const results = s.run({
+      xoxo: {
+        type: 'string',
+        example: 123,
+      },
+    });
+    expect(results.results).toHaveLength(1);
+  });
+
+  test('will pass when complex example is used ', () => {
+    const results = s.run({
+      xoxo: {
+        type: 'object',
+        properties: {
+          url: {
+            type: 'string',
+          },
+          width: {
+            type: 'integer',
+          },
+          height: {
+            type: 'integer',
+          },
+        },
+        required: ['url'],
+        example: {
+          url: 'images/38.png',
+          width: 100,
+          height: 100,
+        },
+      },
+    });
+
+    expect(results.results).toHaveLength(0);
+  });
+
+  test('will error with totally invalid input', () => {
+    const results = s.run({
+      xoxo: {
+        type: 'object',
+        properties: {
+          url: {
+            type: 'string',
+          },
+          width: {
+            type: 'integer',
+          },
+          height: {
+            type: 'integer',
+          },
+        },
+        required: ['url'],
+        example: {
+          url2: 'images/38.png',
+          width: 'coffee',
+          height: false,
+        },
+      },
+    });
+
+    expect(results.results).toHaveLength(1);
+  });
+
+  test('will error with totally invalid input', () => {
+    const results = s.run({
+      swagger: '2.0',
+      info: {
+        version: '1.0.0',
+        title: 'Swagger Petstore',
+      },
+      host: 'petstore.swagger.io',
+      paths: {
+        '/pet': {
+          post: {
+            parameters: [
+              {
+                in: 'body',
+                name: 'body',
+                required: true,
+                schema: {
+                  type: 'string',
+                  format: 'email',
+                  example: 'hello',
+                },
+              },
+            ],
+            responses: {
+              '200': {
+                description: 'OK',
+              },
+            },
+          },
+        },
+      },
+    });
+
+    expect(results.results).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "message": "should match format \\"email\\"",
+    "name": "valid-example",
+    "path": Array [
+      "paths",
+      "/pet",
+      "post",
+      "parameters",
+      0,
+      "schema",
+    ],
+    "severity": 40,
+    "severityLabel": "warn",
+    "summary": "Examples must be valid against their defined schema.",
+  },
+]
+`);
+  });
+});
diff --git a/src/rulesets/oas3/__tests__/valid-example.ts b/src/rulesets/oas3/__tests__/valid-example.ts
new file mode 100644
index 00000000..6c06e617
--- /dev/null
+++ b/src/rulesets/oas3/__tests__/valid-example.ts
@@ -0,0 +1,142 @@
+import { Spectral } from '../../../spectral';
+import { oas3Rules } from '../index';
+
+const ruleset = { rules: oas3Rules() };
+
+describe('valid-example', () => {
+  const s = new Spectral();
+  s.addRules({
+    'valid-example': Object.assign(ruleset.rules['valid-example'], {
+      enabled: true,
+    }),
+  });
+
+  test('will pass when simple example is valid', () => {
+    const results = s.run({
+      xoxo: {
+        type: 'string',
+        example: 'doggie',
+      },
+    });
+    expect(results.results).toHaveLength(0);
+  });
+
+  test('will fail when simple example is invalid', () => {
+    const results = s.run({
+      xoxo: {
+        type: 'string',
+        example: 123,
+      },
+    });
+    expect(results.results).toHaveLength(1);
+  });
+
+  test('will pass when complex example is used ', () => {
+    const results = s.run({
+      xoxo: {
+        type: 'object',
+        properties: {
+          url: {
+            type: 'string',
+          },
+          width: {
+            type: 'integer',
+          },
+          height: {
+            type: 'integer',
+          },
+        },
+        required: ['url'],
+        example: {
+          url: 'images/38.png',
+          width: 100,
+          height: 100,
+        },
+      },
+    });
+
+    expect(results.results).toHaveLength(0);
+  });
+
+  test('will error with totally invalid input', () => {
+    const results = s.run({
+      xoxo: {
+        type: 'object',
+        properties: {
+          url: {
+            type: 'string',
+          },
+          width: {
+            type: 'integer',
+          },
+          height: {
+            type: 'integer',
+          },
+        },
+        required: ['url'],
+        example: {
+          url2: 'images/38.png',
+          width: 'coffee',
+          height: false,
+        },
+      },
+    });
+
+    expect(results.results).toHaveLength(1);
+  });
+
+  test('will error with totally invalid input', () => {
+    const results = s.run({
+      openapi: '3.0.1',
+      info: {
+        title: 'OpenAPI Petstore',
+        version: '1.0.0',
+      },
+      paths: {
+        '/pet': {
+          post: {
+            requestBody: {
+              content: {
+                '*/*': {
+                  schema: {
+                    type: 'string',
+                    format: 'email',
+                    example: 'hello',
+                  },
+                },
+              },
+              required: true,
+            },
+            responses: {
+              '405': {
+                description: 'Invalid input',
+                content: {},
+              },
+            },
+          },
+        },
+      },
+    });
+
+    expect(results.results).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "message": "should match format \\"email\\"",
+    "name": "valid-example",
+    "path": Array [
+      "paths",
+      "/pet",
+      "post",
+      "requestBody",
+      "content",
+      "*/*",
+      "schema",
+    ],
+    "severity": 40,
+    "severityLabel": "warn",
+    "summary": "Examples must be valid against their defined schema.",
+  },
+]
+`);
+  });
+});