Repository: stoplightio/spectral
Pull request #671 (2019-10-11 13:42:30)

feat: static assets
Change driven by Studio.
~Will describe it in a great detail soon, once it's ready to review.~
Long story short - Studio will provide its own assets in the following manner
```ts
Spectral.registerStaticAssets(require('@stoplight/spectral/rulesets/assets/assets.json'));
```



Patch:
diff --git a/package.json b/package.json
index c74e0c94..e7b09c43 100644
--- a/package.json
+++ b/package.json
@@ -38,15 +38,16 @@
     "build": "tsc -p ./tsconfig.build.json",
     "cli": "node -r ts-node/register -r tsconfig-paths/register src/cli/index.ts",
     "cli:debug": "node -r ts-node/register -r tsconfig-paths/register --inspect-brk src/cli/index.ts",
-    "compile-rulesets": "node ./scripts/compile-rulesets.js",
+    "generate-assets": "node ./scripts/generate-assets.js",
     "inline-version": "./scripts/inline-version.js",
     "lint.fix": "yarn lint --fix",
     "lint": "tsc --noEmit && tslint 'src/**/*.ts'",
     "postbuild.oas-functions": "copyfiles -u 1 \"dist/rulesets/oas*/functions/*.js\" ./",
-    "postbuild": "yarn build.oas-functions && yarn compile-rulesets",
+    "postbuild": "yarn build.oas-functions && yarn generate-assets",
     "prebuild": "copyfiles -u 1 \"src/rulesets/oas*/**/*.json\" dist && copyfiles -u 1 \"src/rulesets/oas*/**/*.json\" ./",
     "prebuild.binary": "yarn build",
-    "pretest.karma": "node ./scripts/generate-karma-fixtures.js",
+    "pretest.karma": "node ./scripts/generate-karma-fixtures.js && yarn pretest",
+    "pretest": "node ./scripts/generate-assets.js",
     "schema.update": "yarn typescript-json-schema --id \"http://stoplight.io/schemas/rule.schema.json\" --required tsconfig.json IRule --out ./src/meta/rule.schema.json",
     "test.harness": "jest -c ./jest.harness.config.js",
     "test.karma": "karma start",
diff --git a/rollup.config.js b/rollup.config.js
index 910e7643..3d16c18c 100644
--- a/rollup.config.js
+++ b/rollup.config.js
@@ -1,13 +1,29 @@
 import typescript from 'rollup-plugin-typescript2';
 import * as path from 'path';
+import * as fs from 'fs';
 import resolve from 'rollup-plugin-node-resolve';
 import commonjs from 'rollup-plugin-commonjs';
 import { terser } from 'rollup-plugin-terser';
 
 const BASE_PATH = process.cwd();
+const functions = [];
 
-const processor = (folder, array) => array.map(fn => ({
-  input: path.resolve(BASE_PATH, folder, `${fn}.js`),
+for (const directory of ['dist/rulesets/oas/functions','dist/rulesets/oas2/functions', 'dist/rulesets/oas3/functions']) {
+  const targetDir = path.join(BASE_PATH, directory);
+  if (!fs.existsSync(targetDir)) continue;
+  for (const file of fs.readdirSync(targetDir)) {
+    const targetFile = path.join(targetDir, file);
+    const stat = fs.statSync(targetFile);
+    if (!stat.isFile()) continue;
+    const ext = path.extname(targetFile);
+    if (ext !== '.js') continue;
+
+    functions.push(targetFile);
+  }
+}
+
+module.exports = functions.map(fn => ({
+  input: fn,
   plugins: [
     typescript({
       tsconfig: path.join(BASE_PATH, './tsconfig.rollup.json'),
@@ -18,24 +34,8 @@ const processor = (folder, array) => array.map(fn => ({
     terser(),
   ],
   output: {
-    file: path.resolve(BASE_PATH, folder, `${fn}.js`),
+    file: fn,
     format: 'cjs',
     exports: 'named'
   },
 }));
-
-module.exports = processor('dist/rulesets/oas/functions', [
-  'oasOp2xxResponse',
-  'oasOpFormDataConsumeCheck',
-  'oasOpIdUnique',
-  'oasOpParams',
-  'oasOpSecurityDefined',
-  'oasPathParam',
-  'refSiblings',
-])
-.concat(processor('dist/rulesets/oas2/functions', [
-  // Add here the oas2 specific functions
-]))
-.concat(processor('dist/rulesets/oas3/functions', [
-  // Add here the oas3 specific functions
-]));
diff --git a/scripts/compile-rulesets.js b/scripts/compile-rulesets.js
deleted file mode 100755
index 9b8202da..00000000
--- a/scripts/compile-rulesets.js
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env node
-const path = require('path');
-const fs = require('fs');
-const { readRuleset } = require('../dist/rulesets');
-
-const dist = path.join(__dirname, '../rulesets/precompiled/');
-
-const rulesetNames = ['oas2', 'oas3', 'oas'];
-
-try {
-  fs.mkdirSync(dist);
-} catch (ex) {
-
-}
-
-Promise.all(
-  rulesetNames.map(ruleset => readRuleset(`spectral:${ruleset}`)),
-).then(rulesets => {
-  for (const [i, ruleset] of rulesets.entries()) {
-    fs.writeFileSync(
-      path.join(dist, `${rulesetNames[i]}.json`),
-      JSON.stringify(ruleset),
-    );
-  }
-});
-
diff --git a/scripts/generate-assets.js b/scripts/generate-assets.js
new file mode 100755
index 00000000..914c2cb3
--- /dev/null
+++ b/scripts/generate-assets.js
@@ -0,0 +1,61 @@
+#!/usr/bin/env node
+/**
+ * This script generates a list of assets that are needed to load spectral:oas ruleset.
+ * It contains all OAS custom functions and *resolved* rulesets.
+ * The assets are stores in a single filed call assets.json in the following format:
+ * `<require-call-path>: <content>`
+ * where the `require-call-path` is the path you'd normally pass to require(), i.e. `@stoplight/spectral/rulesets/oas/index.js` and `content` is the text data.
+ * Assets can be loaded using Spectral#registerStaticAssets statc method, i.e. `Spectral.registerStaticAssets(require('@stoplight/spectral/rulesets/assets/assets.json'))`;
+ * If you execute the code above, ruleset will be loaded fully offline, without a need to make any request.
+ */
+
+const path = require('@stoplight/path');
+const fs = require('fs');
+const { promisify } = require('util');
+const { parse } = require('@stoplight/yaml');
+const { httpAndFileResolver } = require('../dist/resolvers/http-and-file');
+
+const readFileAsync = promisify(fs.readFile);
+const writeFileAsync = promisify(fs.writeFile);
+const readdirAsync = promisify(fs.readdir);
+const statAsync = promisify(fs.stat);
+
+const baseDir = path.join(__dirname, '../rulesets/assets/');
+
+if (!fs.existsSync(baseDir)) {
+  fs.mkdirSync(baseDir);
+}
+
+const target = path.join(baseDir, `assets.json`);
+const assets = {};
+
+(async () => {
+  await Promise.all(['', '2', '3'].map(spec => processDirectory(assets, path.join(__dirname, `../rulesets/oas${spec}`))));
+  await writeFileAsync(target, JSON.stringify(assets, null, 2));
+})();
+
+async function processDirectory(assets, dir) {
+  await Promise.all((await readdirAsync(dir)).map(async name => {
+    if (name === 'schemas') return;
+    const target = path.join(dir, name);
+    const stats = await statAsync(target);
+    if (stats.isDirectory()) {
+      return processDirectory(assets, target);
+    } else {
+      let content = await readFileAsync(target, 'utf8');
+      if (path.extname(name) === '.json') {
+        content = JSON.stringify((await httpAndFileResolver.resolve(JSON.parse(content), {
+          dereferenceRemote: true,
+          dereferenceInline: false,
+          baseUri: target,
+          parseResolveResult(opts) {
+            opts.result = parse(opts.result);
+            return opts;
+          },
+        })).result);
+      }
+
+      assets[path.join('@stoplight/spectral', path.relative(path.join(__dirname, '..'), target))] = content;
+    }
+  }));
+}
diff --git a/src/assets.ts b/src/assets.ts
new file mode 100644
index 00000000..cf7aadd9
--- /dev/null
+++ b/src/assets.ts
@@ -0,0 +1,13 @@
+import { Dictionary } from '@stoplight/types';
+
+function resolveSpectralRuleset(ruleset: string) {
+  return `@stoplight/spectral/rulesets/${ruleset}/index.json`;
+}
+
+export const RESOLVE_ALIASES: Dictionary<string, string> = {
+  'spectral:oas': resolveSpectralRuleset('oas'),
+  'spectral:oas2': resolveSpectralRuleset('oas2'),
+  'spectral:oas3': resolveSpectralRuleset('oas3'),
+};
+
+export const STATIC_ASSETS: Dictionary<string> = {};
diff --git a/src/fs/reader.ts b/src/fs/reader.ts
index dcc5a0a9..ba398a3b 100644
--- a/src/fs/reader.ts
+++ b/src/fs/reader.ts
@@ -1,6 +1,7 @@
 import { isURL } from '@stoplight/path';
 import AbortController from 'abort-controller';
 import * as fs from 'fs';
+import { STATIC_ASSETS } from '../assets';
 import request from '../request';
 
 export interface IReadOptions {
@@ -9,6 +10,10 @@ export interface IReadOptions {
 }
 
 export async function readFile(name: string, opts: IReadOptions): Promise<string> {
+  if (name in STATIC_ASSETS) {
+    return STATIC_ASSETS[name];
+  }
+
   if (isURL(name)) {
     let response;
     let timeout: NodeJS.Timeout | number | null = null;
diff --git a/src/rulesets/finder.ts b/src/rulesets/finder.ts
index c15c7c5c..31cee83a 100644
--- a/src/rulesets/finder.ts
+++ b/src/rulesets/finder.ts
@@ -1,9 +1,8 @@
 import * as path from '@stoplight/path';
 import * as fs from 'fs';
-import { filesMap } from './map';
+import { RESOLVE_ALIASES, STATIC_ASSETS } from '../assets';
 
 const SPECTRAL_SRC_ROOT = path.join(__dirname, '..');
-
 // DON'T RENAME THIS FUNCTION, you can move it within this file, but it must be kept as top-level declaration
 // parameter can be renamed, but don't this if you don't need to
 function resolveSpectralVersion(pkg: string) {
@@ -32,6 +31,12 @@ async function resolveFromFS(from: string, to: string) {
   }
 
   targetPath = path.resolve(from, to);
+
+  // if found in static assets, it's fine, as readParsable will handle it just fine
+  if (targetPath in STATIC_ASSETS) {
+    return targetPath;
+  }
+
   // if it's not a built-in ruleset, try to resolve the file according to the provided path
   if (await exists(targetPath)) {
     return targetPath;
@@ -41,11 +46,16 @@ async function resolveFromFS(from: string, to: string) {
 }
 
 export async function findFile(from: string, to: string) {
-  const mapped = filesMap.get(to);
+  const mapped = RESOLVE_ALIASES[to];
+
   if (mapped !== void 0) {
     to = mapped;
   }
 
+  if (to in STATIC_ASSETS) {
+    return to;
+  }
+
   if (path.isAbsolute(to)) {
     return to;
   }
diff --git a/src/rulesets/map.ts b/src/rulesets/map.ts
deleted file mode 100644
index fe8f980b..00000000
--- a/src/rulesets/map.ts
+++ /dev/null
@@ -1,9 +0,0 @@
-function resolveSpectralRuleset(ruleset: string) {
-  return `@stoplight/spectral/rulesets/${ruleset}/index.json`;
-}
-
-export const filesMap = new Map<string, string>([
-  ['spectral:oas', resolveSpectralRuleset('oas')],
-  ['spectral:oas2', resolveSpectralRuleset('oas2')],
-  ['spectral:oas3', resolveSpectralRuleset('oas3')],
-]);
diff --git a/src/spectral.ts b/src/spectral.ts
index 3c4fc545..38ebdc82 100644
--- a/src/spectral.ts
+++ b/src/spectral.ts
@@ -7,16 +7,16 @@ import {
 import { Resolver } from '@stoplight/json-ref-resolver';
 import { ICache, IUriParser } from '@stoplight/json-ref-resolver/types';
 import { extname } from '@stoplight/path';
-import { Dictionary } from '@stoplight/types';
+import { Dictionary, IDiagnostic } from '@stoplight/types';
 import {
   getLocationForJsonPath as getLocationForJsonPathYAML,
   parseWithPointers as parseYAMLWithPointers,
   YamlParserResult,
 } from '@stoplight/yaml';
+import deprecated from 'deprecated-decorator';
 import { merge, set } from 'lodash';
 
-import { IDiagnostic } from '@stoplight/types/dist';
-import deprecated from 'deprecated-decorator';
+import { STATIC_ASSETS } from './assets';
 import { formatParserDiagnostics, formatResolverErrors } from './error-messages';
 import { functions as defaultFunctions } from './functions';
 import { Resolved } from './resolved';
@@ -40,6 +40,7 @@ import {
   RunRuleCollection,
 } from './types';
 import { IRuleset } from './types/ruleset';
+import { empty } from './utils';
 
 export * from './types';
 
@@ -71,6 +72,11 @@ export class Spectral {
     }
   }
 
+  public static registerStaticAssets(assets: Dictionary<string, string>) {
+    empty(STATIC_ASSETS);
+    Object.assign(STATIC_ASSETS, assets);
+  }
+
   public async runWithResolved(
     target: IParsedResult | object | string,
     opts: IRunOpts = {},
@@ -136,10 +142,7 @@ export class Spectral {
   }
 
   public setFunctions(functions: FunctionCollection) {
-    for (const key in this.functions) {
-      if (!Object.hasOwnProperty.call(this.functions, key)) continue;
-      delete this.functions[key];
-    }
+    empty(this.functions);
 
     this._addFunctions({ ...defaultFunctions, ...functions });
   }
@@ -150,10 +153,7 @@ export class Spectral {
   }
 
   public setRules(rules: RuleCollection) {
-    for (const key in this.rules) {
-      if (!Object.hasOwnProperty.call(this.rules, key)) continue;
-      delete this.rules[key];
-    }
+    empty(this.rules);
 
     this._addRules({ ...rules });
   }
diff --git a/src/utils/empty.ts b/src/utils/empty.ts
new file mode 100644
index 00000000..cf80a31a
--- /dev/null
+++ b/src/utils/empty.ts
@@ -0,0 +1,10 @@
+import { Dictionary } from '@stoplight/types';
+
+export const empty = (obj: Dictionary<unknown>) => {
+  for (const key in obj) {
+    if (!Object.hasOwnProperty.call(obj, key)) continue;
+    delete obj[key];
+  }
+
+  return obj;
+};
diff --git a/src/utils/index.ts b/src/utils/index.ts
index 85f1d7b0..d075e157 100644
--- a/src/utils/index.ts
+++ b/src/utils/index.ts
@@ -1,3 +1,4 @@
+export * from './empty';
 export * from './hasRef';
 export * from './hasIntersectingElement';
 export * from './isObject';


Test patch:
diff --git a/src/rulesets/__tests__/reader.jest.test.ts b/src/rulesets/__tests__/reader.jest.test.ts
index c6de4c2e..07290a5b 100644
--- a/src/rulesets/__tests__/reader.jest.test.ts
+++ b/src/rulesets/__tests__/reader.jest.test.ts
@@ -3,11 +3,13 @@ import { Dictionary } from '@stoplight/types';
 import { DiagnosticSeverity } from '@stoplight/types';
 import * as fs from 'fs';
 import * as nock from 'nock';
+import { Spectral } from '../../spectral';
 import { IRule, Rule } from '../../types';
 import { readRuleset } from '../reader';
 const nanoid = require('nanoid');
 
 jest.mock('nanoid');
+jest.mock('fs');
 
 const validFlatRuleset = path.join(__dirname, './__fixtures__/valid-flat-ruleset.json');
 const validRequireInfo = path.join(__dirname, './__fixtures__/valid-require-info-ruleset.yaml');
@@ -44,6 +46,7 @@ describe('Rulesets reader', () => {
 
   afterEach(() => {
     nock.cleanAll();
+    nock.enableNetConnect();
   });
 
   it('given flat, valid ruleset file should return rules', async () => {
@@ -594,4 +597,41 @@ describe('Rulesets reader', () => {
   it('given invalid ruleset should output errors', () => {
     return expect(readRuleset(invalidRuleset)).rejects.toThrowError(/should have required property/);
   });
+
+  it('is able to load the whole ruleset from static file', async () => {
+    nock.disableNetConnect();
+
+    const readFileSpy = jest.spyOn(fs, 'readFile');
+
+    Spectral.registerStaticAssets(require('../../../rulesets/assets/assets.json'));
+
+    const { rules, functions } = await readRuleset('spectral:oas');
+
+    expect(rules).toMatchObject({
+      'openapi-tags': expect.objectContaining({
+        description: 'OpenAPI object should have non-empty `tags` array.',
+        formats: ['oas2', 'oas3'],
+      }),
+      'oas2-schema': expect.objectContaining({
+        description: 'Validate structure of OpenAPI v2 specification.',
+        formats: ['oas2'],
+      }),
+      'oas3-schema': expect.objectContaining({
+        description: 'Validate structure of OpenAPI v3 specification.',
+        formats: ['oas3'],
+      }),
+    });
+
+    expect(functions).toMatchObject({
+      oasOp2xxResponse: expect.any(Object),
+      oasOpFormDataConsumeCheck: expect.any(Object),
+      oasOpIdUnique: expect.any(Object),
+      oasOpParams: expect.any(Object),
+      oasOpSecurityDefined: expect.any(Object),
+      oasPathParam: expect.any(Object),
+    });
+
+    expect(readFileSpy).not.toBeCalled();
+    readFileSpy.mockRestore();
+  });
 });
diff --git a/src/rulesets/__tests__/reader.karma.test.ts b/src/rulesets/__tests__/reader.karma.test.ts
new file mode 100644
index 00000000..3f1839e4
--- /dev/null
+++ b/src/rulesets/__tests__/reader.karma.test.ts
@@ -0,0 +1,47 @@
+import { FetchMockSandbox } from 'fetch-mock';
+import { Spectral } from '../../spectral';
+import { readRuleset } from '../reader';
+
+declare const fetch: FetchMockSandbox;
+
+describe('Rulesets reader', () => {
+  afterEach(() => {
+    Spectral.registerStaticAssets({});
+  });
+
+  it('is able to load the whole ruleset from static file', async () => {
+    fetch.resetBehavior();
+    fetch.get('https://unpkg.com/@stoplight/spectral/rulesets/oas/index.json', {
+      status: 404,
+      body: {},
+    });
+
+    Spectral.registerStaticAssets(require('../../../rulesets/assets/assets.json'));
+
+    const { rules, functions } = await readRuleset('spectral:oas');
+
+    expect(rules).toMatchObject({
+      'openapi-tags': expect.objectContaining({
+        description: 'OpenAPI object should have non-empty `tags` array.',
+        formats: ['oas2', 'oas3'],
+      }),
+      'oas2-schema': expect.objectContaining({
+        description: 'Validate structure of OpenAPI v2 specification.',
+        formats: ['oas2'],
+      }),
+      'oas3-schema': expect.objectContaining({
+        description: 'Validate structure of OpenAPI v3 specification.',
+        formats: ['oas3'],
+      }),
+    });
+
+    expect(functions).toMatchObject({
+      oasOp2xxResponse: expect.any(Object),
+      oasOpFormDataConsumeCheck: expect.any(Object),
+      oasOpIdUnique: expect.any(Object),
+      oasOpParams: expect.any(Object),
+      oasOpSecurityDefined: expect.any(Object),
+      oasPathParam: expect.any(Object),
+    });
+  });
+});