Repository: stoplightio/spectral (ts)

Issue:
Respect HTTP_PROXY and HTTPS_PROXY
**User story.**
As a user running behind a corporate proxy, I can use Spectral from JS, so that I can reap its benefits.

**Is your feature request related to a problem?**
Spectral doesn't work behind a corporate proxy if you're not using the CLI.

**Describe the solution you'd like**
Spectral should try to use a proxy when it is instantiated [from code](https://github.com/stoplightio/spectral/blob/b893654856f3e222678806070128c12dedc66003/src/spectral.ts#L50) as documented [here](https://stoplight.io/p/docs/gh/stoplightio/spectral/docs/guides/javascript.md) and not only when it's called [via CLI](https://github.com/stoplightio/spectral/blob/b893654856f3e222678806070128c12dedc66003/src/cli/index.ts#L10).

AFAIK, HTTPS_PROXY, HTTP_PROXY and their lowercase cousins are standard environment variables for setting proxies. Many people will already have them set if they are using a proxy. In order, Spectral should try to use:
- HTTPS_PROXY
- https_proxy
- HTTP_PROXY
- http_proxy
- PROXY
- proxy

**Additional context**
I'm calling Spectral within a TS project and calls to loadRuleset were taking > 2 minutes because requests to http://json-schema.org were timing out when I extended the `spectral:oas` ruleset.

Happy to try my hand at a PR if you agree with the proposed solution?

Pull request:
feat: support proxies via JS
Fixes #1081.

**Checklist**

- [x] Tests added / updated
- [x] Docs added / updated

**Does this PR introduce a breaking change?**

- [ ] Yes
- [x] No

**Additional context**

This fixes #1081 (after a long delay -- my bad) by allowing a proxy to be passed to the `Spectral` constructor. It allows remote $refs and rulesets to be loaded from behind a proxy.

Patch:
diff --git a/docs/guides/3-javascript.md b/docs/guides/3-javascript.md
index defcb437..7264aa37 100644
--- a/docs/guides/3-javascript.md
+++ b/docs/guides/3-javascript.md
@@ -192,6 +192,19 @@ spectral
   });
 ```
 
+### Using a proxy
+
+Spectral supports http(s) proxies when fetching remote schemas and rulesets.
+
+```js
+const { Spectral } = require('@stoplight/spectral');
+
+const spectral = new Spectral({ proxyUri: 'http://my-proxy:3000' });
+spectral.loadRuleset('https://example.org/my-rules')
+
+// lint as usual - $refs and rules will be requested using the proxy
+```
+
 ### Using custom resolver
 
 Spectral lets you provide any custom $ref resolver. By default, http(s) and file protocols are resolved, relatively to
diff --git a/src/cli/services/linter/utils/getResolver.ts b/src/cli/services/linter/utils/getResolver.ts
index fd3322a3..0c984cbd 100644
--- a/src/cli/services/linter/utils/getResolver.ts
+++ b/src/cli/services/linter/utils/getResolver.ts
@@ -1,6 +1,6 @@
 import { isAbsolute, join } from '@stoplight/path';
 import { Optional } from '@stoplight/types';
-import { httpAndFileResolver } from '../../../../resolvers/http-and-file';
+import { createHttpAndFileResolver } from '../../../../resolvers/http-and-file';
 
 export const getResolver = (resolver: Optional<string>) => {
   if (resolver) {
@@ -11,7 +11,7 @@ export const getResolver = (resolver: Optional<string>) => {
     }
   }
 
-  return httpAndFileResolver;
+  return createHttpAndFileResolver();
 };
 
 function formatMessage(message: string): Optional<string> {
diff --git a/src/fs/reader.ts b/src/fs/reader.ts
index d413178d..56e03727 100644
--- a/src/fs/reader.ts
+++ b/src/fs/reader.ts
@@ -1,12 +1,15 @@
 import { isURL } from '@stoplight/path';
 import AbortController from 'abort-controller';
 import * as fs from 'fs';
+import { RequestInit } from 'node-fetch';
 import { STATIC_ASSETS } from '../assets';
 import request from '../request';
+import { Agent } from 'http';
 
 export interface IReadOptions {
   encoding: string;
   timeout?: number;
+  agent?: Agent;
 }
 
 export async function readFile(name: string | number, opts: IReadOptions): Promise<string> {
@@ -36,16 +39,18 @@ export async function readFile(name: string | number, opts: IReadOptions): Promi
     let response;
     let timeout: NodeJS.Timeout | number | null = null;
     try {
+      const requestOpts: RequestInit = {};
+      requestOpts.agent = opts.agent;
       if (opts.timeout) {
         const controller = new AbortController();
         timeout = setTimeout(() => {
           controller.abort();
         }, opts.timeout);
-        response = await request(name, { signal: controller.signal });
-      } else {
-        response = await request(name);
+        requestOpts.signal = controller.signal;
       }
 
+      response = await request(name, requestOpts);
+
       if (!response.ok) throw new Error(response.statusText);
       return await response.text();
     } catch (ex) {
diff --git a/src/resolvers/http-and-file.ts b/src/resolvers/http-and-file.ts
index a38c9252..a1487f2d 100644
--- a/src/resolvers/http-and-file.ts
+++ b/src/resolvers/http-and-file.ts
@@ -1,14 +1,23 @@
 import { createResolveHttp, resolveFile } from '@stoplight/json-ref-readers';
 import { Resolver } from '@stoplight/json-ref-resolver';
 import { DEFAULT_REQUEST_OPTIONS } from '../request';
+import { Agent } from 'http';
 
-const resolveHttp = createResolveHttp(DEFAULT_REQUEST_OPTIONS);
+export interface IHttpAndFileResolverOptions {
+  agent?: Agent;
+}
+
+export const httpAndFileResolver = createHttpAndFileResolver();
 
 // resolves files, http and https $refs, and internal $refs
-export const httpAndFileResolver = new Resolver({
-  resolvers: {
-    https: { resolve: resolveHttp },
-    http: { resolve: resolveHttp },
-    file: { resolve: resolveFile },
-  },
-});
+export function createHttpAndFileResolver(opts?: IHttpAndFileResolverOptions): Resolver {
+  const resolveHttp = createResolveHttp({ ...DEFAULT_REQUEST_OPTIONS, ...opts });
+
+  return new Resolver({
+    resolvers: {
+      https: { resolve: resolveHttp },
+      http: { resolve: resolveHttp },
+      file: { resolve: resolveFile },
+    },
+  });
+}
diff --git a/src/rulesets/reader.ts b/src/rulesets/reader.ts
index c6e4126b..6015d2ec 100644
--- a/src/rulesets/reader.ts
+++ b/src/rulesets/reader.ts
@@ -4,14 +4,14 @@ import { extname, join } from '@stoplight/path';
 import { Optional } from '@stoplight/types';
 import { parse } from '@stoplight/yaml';
 import { readFile, readParsable } from '../fs/reader';
-import { httpAndFileResolver } from '../resolvers/http-and-file';
+import { createHttpAndFileResolver, IHttpAndFileResolverOptions } from '../resolvers/http-and-file';
 import { FileRulesetSeverity, IRuleset, RulesetFunctionCollection } from '../types/ruleset';
 import { findFile, isNPMSource } from './finder';
 import { mergeFormats, mergeFunctions, mergeRules } from './mergers';
 import { mergeExceptions } from './mergers/exceptions';
 import { assertValidRuleset } from './validation';
 
-export interface IRulesetReadOptions {
+export interface IRulesetReadOptions extends IHttpAndFileResolverOptions {
   timeout?: number;
 }
 
@@ -61,11 +61,12 @@ const createRulesetProcessor = (
     }
 
     processedRulesets.add(rulesetUri);
-    const { result } = await httpAndFileResolver.resolve(
+    const { result } = await createHttpAndFileResolver({ agent: readOpts?.agent }).resolve(
       parseContent(
         await readParsable(rulesetUri, {
           timeout: readOpts?.timeout,
           encoding: 'utf8',
+          agent: readOpts?.agent,
         }),
         rulesetUri,
       ),
@@ -145,6 +146,7 @@ const createRulesetProcessor = (
               code: await readFile(source, {
                 timeout: readOpts?.timeout,
                 encoding: 'utf8',
+                agent: readOpts?.agent,
               }),
               schema: fnSchema,
               source,
diff --git a/src/spectral.ts b/src/spectral.ts
index 1489dafc..b407be93 100644
--- a/src/spectral.ts
+++ b/src/spectral.ts
@@ -3,6 +3,7 @@ import { Resolver } from '@stoplight/json-ref-resolver';
 import { DiagnosticSeverity, Dictionary, Optional } from '@stoplight/types';
 import { YamlParserResult } from '@stoplight/yaml';
 import { memoize, merge } from 'lodash';
+import type { Agent } from 'http';
 
 import { STATIC_ASSETS } from './assets';
 import { Document, IDocument, IParsedResult, isParsedResult, ParsedDocument, normalizeSource } from './document';
@@ -10,6 +11,7 @@ import { DocumentInventory } from './documentInventory';
 import { CoreFunctions, functions as coreFunctions } from './functions';
 import * as Parsers from './parsers';
 import request from './request';
+import { createHttpAndFileResolver } from './resolvers/http-and-file';
 import { OptimizedRule, Rule } from './rule';
 import { readRuleset } from './rulesets';
 import { compileExportedFunction, setFunctionContext } from './rulesets/evaluators';
@@ -40,6 +42,7 @@ export * from './types';
 
 export class Spectral {
   private readonly _resolver: IResolver;
+  private readonly agent: Agent | undefined;
 
   public functions: FunctionCollection & CoreFunctions = { ...coreFunctions };
   public rules: RunRuleCollection = {};
@@ -52,7 +55,19 @@ export class Spectral {
 
   constructor(protected readonly opts?: IConstructorOpts) {
     this._computeFingerprint = memoize(opts?.computeFingerprint ?? defaultComputeResultFingerprint);
-    this._resolver = opts?.resolver ?? new Resolver();
+
+    if (opts?.proxyUri) {
+      // using eval so bundlers do not include proxy-agent when Spectral is used in the browser
+      const ProxyAgent = eval('require')('proxy-agent');
+      this.agent = new ProxyAgent(opts.proxyUri);
+    }
+    if (opts?.resolver) {
+      this._resolver = opts.resolver;
+    } else {
+      this._resolver =
+        typeof window === 'undefined' ? createHttpAndFileResolver({ agent: this.agent }) : new Resolver();
+    }
+
     this.formats = {};
     this.runtime = new RunnerRuntime();
   }
@@ -161,7 +176,7 @@ export class Spectral {
   }
 
   public async loadRuleset(uris: string[] | string, options?: IRulesetReadOptions) {
-    this.setRuleset(await readRuleset(Array.isArray(uris) ? uris : [uris], options));
+    this.setRuleset(await readRuleset(Array.isArray(uris) ? uris : [uris], { agent: this.agent, ...options }));
   }
 
   public setRuleset(ruleset: IRuleset) {
diff --git a/src/types/spectral.ts b/src/types/spectral.ts
index 2bd4c333..bc93987c 100644
--- a/src/types/spectral.ts
+++ b/src/types/spectral.ts
@@ -23,6 +23,7 @@ export interface IConstructorOpts {
   resolver?: IResolver;
   computeFingerprint?: ComputeFingerprintFunc;
   useNimma?: boolean;
+  proxyUri?: string;
 }
 
 export interface IRunOpts {


Test patch:
diff --git a/src/__tests__/linter.karma.test.ts b/src/__tests__/linter.karma.test.ts
index 89d5101c..13e0ad95 100644
--- a/src/__tests__/linter.karma.test.ts
+++ b/src/__tests__/linter.karma.test.ts
@@ -15,6 +15,18 @@ describe('Linter', () => {
     window.fetch = fetch;
   });
 
+  describe('default resolver', () => {
+    it('should not use a resolver that depends on the Node.js "fs" module', async () => {
+      const result = await spectral.run({
+        info: {
+          $ref: './foo/bar.json',
+        },
+      });
+
+      expect(result[0]).toHaveProperty('message', "No resolver defined for scheme 'file' in ref ./foo/bar.json");
+    });
+  });
+
   describe('custom functions', () => {
     it('should be able to make a request using fetch', async () => {
       fetchMock.mock('https://stoplight.io', {
diff --git a/src/__tests__/linter.test.ts b/src/__tests__/linter.test.ts
index 5a41383e..66ec4bd2 100644
--- a/src/__tests__/linter.test.ts
+++ b/src/__tests__/linter.test.ts
@@ -713,8 +713,11 @@ responses:: !!foo
     );
   });
 
-  test('should report invalid $refs', async () => {
-    const result = await spectral.run(invalidSchema);
+  test('should report when a resolver is not defined for a given $ref type', async () => {
+    const s = new Spectral({ resolver: new Resolver() });
+    s.registerFormat('oas3', isOpenApiv3);
+
+    const result = await s.run(invalidSchema);
 
     expect(result).toEqual([
       expect.objectContaining({
diff --git a/src/__tests__/request.jest.test.ts b/src/__tests__/request.jest.test.ts
index d6a5b671..cf883b47 100644
--- a/src/__tests__/request.jest.test.ts
+++ b/src/__tests__/request.jest.test.ts
@@ -1,46 +1,46 @@
 import * as http from 'http';
 import * as url from 'url';
 import { DEFAULT_REQUEST_OPTIONS } from '../request';
-import { httpAndFileResolver } from '../resolvers/http-and-file';
+import { createHttpAndFileResolver } from '../resolvers/http-and-file';
 import { Spectral } from '../spectral';
 const ProxyAgent = require('proxy-agent');
 
 const PORT = 4001;
 
 describe('request', () => {
-  describe('when agent is set', () => {
-    let server: http.Server;
-
-    beforeAll(() => {
-      // nock cannot mock proxied requests
-      server = http
-        .createServer((req, res) => {
-          const { pathname } = url.parse(String(req.url));
-          if (pathname === '/custom-ruleset') {
-            res.writeHead(403);
-          } else if (pathname === '/ok.json') {
-            res.writeHead(200, { 'Content-Type': 'application/json' });
-            res.write(
-              JSON.stringify({
-                info: {
-                  title: '',
-                  description: 'Foo',
-                },
-              }),
-            );
-          } else {
-            res.writeHead(404);
-          }
-
-          res.end();
-        })
-        .listen(PORT, '0.0.0.0');
-    });
+  let server: http.Server;
 
-    afterAll(() => {
-      server.close();
-    });
+  beforeAll(() => {
+    // nock cannot mock proxied requests
+    server = http
+      .createServer((req, res) => {
+        const { pathname } = url.parse(String(req.url));
+        if (pathname === '/custom-ruleset') {
+          res.writeHead(403);
+        } else if (pathname === '/ok.json') {
+          res.writeHead(200, { 'Content-Type': 'application/json' });
+          res.write(
+            JSON.stringify({
+              info: {
+                title: '',
+                description: 'Foo',
+              },
+            }),
+          );
+        } else {
+          res.writeHead(404);
+        }
+
+        res.end();
+      })
+      .listen(PORT, '0.0.0.0');
+  });
 
+  afterAll(() => {
+    server.close();
+  });
+
+  describe('when agent is set', () => {
     beforeEach(() => {
       DEFAULT_REQUEST_OPTIONS.agent = new ProxyAgent(`http://localhost:${PORT}`);
     });
@@ -68,7 +68,39 @@ describe('request', () => {
           },
         };
 
-        const s = new Spectral({ resolver: httpAndFileResolver });
+        const s = new Spectral({ resolver: createHttpAndFileResolver() });
+
+        return expect(s.runWithResolved(doc)).resolves.toHaveProperty('resolved', {
+          info: {
+            title: '',
+            description: 'Foo',
+          },
+        });
+      });
+    });
+  });
+
+  describe('when proxy uri is passed to Spectral constructor', () => {
+    describe('loading a ruleset', () => {
+      it('proxies the request', () => {
+        const s = new Spectral({ proxyUri: `http://localhost:${PORT}` });
+
+        return expect(s.loadRuleset('http://localhost:4000/custom-ruleset')).rejects.toHaveProperty(
+          'message',
+          'Could not parse http://localhost:4000/custom-ruleset: Forbidden',
+        );
+      });
+    });
+
+    describe('loading a $ref', () => {
+      it('proxies the request', () => {
+        const doc = {
+          info: {
+            $ref: 'http://localhost:8089/ok.json#/info',
+          },
+        };
+
+        const s = new Spectral({ proxyUri: `http://localhost:${PORT}` });
 
         return expect(s.runWithResolved(doc)).resolves.toHaveProperty('resolved', {
           info: {
diff --git a/src/cli/services/linter/utils/__tests__/getResolver.test.ts b/src/cli/services/linter/utils/__tests__/getResolver.test.ts
index 22d65fe5..a1da0512 100644
--- a/src/cli/services/linter/utils/__tests__/getResolver.test.ts
+++ b/src/cli/services/linter/utils/__tests__/getResolver.test.ts
@@ -1,5 +1,5 @@
 import { join, relative } from '@stoplight/path';
-import { httpAndFileResolver } from '../../../../../resolvers/http-and-file';
+import { createHttpAndFileResolver } from '../../../../../resolvers/http-and-file';
 import { getResolver } from '../getResolver';
 
 const customResolver = require('./__fixtures__/resolver');
@@ -21,6 +21,6 @@ describe('getResolver', () => {
   });
 
   it('given no path, returns default resolver', () => {
-    expect(getResolver(void 0)).toBe(httpAndFileResolver);
+    expect(JSON.stringify(getResolver(void 0))).toEqual(JSON.stringify(createHttpAndFileResolver()));
   });
 });