Repository: stoplightio/spectral (ts)

Issue:
Warnings are not being shown in junit output
> **Update from Stoplight/Phil:** Let's go with the "Convert anything that meets the fail-severity to an error in the case of JUnit" option. It makes sense.

We use the junit output to show errors in Gitlab's Merge Requests, and it's worked great until now.  After upgrading to v5, we were trying to setup our builds to fail when spectral catches an `error`, but would also like to show warnings.  However, it appears that the `junit` output doesn't display warnings, but other formatters do (checked with `stylish` and `json`, but haven't rigorously tested all the others).

With `stylish` formatter, warnings show:
![image](https://user-images.githubusercontent.com/3663141/72750150-a8fa0900-3b81-11ea-826e-9359bafa862f.png)

With `junit` formatter, no warnings shown:
![image](https://user-images.githubusercontent.com/3663141/72751096-3cccd480-3b84-11ea-93ee-6dcbde37e20a.png)

I'd be happy to try and cook up an example if it's needed, but my hope is that it was just either a) an oversight on our part, b) something that's easily spotted to fix or c) intentional but undocumented behavior.

Let me know if you need more information!
Thanks!


Pull request:
 fix(cli): JUnit formatter respects fail-severity 
Fixes #935 and the lack of choices as reported in https://github.com/stoplightio/spectral/issues/935#issuecomment-698332260 by @fgabolde

**Checklist**

- [x] Tests added / updated
- [x] Docs added / updated

**Does this PR introduce a breaking change?**

- [ ] Yes
- [x] No



Patch:
diff --git a/docs/guides/2-cli.md b/docs/guides/2-cli.md
index 872758ba..e85d462c 100644
--- a/docs/guides/2-cli.md
+++ b/docs/guides/2-cli.md
@@ -24,13 +24,14 @@ Other options include:
   --version                    Show version number                                          [boolean]
   --help                       Show help                                                    [boolean]
   --encoding, -e               text encoding to use                        [string] [default: "utf8"]
-  --format, -f                 formatter to use for outputting results  [string] [default: "stylish"]
+  --format, -f                 formatter to use for outputting results
+      [string] [choices: "json", "stylish", "junit", "html", "text", "teamcity"] [default: "stylish"]
   --output, -o                 output to a file instead of stdout                            [string]
   --resolver                   path to custom json-ref-resolver instance                     [string]
   --ruleset, -r                path/URL to a ruleset file                                    [string]
   --skip-rule, -s              ignore certain rules if they are causing trouble              [string]
   --fail-severity, -F          results of this level or above will trigger a failure exit code
-                                [string] [choices: "error", "warn", "info", "hint"] [default: "warn"]
+                               [string] [choices: "error", "warn", "info", "hint"] [default: "error"]
   --display-only-failures, -D  only output results equal to or greater than --fail-severity
                                                                            [boolean] [default: false]
   --ignore-unknown-format      do not warn about unmatched formats         [boolean] [default: false]
diff --git a/package.json b/package.json
index f27a5da0..64b6c45e 100644
--- a/package.json
+++ b/package.json
@@ -43,10 +43,10 @@
     "inline-version": "node ./scripts/inline-version.js",
     "prelint": "tsc --noEmit",
     "lint.fix": "yarn lint.prettier --write && yarn lint.eslint --fix",
-    "lint.eslint":" eslint --cache --cache-location .cache/ --ext=.js,.ts src test-harness",
+    "lint.eslint": "eslint --cache --cache-location .cache/ --ext=.js,.ts src test-harness",
     "lint.prettier": "prettier --ignore-path .eslintignore --check {src,test-harness}/**/*.{json,yaml} docs/**/*.md README.md",
     "lint": "yarn lint.prettier && yarn lint.eslint",
-    "copy.html-templates": "copyfiles -u 1 \"./src/formatters/html/*.html\" \"./dist/\"",
+    "copy.html-templates": "copyfiles -u 1 \"./src/cli/formatters/html/*.html\" \"./dist/\"",
     "postbuild.functions": "copyfiles -u 1 \"dist/rulesets/{oas,asyncapi}/functions/*.js\" ./",
     "postbuild": "yarn build.functions && yarn generate-assets",
     "prebuild": "yarn build.clean && copyfiles -u 1 \"src/rulesets/{oas,asyncapi}/**/*.json\" dist && copyfiles -u 1 \"src/rulesets/{oas,asyncapi}/**/*.json\" ./ && yarn copy.html-templates",
diff --git a/src/cli/commands/lint.ts b/src/cli/commands/lint.ts
index 51978780..8e8994b2 100644
--- a/src/cli/commands/lint.ts
+++ b/src/cli/commands/lint.ts
@@ -62,7 +62,7 @@ const lintCommand: CommandModule = {
         format: {
           alias: 'f',
           description: 'formatter to use for outputting results',
-          options: formatOptions,
+          choices: formatOptions,
           default: OutputFormat.STYLISH,
           type: 'string',
         },
@@ -161,7 +161,7 @@ const lintCommand: CommandModule = {
         } else if (!config.quiet) {
           console.log(`No results with a severity of '${failSeverity}' or higher found!`);
         }
-        const formattedOutput = formatOutput(results, format);
+        const formattedOutput = formatOutput(results, format, { failSeverity: getDiagnosticSeverity(failSeverity) });
         return writeOutput(formattedOutput, output);
       })
       .catch(fail);
diff --git a/src/formatters/html/html-template-message.html b/src/cli/formatters/html/html-template-message.html
similarity index 100%
rename from src/formatters/html/html-template-message.html
rename to src/cli/formatters/html/html-template-message.html
diff --git a/src/formatters/html/html-template-page.html b/src/cli/formatters/html/html-template-page.html
similarity index 100%
rename from src/formatters/html/html-template-page.html
rename to src/cli/formatters/html/html-template-page.html
diff --git a/src/formatters/html/html-template-result.html b/src/cli/formatters/html/html-template-result.html
similarity index 100%
rename from src/formatters/html/html-template-result.html
rename to src/cli/formatters/html/html-template-result.html
diff --git a/src/formatters/html/index.ts b/src/cli/formatters/html/index.ts
similarity index 98%
rename from src/formatters/html/index.ts
rename to src/cli/formatters/html/index.ts
index 505df7e6..0ba348d2 100644
--- a/src/formatters/html/index.ts
+++ b/src/cli/formatters/html/index.ts
@@ -27,7 +27,7 @@ import { Dictionary } from '@stoplight/types';
 import * as eol from 'eol';
 import * as fs from 'fs';
 import { template } from 'lodash';
-import { IRuleResult } from '../../types';
+import { IRuleResult } from '../../../types';
 import { Formatter } from '../types';
 import { getHighestSeverity, getSeverityName, getSummary, getSummaryForSource, groupBySource } from '../utils';
 
diff --git a/src/formatters/index.ts b/src/cli/formatters/index.ts
similarity index 100%
rename from src/formatters/index.ts
rename to src/cli/formatters/index.ts
diff --git a/src/formatters/json.ts b/src/cli/formatters/json.ts
similarity index 100%
rename from src/formatters/json.ts
rename to src/cli/formatters/json.ts
diff --git a/src/formatters/junit.ts b/src/cli/formatters/junit.ts
similarity index 92%
rename from src/formatters/junit.ts
rename to src/cli/formatters/junit.ts
index 5acfe38f..c91e4ca5 100644
--- a/src/formatters/junit.ts
+++ b/src/cli/formatters/junit.ts
@@ -24,13 +24,12 @@
  */
 
 import { extname } from '@stoplight/path';
-import { DiagnosticSeverity } from '@stoplight/types';
 import { escapeRegExp } from 'lodash';
-import { printPath, PrintStyle } from '../utils';
+import { printPath, PrintStyle } from '../../utils';
 import { Formatter } from './types';
 import { groupBySource, xmlEscape } from './utils';
 
-export const junit: Formatter = results => {
+export const junit: Formatter = (results, { failSeverity }) => {
   let output = '';
 
   output += '<?xml version="1.0" encoding="utf-8"?>\n';
@@ -42,9 +41,7 @@ export const junit: Formatter = results => {
     const classname = source.replace(new RegExp(`${escapeRegExp(extname(source))}$`), '');
 
     if (validationResults.length > 0) {
-      const filteredValidationResults = validationResults.filter(
-        result => result.severity === DiagnosticSeverity.Error,
-      );
+      const filteredValidationResults = validationResults.filter(result => result.severity <= failSeverity);
 
       output += `<testsuite package="org.spectral" time="0" tests="${filteredValidationResults.length}" errors="0" failures="${filteredValidationResults.length}" name="${source}">\n`;
 
diff --git a/src/formatters/stylish.ts b/src/cli/formatters/stylish.ts
similarity index 96%
rename from src/formatters/stylish.ts
rename to src/cli/formatters/stylish.ts
index 20a5d0ad..b81066fb 100644
--- a/src/formatters/stylish.ts
+++ b/src/cli/formatters/stylish.ts
@@ -28,7 +28,7 @@ import * as chalk from 'chalk';
 import stripAnsi = require('strip-ansi');
 import * as table from 'text-table';
 
-import { IRuleResult } from '../types';
+import { IRuleResult } from '../../types';
 import { Formatter } from './types';
 import { getColorForSeverity, getHighestSeverity, getSeverityName, getSummary, groupBySource } from './utils';
 
@@ -42,7 +42,7 @@ function formatRange(range?: IRange): string {
   return ` ${range.start.line + 1}:${range.start.character + 1}`;
 }
 
-function getMessageType(severity: DiagnosticSeverity) {
+function getMessageType(severity: DiagnosticSeverity): string {
   const color = getColorForSeverity(severity);
   const name = getSeverityName(severity);
 
diff --git a/src/formatters/teamcity.ts b/src/cli/formatters/teamcity.ts
similarity index 97%
rename from src/formatters/teamcity.ts
rename to src/cli/formatters/teamcity.ts
index eee95e4a..4f4ea67e 100644
--- a/src/formatters/teamcity.ts
+++ b/src/cli/formatters/teamcity.ts
@@ -1,5 +1,5 @@
 import { Dictionary, Optional } from '@stoplight/types';
-import { IRuleResult } from '../types';
+import { IRuleResult } from '../../types';
 import { Formatter } from './types';
 import { getSeverityName, groupBySource } from './utils';
 
diff --git a/src/formatters/text.ts b/src/cli/formatters/text.ts
similarity index 95%
rename from src/formatters/text.ts
rename to src/cli/formatters/text.ts
index 9cf53d74..9730dbee 100644
--- a/src/formatters/text.ts
+++ b/src/cli/formatters/text.ts
@@ -1,5 +1,5 @@
 import { Dictionary } from '@stoplight/types';
-import { IRuleResult } from '../types';
+import { IRuleResult } from '../../types';
 import { Formatter } from './types';
 import { getSeverityName, groupBySource } from './utils';
 
diff --git a/src/cli/formatters/types.ts b/src/cli/formatters/types.ts
new file mode 100644
index 00000000..e1049a14
--- /dev/null
+++ b/src/cli/formatters/types.ts
@@ -0,0 +1,8 @@
+import { IRuleResult } from '../../types';
+import type { DiagnosticSeverity } from '@stoplight/types';
+
+export type FormatterOptions = {
+  failSeverity: DiagnosticSeverity;
+};
+
+export type Formatter = (results: IRuleResult[], options: FormatterOptions) => string;
diff --git a/src/formatters/utils/getColorForSeverity.ts b/src/cli/formatters/utils/getColorForSeverity.ts
similarity index 100%
rename from src/formatters/utils/getColorForSeverity.ts
rename to src/cli/formatters/utils/getColorForSeverity.ts
diff --git a/src/formatters/utils/getHighestSeverity.ts b/src/cli/formatters/utils/getHighestSeverity.ts
similarity index 82%
rename from src/formatters/utils/getHighestSeverity.ts
rename to src/cli/formatters/utils/getHighestSeverity.ts
index 46672dc8..97d42b03 100644
--- a/src/formatters/utils/getHighestSeverity.ts
+++ b/src/cli/formatters/utils/getHighestSeverity.ts
@@ -1,5 +1,5 @@
 import { DiagnosticSeverity } from '@stoplight/types';
-import { IRuleResult } from '../../types';
+import { IRuleResult } from '../../../types';
 
 export const getHighestSeverity = (results: IRuleResult[]) =>
   results.length === 0 ? DiagnosticSeverity.Hint : Math.min(...results.map(({ severity }) => severity));
diff --git a/src/formatters/utils/getSeverityName.ts b/src/cli/formatters/utils/getSeverityName.ts
similarity index 100%
rename from src/formatters/utils/getSeverityName.ts
rename to src/cli/formatters/utils/getSeverityName.ts
diff --git a/src/formatters/utils/getSummary.ts b/src/cli/formatters/utils/getSummary.ts
similarity index 97%
rename from src/formatters/utils/getSummary.ts
rename to src/cli/formatters/utils/getSummary.ts
index 5a47cc27..c3d5f170 100644
--- a/src/formatters/utils/getSummary.ts
+++ b/src/cli/formatters/utils/getSummary.ts
@@ -1,5 +1,5 @@
 import { DiagnosticSeverity, Dictionary } from '@stoplight/types';
-import { IRuleResult } from '../../types';
+import { IRuleResult } from '../../../types';
 import { groupBySeverity } from './groupBySeverity';
 import { pluralize } from './pluralize';
 
diff --git a/src/formatters/utils/groupBySeverity.ts b/src/cli/formatters/utils/groupBySeverity.ts
similarity index 91%
rename from src/formatters/utils/groupBySeverity.ts
rename to src/cli/formatters/utils/groupBySeverity.ts
index f1d5f85b..2b8dd18d 100644
--- a/src/formatters/utils/groupBySeverity.ts
+++ b/src/cli/formatters/utils/groupBySeverity.ts
@@ -1,5 +1,5 @@
 import { DiagnosticSeverity, Dictionary } from '@stoplight/types';
-import { IRuleResult } from '../../types';
+import { IRuleResult } from '../../../types';
 
 export const groupBySeverity = (results: IRuleResult[]) =>
   results.reduce<Dictionary<IRuleResult[], DiagnosticSeverity>>(
diff --git a/src/formatters/utils/groupBySource.ts b/src/cli/formatters/utils/groupBySource.ts
similarity index 87%
rename from src/formatters/utils/groupBySource.ts
rename to src/cli/formatters/utils/groupBySource.ts
index e285dd47..7a302144 100644
--- a/src/formatters/utils/groupBySource.ts
+++ b/src/cli/formatters/utils/groupBySource.ts
@@ -1,5 +1,5 @@
 import { Dictionary } from '@stoplight/types';
-import { IRuleResult } from '../../types';
+import { IRuleResult } from '../../../types';
 
 export const groupBySource = (results: IRuleResult[]): Dictionary<IRuleResult[]> => {
   return results.reduce<Dictionary<IRuleResult[]>>((grouped, result) => {
diff --git a/src/cli/services/output.ts b/src/cli/services/output.ts
index 6ec81f4b..e1fdf7ee 100644
--- a/src/cli/services/output.ts
+++ b/src/cli/services/output.ts
@@ -1,8 +1,8 @@
 import { Dictionary } from '@stoplight/types';
 import { writeFile } from 'fs';
 import { promisify } from 'util';
-import { html, json, junit, stylish, teamcity, text } from '../../formatters';
-import { Formatter } from '../../formatters/types';
+import { html, json, junit, stylish, teamcity, text } from '../formatters';
+import { Formatter, FormatterOptions } from '../formatters/types';
 import { IRuleResult } from '../../types';
 import { OutputFormat } from '../../types/config';
 
@@ -17,8 +17,8 @@ const formatters: Dictionary<Formatter, OutputFormat> = {
   teamcity,
 };
 
-export function formatOutput(results: IRuleResult[], format: OutputFormat): string {
-  return formatters[format](results);
+export function formatOutput(results: IRuleResult[], format: OutputFormat, formatOptions: FormatterOptions): string {
+  return formatters[format](results, formatOptions);
 }
 
 export async function writeOutput(outputStr: string, outputFile?: string) {
diff --git a/src/formatters/types.ts b/src/formatters/types.ts
deleted file mode 100644
index 3632c747..00000000
--- a/src/formatters/types.ts
+++ /dev/null
@@ -1,3 +0,0 @@
-import { IRuleResult } from '../types';
-
-export type Formatter = (results: IRuleResult[]) => string;


Test patch:
diff --git a/src/cli/commands/__tests__/lint.test.ts b/src/cli/commands/__tests__/lint.test.ts
index 0b5d35b5..9693a3c1 100644
--- a/src/cli/commands/__tests__/lint.test.ts
+++ b/src/cli/commands/__tests__/lint.test.ts
@@ -1,6 +1,6 @@
 import * as yargs from 'yargs';
 
-import { DiagnosticSeverity } from '@stoplight/types/dist';
+import { DiagnosticSeverity } from '@stoplight/types';
 import { IRuleResult } from '../../../types';
 import { lint } from '../../services/linter';
 import { formatOutput, writeOutput } from '../../services/output';
@@ -150,7 +150,7 @@ describe('lint', () => {
     // needed by Node 8 (different ticking?) - can be simplified once we drop support for version 8
     await new Promise(resolve => {
       setImmediate(() => {
-        expect(formatOutput).toBeCalledWith(results, format);
+        expect(formatOutput).toBeCalledWith(results, format, { failSeverity: DiagnosticSeverity.Error });
         resolve();
       });
     });
diff --git a/src/formatters/__tests__/__fixtures__/mixed-errors-with-different-paths.json b/src/cli/formatters/__tests__/__fixtures__/mixed-errors-with-different-paths.json
similarity index 100%
rename from src/formatters/__tests__/__fixtures__/mixed-errors-with-different-paths.json
rename to src/cli/formatters/__tests__/__fixtures__/mixed-errors-with-different-paths.json
diff --git a/src/formatters/__tests__/__fixtures__/mixed-errors.json b/src/cli/formatters/__tests__/__fixtures__/mixed-errors.json
similarity index 100%
rename from src/formatters/__tests__/__fixtures__/mixed-errors.json
rename to src/cli/formatters/__tests__/__fixtures__/mixed-errors.json
diff --git a/src/formatters/__tests__/__fixtures__/oas3-schema-errors.json b/src/cli/formatters/__tests__/__fixtures__/oas3-schema-errors.json
similarity index 100%
rename from src/formatters/__tests__/__fixtures__/oas3-schema-errors.json
rename to src/cli/formatters/__tests__/__fixtures__/oas3-schema-errors.json
diff --git a/src/formatters/__tests__/html.test.ts b/src/cli/formatters/__tests__/html.test.ts
similarity index 89%
rename from src/formatters/__tests__/html.test.ts
rename to src/cli/formatters/__tests__/html.test.ts
index ec67396a..66b29999 100644
--- a/src/formatters/__tests__/html.test.ts
+++ b/src/cli/formatters/__tests__/html.test.ts
@@ -1,12 +1,13 @@
+import { DiagnosticSeverity } from '@stoplight/types';
 import { HTMLElement, parse } from 'node-html-parser';
-import { sortResults } from '../../utils';
+import { sortResults } from '../../../utils';
 import { html } from '../html';
 
 const mixedErrors = sortResults(require('./__fixtures__/mixed-errors.json'));
 
 describe('HTML formatter', () => {
   test('should display proper severity levels', () => {
-    const result = parse(html(mixedErrors)) as HTMLElement;
+    const result = parse(html(mixedErrors, { failSeverity: DiagnosticSeverity.Error })) as HTMLElement;
     const table = result.querySelector('table tbody');
     expect(table.innerHTML.trim()).toEqual(`<tr class="bg-error" data-group="f-0">
     <th colspan="4">
diff --git a/src/formatters/__tests__/json.test.ts b/src/cli/formatters/__tests__/json.test.ts
similarity index 83%
rename from src/formatters/__tests__/json.test.ts
rename to src/cli/formatters/__tests__/json.test.ts
index 8aaab451..1b84763e 100644
--- a/src/formatters/__tests__/json.test.ts
+++ b/src/cli/formatters/__tests__/json.test.ts
@@ -1,5 +1,6 @@
-import { IRuleResult } from '../../types';
-import { sortResults } from '../../utils';
+import { DiagnosticSeverity } from '@stoplight/types';
+import { IRuleResult } from '../../../types';
+import { sortResults } from '../../../utils';
 import { json } from '../json';
 
 const results: IRuleResult[] = sortResults([
@@ -41,7 +42,7 @@ const results: IRuleResult[] = sortResults([
 
 describe('JSON formatter', () => {
   test('should include ranges', () => {
-    expect(JSON.parse(json(results))).toEqual([
+    expect(JSON.parse(json(results, { failSeverity: DiagnosticSeverity.Error }))).toEqual([
       expect.objectContaining({
         range: {
           start: {
@@ -70,7 +71,7 @@ describe('JSON formatter', () => {
   });
 
   test('should include message', () => {
-    expect(JSON.parse(json(results))).toEqual([
+    expect(JSON.parse(json(results, { failSeverity: DiagnosticSeverity.Error }))).toEqual([
       expect.objectContaining({
         message: 'paths./pets.get.description is not truthy',
       }),
diff --git a/src/formatters/__tests__/junit.test.ts b/src/cli/formatters/__tests__/junit.test.ts
similarity index 62%
rename from src/formatters/__tests__/junit.test.ts
rename to src/cli/formatters/__tests__/junit.test.ts
index 0bbff220..e5efd970 100644
--- a/src/formatters/__tests__/junit.test.ts
+++ b/src/cli/formatters/__tests__/junit.test.ts
@@ -1,6 +1,7 @@
 import { Parser } from 'xml2js';
-import { sortResults } from '../../utils';
+import { sortResults } from '../../../utils';
 import { junit } from '../junit';
+import { DiagnosticSeverity } from '@stoplight/types';
 
 const oas3SchemaErrors = sortResults(require('./__fixtures__/oas3-schema-errors.json'));
 const mixedErrors = sortResults(require('./__fixtures__/mixed-errors.json'));
@@ -14,7 +15,7 @@ describe('JUnit formatter', () => {
   });
 
   test('should produce valid report', async () => {
-    const result = await parse(junit(oas3SchemaErrors));
+    const result = await parse(junit(oas3SchemaErrors, { failSeverity: DiagnosticSeverity.Error }));
     expect(result).toEqual({
       testsuites: {
         testsuite: [
@@ -83,8 +84,8 @@ describe('JUnit formatter', () => {
     });
   });
 
-  test('should filter out non-error validation results', async () => {
-    const result = await parse(junit(mixedErrors));
+  test('given failSeverity set to error, should filter out non-error validation results', async () => {
+    const result = await parse(junit(mixedErrors, { failSeverity: DiagnosticSeverity.Error }));
     expect(result).toEqual({
       testsuites: {
         testsuite: [
@@ -119,4 +120,57 @@ describe('JUnit formatter', () => {
       },
     });
   });
+
+  test('given failSeverity set to other value than error, should filter treat all validation results matching the severity as errors', async () => {
+    const result = await parse(junit(mixedErrors, { failSeverity: DiagnosticSeverity.Warning }));
+    expect(result).toEqual({
+      testsuites: {
+        testsuite: [
+          {
+            $: {
+              errors: '0',
+              failures: '2',
+              name: '/home/Stoplight/spectral/src/__tests__/__fixtures__/petstore.oas3.json',
+              package: 'org.spectral',
+              tests: '2',
+              time: '0',
+            },
+            testcase: [
+              {
+                $: {
+                  classname: '/home/Stoplight/spectral/src/__tests__/__fixtures__/petstore.oas3',
+                  name: 'org.spectral.info-description',
+                  time: '0',
+                },
+                failure: [
+                  {
+                    $: {
+                      message: 'OpenAPI object info `description` must be present and non-empty string.',
+                    },
+                    _:
+                      'line 3, col 10, OpenAPI object info `description` must be present and non-empty string. (info-description) at path #/info',
+                  },
+                ],
+              },
+              {
+                $: {
+                  classname: '/home/Stoplight/spectral/src/__tests__/__fixtures__/petstore.oas3',
+                  name: 'org.spectral.info-matches-stoplight',
+                  time: '0',
+                },
+                failure: [
+                  {
+                    $: {
+                      message: 'Info must contain Stoplight',
+                    },
+                    _: 'line 5, col 14, Info must contain Stoplight (info-matches-stoplight) at path #/info/title',
+                  },
+                ],
+              },
+            ],
+          },
+        ],
+      },
+    });
+  });
 });
diff --git a/src/formatters/__tests__/stylish.test.ts b/src/cli/formatters/__tests__/stylish.test.ts
similarity index 84%
rename from src/formatters/__tests__/stylish.test.ts
rename to src/cli/formatters/__tests__/stylish.test.ts
index b262c3a9..7593e10e 100644
--- a/src/formatters/__tests__/stylish.test.ts
+++ b/src/cli/formatters/__tests__/stylish.test.ts
@@ -1,5 +1,6 @@
+import { DiagnosticSeverity } from '@stoplight/types';
 import * as chalk from 'chalk';
-import { sortResults } from '../../utils';
+import { sortResults } from '../../../utils';
 import { stylish } from '../stylish';
 
 const oas3SchemaErrors = sortResults(require('./__fixtures__/oas3-schema-errors.json'));
@@ -7,14 +8,14 @@ const mixedErrors = sortResults(require('./__fixtures__/mixed-errors.json'));
 
 describe('Stylish formatter', () => {
   test('should prefer message for oas-schema errors', () => {
-    const result = stylish(oas3SchemaErrors);
+    const result = stylish(oas3SchemaErrors, { failSeverity: DiagnosticSeverity.Error });
     expect(result).toContain('oas3-schema  should NOT have additional properties: type\n');
     expect(result).toContain('oas3-schema  should match exactly one schema in oneOf');
     expect(result).toContain("oas3-schema  should have required property '$ref'\n");
   });
 
   test('should display proper severity level', () => {
-    const result = stylish(mixedErrors);
+    const result = stylish(mixedErrors, { failSeverity: DiagnosticSeverity.Error });
     expect(result).toContain(`
   3:10         ${chalk.white('hint')}  info-contact            Info object should contain \`contact\` object.
   3:10      ${chalk.yellow(
diff --git a/src/formatters/__tests__/teamcity.test.ts b/src/cli/formatters/__tests__/teamcity.test.ts
similarity index 93%
rename from src/formatters/__tests__/teamcity.test.ts
rename to src/cli/formatters/__tests__/teamcity.test.ts
index d3b4f702..37f7b634 100644
--- a/src/formatters/__tests__/teamcity.test.ts
+++ b/src/cli/formatters/__tests__/teamcity.test.ts
@@ -1,11 +1,12 @@
-import { sortResults } from '../../utils';
+import { DiagnosticSeverity } from '@stoplight/types';
+import { sortResults } from '../../../utils';
 import { teamcity } from '../teamcity';
 
 const mixedErrors = sortResults(require('./__fixtures__/mixed-errors.json'));
 
 describe('Teamcity formatter', () => {
   test('should format messages', () => {
-    const result = teamcity(mixedErrors);
+    const result = teamcity(mixedErrors, { failSeverity: DiagnosticSeverity.Error });
     expect(result)
       .toContain(`##teamcity[inspectionType category='openapi' id='info-contact' name='info-contact' description='hint -- Info object should contain \`contact\` object.']
 ##teamcity[inspection typeId='info-contact' file='/home/Stoplight/spectral/src/__tests__/__fixtures__/petstore.oas3.json' line='3' message='hint -- Info object should contain \`contact\` object.']
diff --git a/src/formatters/__tests__/text.test.ts b/src/cli/formatters/__tests__/text.test.ts
similarity index 86%
rename from src/formatters/__tests__/text.test.ts
rename to src/cli/formatters/__tests__/text.test.ts
index 7afcd452..8f7b8e6d 100644
--- a/src/formatters/__tests__/text.test.ts
+++ b/src/cli/formatters/__tests__/text.test.ts
@@ -1,11 +1,12 @@
-import { sortResults } from '../../utils';
+import { DiagnosticSeverity } from '@stoplight/types';
+import { sortResults } from '../../../utils';
 import { text } from '../text';
 
 const mixedErrors = sortResults(require('./__fixtures__/mixed-errors.json'));
 
 describe('Text formatter', () => {
   test('should format messages', () => {
-    const result = text(mixedErrors);
+    const result = text(mixedErrors, { failSeverity: DiagnosticSeverity.Error });
     expect(result)
       .toContain(`/home/Stoplight/spectral/src/__tests__/__fixtures__/petstore.oas3.json:3:10 hint info-contact "Info object should contain \`contact\` object."
 /home/Stoplight/spectral/src/__tests__/__fixtures__/petstore.oas3.json:3:10 warning info-description "OpenAPI object info \`description\` must be present and non-empty string."
diff --git a/src/formatters/utils/index.ts b/src/cli/formatters/utils/index.ts
similarity index 100%
rename from src/formatters/utils/index.ts
rename to src/cli/formatters/utils/index.ts
diff --git a/src/formatters/utils/pluralize.ts b/src/cli/formatters/utils/pluralize.ts
similarity index 100%
rename from src/formatters/utils/pluralize.ts
rename to src/cli/formatters/utils/pluralize.ts
diff --git a/src/formatters/utils/xmlEscape.ts b/src/cli/formatters/utils/xmlEscape.ts
similarity index 100%
rename from src/formatters/utils/xmlEscape.ts
rename to src/cli/formatters/utils/xmlEscape.ts
diff --git a/src/cli/services/__tests__/output.test.ts b/src/cli/services/__tests__/output.test.ts
index 66854f99..fda4018f 100644
--- a/src/cli/services/__tests__/output.test.ts
+++ b/src/cli/services/__tests__/output.test.ts
@@ -1,10 +1,10 @@
 import { DiagnosticSeverity } from '@stoplight/types';
 import * as fs from 'fs';
-import * as formatters from '../../../formatters';
+import * as formatters from '../../formatters';
 import { OutputFormat } from '../../../types/config';
 import { formatOutput, writeOutput } from '../output';
 
-jest.mock('../../../formatters');
+jest.mock('../../formatters');
 jest.mock('fs');
 
 describe('Output service', () => {
@@ -32,7 +32,7 @@ describe('Output service', () => {
 
       const output = `value for ${format}`;
       (formatters[format] as jest.Mock).mockReturnValueOnce(output);
-      expect(formatOutput(results, format as OutputFormat)).toEqual(output);
+      expect(formatOutput(results, format as OutputFormat, { failSeverity: DiagnosticSeverity.Error })).toEqual(output);
     });
   });
 
diff --git a/test-harness/scenarios/help-no-document.scenario b/test-harness/scenarios/help-no-document.scenario
index ebb20053..a38612dc 100644
--- a/test-harness/scenarios/help-no-document.scenario
+++ b/test-harness/scenarios/help-no-document.scenario
@@ -14,7 +14,7 @@ Options:
   --version                    Show version number  [boolean]
   --help                       Show help  [boolean]
   --encoding, -e               text encoding to use  [string] [default: "utf8"]
-  --format, -f                 formatter to use for outputting results  [string] [default: "stylish"]
+  --format, -f                 formatter to use for outputting results  [string] [choices: "json", "stylish", "junit", "html", "text", "teamcity"] [default: "stylish"]
   --output, -o                 output to a file instead of stdout  [string]
   --resolver                   path to custom json-ref-resolver instance  [string]
   --ruleset, -r                path/URL to a ruleset file  [string]