Repository: stoplightio/spectral
Pull request #621 (2019-10-02 01:43:42)

fix(junit): make formatter more spec-compliant
Fixes https://github.com/stoplightio/spectral/issues/607

**Checklist**

- [x] Tests added / updated
- [x] Docs added / updated

**Does this PR introduce a breaking change?**

- [ ] Yes
- [x] No

**Additional context**

Let's double-check whether everything is alright this time around.


philsturgeon: Ok this is good but we lost the path, which will undo the done-ness of that previous issue. Seeing as JUnit does not care about Path, what can we recommend as a feature to follow in the mean time? #389 ?
P0lip: @philsturgeon we didn't lose path. Path is now inlined and stands next to the location and message. 

Patch:
diff --git a/src/formatters/junit.ts b/src/formatters/junit.ts
index 92a57daf..543da66e 100644
--- a/src/formatters/junit.ts
+++ b/src/formatters/junit.ts
@@ -25,25 +25,12 @@
 
 import { encodePointerFragment } from '@stoplight/json';
 import { extname } from '@stoplight/path';
-import { DiagnosticSeverity, Dictionary, JsonPath } from '@stoplight/types';
+import { DiagnosticSeverity, JsonPath } from '@stoplight/types';
 import { escapeRegExp } from 'lodash';
-import { IRuleResult, SpectralDiagnosticSeverity } from '../types';
 import { Formatter } from './types';
 import { groupBySource } from './utils/groupBySource';
 import { xmlEscape } from './utils/xmlEscape';
 
-const SEVERITY_MAP: Dictionary<string, SpectralDiagnosticSeverity> = {
-  [DiagnosticSeverity.Error]: 'Error',
-  [DiagnosticSeverity.Warning]: 'Warning',
-  [DiagnosticSeverity.Hint]: 'Hint',
-  [DiagnosticSeverity.Information]: 'Information',
-  [-1]: 'off',
-};
-
-function getMessageType(result: IRuleResult) {
-  return SEVERITY_MAP[result.severity];
-}
-
 function stringifyPath(path: JsonPath) {
   return ['#', ...path.map(encodePointerFragment)].join('/');
 }
@@ -60,21 +47,23 @@ export const junit: Formatter = results => {
     const classname = source.replace(new RegExp(`${escapeRegExp(extname(source))}$`), '');
 
     if (validationResults.length > 0) {
-      output += `<testsuite package="org.spectral" time="0" tests="${validationResults.length}" errors="${
-        validationResults.length
-      }" name="${source}">\n`;
+      const filteredValidationResults = validationResults.filter(
+        result => result.severity === DiagnosticSeverity.Error,
+      );
+
+      output += `<testsuite package="org.spectral" time="0" tests="${
+        filteredValidationResults.length
+      }" errors="0" failures="${filteredValidationResults.length}" name="${source}">\n`;
 
-      for (const result of validationResults) {
-        const tag = getMessageType(result).toLowerCase();
+      for (const result of filteredValidationResults) {
         output += `<testcase time="0" name="org.spectral.${result.code || 'unknown'}" classname="${classname}">`;
-        output += `<${tag} message="${xmlEscape(result.message)}" path="${stringifyPath(result.path)}">`;
+        output += `<failure message="${xmlEscape(result.message)}">`;
         output += '<![CDATA[';
-        output += `line ${result.range.start.line + 1}, col `;
-        output += `${result.range.start.character + 1}, ${getMessageType(result)}`;
-        output += ` - ${xmlEscape(result.message)}`;
-        output += ` (${result.code})`;
+        output += `line ${result.range.start.line + 1}, col ${result.range.start.character + 1}, `;
+        output += `${xmlEscape(result.message)} (${result.code}) `;
+        output += `at path ${xmlEscape(stringifyPath(result.path))}`;
         output += ']]>';
-        output += `</${tag}>`;
+        output += `</failure>`;
         output += '</testcase>\n';
       }
 


Test patch:
diff --git a/src/formatters/__tests__/junit.test.ts b/src/formatters/__tests__/junit.test.ts
index 5cec447b..1019b62a 100644
--- a/src/formatters/__tests__/junit.test.ts
+++ b/src/formatters/__tests__/junit.test.ts
@@ -20,7 +20,8 @@ describe('JUnit formatter', () => {
         testsuite: [
           {
             $: {
-              errors: '3',
+              errors: '0',
+              failures: '3',
               name: '/home/Stoplight/spectral/src/__tests__/__fixtures__/petstore.invalid-schema.oas3.yaml',
               package: 'org.spectral',
               tests: '3',
@@ -33,13 +34,13 @@ describe('JUnit formatter', () => {
                   name: 'org.spectral.oas3-schema',
                   time: '0',
                 },
-                error: [
+                failure: [
                   {
                     $: {
                       message: 'should NOT have additional properties: type',
-                      path: '#/paths/~1pets/get/responses/200/headers/header-1',
                     },
-                    _: 'line 36, col 22, Error - should NOT have additional properties: type (oas3-schema)',
+                    _:
+                      'line 36, col 22, should NOT have additional properties: type (oas3-schema) at path #/paths/~1pets/get/responses/200/headers/header-1',
                   },
                 ],
               },
@@ -49,13 +50,13 @@ describe('JUnit formatter', () => {
                   name: 'org.spectral.oas3-schema',
                   time: '0',
                 },
-                error: [
+                failure: [
                   {
                     $: {
                       message: 'should match exactly one schema in oneOf',
-                      path: '#/paths/~1pets/get/responses/200/headers/header-1',
                     },
-                    _: 'line 36, col 22, Error - should match exactly one schema in oneOf (oas3-schema)',
+                    _:
+                      'line 36, col 22, should match exactly one schema in oneOf (oas3-schema) at path #/paths/~1pets/get/responses/200/headers/header-1',
                   },
                 ],
               },
@@ -65,13 +66,13 @@ describe('JUnit formatter', () => {
                   name: 'org.spectral.oas3-schema',
                   time: '0',
                 },
-                error: [
+                failure: [
                   {
                     $: {
                       message: "should have required property '$ref'",
-                      path: '#/paths/~1pets/get/responses/200/headers/header-1',
                     },
-                    _: 'line 36, col 22, Error - should have required property &apos;$ref&apos; (oas3-schema)',
+                    _:
+                      'line 36, col 22, should have required property &apos;$ref&apos; (oas3-schema) at path #/paths/~1pets/get/responses/200/headers/header-1',
                   },
                 ],
               },
@@ -82,70 +83,34 @@ describe('JUnit formatter', () => {
     });
   });
 
-  test('should produce valid paths', async () => {
+  test('should filter out non-error validation results', async () => {
     const result = await parse(junit(mixedErrors));
     expect(result).toEqual({
       testsuites: {
         testsuite: [
           {
             $: {
-              errors: '4',
+              errors: '0',
+              failures: '1',
               name: '/home/Stoplight/spectral/src/__tests__/__fixtures__/petstore.oas3.json',
               package: 'org.spectral',
-              tests: '4',
+              tests: '1',
               time: '0',
             },
             testcase: [
               {
-                $: expect.objectContaining({
-                  name: 'org.spectral.info-contact',
-                }),
-                hint: [
-                  expect.objectContaining({
-                    $: {
-                      message: 'Info object should contain `contact` object.',
-                      path: '#',
-                    },
-                  }),
-                ],
-              },
-              {
-                $: expect.objectContaining({
-                  name: 'org.spectral.info-description',
-                }),
-                warning: [
-                  expect.objectContaining({
-                    $: {
-                      message: 'OpenAPI object info `description` must be present and non-empty string.',
-                      path: '#/',
-                    },
-                  }),
-                ],
-              },
-              {
-                $: expect.objectContaining({
-                  name: 'org.spectral.operation-description',
-                }),
-                information: [
-                  expect.objectContaining({
-                    $: {
-                      message: 'Operation `description` must be present and non-empty string.',
-                      path: '#/paths/~1pets/post',
-                    },
-                  }),
-                ],
-              },
-              {
-                $: expect.objectContaining({
+                $: {
+                  classname: '/home/Stoplight/spectral/src/__tests__/__fixtures__/petstore.oas3',
                   name: 'org.spectral.info-matches-stoplight',
-                }),
-                error: [
-                  expect.objectContaining({
+                  time: '0',
+                },
+                failure: [
+                  {
                     $: {
                       message: 'Info must contain Stoplight',
-                      path: '#/info/title',
                     },
-                  }),
+                    _: 'line 5, col 14, Info must contain Stoplight (info-matches-stoplight) at path #/info/title',
+                  },
                 ],
               },
             ],
diff --git a/test-harness/scenarios/results-format-junit.oas3.scenario b/test-harness/scenarios/results-format-junit.oas3.scenario
index 1a79b64a..948dc69e 100644
--- a/test-harness/scenarios/results-format-junit.oas3.scenario
+++ b/test-harness/scenarios/results-format-junit.oas3.scenario
@@ -13,10 +13,7 @@ lint {document} --format=junit
 OpenAPI 3.x detected
 <?xml version="1.0" encoding="utf-8"?>
 <testsuites>
-<testsuite package="org.spectral" time="0" tests="4" errors="4" name="{document}">
-<testcase time="0" name="org.spectral.api-servers" classname="{documentWithoutExt}"><warning message="OpenAPI `servers` must be present and non-empty array." path="#"><![CDATA[line 1, col 1, Warning - OpenAPI `servers` must be present and non-empty array. (api-servers)]]></warning></testcase>
-<testcase time="0" name="org.spectral.oas3-schema" classname="{documentWithoutExt}"><error message="/info Property foo is not expected to be here" path="#/info"><![CDATA[line 2, col 6, Error - /info Property foo is not expected to be here (oas3-schema)]]></error></testcase>
-<testcase time="0" name="org.spectral.info-contact" classname="{documentWithoutExt}"><warning message="Info object should contain `contact` object." path="#/info"><![CDATA[line 2, col 6, Warning - Info object should contain `contact` object. (info-contact)]]></warning></testcase>
-<testcase time="0" name="org.spectral.info-description" classname="{documentWithoutExt}"><warning message="OpenAPI object info `description` must be present and non-empty string." path="#/info"><![CDATA[line 2, col 6, Warning - OpenAPI object info `description` must be present and non-empty string. (info-description)]]></warning></testcase>
+<testsuite package="org.spectral" time="0" tests="1" errors="0" failures="1" name="{document}">
+<testcase time="0" name="org.spectral.oas3-schema" classname="{documentWithoutExt}"><failure message="/info Property foo is not expected to be here"><![CDATA[line 2, col 6, /info Property foo is not expected to be here (oas3-schema) at path #/info]]></failure></testcase>
 </testsuite>
 </testsuites>