Repository: elastic/synthetics (ts)

Issue:
Add support for junit file output
We should add support for junit file output, that will make it easy to integerate synthetics test into CI

Pull request:
feat: add junit reporter
+ fix #149 
+ Can be invoked via the `reporter flag` from CLI
```sh
npx @elastic/synthetics suites --reporter=junit
// OR
npx @elastic/synthetics suites --reporter junit
```

```xml
<testsuites name=\\"\\" tests=\\"2\\" failures=\\"1\\" skipped=\\"1\\" errors=\\"0\\" time=\\"0\\">
   <testsuite name=\\"j1\\" tests=\\"2\\" failures=\\"1\\" skipped=\\"1\\" errors=\\"0\\">
      <testcase name=\\"s1\\" classname=\\"j1 s1\\" time=\\"1\\">
         <failure message=\\"Boom\\" type=\\"Error\\">
         Error: Boom
         </failure>
      </testcase>
      <testcase name=\\"s2\\" classname=\\"j1 s2\\" time=\\"1\\">
         <skipped message=\\"previous step failed\\">
         </skipped>
      </testcase>
   </testsuite>
</testsuites>

```

+ Extract the test case to a separate file using the env flag

```
SYNTHETICS_JUNIT_FILE='/path/to/junit.xml' npx @elastic/synthetics suites --reporter junit
```

Patch:
diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
index 9b5337e..761e846 100644
--- a/.ci/Jenkinsfile
+++ b/.ci/Jenkinsfile
@@ -121,6 +121,9 @@ pipeline {
       }
     }
     stage('E2e Test') {
+      environment {
+        E2E_FOLDER = "__tests__/e2e"
+      }
       options {
         timeout(15)
       }
@@ -132,15 +135,19 @@ pipeline {
               sh(label: 'set permissions', script: '''
                 chmod -R ugo+rw examples
               ''')
-              dir("__tests__/e2e"){
-                timeout(time: 10, unit: 'MINUTES') {
-                  sh(label: 'run e2e tests',script: 'npm run test')
-                }
+              dir("${E2E_FOLDER}"){
+                sh(label: 'run e2e tests',script: 'npm run test')
               }
             }
           }
         }
       }
+      post {
+        always {
+          archiveArtifacts(allowEmptyArchive: true, artifacts: "${BASE_DIR}/${E2E_FOLDER}/junit.xml")
+          junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/${E2E_FOLDER}/junit.xml")
+        }
+      }
     }
     /**
      Publish Docker images.
diff --git a/.gitignore b/.gitignore
index 0e6b6f2..a650b75 100644
--- a/.gitignore
+++ b/.gitignore
@@ -109,6 +109,7 @@ tmp
 *.swp
 
 __tests__/e2e/tmp/
+__tests__/e2e/junit.xml
 
 .idea/
-seccomp/build
\ No newline at end of file
+seccomp/build
diff --git a/src/cli.ts b/src/cli.ts
index bbb1591..39d79ac 100644
--- a/src/cli.ts
+++ b/src/cli.ts
@@ -160,7 +160,7 @@ async function prepareSuites(inputs: string[]) {
   /**
    * use JSON reporter if json flag is enabled
    */
-  const reporter = options.json ? 'json' : 'default';
+  const reporter = options.json ? 'json' : options.reporter;
 
   const results = await run({
     params: JSON.parse(options.suiteParams),
diff --git a/src/common_types.ts b/src/common_types.ts
index 6f960c7..c9943d3 100644
--- a/src/common_types.ts
+++ b/src/common_types.ts
@@ -25,10 +25,11 @@
 
 import { Protocol } from 'playwright-chromium/types/protocol';
 import { Step } from './dsl';
+import { reporters } from './reporters';
 
 export type VoidCallback = () => void;
-
 export type StatusValue = 'succeeded' | 'failed' | 'skipped';
+export type Reporters = keyof typeof reporters;
 
 export type FilmStrip = {
   snapshot: string;
@@ -83,7 +84,7 @@ export type CliArgs = {
   journeyName?: string;
   network?: boolean;
   pauseOnError?: boolean;
-  reporter?: string;
+  reporter?: Reporters;
   wsEndpoint?: string;
   sandbox?: boolean;
   json?: boolean;
diff --git a/src/core/runner.ts b/src/core/runner.ts
index ac7416c..2ff76a7 100644
--- a/src/core/runner.ts
+++ b/src/core/runner.ts
@@ -33,31 +33,22 @@ import {
   FilmStrip,
   NetworkInfo,
   VoidCallback,
+  CliArgs,
 } from '../common_types';
 import { BrowserMessage, PluginManager } from '../plugins';
 import { PerformanceManager, Metrics } from '../plugins';
 import { Driver, Gatherer } from './gatherer';
 import { log } from './logger';
 
-type RunParamaters = Record<string, any>;
-
-export type RunOptions = {
+export type RunOptions = Omit<
+  CliArgs,
+  'debug' | 'json' | 'pattern' | 'inline' | 'require' | 'suiteParams'
+> & {
   params?: RunParamaters;
-  environment?: string;
-  reporter?: 'default' | 'json';
-  headless?: boolean;
-  screenshots?: boolean;
-  filmstrips?: boolean;
-  dryRun?: boolean;
-  journeyName?: string;
-  pauseOnError?: boolean;
-  network?: boolean;
-  outfd?: number;
-  metrics?: boolean;
-  wsEndpoint?: string;
-  sandbox?: boolean;
 };
 
+type RunParamaters = Record<string, unknown>;
+
 type BaseContext = {
   params?: RunParamaters;
   start: number;
@@ -306,11 +297,11 @@ export default class Runner {
     }
     this.active = true;
     log(`Runner: run ${this.journeys.length} journeys`);
-    const { reporter = 'default', journeyName, outfd } = options;
+    const { reporter, journeyName, outfd } = options;
     /**
-     * Set up the corresponding reporter
+     * Set up the corresponding reporter and fallback
      */
-    const Reporter = reporters[reporter];
+    const Reporter = reporters[reporter] || reporters['default'];
     new Reporter(this, { fd: outfd });
     this.emit('start', { numJourneys: this.journeys.length });
     await this.runBeforeAllHook();
diff --git a/src/parse_args.ts b/src/parse_args.ts
index 91932fd..b03cfa0 100644
--- a/src/parse_args.ts
+++ b/src/parse_args.ts
@@ -24,6 +24,11 @@
  */
 
 import { program } from 'commander';
+import { reporters } from './reporters';
+
+const availableReporters = Object.keys(reporters)
+  .map(r => String(r))
+  .join();
 
 /* eslint-disable-next-line @typescript-eslint/no-var-requires */
 const { name, version } = require('../package.json');
@@ -33,6 +38,10 @@ program
   .option('-s, --suite-params <jsonstring>', 'Variables', '{}')
   .option('-e, --environment <envname>', 'e.g. production', 'development')
   .option('-j, --json', 'output newline delimited JSON')
+  .option(
+    '--reporter <reporter>',
+    `output repoter format, can be one of ${availableReporters}`
+  )
   .option('-d, --debug', 'print debug logs info')
   .option(
     '--pattern <pattern>',
diff --git a/src/reporters/base.ts b/src/reporters/base.ts
index 0594c0e..8a064c5 100644
--- a/src/reporters/base.ts
+++ b/src/reporters/base.ts
@@ -65,8 +65,8 @@ function renderDuration(durationMs) {
 export default class BaseReporter {
   stream: SonicBoom;
   fd: number;
-  constructor(public runner: Runner, public options: ReporterOptions = {}) {
-    this.runner = runner;
+
+  constructor(public runner: Runner, options: ReporterOptions = {}) {
     this.fd = options.fd || process.stdout.fd;
     this.stream = new SonicBoom({ fd: this.fd, sync: true });
     this._registerListeners();
diff --git a/src/reporters/index.ts b/src/reporters/index.ts
index 6e887f6..0e7d967 100644
--- a/src/reporters/index.ts
+++ b/src/reporters/index.ts
@@ -25,8 +25,10 @@
 
 import BaseReporter from './base';
 import JSONReporter from './json';
+import JUnitReporter from './junit';
 
 export const reporters = {
   default: BaseReporter,
   json: JSONReporter,
+  junit: JUnitReporter,
 };
diff --git a/src/reporters/junit.ts b/src/reporters/junit.ts
new file mode 100644
index 0000000..bc3041f
--- /dev/null
+++ b/src/reporters/junit.ts
@@ -0,0 +1,178 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import { mkdirSync, writeFileSync } from 'fs';
+import { dirname } from 'path';
+import { formatError, indent, now } from '../helpers';
+import BaseReporter from './base';
+
+type XMLEntry = {
+  name: string;
+  attributes?: {
+    name: string;
+    timestamp?: number;
+    classname?: string;
+    time?: number;
+    tests?: number;
+    failures?: number;
+    skipped?: number;
+    errors?: number;
+  };
+  children?: XMLEntry[];
+  text?: string;
+};
+
+/**
+ * JUnit Reporting Format - https://llg.cubic.org/docs/junit/
+ */
+export default class JUnitReporter extends BaseReporter {
+  private totalTests = 0;
+  private totalFailures = 0;
+  private totalSkipped = 0;
+
+  _registerListeners() {
+    const journeyMap = new Map<string, XMLEntry>();
+
+    this.runner.on('journey:start', ({ journey }) => {
+      if (!journeyMap.has(journey.name)) {
+        const entry = {
+          name: 'testsuite',
+          attributes: {
+            name: journey.name,
+            tests: 0,
+            failures: 0,
+            skipped: 0,
+            errors: 0,
+          },
+          children: [],
+        };
+        journeyMap.set(journey.name, entry);
+      }
+    });
+
+    this.runner.on(
+      'step:end',
+      ({ journey, step, status, error, start, end }) => {
+        if (!journeyMap.has(journey.name)) {
+          return;
+        }
+        const entry = journeyMap.get(journey.name);
+        const caseEntry = {
+          name: 'testcase',
+          attributes: {
+            name: step.name,
+            classname: journey.name + ' ' + step.name,
+            time: end - start,
+          },
+          children: [],
+        };
+
+        entry.attributes.tests++;
+        if (status === 'failed') {
+          const { name, message, stack } = formatError(error);
+          caseEntry.children.push({
+            name: 'failure',
+            attributes: {
+              message,
+              type: name,
+            },
+            text: stack,
+          });
+          entry.attributes.failures++;
+        } else if (status === 'skipped') {
+          caseEntry.children.push({
+            name: 'skipped',
+            attributes: {
+              message: 'previous step failed',
+            },
+          });
+          entry.attributes.skipped++;
+        }
+        entry.children.push(caseEntry);
+      }
+    );
+
+    this.runner.on('journey:end', ({ journey }) => {
+      if (!journeyMap.has(journey.name)) {
+        return;
+      }
+      const { attributes } = journeyMap.get(journey.name);
+      this.totalTests += attributes.tests;
+      this.totalFailures += attributes.failures;
+      this.totalSkipped += attributes.skipped;
+    });
+
+    this.runner.on('end', () => {
+      const root: XMLEntry = {
+        name: 'testsuites',
+        attributes: {
+          name: '',
+          tests: this.totalTests,
+          failures: this.totalFailures,
+          skipped: this.totalSkipped,
+          errors: 0,
+          time: parseInt(String(now())) / 1000,
+        },
+        children: [...journeyMap.values()],
+      };
+      const output = serializeEntries(root).join('\n');
+      /**
+       * write the xml output to a file if specified via env flag
+       */
+      const fileName = process.env['SYNTHETICS_JUNIT_FILE'];
+      if (fileName) {
+        mkdirSync(dirname(fileName), { recursive: true });
+        writeFileSync(fileName, output);
+      } else {
+        this.write(output);
+      }
+    });
+  }
+}
+
+function escape(text: string): string {
+  return text
+    .replace(/"/g, '&quot;')
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;');
+}
+
+function serializeEntries(entry: XMLEntry, tokens: string[] = [], space = '') {
+  tokens.push(
+    indent(
+      `<${entry.name} ${Object.entries(entry.attributes || {})
+        .map(([key, value]) => `${key}="${escape(String(value))}"`)
+        .join(' ')}>`,
+      space
+    )
+  );
+  for (const child of entry.children || []) {
+    serializeEntries(child, tokens, space + '   ');
+  }
+  if (entry.text) tokens.push(indent(escape(entry.text), space));
+  tokens.push(indent(`</${entry.name}>`, space));
+  return tokens;
+}


Test patch:
diff --git a/__tests__/e2e/docker-compose.yml b/__tests__/e2e/docker-compose.yml
index f7c0d91..4f738e6 100644
--- a/__tests__/e2e/docker-compose.yml
+++ b/__tests__/e2e/docker-compose.yml
@@ -30,7 +30,7 @@ services:
       - elasticsearch
       - synthetic
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:5601/api/status"]
+      test: ["CMD", "curl", "-s", "-f", "http://localhost:5601/api/status"]
       interval: 30s
       retries: 10
       start_period: 30s
diff --git a/__tests__/e2e/scripts/setup.sh b/__tests__/e2e/scripts/setup.sh
index 38fc207..b5ca174 100755
--- a/__tests__/e2e/scripts/setup.sh
+++ b/__tests__/e2e/scripts/setup.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -xe
+set -e
 
 # variables
 
diff --git a/__tests__/e2e/scripts/test.sh b/__tests__/e2e/scripts/test.sh
index c8d005d..5018eea 100755
--- a/__tests__/e2e/scripts/test.sh
+++ b/__tests__/e2e/scripts/test.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -xe
+set -eo pipefail
 
 if [ -z "${JENKINS_URL}" ]; then
   # formatting
@@ -14,7 +14,7 @@ fi
 ##################################################
 echo "" # newline
 echo "${bold}Waiting for synthetics docker to start...${normal}"
-until [ "$(docker inspect -f {{.State.Running}} synthetics)" == "true" ]; do
+until [ "$(docker inspect -f '{{.State.Running}}' synthetics)" == "true" ]; do
   sleep ${SLEEP_TIME};
 done;
 
@@ -23,5 +23,4 @@ echo "✅ Setup completed successfully. Running e2e tests..."
 #
 # run e2e tests journey
 ##################################################
-
-npx @elastic/synthetics uptime.journey.ts
+SYNTHETICS_JUNIT_FILE='junit.xml' npx @elastic/synthetics uptime.journey.ts --reporter junit
diff --git a/__tests__/e2e/uptime.journey.ts b/__tests__/e2e/uptime.journey.ts
index 9ec079e..27acda4 100644
--- a/__tests__/e2e/uptime.journey.ts
+++ b/__tests__/e2e/uptime.journey.ts
@@ -43,10 +43,13 @@ journey('E2e test synthetics', async ({ page }) => {
 
   step('Go to kibana uptime app', async () => {
     await page.goto('http://localhost:5620/app/uptime');
+    await page.waitForTimeout(30 * 1000);
   });
 
   step('Check if there is table data', async () => {
-    await page.click('[data-test-subj=uptimeOverviewPage]');
+    await page.click('[data-test-subj=uptimeOverviewPage]', {
+      timeout: 60 * 1000,
+    });
     await refreshUptimeApp();
     await page.click('div.euiBasicTable', { timeout: 60 * 1000 });
   });
diff --git a/__tests__/reporters/__snapshots__/junit.test.ts.snap b/__tests__/reporters/__snapshots__/junit.test.ts.snap
new file mode 100644
index 0000000..2d278c5
--- /dev/null
+++ b/__tests__/reporters/__snapshots__/junit.test.ts.snap
@@ -0,0 +1,29 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`junit reporter writes the output to a file 1`] = `
+"<testsuites name=\\"\\" tests=\\"1\\" failures=\\"0\\" skipped=\\"1\\" errors=\\"0\\" time=\\"0\\">
+   <testsuite name=\\"j1\\" tests=\\"1\\" failures=\\"0\\" skipped=\\"1\\" errors=\\"0\\">
+      <testcase name=\\"s1\\" classname=\\"j1 s1\\" time=\\"1\\">
+         <skipped message=\\"previous step failed\\">
+         </skipped>
+      </testcase>
+   </testsuite>
+</testsuites>"
+`;
+
+exports[`junit reporter writes the output to fd 1`] = `
+"<testsuites name=\\"\\" tests=\\"2\\" failures=\\"1\\" skipped=\\"1\\" errors=\\"0\\" time=\\"0\\">
+   <testsuite name=\\"j1\\" tests=\\"2\\" failures=\\"1\\" skipped=\\"1\\" errors=\\"0\\">
+      <testcase name=\\"s1\\" classname=\\"j1 s1\\" time=\\"1\\">
+         <failure message=\\"Boom\\" type=\\"Error\\">
+         at /__tests/reporters/junit.test.ts
+         </failure>
+      </testcase>
+      <testcase name=\\"s2\\" classname=\\"j1 s2\\" time=\\"1\\">
+         <skipped message=\\"previous step failed\\">
+         </skipped>
+      </testcase>
+   </testsuite>
+</testsuites>
+"
+`;
diff --git a/__tests__/reporters/junit.test.ts b/__tests__/reporters/junit.test.ts
new file mode 100644
index 0000000..9cb71f2
--- /dev/null
+++ b/__tests__/reporters/junit.test.ts
@@ -0,0 +1,124 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import fs from 'fs';
+import { join } from 'path';
+import { step, journey } from '../../src/core';
+import Runner from '../../src/core/runner';
+import JUnitReporter from '../../src/reporters/junit';
+import * as helpers from '../../src/helpers';
+
+describe('junit reporter', () => {
+  beforeEach(() => {});
+  let dest: string;
+  let runner: Runner;
+  let stream;
+  const timestamp = 1600300800000000;
+  const j1 = journey('j1', async () => {});
+  const s1 = step('s1', async () => {});
+
+  beforeAll(() => {
+    runner = new Runner();
+  });
+
+  beforeEach(() => {
+    runner = new Runner();
+    dest = helpers.generateTempPath();
+    stream = new JUnitReporter(runner, { fd: fs.openSync(dest, 'w') }).stream;
+    jest.spyOn(helpers, 'now').mockImplementation(() => 0);
+  });
+
+  afterAll(() => fs.unlinkSync(dest));
+
+  it('writes the output to fd', async () => {
+    runner.emit('journey:start', {
+      journey: j1,
+      params: {},
+      timestamp,
+    });
+    const error = new Error('Boom');
+    /**
+     * Snapshots would be different for everyone
+     * so keep it simple
+     */
+    error.stack = 'at /__tests/reporters/junit.test.ts';
+    runner.emit('step:end', {
+      journey: j1,
+      status: 'failed',
+      error,
+      step: s1,
+      start: 0,
+      end: 1,
+    });
+    runner.emit('step:end', {
+      journey: j1,
+      status: 'skipped',
+      step: step('s2', async () => {}),
+      start: 0,
+      end: 1,
+    });
+    runner.emit('journey:end', {
+      journey: j1,
+      start: 0,
+      status: 'failed',
+    });
+    runner.emit('end', 'done');
+    /**
+     * Close the underyling stream writing to FD to read all its contents
+     */
+    stream.end();
+    await new Promise(resolve => stream.once('finish', resolve));
+    const fd = fs.openSync(dest, 'r');
+    const buffer = fs.readFileSync(fd);
+    expect(buffer.toString()).toMatchSnapshot();
+  });
+
+  it('writes the output to a file', async () => {
+    const filepath = join(__dirname, '../../tmp', 'junit.xml');
+    process.env.SYNTHETICS_JUNIT_FILE = filepath;
+    runner.emit('journey:start', {
+      journey: j1,
+      params: {},
+      timestamp,
+    });
+    runner.emit('step:end', {
+      journey: j1,
+      status: 'skipped',
+      step: s1,
+      start: 0,
+      end: 1,
+    });
+    runner.emit('journey:end', {
+      journey: j1,
+      start: 0,
+      status: 'failed',
+    });
+    runner.emit('end', 'done');
+    stream.end();
+    expect(fs.readFileSync(filepath, 'utf-8')).toMatchSnapshot();
+    fs.unlinkSync(filepath);
+    process.env.SYNTHETICS_JUNIT_FILE = '';
+  });
+});