Repository: elastic/synthetics
Pull request #506 (2022-05-20 18:11:13)

chore: handle 404 erorrs on push endpoint
+ Part of #470 
+ Handle cases when invalid Kibana endpoint was provided. Most likely happens when development when there is more chance of base path being different between every restart. 

apmmachine: 
## :green_heart: Build Succeeded


<!-- BUILD BADGES-->
> _the below badges are clickable and redirect to their specific view in the CI or DOCS_
[![Pipeline View](https://img.shields.io/badge/pipeline-pipeline%20-green)](https://apm-ci.elastic.co/blue/organizations/jenkins/apm-agent-rum%2Felastic-synthetics%2FPR-506/detail/PR-506/1//pipeline) [![Test View](https://img.shields.io/badge/test-test-green)](https://apm-ci.elastic.co/blue/organizations/jenkins/apm-agent-rum%2Felastic-synthetics%2FPR-506/detail/PR-506/1//tests) [![Changes](https://img.shields.io/badge/changes-changes-green)](https://apm-ci.elastic.co/blue/organizations/jenkins/apm-agent-rum%2Felastic-synthetics%2FPR-506/detail/PR-506/1//changes) [![Artifacts](https://img.shields.io/badge/artifacts-artifacts-yellow)](https://apm-ci.elastic.co/blue/organizations/jenkins/apm-agent-rum%2Felastic-synthetics%2FPR-506/detail/PR-506/1//artifacts) [![preview](https://img.shields.io/badge/docs-preview-yellowgreen)](http://synthetics_506.docs-preview.app.elstc.co/diff)  [![preview](https://img.shields.io/badge/elastic-observability-blue)](https://ci-stats.elastic.co/app/apm/services/apm-ci/transactions/view?rangeFrom=2022-05-20T18:01:28.122Z&rangeTo=2022-05-20T18:21:28.122Z&transactionName=BUILD+apm-agent-rum%2Felastic-synthetics%2FPR-%7Bnumber%7D&transactionType=job&latencyAggregationType=avg&traceId=d098635e58d0937142c511c985974a85&transactionId=20c06f1a437e516a)

<!-- BUILD SUMMARY-->
<details><summary>Expand to view the summary</summary>
<p>

#### Build stats




* Start Time: 2022-05-20T18:11:28.122+0000


* Duration: 14 min 20 sec




#### Test stats :test_tube:

| Test         | Results                         |
| ------------ | :-----------------------------: |
| Failed       | 0  |
| Passed       | 160  |
| Skipped      | 2 |
| Total        | 162   |


</p>
</details>

<!-- TEST RESULTS IF ANY-->


<!-- STEPS ERRORS IF ANY -->












## :robot: GitHub comments

To re-run your PR in the CI, just comment with:



- `/test` : Re-trigger the build.




<!--COMMENT_GENERATED_WITH_ID_comment.id-->

Patch:
diff --git a/.eslintignore b/.eslintignore
index 6ac1748..d83b8ee 100644
--- a/.eslintignore
+++ b/.eslintignore
@@ -1,4 +1,5 @@
 dist
 examples
+templates
 __tests__/e2e/tmp/
 src/sdk/lh-trace-processor.ts
diff --git a/package.json b/package.json
index 7e675ae..5992f46 100644
--- a/package.json
+++ b/package.json
@@ -22,7 +22,8 @@
   },
   "files": [
     "dist",
-    "src"
+    "src",
+    "templates"
   ],
   "lint-staged": {
     "*.{js,ts}": [
diff --git a/src/cli.ts b/src/cli.ts
index 1079c75..c5a9b16 100644
--- a/src/cli.ts
+++ b/src/cli.ts
@@ -34,6 +34,9 @@ import { run } from './';
 import { runner } from './core';
 import { SyntheticsLocations } from './dsl/monitor';
 import { push } from './push';
+import { resolve } from 'path';
+import { Generator } from './generator';
+import { error } from './helpers';
 
 /* eslint-disable-next-line @typescript-eslint/no-var-requires */
 const { name, version } = require('../package.json');
@@ -141,6 +144,7 @@ program
     }
   });
 
+// Push command
 program
   .command('push [files...]')
   .description(
@@ -185,4 +189,18 @@ program
     }
   });
 
+// Init command
+program
+  .command('init <project-dir>')
+  .description('Initalize Elastic synthetics project')
+  .action(async (dir: string) => {
+    try {
+      const generator = await new Generator(resolve(process.cwd(), dir));
+      await generator.setup();
+    } catch (e) {
+      error(e);
+      process.exit(1);
+    }
+  });
+
 program.parse(process.argv);
diff --git a/src/generator/index.ts b/src/generator/index.ts
new file mode 100644
index 0000000..602a5c0
--- /dev/null
+++ b/src/generator/index.ts
@@ -0,0 +1,159 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+import { execSync } from 'child_process';
+import { existsSync } from 'fs';
+import { mkdir, readFile, writeFile } from 'fs/promises';
+import { bold, cyan, yellow } from 'kleur/colors';
+import { join, relative, dirname, basename } from 'path';
+import { progress, write as stdWrite } from '../helpers';
+import { getPackageManager, runCommand } from './utils';
+
+// Templates that are required for setting up new
+// synthetics project
+const templateDir = join(__dirname, '..', '..', 'templates');
+
+export class Generator {
+  pkgManager = 'npm';
+  constructor(public projectDir: string) {}
+
+  async directory() {
+    if (!existsSync(this.projectDir)) {
+      await mkdir(this.projectDir);
+    }
+  }
+
+  async files() {
+    const fileMap = new Map<string, string>();
+    // Setup Synthetics config file
+    const configFile = 'synthetics.config.ts';
+    fileMap.set(
+      configFile,
+      await readFile(join(templateDir, configFile), 'utf-8')
+    );
+
+    // Setup example journey file
+    const journeyFile = 'journeys/example.journey.ts';
+    fileMap.set(
+      journeyFile,
+      await readFile(join(templateDir, journeyFile), 'utf-8')
+    );
+    // Setup gitignore
+    fileMap.set(
+      '.gitignore',
+      await readFile(join(templateDir, '.gitignore'), 'utf-8')
+    );
+
+    // Create files
+    for (const [relativePath, content] of fileMap) {
+      await this.createFile(relativePath, content);
+    }
+  }
+
+  async createFile(relativePath: string, content: string, override = false) {
+    const absolutePath = join(this.projectDir, relativePath);
+    if (override || !existsSync(absolutePath)) {
+      progress(`Writing ${relative(process.cwd(), absolutePath)}.`);
+      await mkdir(dirname(absolutePath), { recursive: true });
+      await writeFile(absolutePath, content, 'utf-8');
+    }
+  }
+
+  async package() {
+    this.pkgManager = await getPackageManager(this.projectDir);
+    const commands = new Map<string, string>();
+    commands.set(
+      `Initializing Synthetics project using ${
+        this.pkgManager == 'yarn' ? 'Yarn' : 'NPM'
+      }`,
+      this.pkgManager == 'yarn' ? 'yarn init -y' : 'npm init -y'
+    );
+
+    const pkgName = '@elastic/synthetics';
+    commands.set(
+      `Installing @elastic/synthetics library`,
+      this.pkgManager == 'yarn'
+        ? `yarn add -dev ${pkgName}`
+        : `npm i -d ${pkgName}`
+    );
+
+    // Execute commands
+    for (const [name, command] of commands) {
+      progress(`${name}...`);
+      execSync(command, {
+        stdio: 'inherit',
+        cwd: this.projectDir,
+      });
+    }
+  }
+
+  async patchPkgJSON() {
+    const filename = 'package.json';
+    const pkgJSON = JSON.parse(
+      await readFile(join(this.projectDir, filename), 'utf-8')
+    );
+
+    if (!pkgJSON.scripts) {
+      pkgJSON.scripts = {};
+    }
+    // Add test command
+    pkgJSON.scripts.test = 'npx @elastic/synthetics journeys';
+
+    // Add push command
+    const project = basename(this.projectDir);
+    pkgJSON.scripts.push = `npx @elastic/synthetics push journeys --project ${project} --url http://localhost:5601 --auth basic-auth`;
+
+    await this.createFile(
+      filename,
+      JSON.stringify(pkgJSON, null, 2) + '\n',
+      true
+    );
+  }
+
+  banner() {
+    stdWrite(
+      bold(`
+All set, you can run below commands inside: ${this.projectDir}:
+
+  Run synthetic tests: ${cyan(runCommand(this.pkgManager, 'test'))}
+
+  Push monitors to Kibana: ${cyan(runCommand(this.pkgManager, 'push'))}
+
+  ${yellow(
+    'Make sure to update the Kibana url and authentication info before pushing monitors to Kibana.'
+  )}
+
+Visit https://www.elastic.co/guide/en/observability/master/synthetics-journeys.html to learn more.
+    `)
+    );
+  }
+
+  async setup() {
+    await this.directory();
+    await this.package();
+    await this.files();
+    await this.patchPkgJSON();
+    this.banner();
+  }
+}
diff --git a/src/generator/utils.ts b/src/generator/utils.ts
new file mode 100644
index 0000000..5745dc6
--- /dev/null
+++ b/src/generator/utils.ts
@@ -0,0 +1,41 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import { existsSync } from 'fs';
+import { join } from 'path';
+
+export async function getPackageManager(dir: string) {
+  if (existsSync(join(dir, 'yarn.lock'))) {
+    return 'yarn';
+  }
+  return 'npm';
+}
+
+export function runCommand(pkgManager: string, command: string) {
+  if (pkgManager === 'yarn') {
+    return `yarn ${command}`;
+  }
+  return `npm run ${command}`;
+}
diff --git a/src/helpers.ts b/src/helpers.ts
index fca27e5..abf09b0 100644
--- a/src/helpers.ts
+++ b/src/helpers.ts
@@ -23,7 +23,7 @@
  *
  */
 
-import { red, green, yellow, cyan } from 'kleur/colors';
+import { red, green, yellow, cyan, bold } from 'kleur/colors';
 import os from 'os';
 import { resolve, join, dirname } from 'path';
 import fs from 'fs';
@@ -321,3 +321,20 @@ export function wrapFnWithLocation<A extends unknown[], R>(
     return func(location, ...args);
   };
 }
+
+// Console helpers
+export function write(message: string) {
+  process.stderr.write(message + '\n');
+}
+
+export function progress(message: string) {
+  write(cyan(bold(`${symbols.progress} ${message}`)));
+}
+
+export function error(message: string) {
+  write(red(message));
+}
+
+export function done(message: string) {
+  write(bold(green(`${symbols['succeeded']} ${message}`)));
+}
diff --git a/src/push/index.ts b/src/push/index.ts
index 97b6d12..897fd2f 100644
--- a/src/push/index.ts
+++ b/src/push/index.ts
@@ -23,34 +23,18 @@
  *
  */
 
-import { bold, cyan, green, red } from 'kleur/colors';
 import {
   createMonitors,
   ok,
   formatAPIError,
   formatFailedMonitors,
   formatStaleMonitors,
+  formatNotFoundError,
 } from './request';
 import { Monitor } from '../dsl/monitor';
 import { PushOptions } from '../common_types';
 import { buildMonitorSchema } from './monitor';
-import { symbols } from '../helpers';
-
-function write(message: string) {
-  process.stderr.write(message + '\n');
-}
-
-function progress(message: string) {
-  write(cyan(bold(`${symbols.progress} ${message}`)));
-}
-
-function error(message: string) {
-  write(red(message));
-}
-
-function done(message: string) {
-  write(bold(green(`${symbols['succeeded']} ${message}`)));
-}
+import { progress, error, done, write } from '../helpers';
 
 export async function push(monitors: Monitor[], options: PushOptions) {
   progress(`preparing all monitors`);
@@ -58,6 +42,9 @@ export async function push(monitors: Monitor[], options: PushOptions) {
   try {
     progress(`creating all monitors`);
     const { body, statusCode } = await createMonitors(schemas, options);
+    if (statusCode === 404) {
+      throw formatNotFoundError(await body.text());
+    }
     if (!ok(statusCode)) {
       const { error, message } = await body.json();
       throw formatAPIError(statusCode, error, message);
diff --git a/src/push/request.ts b/src/push/request.ts
index 6de663d..f934760 100644
--- a/src/push/request.ts
+++ b/src/push/request.ts
@@ -83,6 +83,12 @@ export type APIMonitorError = {
   details: string;
 };
 
+export function formatNotFoundError(message: string) {
+  return bold(
+    `${symbols['failed']} Please check your kibana url and try again - 404:${message}`
+  );
+}
+
 export function formatAPIError(
   statuCode: number,
   error: string,
diff --git a/templates/.gitignore b/templates/.gitignore
new file mode 100644
index 0000000..36c1c3a
--- /dev/null
+++ b/templates/.gitignore
@@ -0,0 +1,2 @@
+node_modules
+.synthetics
diff --git a/templates/journeys/example.journey.ts b/templates/journeys/example.journey.ts
new file mode 100644
index 0000000..a72c110
--- /dev/null
+++ b/templates/journeys/example.journey.ts
@@ -0,0 +1,18 @@
+import { journey, step, monitor, expect } from '@elastic/synthetics';
+
+journey('My Example Journey', ({ page, params }) => {
+  // Only relevant for the push command to create
+  // monitors in Kibana
+  monitor.use({
+    id: 'example-monitor',
+    schedule: 10,
+  });
+  step('launch application', async () => {
+    await page.goto(params.url);
+  });
+
+  step('assert title', async () => {
+    const header = await page.$('h1');
+    expect(await header.textContent()).toBe('todos');
+  });
+});
diff --git a/templates/synthetics.config.ts b/templates/synthetics.config.ts
new file mode 100644
index 0000000..4109f86
--- /dev/null
+++ b/templates/synthetics.config.ts
@@ -0,0 +1,19 @@
+import type { SyntheticsConfig } from '@elastic/synthetics';
+
+export default env => {
+  const config: SyntheticsConfig = {
+    params: {
+      url: 'https://elastic.github.io/synthetics-demo/',
+    },
+    playwrightOptions: {
+      ignoreHTTPSErrors: false,
+    },
+  };
+  if (env !== 'development') {
+    /**
+     * Override configuration specific to environment
+     * Ex: config.params.url = ""
+     */
+  }
+  return config;
+};


Test patch:
diff --git a/__tests__/cli.test.ts b/__tests__/cli.test.ts
index a887b1a..b7a4872 100644
--- a/__tests__/cli.test.ts
+++ b/__tests__/cli.test.ts
@@ -23,10 +23,10 @@
  *
  */
 
-import { ChildProcess, spawn } from 'child_process';
 import { join } from 'path';
 import { devices } from 'playwright-chromium';
 import { Server } from './utils/server';
+import { CLIMock } from './utils/test-config';
 import {
   DEFAULT_THROTTLING_OPTIONS,
   getNetworkConditions,
@@ -521,82 +521,3 @@ describe('CLI', () => {
     });
   });
 });
-
-class CLIMock {
-  private process: ChildProcess;
-  private data = '';
-  private chunks: Array<string> = [];
-  private waitForText: string;
-  private waitForPromise: () => void;
-  private cliArgs: string[];
-  private stdinStr?: string;
-  exitCode: Promise<number>;
-
-  constructor() {}
-
-  args(a: string[]): CLIMock {
-    this.cliArgs = a;
-    return this;
-  }
-
-  stdin(s: string): CLIMock {
-    this.stdinStr = s;
-    return this;
-  }
-
-  run(spawnOverrides?: { cwd?: string }): CLIMock {
-    this.process = spawn(
-      'node',
-      [join(__dirname, '..', 'dist', 'cli.js'), ...this.cliArgs],
-      {
-        env: process.env,
-        stdio: 'pipe',
-        ...spawnOverrides,
-      }
-    );
-
-    if (this.stdinStr) {
-      this.process.stdin.setDefaultEncoding('utf8');
-      this.process.stdin.write(this.stdinStr);
-      this.process.stdin.end();
-    }
-
-    const dataListener = data => {
-      this.data = data.toString();
-      // Uncomment the line below if the process is blocked and you need to see its output
-      // console.log('CLIMock.stdout:', this.data);
-      this.chunks.push(this.data);
-      if (this.waitForPromise && this.data.includes(this.waitForText)) {
-        this.process.stdout.off('data', dataListener);
-        this.waitForPromise();
-      }
-    };
-    this.process.stdout.on('data', dataListener);
-
-    this.exitCode = new Promise(res => {
-      // Uncomment to debug stderr
-      // this.process.stderr.on('data', data => {
-      //   console.log('CLIMock.stderr:', data.toString());
-      // });
-      this.process.on('exit', code => res(code));
-    });
-
-    return this;
-  }
-
-  async waitFor(text: string): Promise<void> {
-    this.waitForText = text;
-    return new Promise(r => (this.waitForPromise = r));
-  }
-
-  output() {
-    return this.data;
-  }
-
-  buffer() {
-    // Merge all the interleaved chunks from stdout and
-    // split them on new line as synthetics runner writes the
-    // JSON output in separate lines for every event.
-    return this.chunks.join('').split('\n').filter(Boolean);
-  }
-}
diff --git a/__tests__/generator/index.test.ts b/__tests__/generator/index.test.ts
new file mode 100644
index 0000000..761a713
--- /dev/null
+++ b/__tests__/generator/index.test.ts
@@ -0,0 +1,63 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import { existsSync } from 'fs';
+import { rm } from 'fs/promises';
+import { join } from 'path';
+import { CLIMock } from '../utils/test-config';
+
+describe('Generator', () => {
+  const scaffoldDir = join(__dirname, 'scaffold-test');
+  beforeAll(() => {
+    // Not useful for this particular test and delays the test itself
+    process.env.PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD = '1';
+  });
+  afterAll(async () => {
+    process.env.PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD = '0';
+    await rm(scaffoldDir, {
+      recursive: true,
+      force: true,
+    });
+  });
+  it('generate synthetics project - NPM', async () => {
+    const cli = new CLIMock().args(['init', scaffoldDir]).run();
+    expect(await cli.exitCode).toBe(0);
+
+    // Verify files
+    expect(existsSync(join(scaffoldDir, 'package.json'))).toBeTruthy();
+    expect(existsSync(join(scaffoldDir, 'package-lock.json'))).toBeTruthy();
+    expect(existsSync(join(scaffoldDir, '.gitignore'))).toBeTruthy();
+    expect(
+      existsSync(join(scaffoldDir, 'journeys', 'example.journey.ts'))
+    ).toBeTruthy();
+    expect(existsSync(join(scaffoldDir, 'synthetics.config.ts'))).toBeTruthy();
+
+    // Verify stdout
+    const stderr = cli.stderr();
+    expect(stderr).toContain('Initializing Synthetics project using NPM');
+    expect(stderr).toContain('Installing @elastic/synthetics library');
+    expect(stderr).toContain('All set, you can run below commands');
+  }, 30000);
+});
diff --git a/__tests__/push/__snapshots__/request.test.ts.snap b/__tests__/push/__snapshots__/request.test.ts.snap
index f2c4be4..07994bf 100644
--- a/__tests__/push/__snapshots__/request.test.ts.snap
+++ b/__tests__/push/__snapshots__/request.test.ts.snap
@@ -1,5 +1,7 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`Push api request format 404 error 1`] = `"✖ Please check your kibana url and try again - 404:Not Found"`;
+
 exports[`Push api request format api error 1`] = `
 "✖ Error
    ✖ monitor creation failed - 401:Unauthorized
diff --git a/__tests__/push/request.test.ts b/__tests__/push/request.test.ts
index d082b04..b68c6fa 100644
--- a/__tests__/push/request.test.ts
+++ b/__tests__/push/request.test.ts
@@ -31,6 +31,7 @@ import {
   createMonitors,
   formatAPIError,
   formatFailedMonitors,
+  formatNotFoundError,
   formatStaleMonitors,
 } from '../../src/push/request';
 import { Server } from '../utils/server';
@@ -90,6 +91,11 @@ describe('Push api request', () => {
     expect(formatAPIError(statusCode, error, message)).toMatchSnapshot();
   });
 
+  it('format 404 error', () => {
+    const message = 'Not Found';
+    expect(formatNotFoundError(message)).toMatchSnapshot();
+  });
+
   it('format failed monitors', () => {
     const errors: APIMonitorError[] = [
       {
diff --git a/__tests__/utils/test-config.ts b/__tests__/utils/test-config.ts
index 13236a8..010e5e1 100644
--- a/__tests__/utils/test-config.ts
+++ b/__tests__/utils/test-config.ts
@@ -23,6 +23,7 @@
  *
  */
 
+import { ChildProcess, spawn } from 'child_process';
 import { join } from 'path';
 import { Monitor } from '../../dist/dsl/monitor';
 
@@ -45,3 +46,90 @@ export function createTestMonitor(filename: string) {
   monitor.setFilter({ match: 'test' });
   return monitor;
 }
+
+export class CLIMock {
+  private process: ChildProcess;
+  private data = '';
+  private chunks: Array<string> = [];
+  private waitForText: string;
+  private waitForPromise: () => void;
+  private cliArgs: string[];
+  private stdinStr?: string;
+  private stderrStr: string;
+  exitCode: Promise<number>;
+
+  constructor(public debug: boolean = false) {}
+
+  args(a: string[]): CLIMock {
+    this.cliArgs = a;
+    return this;
+  }
+
+  stdin(s: string): CLIMock {
+    this.stdinStr = s;
+    return this;
+  }
+
+  run(spawnOverrides?: { cwd?: string }): CLIMock {
+    this.process = spawn(
+      'node',
+      [join(__dirname, '..', '..', 'dist', 'cli.js'), ...this.cliArgs],
+      {
+        env: process.env,
+        stdio: 'pipe',
+        ...spawnOverrides,
+      }
+    );
+
+    if (this.stdinStr) {
+      this.process.stdin.setDefaultEncoding('utf8');
+      this.process.stdin.write(this.stdinStr);
+      this.process.stdin.end();
+    }
+
+    const dataListener = data => {
+      this.data = data.toString();
+      if (this.debug) {
+        console.log('CLIMock.stdout:', this.data);
+      }
+      this.chunks.push(this.data);
+      if (this.waitForPromise && this.data.includes(this.waitForText)) {
+        this.process.stdout.off('data', dataListener);
+        this.waitForPromise();
+      }
+    };
+    this.process.stdout.on('data', dataListener);
+
+    this.exitCode = new Promise(res => {
+      this.process.stderr.on('data', data => {
+        this.stderrStr += data;
+        if (this.debug) {
+          console.log('CLIMock.stderr:', data.toString());
+        }
+      });
+      this.process.on('exit', code => res(code));
+    });
+
+    return this;
+  }
+
+  async waitFor(text: string): Promise<void> {
+    this.waitForText = text;
+    return new Promise(r => (this.waitForPromise = r));
+  }
+
+  output() {
+    return this.data;
+  }
+
+  buffer() {
+    // Merge all the interleaved chunks from stdout and
+    // split them on new line as synthetics runner writes the
+    // JSON output in separate lines for every event.
+    return this.chunks.join('').split('\n').filter(Boolean);
+  }
+
+  stderr() {
+    return this.stderrStr.toString();
+  }
+}