Repository: elastic/synthetics (ts)

Issue:
Discovery option for public locations
As a user of Synthetic Project Monitors
I want to be able to determine the public locations I can use for running tests from (via the service)
So that I know what configuration to add to my scripts

ACs:
* A mechanism in Synthetics to get the locations from the command line (e.g. `npx @elastic/synthetics locations --list`)
* This will return a list of locations
  * Parsed from https://github.com/elastic/synthetics/blob/main/src/dsl/locations.ts (i.e. is based on what the current version of the Synthetics package installed, knows about)
  * It must be clear from the list, what string the user needs to include in `monitor.use.locations: []`

Pull request description:
feat: discovery command for locations
+ fix #551 
+ Support discovery option for both Public and Private locations. 
+ By default we render all default public locations, and when user provides `--auth` and `--url` options, we also render the private locations which comes in handy without doing project init setup. 

### Testing
- Build the local project using - npm run build
- List Public locations via - node dist/cli.js locations
- List public and private locations - node dist/cli.js locations --auth apiKey --url kibana-url



Patch:
diff --git a/src/cli.ts b/src/cli.ts
index 2554882..b02cd92 100644
--- a/src/cli.ts
+++ b/src/cli.ts
@@ -29,7 +29,11 @@ import { program, Option } from 'commander';
 import { cwd } from 'process';
 import { CliArgs, PushOptions } from './common_types';
 import { reporters } from './reporters';
-import { normalizeOptions, parseThrottling } from './options';
+import {
+  normalizeOptions,
+  parseThrottling,
+  getCommonCommandOpts,
+} from './options';
 import { loadTestFiles } from './loader';
 import { run } from './';
 import { runner } from './core';
@@ -40,14 +44,22 @@ import {
   validateSettings,
   catchIncorrectSettings,
 } from './push';
-import { getLocations, LocationCmdOptions } from './locations';
+import {
+  formatLocations,
+  getLocations,
+  renderLocations,
+  LocationCmdOptions,
+} from './locations';
 import { resolve } from 'path';
 import { Generator } from './generator';
 import { error } from './helpers';
+import { LocationsMap } from './locations/public-locations';
 
 /* eslint-disable-next-line @typescript-eslint/no-var-requires */
 const { name, version } = require('../package.json');
 
+const { params, pattern, tags, match, playwrightOpts } = getCommonCommandOpts();
+
 program
   .name(`npx ${name}`)
   .usage('[options] [dir] [files] file')
@@ -55,20 +67,15 @@ program
     '-c, --config <path>',
     'configuration path (default: synthetics.config.js)'
   )
-  .option(
-    '-p, --params <jsonstring>',
-    'JSON object that gets injected to all journeys',
-    JSON.parse
-  )
+  .addOption(pattern)
+  .addOption(tags)
+  .addOption(match)
+  .addOption(params)
   .addOption(
     new Option('--reporter <value>', `output repoter format`).choices(
       Object.keys(reporters)
     )
   )
-  .option(
-    '--pattern <pattern>',
-    'RegExp file patterns to search inside directory'
-  )
   .option('--inline', 'Run inline journeys from heartbeat')
   .option('-r, --require <modules...>', 'module(s) to preload')
   .option('--no-headless', 'run browser in headful mode')
@@ -87,14 +94,6 @@ program
     '--dry-run',
     "don't actually execute anything, report only registered journeys"
   )
-  .option(
-    '--tags <name...>',
-    'run only journeys with a tag that matches the glob'
-  )
-  .option(
-    '--match <name>',
-    'run only journeys with a name or tag that matches the glob'
-  )
   .option(
     '--outfd <fd>',
     'specify a file descriptor for logs. Default is stdout',
@@ -123,11 +122,7 @@ program
     {}
   )
   .option('--no-throttling', 'Turns off default network throttling.')
-  .option(
-    '--playwright-options <jsonstring>',
-    'JSON object to pass in custom Playwright options for the agent. Options passed will be merged with Playwright options defined in your synthetics.config.js file. Options defined via --playwright-options take precedence.',
-    JSON.parse
-  )
+  .addOption(playwrightOpts)
   .version(version)
   .description('Run synthetic tests')
   .action(async (cliArgs: CliArgs) => {
@@ -155,24 +150,12 @@ program
 program
   .command('push')
   .description(
-    'Push journeys in the current directory to create monitors within the Kibana monitor management UI'
+    'Push all journeys in the current directory to create monitors within the Kibana monitor management UI'
   )
   .requiredOption(
     '--auth <auth>',
     'API key used for Kibana authentication(https://www.elastic.co/guide/en/kibana/master/api-keys.html).'
   )
-  .option(
-    '--pattern <pattern>',
-    'RegExp file patterns to push inside current directory'
-  )
-  .option(
-    '--tags <name...>',
-    'push only journeys with a tag that matches the glob'
-  )
-  .option(
-    '--match <name>',
-    'push only journeys with a name or tag that matches the glob'
-  )
   .option(
     '--schedule <time-in-minutes>',
     "schedule in minutes for the pushed monitors. Setting `10`, for example, configures monitors which don't have an interval defined to run every 10 minutes.",
@@ -184,13 +167,11 @@ program
       'default list of locations from which your monitors will run.'
     ).choices(SyntheticsLocations)
   )
-  .addOption(
-    new Option(
-      '--private-locations <locations...>',
-      'default list of private locations from which your monitors will run.'
-    )
+  .option(
+    '--private-locations <locations...>',
+    'default list of private locations from which your monitors will run.'
   )
-  .option('--url <url>', 'kibana URL to upload the monitors')
+  .option('--url <url>', 'Kibana URL to upload the monitors')
   .option(
     '--project <id>',
     'id that will be used for logically grouping monitors'
@@ -199,6 +180,11 @@ program
     '--space <space>',
     'the target Kibana spaces for the pushed monitors — spaces help you organise pushed monitors.'
   )
+  .addOption(pattern)
+  .addOption(tags)
+  .addOption(match)
+  .addOption(params)
+  .addOption(playwrightOpts)
   .action(async (cmdOpts: PushOptions) => {
     try {
       const settings = await loadSettings();
@@ -224,7 +210,6 @@ program
   .description('Initialize Elastic synthetics project')
   .action(async (dir = '') => {
     try {
-      console.log(dir);
       const generator = await new Generator(resolve(process.cwd(), dir));
       await generator.setup();
     } catch (e) {
@@ -236,16 +221,22 @@ program
 // Locations command
 program
   .command('locations')
-  .description('List all locations to run the synthetics monitors')
-  .option('--url <url>', 'kibana URL to upload the monitors')
-  .requiredOption(
+  .description(
+    `List all locations to run the synthetics monitors. Pass optional '--url' and '--auth' to list private locations.`
+  )
+  .option('--url <url>', 'Kibana URL to fetch all public and private locations')
+  .option(
     '--auth <auth>',
     'API key used for Kibana authentication(https://www.elastic.co/guide/en/kibana/master/api-keys.html).'
   )
   .action(async (cmdOpts: LocationCmdOptions) => {
     try {
-      console.log(await getLocations(cmdOpts));
-      // TODO: Display list of locations
+      let locations = Object.keys(LocationsMap);
+      if (cmdOpts.auth && cmdOpts.url) {
+        const allLocations = await getLocations(cmdOpts);
+        locations = formatLocations(allLocations);
+      }
+      renderLocations(locations);
     } catch (e) {
       e && error(e);
       process.exit(1);
diff --git a/src/dsl/monitor.ts b/src/dsl/monitor.ts
index 261deda..195ba33 100644
--- a/src/dsl/monitor.ts
+++ b/src/dsl/monitor.ts
@@ -31,7 +31,7 @@ import {
   Params,
   PlaywrightOptions,
 } from '../common_types';
-import { LocationsMap } from './locations';
+import { LocationsMap } from '../locations/public-locations';
 
 export type SyntheticsLocationsType = keyof typeof LocationsMap;
 export const SyntheticsLocations = Object.keys(
diff --git a/src/generator/index.ts b/src/generator/index.ts
index e9d3669..df14317 100644
--- a/src/generator/index.ts
+++ b/src/generator/index.ts
@@ -54,13 +54,18 @@ type PromptOptions = ProjectSettings & {
 };
 
 // exported for testing
-export const regularFiles = [
-  '.github/workflows/run-synthetics.yml',
-  'README.md',
+export const REGULAR_FILES_PATH = [
   'journeys/example.journey.ts',
-  'journeys/advanced-example.journey.ts',
   'journeys/advanced-example-helpers.ts',
+  'journeys/advanced-example.journey.ts',
+  '.github/workflows/run-synthetics.yml',
+  'README.md',
 ];
+export const SETTINGS_PATH = '.synthetics/project.json';
+export const CONFIG_PATH = 'synthetics.config.ts';
+
+// Files to be overriden by default if the project is initialized multiple times
+const DEFAULT_OVERRIDES = [SETTINGS_PATH, CONFIG_PATH];
 
 export class Generator {
   pkgManager = 'npm';
@@ -169,24 +174,20 @@ export class Generator {
       space: answers.space,
       project: answers.project,
     };
-    fileMap.set(
-      '.synthetics/project.json',
-      JSON.stringify(projectJSON, null, 2) + '\n'
-    );
+    fileMap.set(SETTINGS_PATH, JSON.stringify(projectJSON, null, 2) + '\n');
 
     // Setup Synthetics config file
-    const configFile = 'synthetics.config.ts';
     fileMap.set(
-      configFile,
+      CONFIG_PATH,
       replaceTemplates(
-        await readFile(join(templateDir, configFile), 'utf-8'),
+        await readFile(join(templateDir, CONFIG_PATH), 'utf-8'),
         answers
       )
     );
 
     // Setup non-templated files
     Promise.all(
-      regularFiles.map(async file => {
+      REGULAR_FILES_PATH.map(async file => {
         fileMap.set(file, await readFile(join(templateDir, file), 'utf-8'));
       })
     ).catch(e => {
@@ -202,7 +203,11 @@ export class Generator {
   async createFile(relativePath: string, content: string, override = false) {
     const absolutePath = join(this.projectDir, relativePath);
 
-    if (!override && existsSync(absolutePath)) {
+    if (
+      !override &&
+      !DEFAULT_OVERRIDES.includes(relativePath) &&
+      existsSync(absolutePath)
+    ) {
       const { override } = await prompt<{ override: boolean }>({
         type: 'confirm',
         name: 'override',
diff --git a/src/locations/index.ts b/src/locations/index.ts
index 4c6a9ce..a0296b7 100644
--- a/src/locations/index.ts
+++ b/src/locations/index.ts
@@ -29,7 +29,7 @@ import {
   sendRequest,
   ok,
 } from '../push/request';
-import { removeTrailingSlash } from '../helpers';
+import { indent, removeTrailingSlash, write } from '../helpers';
 
 export type LocationCmdOptions = {
   auth: string;
@@ -71,7 +71,7 @@ export function formatLocations(locations: Array<LocationMetadata>) {
   for (const location of locations) {
     let name = location.label;
     if (location.isServiceManaged) {
-      [, name] = name.split('-');
+      [, name] = name.includes('-') ? name.split('-') : [, name];
       name = name.toLowerCase().trim().split(' ').join('_');
     } else {
       name = `${name}${PRIVATE_KEYWORD}`;
@@ -92,3 +92,17 @@ export function groupLocations(locations: Array<string>) {
   }
   return grouped;
 }
+
+export function renderLocations(locations: Array<string>) {
+  let outer = 'Available locations: \n';
+  let inner = '';
+  for (const location of locations) {
+    inner += `* ${location}\n`;
+  }
+  outer += indent(inner);
+  outer += `\nSet default location for monitors via
+  - Synthetics config file 'monitors.locations' | 'monitors.privateLocations' field
+  - Monitor API 'monitor.use({ locations: ["japan"], privateLocations: ["custom-location"] }')`;
+
+  write(outer);
+}
diff --git a/src/dsl/locations.ts b/src/locations/public-locations.ts
similarity index 100%
rename from src/dsl/locations.ts
rename to src/locations/public-locations.ts
diff --git a/src/options.ts b/src/options.ts
index e50066b..0ba2cef 100644
--- a/src/options.ts
+++ b/src/options.ts
@@ -24,6 +24,7 @@
  */
 
 import merge from 'deepmerge';
+import { createOption } from 'commander';
 import { readConfig } from './config';
 import { getNetworkConditions, DEFAULT_THROTTLING_OPTIONS } from './helpers';
 import type { CliArgs, RunOptions, ThrottlingOptions } from './common_types';
@@ -173,3 +174,38 @@ export function parseThrottling(value: string, prev?: string) {
   }
   return JSON.parse(value || prev);
 }
+
+export function getCommonCommandOpts() {
+  const params = createOption(
+    '-p, --params <jsonstring>',
+    'JSON object that gets injected to all journeys'
+  );
+  params.argParser(JSON.parse);
+
+  const playwrightOpts = createOption(
+    '--playwright-options <jsonstring>',
+    'JSON object to pass in custom Playwright options for the agent. Options passed will be merged with Playwright options defined in your synthetics.config.js file.'
+  );
+  playwrightOpts.argParser(JSON.parse);
+
+  const pattern = createOption(
+    '--pattern <pattern>',
+    'RegExp file patterns to search inside directory'
+  );
+  const tags = createOption(
+    '--tags <name...>',
+    'run only journeys with a tag that matches the glob'
+  );
+  const match = createOption(
+    '--match <name>',
+    'run only journeys with a name or tag that matches the glob'
+  );
+
+  return {
+    params,
+    playwrightOpts,
+    pattern,
+    tags,
+    match,
+  };
+}
diff --git a/src/push/monitor.ts b/src/push/monitor.ts
index f8cdc4b..137c208 100644
--- a/src/push/monitor.ts
+++ b/src/push/monitor.ts
@@ -28,7 +28,7 @@ import { join } from 'path';
 import { Bundler } from './bundler';
 import { sendRequest } from './request';
 import { removeTrailingSlash, SYNTHETICS_PATH } from '../helpers';
-import { LocationsMap } from '../dsl/locations';
+import { LocationsMap } from '../locations/public-locations';
 import { Monitor, MonitorConfig } from '../dsl/monitor';
 import { PushOptions } from '../common_types';
 
@@ -46,7 +46,7 @@ type APISchema = {
 
 function translateLocation(locations?: MonitorConfig['locations']) {
   if (!locations) return [];
-  return locations.map(loc => LocationsMap[loc]).filter(Boolean);
+  return locations.map(loc => LocationsMap[loc] || loc).filter(Boolean);
 }
 
 export async function buildMonitorSchema(monitors: Monitor[]) {
diff --git a/utils/update_locations.sh b/utils/update_locations.sh
index e503a38..41ac3d6 100755
--- a/utils/update_locations.sh
+++ b/utils/update_locations.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 set -ex
-TARGET=src/dsl/locations.ts
+TARGET=src/locations/public-locations.ts
 echo '// DO NOT EDIT - UPDATE WITH `npm run build:locations`' > $TARGET
 echo 'export const LocationsMap =' >> $TARGET
 curl -s https://manifest.synthetics.elastic-cloud.com/v1/manifest.json | jq '.locations | to_entries | map({(.value.geo.name | split(" - ")[1] | ascii_downcase | split(" ") | join("_") ) : .key } ) | add' >> $TARGET


Test patch:
diff --git a/__tests__/e2e/versions b/__tests__/e2e/versions
index 268c34a..339603a 100644
--- a/__tests__/e2e/versions
+++ b/__tests__/e2e/versions
@@ -1,3 +1,3 @@
 7.17.6-SNAPSHOT
-8.3.1-SNAPSHOT
 8.4.0-SNAPSHOT
+8.5.0-SNAPSHOT
diff --git a/__tests__/fixtures/locationinfo.ts b/__tests__/fixtures/locationinfo.ts
index 75b3d0e..82d1d17 100644
--- a/__tests__/fixtures/locationinfo.ts
+++ b/__tests__/fixtures/locationinfo.ts
@@ -31,6 +31,11 @@ export const LOCATIONS: LocationAPIResponse['locations'] = [
     label: 'Asia/Pacific - Japan',
     isServiceManaged: true,
   },
+  {
+    id: 'new-location',
+    label: 'New Location',
+    isServiceManaged: true,
+  },
   {
     id: 'private-node-1',
     label: 'custom location 1',
diff --git a/__tests__/generator/index.test.ts b/__tests__/generator/index.test.ts
index 973c9e8..38eda88 100644
--- a/__tests__/generator/index.test.ts
+++ b/__tests__/generator/index.test.ts
@@ -27,7 +27,11 @@ import { existsSync } from 'fs';
 import { readFile, rm } from 'fs/promises';
 import { join } from 'path';
 import { CLIMock } from '../utils/test-config';
-import { regularFiles } from '../../src/generator';
+import {
+  REGULAR_FILES_PATH,
+  CONFIG_PATH,
+  SETTINGS_PATH,
+} from '../../src/generator';
 
 describe('Generator', () => {
   const scaffoldDir = join(__dirname, 'scaffold-test');
@@ -64,9 +68,7 @@ describe('Generator', () => {
     }
 
     // Project setup file
-    expect(
-      existsSync(join(scaffoldDir, '.synthetics', 'project.json'))
-    ).toBeTruthy();
+    expect(existsSync(join(scaffoldDir, SETTINGS_PATH))).toBeTruthy();
 
     // Verify files
     expect(existsSync(join(scaffoldDir, 'package.json'))).toBeTruthy();
@@ -81,15 +83,12 @@ describe('Generator', () => {
       "
     `);
 
-    regularFiles.forEach(fn => {
+    REGULAR_FILES_PATH.forEach(fn => {
       expect(existsSync(join(scaffoldDir, fn))).toBeTruthy();
     });
-    expect(existsSync(join(scaffoldDir, 'synthetics.config.ts'))).toBeTruthy();
+    expect(existsSync(join(scaffoldDir, CONFIG_PATH))).toBeTruthy();
     // Verify schedule and locations
-    const configFile = await readFile(
-      join(scaffoldDir, 'synthetics.config.ts'),
-      'utf-8'
-    );
+    const configFile = await readFile(join(scaffoldDir, CONFIG_PATH), 'utf-8');
     expect(configFile).toContain(`locations: ['us_east']`);
     expect(configFile).toContain(`privateLocations: ['custom']`);
     expect(configFile).toContain(`schedule: 30`);
diff --git a/__tests__/locations/index.test.ts b/__tests__/locations/index.test.ts
index d2d17d0..d944e62 100644
--- a/__tests__/locations/index.test.ts
+++ b/__tests__/locations/index.test.ts
@@ -23,15 +23,17 @@
  *
  */
 
-import { Server } from '../utils/server';
 import {
   formatLocations,
   getLocations,
   groupLocations,
 } from '../../src/locations';
 import { LOCATIONS } from '../fixtures/locationinfo';
+import { Server } from '../utils/server';
+import { CLIMock } from '../utils/test-config';
 
 describe('Locations', () => {
+  const apiKey = 'foo';
   let server: Server;
   beforeAll(async () => {
     server = await Server.create();
@@ -47,27 +49,52 @@ describe('Locations', () => {
   it('get locations', async () => {
     const locations = await getLocations({
       url: `${server.PREFIX}`,
-      auth: 'apiKey',
+      auth: apiKey,
     });
-    expect(locations.length).toBe(4);
+    expect(locations.length).toBe(5);
   });
 
   it('format and group locations by labels', async () => {
     const locations = await getLocations({
       url: `${server.PREFIX}`,
-      auth: 'apiKey',
+      auth: apiKey,
     });
     const formatted = formatLocations(locations);
     expect(formatted).toEqual([
       'japan',
+      'new_location',
       'custom location 1(private)',
       'custom location 2(private)',
       'us_west',
     ]);
 
     expect(groupLocations(formatted)).toEqual({
-      locations: ['japan', 'us_west'],
+      locations: ['japan', 'new_location', 'us_west'],
       privateLocations: ['custom location 1', 'custom location 2'],
     });
   });
+
+  describe('CLI command', () => {
+    const runLocations = async (args: Array<string> = []) => {
+      const cli = new CLIMock()
+        .args(['locations', ...args])
+        .run({ cwd: __dirname });
+      expect(await cli.exitCode).toBe(0);
+      return cli.stderr();
+    };
+    it('render public locations by default', async () => {
+      const output = await runLocations();
+      expect(output).not.toContain(`custom location`);
+    });
+
+    it('render private locations when options are provided', async () => {
+      const output = await runLocations([
+        '--url',
+        server.PREFIX,
+        '--auth',
+        apiKey,
+      ]);
+      expect(output).toContain(`custom location 1(private)`);
+    });
+  });
 });