Repository: elastic/synthetics
Pull request #880 (2024-01-04 11:04:36)

feat: added journey and step annotations
+ fix #404 
Support syntax like

Following annotation have been added

`journey.only` will only run this journey
`journey.skip` will skip this journey
`step.skip`
`step.soft` failure of this step will not abort rest of the journey
`step.only`

`-f --force` if journey is skipped , then while pushing it will error out, but can be ignore with this flag

```
  step.skip('launch application again', async () => {
    expect(page.textContent('body')).toContain('Google');
  });
```

example result 

<img width="808" alt="image" src="https://github.com/elastic/synthetics/assets/3505601/c8c32f07-003b-46c4-8b36-e9888ac6a534">


paulb-elastic: @shahzad31 what's the use case for skipping a step like this (as opposed to deleting it, commenting it out)?  Is it to keep the structure as it was before a change (so step n is always step n, even if step n-1 is removed)?  Do you envisage this being a temporary thing a user would do, or permanent?
shahzad31: > @shahzad31 what's the use case for skipping a step like this (as opposed to deleting it, commenting it out)? Is it to keep the structure as it was before a change (so step n is always step n, even if step n-1 is removed)? Do you envisage this being a temporary thing a user would do, or permanent?

@paulb-elastic it's the most familiar way to in testing frameworks to provide an option like this instead of commenting out , it's especially useful when testing journeys locally. It's a very familiar syntax for especially jest developers. 
vigneshshanmugam: @shahzad31 I pushed a fix that deals with few things
- Multiple .only on journey/steps work properly
- Reporter had issues and was displaying displayed journeys where our counts are based on steps. Ignored journeys are never reported to keep it simpler. 
- Step level only journeys are run even if they are failling. Last error will be added to the journey level error. 

Patch:
diff --git a/src/common_types.ts b/src/common_types.ts
index 8fd96f9..220be52 100644
--- a/src/common_types.ts
+++ b/src/common_types.ts
@@ -276,6 +276,7 @@ export type JourneyResult = {
   error?: Error;
   networkinfo?: PluginOutput['networkinfo'];
   browserconsole?: PluginOutput['browserconsole'];
+  steps?: Array<StepResult>;
 };
 
 export type TestError = {
diff --git a/src/core/index.ts b/src/core/index.ts
index b6bb81c..affe8a8 100644
--- a/src/core/index.ts
+++ b/src/core/index.ts
@@ -23,7 +23,13 @@
  *
  */
 
-import { Journey, JourneyCallback, JourneyOptions } from '../dsl';
+import {
+  Journey,
+  JourneyCallback,
+  JourneyOptions,
+  JourneyWithAnnotations,
+  StepWithAnnotations,
+} from '../dsl';
 import Runner from './runner';
 import { VoidCallback, HooksCallback, Location } from '../common_types';
 import { wrapFnWithLocation } from '../helpers';
@@ -41,28 +47,48 @@ if (!global[SYNTHETICS_RUNNER]) {
 
 export const runner: Runner = global[SYNTHETICS_RUNNER];
 
-export const journey = wrapFnWithLocation(
-  (
-    location: Location,
-    options: JourneyOptions | string,
-    callback: JourneyCallback
-  ) => {
-    log(`Journey register: ${JSON.stringify(options)}`);
-    if (typeof options === 'string') {
-      options = { name: options, id: options };
+const createJourney = (type?: 'skip' | 'only') =>
+  wrapFnWithLocation(
+    (
+      location: Location,
+      options: JourneyOptions | string,
+      callback: JourneyCallback
+    ) => {
+      log(`Journey register: ${JSON.stringify(options)}`);
+      if (typeof options === 'string') {
+        options = { name: options, id: options };
+      }
+      const j = new Journey(options, callback, location);
+
+      if (type) {
+        j[type] = true;
+      }
+
+      runner.addJourney(j);
+      return j;
     }
-    const j = new Journey(options, callback, location);
-    runner.addJourney(j);
-    return j;
-  }
-);
+  );
 
-export const step = wrapFnWithLocation(
-  (location: Location, name: string, callback: VoidCallback) => {
-    log(`Step register: ${name}`);
-    return runner.currentJourney?.addStep(name, callback, location);
-  }
-);
+export const journey = createJourney() as JourneyWithAnnotations;
+journey.skip = createJourney('skip');
+journey.only = createJourney('only');
+
+const createStep = (type?: 'skip' | 'soft' | 'only') =>
+  wrapFnWithLocation(
+    (location: Location, name: string, callback: VoidCallback) => {
+      log(`Step register: ${name}`);
+      const step = runner.currentJourney?.addStep(name, callback, location);
+      if (type) {
+        step[type] = true;
+      }
+      return step;
+    }
+  );
+
+export const step = createStep() as StepWithAnnotations;
+step.skip = createStep('skip');
+step.soft = createStep('soft');
+step.only = createStep('only');
 
 export const monitor = {
   use: wrapFnWithLocation((location: Location, config: MonitorConfig) => {
diff --git a/src/core/runner.ts b/src/core/runner.ts
index f7d67cf..0cc9f46 100644
--- a/src/core/runner.ts
+++ b/src/core/runner.ts
@@ -256,19 +256,30 @@ export default class Runner {
     options: RunOptions
   ) {
     const results: Array<StepResult> = [];
+    const isOnlyExists = journey.steps.filter(s => s.only).length > 0;
     let skipStep = false;
     for (const step of journey.steps) {
       const start = monotonicTimeInSeconds();
       this.#reporter?.onStepStart?.(journey, step);
       let data: StepResult = { status: 'succeeded' };
-      if (skipStep) {
+      /**
+       * Skip the step
+       * - if the step is marked as skip
+       * - if the previous step fails and the current step is not marked as soft
+       * - if the step is not marked as only and there are steps marked as only
+       */
+      if (
+        step.skip ||
+        (skipStep && !step.only) ||
+        (isOnlyExists && !step.only)
+      ) {
         data.status = 'skipped';
       } else {
         data = await this.runStep(step, context, options);
         /**
          * skip next steps if the previous step returns error
          */
-        if (data.error) skipStep = true;
+        if (data.error && !step.soft) skipStep = true;
       }
       this.#reporter?.onStepEnd?.(journey, step, {
         start,
@@ -366,6 +377,7 @@ export default class Runner {
           result.error = stepResult.error;
         }
       }
+      result.steps = stepResults;
       await this.runAfterHook(journey, hookArgs);
     } catch (e) {
       result.status = 'failed';
@@ -410,6 +422,11 @@ export default class Runner {
     const monitors: Monitor[] = [];
     for (const journey of this.journeys) {
       this.#currentJourney = journey;
+      if (journey.skip) {
+        throw new Error(
+          `Journey ${journey.name} is skipped. Please remove the journey.skip annotation and try again.`
+        );
+      }
       /**
        * Execute dummy callback to get all monitor specific
        * configurations for the current journey
@@ -436,6 +453,14 @@ export default class Runner {
     }).catch(e => (this.hookError = e));
 
     const { dryRun, match, tags } = options;
+    /**
+     * Skip other journeys when using `.only`
+     */
+    const onlyJournerys = this.journeys.filter(j => j.only);
+    if (onlyJournerys.length > 0) {
+      this.journeys = onlyJournerys;
+    }
+
     for (const journey of this.journeys) {
       /**
        * Used by heartbeat to gather all registered journeys
@@ -444,7 +469,7 @@ export default class Runner {
         this.#reporter.onJourneyRegister?.(journey);
         continue;
       }
-      if (!journey.isMatch(match, tags)) {
+      if (!journey.isMatch(match, tags) || journey.skip) {
         continue;
       }
       const journeyResult: JourneyResult = this.hookError
diff --git a/src/dsl/journey.ts b/src/dsl/journey.ts
index b6eea38..15d47e2 100644
--- a/src/dsl/journey.ts
+++ b/src/dsl/journey.ts
@@ -61,6 +61,8 @@ export class Journey {
   steps: Step[] = [];
   hooks: Hooks = { before: [], after: [] };
   monitor: Monitor;
+  skip = false;
+  only = false;
 
   constructor(
     options: JourneyOptions,
@@ -107,3 +109,19 @@ export class Journey {
     return isMatch(this.tags, this.name, tagsPattern, matchPattern);
   }
 }
+
+type JourneyType = (
+  options: string | JourneyOptions,
+  callback: JourneyCallback
+) => Journey;
+
+export type JourneyWithAnnotations = JourneyType & {
+  /**
+   * Skip this journey and all its steps
+   */
+  skip: JourneyType;
+  /**
+   * Run only this journey and skip rest of the journeys
+   */
+  only: JourneyType;
+};
diff --git a/src/dsl/step.ts b/src/dsl/step.ts
index 44148ee..0196379 100644
--- a/src/dsl/step.ts
+++ b/src/dsl/step.ts
@@ -30,6 +30,9 @@ export class Step {
   index: number;
   callback: VoidCallback;
   location?: Location;
+  skip = false;
+  soft = false;
+  only = false;
 
   constructor(
     name: string,
@@ -43,3 +46,20 @@ export class Step {
     this.location = location;
   }
 }
+
+type StepType = (name: string, callback: VoidCallback) => Step;
+
+export type StepWithAnnotations = StepType & {
+  /**
+   * Skip this step on the journey
+   */
+  skip: StepType;
+  /**
+   * Failure of soft step will not skip rest of the steps in the journey
+   */
+  soft: StepType;
+  /**
+   * Run only this step and skip rest of the steps in the journey
+   */
+  only: StepType;
+};
diff --git a/src/helpers.ts b/src/helpers.ts
index d9b892b..47c0673 100644
--- a/src/helpers.ts
+++ b/src/helpers.ts
@@ -312,7 +312,7 @@ export function wrapFnWithLocation<A extends unknown[], R>(
   };
 }
 
-// Safely parse ND JSON (Newline delimitted JSON) chunks
+// Safely parse ND JSON (Newline delimited JSON) chunks
 export function safeNDJSONParse(data: string | string[]) {
   // data may not be at proper newline boundaries, so we make sure everything is split
   // on proper newlines
diff --git a/src/index.ts b/src/index.ts
index d1cb325..df3fcbd 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -73,7 +73,13 @@ export type {
   StartEvent,
 } from './common_types';
 
-export type { Journey, JourneyCallback } from './dsl';
+export type {
+  Journey,
+  JourneyCallback,
+  JourneyWithAnnotations,
+  JourneyOptions,
+  StepWithAnnotations,
+} from './dsl';
 
 export type {
   Action,


Test patch:
diff --git a/__tests__/core/runner.test.ts b/__tests__/core/runner.test.ts
index 363c6b8..5112cff 100644
--- a/__tests__/core/runner.test.ts
+++ b/__tests__/core/runner.test.ts
@@ -130,9 +130,18 @@ describe('runner', () => {
       throw error;
     });
     const result = await runner.runJourney(journey, defaultRunOptions);
-    expect(result).toEqual({
+    expect(result).toMatchObject({
+      error: error,
       status: 'failed',
-      error,
+      steps: [
+        {
+          status: 'succeeded',
+        },
+        {
+          error: error,
+          status: 'failed',
+        },
+      ],
     });
   });
 
@@ -141,7 +150,7 @@ describe('runner', () => {
     journey.addHook('before', noop);
     journey.addHook('after', noop);
     const result = await runner.runJourney(journey, defaultRunOptions);
-    expect(result).toEqual({
+    expect(result).toMatchObject({
       status: 'succeeded',
     });
   });
@@ -154,7 +163,7 @@ describe('runner', () => {
       throw error;
     });
     const result = await runner.runJourney(journey, defaultRunOptions);
-    expect(result).toEqual({
+    expect(result).toMatchObject({
       status: 'failed',
       error,
     });
@@ -318,7 +327,7 @@ describe('runner', () => {
     const journey = new Journey({ name }, noop);
     runner.addJourney(journey);
     const result = await runner.run(defaultRunOptions);
-    expect(result).toEqual({
+    expect(result).toMatchObject({
       [name]: { status: 'succeeded' },
     });
   });
@@ -331,7 +340,7 @@ describe('runner', () => {
         ...defaultRunOptions,
         match: 'j2',
       })
-    ).toEqual({
+    ).toMatchObject({
       j2: { status: 'succeeded' },
     });
   });
@@ -344,7 +353,7 @@ describe('runner', () => {
         ...defaultRunOptions,
         match: 'j*',
       })
-    ).toEqual({
+    ).toMatchObject({
       j1: { status: 'succeeded' },
       tagj2: { status: 'succeeded' },
     });
@@ -362,7 +371,7 @@ describe('runner', () => {
         tags: ['foo*'],
         match: 'j*',
       })
-    ).toEqual({
+    ).toMatchObject({
       j1: { status: 'succeeded' },
       j3: { status: 'succeeded' },
       j5: { status: 'succeeded' },
@@ -378,7 +387,7 @@ describe('runner', () => {
         ...defaultRunOptions,
         tags: ['hello:b*'],
       })
-    ).toEqual({
+    ).toMatchObject({
       j2: { status: 'succeeded' },
       j3: { status: 'succeeded' },
     });
@@ -393,7 +402,7 @@ describe('runner', () => {
         ...defaultRunOptions,
         tags: ['!hello:b*'],
       })
-    ).toEqual({
+    ).toMatchObject({
       j1: { status: 'succeeded' },
     });
   });
@@ -407,7 +416,7 @@ describe('runner', () => {
     });
     runner.addJourney(j2);
     const result = await runner.run(defaultRunOptions);
-    expect(result).toEqual({
+    expect(result).toMatchObject({
       j1: { status: 'succeeded' },
       j2: { status: 'failed', error },
     });
@@ -526,7 +535,7 @@ describe('runner', () => {
       ...defaultRunOptions,
       reporter: CustomReporter,
     });
-    expect(result).toEqual({
+    expect(result).toMatchObject({
       foo: {
         status: 'succeeded',
       },
@@ -793,4 +802,123 @@ describe('runner', () => {
       });
     });
   });
+
+  describe('journey and step annotations', () => {
+    it('skip journey', async () => {
+      runner.addJourney(journey.skip('j1', noop));
+      runner.addJourney(journey('j2', noop));
+      const result = await runner.run(defaultRunOptions);
+      expect(result).toEqual({
+        j2: {
+          status: 'succeeded',
+          steps: [],
+        },
+      });
+    });
+
+    it('only journey', async () => {
+      runner.addJourney(journey.only('j1', noop));
+      runner.addJourney(journey('j2', noop));
+      const result = await runner.run(defaultRunOptions);
+      expect(result).toEqual({
+        j1: {
+          status: 'succeeded',
+          steps: [],
+        },
+      });
+    });
+
+    it('skip step', async () => {
+      runner.addJourney(
+        journey('j1', async () => {
+          step('step1', noop);
+          step.skip('step2', noop);
+        })
+      );
+      const result = await runner.run(defaultRunOptions);
+      expect(result).toMatchObject({
+        j1: {
+          status: 'succeeded',
+          steps: [
+            {
+              status: 'succeeded',
+            },
+            {
+              status: 'skipped',
+            },
+          ],
+        },
+      });
+    });
+    it('soft step failure', async () => {
+      runner.addJourney(
+        journey('j1', async () => {
+          step('step1', noop);
+          step.soft('step2', async () => {
+            throw new Error('step2 soft error');
+          });
+          step.soft('step3', noop);
+        })
+      );
+      const result = await runner.run(defaultRunOptions);
+
+      expect(result).toMatchObject({
+        j1: {
+          error: new Error('step2 soft error'),
+          status: 'failed',
+          steps: [
+            {
+              status: 'succeeded',
+            },
+            {
+              error: new Error('step2 soft error'),
+              status: 'failed',
+            },
+            {
+              status: 'succeeded',
+            },
+          ],
+        },
+      });
+    });
+
+    it('only step', async () => {
+      runner.addJourney(
+        journey('j1', async () => {
+          step('step1', noop);
+          step.only('step2', async () => {
+            throw new Error('step2 only error');
+          });
+          step.only('step3', async () => {
+            throw new Error('step3 only error');
+          });
+          step('step3', noop);
+        })
+      );
+      const result = await runner.run(defaultRunOptions);
+
+      expect(result).toMatchObject({
+        j1: {
+          error: new Error('step3 only error'),
+          status: 'failed',
+          steps: [
+            {
+              status: 'skipped',
+            },
+            {
+              error: new Error('step2 only error'),
+              status: 'failed',
+            },
+            {
+              error: new Error('step3 only error'),
+              status: 'failed',
+            },
+            {
+              status: 'skipped',
+            },
+          ],
+        },
+      });
+    });
+  });
 });
diff --git a/__tests__/index.test.ts b/__tests__/index.test.ts
index e025ec1..70a41a5 100644
--- a/__tests__/index.test.ts
+++ b/__tests__/index.test.ts
@@ -48,7 +48,10 @@ describe('Run', () => {
     const results = await Promise.all(promises);
 
     expect(results).toEqual([
-      { j1: { status: 'succeeded' }, j2: { status: 'succeeded' } },
+      {
+        j1: { status: 'succeeded', steps: [] },
+        j2: { status: 'succeeded', steps: [] },
+      },
       {},
     ]);
   });