Repository: nightwatchjs/nightwatch (js)

Issue:
[Enhancement] Report Test-status to Browserstack when using cucumber runner
## Describe the bug
Right now, the test scenario remains unmarked in Browserstack while using cucumber-runner in Nightwatch.

## Proposed Solution
<img width="1198" alt="Screenshot 2022-04-05 at 3 45 34 PM" src="https://user-images.githubusercontent.com/28780767/161733097-8ee320da-ac59-4051-9cfc-c0544cd2540f.png">

 The scenarios should be marked as passed or failed status and Nightwatch should also send the reason for the failure.  


















Pull request:
Fixed #3132 - Mark test scenarios passed/failed and send reason to Browserstack while using Cucumber as a test-runner
### Changes
- extracted scenarios results in the `After` hook and send to BrowserStack.
- exposed `transport` from Nightwatch `createClient`.
- added tests for the change.

### Impacts
- fixes #3132 

Patch:
diff --git a/.github/workflows/build-node.yaml b/.github/workflows/build-node.yaml
index fa5484c2..5826f553 100644
--- a/.github/workflows/build-node.yaml
+++ b/.github/workflows/build-node.yaml
@@ -16,7 +16,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [12.x, 14.x, 15.x, 16.x]
+        node-version: [12.x, 14.x, 16.x]
         # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
 
     steps:
@@ -34,9 +34,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [12.x, 14.x, 15.x, 16.x]
-        # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
-
+        node-version: [16.x]
     steps:
     - uses: actions/checkout@v2
     - name: Use Node.js ${{ matrix.node-version }}
diff --git a/cucumber-js/_setup_cucumber_runner.js b/cucumber-js/_setup_cucumber_runner.js
index 8cf5e084..62553828 100644
--- a/cucumber-js/_setup_cucumber_runner.js
+++ b/cucumber-js/_setup_cucumber_runner.js
@@ -63,7 +63,15 @@ Before(function({pickle}) {
   }
 });
 
-After(async function() {
+After(async function(testCase) {
+
+  //send test-case result to cloud provider
+  const {result} = testCase;
+  if (this.client && result) {
+    const error = result.status === 'FAILED' ? new Error(result.message) : null;
+    await this.client.transport.testSuiteFinished(error);
+  }
+
   if (this.browser) {
     await this.browser.quit();
   }
diff --git a/lib/api/assertions/domPropertyMatches.js b/lib/api/assertions/domPropertyMatches.js
index fbcb8481..e6945ecf 100644
--- a/lib/api/assertions/domPropertyMatches.js
+++ b/lib/api/assertions/domPropertyMatches.js
@@ -5,15 +5,15 @@
  * @example
  * 
  * ```
- *    this.demoTest = function (browser) {
- *      browser.assert.domPropertyMatches('#main', 'tagName', '(frame)');
- *
+ * this.demoTest = function (browser) {
+ *   browser.assert.domPropertyMatches('#main', 'tagName', /^frame/);
+ * }
  * ```
  * 
  * @method domPropertyMatches
  * @param {string|object} definition The selector (CSS/Xpath) used to locate the element. Can either be a string or an object which specifies [element properties](https://nightwatchjs.org/guide#element-properties).
  * @param {string} domProperty The DOM property name.
- * @param {string|RegExp} regexExpression TRegex expression to match text.
+ * @param {string|RegExp} regexExpression Regex to match against.
  * @param {string} [msg] Optional log message to display in the output. If missing, one is displayed by default.
  * @api assertions
  */
diff --git a/lib/api/assertions/titleContains.js b/lib/api/assertions/titleContains.js
index 70fb5dd8..d0206e02 100644
--- a/lib/api/assertions/titleContains.js
+++ b/lib/api/assertions/titleContains.js
@@ -7,8 +7,8 @@
  *    };
  * ```
  *
- * @method title
- * @param {string} expected The expected page title.
+ * @method titleContains
+ * @param {string} value The value to look for.
  * @param {string} [msg] Optional log message to display in the output. If missing, one is displayed by default.
  * @api assertions
  */
diff --git a/lib/index.js b/lib/index.js
index 6c599d59..741430d1 100644
--- a/lib/index.js
+++ b/lib/index.js
@@ -131,6 +131,11 @@ Nightwatch.createClient = function({
         return client.settings;
       }
     },
+    transport: {
+      get: function() {
+        return client.transport;
+      }
+    },
     nightwatch_client: {
       configurable: true,
       get: function() {
diff --git a/lib/runner/cli/nightwatch.conf.ejs b/lib/runner/cli/nightwatch.conf.ejs
index cb7853ce..1dc2cd0c 100644
--- a/lib/runner/cli/nightwatch.conf.ejs
+++ b/lib/runner/cli/nightwatch.conf.ejs
@@ -253,6 +253,61 @@ module.exports = {
       }
     },
     //////////////////////////////////////////////////////////////////////////////////
+    // Configuration for when using the SauceLabs cloud service                      |
+    //                                                                               |
+    // Please set the username and access key by setting the environment variables:  |
+    // - SAUCE_USERNAME                                                              |
+    // - SAUCE_ACCESS_KEY                                                            |
+    //////////////////////////////////////////////////////////////////////////////////
+    saucelabs: {
+      selenium: {
+        host: 'ondemand.saucelabs.com',
+        port: 443
+      },
+      // More info on configuring capabilities can be found on:
+      // https://wiki.saucelabs.com/display/DOCS/Test+Configuration+Options
+      desiredCapabilities: {
+        'sauce:options' : {
+          username: '${SAUCE_USERNAME}',
+          accessKey: '${SAUCE_ACCESS_KEY}',
+          // https://docs.saucelabs.com/dev/cli/sauce-connect-proxy/#--region
+          // region: 'us-west-1'
+          // https://docs.saucelabs.com/dev/test-configuration-options/#tunnelidentifier
+          // parentTunnel: '',
+          // tunnelIdentifier: '',
+        }
+      },
+      disable_error_log: false,
+      webdriver: {
+        start_process: false
+      }
+    },
+    'saucelabs.chrome': {
+      extends: 'saucelabs',
+      desiredCapabilities: {
+        browserName: 'chrome',
+        screenResolution: '1280x1024',
+        browserVersion: 'latest',
+        javascriptEnabled: true,
+        acceptSslCerts: true,
+        timeZone: 'London',
+        chromeOptions : {
+          w3c: true
+        }
+      }
+    },
+    'saucelabs.firefox': {
+      extends: 'saucelabs',
+      desiredCapabilities: {
+        browserName: 'firefox',
+        screenResolution: '1280x1024',
+        browserVersion: 'latest',
+        javascriptEnabled: true,
+        acceptSslCerts: true,
+        timeZone: 'London'
+      }
+    },
+    //////////////////////////////////////////////////////////////////////////////////
     // Configuration for when using the Selenium service, either locally or remote,  |
     //  like Selenium Grid                                                           |
     //////////////////////////////////////////////////////////////////////////////////
diff --git a/lib/transport/selenium-webdriver/browserstack.js b/lib/transport/selenium-webdriver/browserstack.js
index 4a21fade..9ffb63f8 100644
--- a/lib/transport/selenium-webdriver/browserstack.js
+++ b/lib/transport/selenium-webdriver/browserstack.js
@@ -98,29 +98,29 @@ class Browserstack extends Selenium {
     }
   }
 
-  async testSuiteFinished(failures) {
-    try {
-      let reason = failures instanceof Error;
-      if (reason) {
-        reason = stripAnsi(`${failures.name}: ${failures.message}`);
-      } else {
-        reason = '';
+  async sendReasonToBrowserstack(isFailure = false, reason = '') {
+    reason = stripAnsi(reason);
+    await this.sendHttpRequest({
+      url: `${Browserstack.ApiUrl}/sessions/${this.sessionId}.json`,
+      method: 'PUT',
+      use_ssl: true,
+      data: {
+        status: isFailure ? 'failed' : 'passed',
+        reason
+      },
+      auth: {
+        user: this.username,
+        pass: this.accessKey
       }
+    });
 
-      await this.sendHttpRequest({
-        url: `${Browserstack.ApiUrl}/sessions/${this.sessionId}.json`,
-        method: 'PUT',
-        use_ssl: true,
-        data: {
-          status: !!failures ? 'failed' : 'passed',
-          reason
-        },
-        auth: {
-          user: this.username,
-          pass: this.accessKey
-        }
-      });
+  }
 
+  async testSuiteFinished(failures) {
+    
+    try {
+      const reason = failures instanceof Error ? `${failures.name}: ${failures.message}` : '';
+      await this.sendReasonToBrowserstack(!!failures, reason);
       // eslint-disable-next-line no-console
       console.log('\n  ' + 'See more info, video, & screenshots on Browserstack:\n' +
         '  ' + Logger.colors.light_cyan(`https://automate.browserstack.com/builds/${this.buildId}/sessions/${this.sessionId}`));
diff --git a/package-lock.json b/package-lock.json
index eeef6232..c2549d75 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,12 +1,12 @@
 {
   "name": "nightwatch",
-  "version": "2.1.3",
+  "version": "2.1.4",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "nightwatch",
-      "version": "2.1.3",
+      "version": "2.1.4",
       "license": "MIT",
       "dependencies": {
         "@nightwatch/chai": "5.0.2",
@@ -865,14 +865,6 @@
         "node": ">=12"
       }
     },
-    "node_modules/@nightwatch/chai/node_modules/pathval": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz",
-      "integrity": "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==",
-      "engines": {
-        "node": "*"
-      }
-    },
     "node_modules/@nightwatch/ejs": {
       "version": "3.2.0",
       "resolved": "https://registry.npmjs.org/@nightwatch/ejs/-/ejs-3.2.0.tgz",
@@ -5366,10 +5358,9 @@
       }
     },
     "node_modules/pathval": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz",
-      "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=",
-      "dev": true,
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz",
+      "integrity": "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==",
       "engines": {
         "node": "*"
       }
@@ -7414,13 +7405,6 @@
         "loupe": "2.3.4",
         "pathval": "1.1.1",
         "type-detect": "4.0.8"
-      },
-      "dependencies": {
-        "pathval": {
-          "version": "1.1.1",
-          "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz",
-          "integrity": "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ=="
-        }
       }
     },
     "@nightwatch/ejs": {
@@ -10860,10 +10844,9 @@
       "peer": true
     },
     "pathval": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz",
-      "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=",
-      "dev": true
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz",
+      "integrity": "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ=="
     },
     "pend": {
       "version": "1.2.0",
diff --git a/package.json b/package.json
index 6ad869d6..2681b05f 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "nightwatch",
   "description": "Easy to use Node.js based end-to-end testing solution for web applications using the W3C WebDriver API.",
-  "version": "2.1.3",
+  "version": "2.1.4",
   "author": "Andrei Rusu",
   "homepage": "https://nightwatchjs.org",
   "main": "./lib/index.js",


Test patch:
diff --git a/test/extra/cucumber-config.js b/test/extra/cucumber-config.js
index 8d637537..df952e24 100644
--- a/test/extra/cucumber-config.js
+++ b/test/extra/cucumber-config.js
@@ -26,6 +26,17 @@ module.exports = {
       feature_path: path.join(__dirname, '../cucumber-integration-tests/sample_cucumber_tests/integration/sample.feature')
     }
   },
-  output: false,
-  silent: false
+  test_settings: {
+    browserstack: {
+      selenium: {
+        host: 'hub-cloud.browserstack.com',
+        port: 443
+      },
+      desiredCapabilities: {
+        'browserstack.user': 'test-access-user',
+        'browserstack.key': 'test-access-key',
+        browserName: 'chrome'
+      }
+    }
+  }
 };
\ No newline at end of file
diff --git a/test/src/cli/testCliRunnerGenerate.js b/test/src/cli/testCliRunnerGenerate.js
index e3c0d110..e8898a05 100644
--- a/test/src/cli/testCliRunnerGenerate.js
+++ b/test/src/cli/testCliRunnerGenerate.js
@@ -167,6 +167,57 @@ describe('Test CLI Runner Generate', function() {
           }
         });
 
+        assert.deepStrictEqual(configData.test_settings.saucelabs, {
+          desiredCapabilities: {
+            'sauce:options': {
+              username: '${SAUCE_USERNAME}',
+              accessKey: '${SAUCE_ACCESS_KEY}'
+              // https://docs.saucelabs.com/dev/cli/sauce-connect-proxy/#--region
+              // region: 'us-west-1'
+              // https://docs.saucelabs.com/dev/test-configuration-options/#tunnelidentifier
+              // parentTunnel: ''
+              // tunnelIdentifier: ''
+            }
+          },
+
+          selenium: {
+            host: 'ondemand.saucelabs.com',
+            port: 443
+          },
+
+          disable_error_log: false,
+          webdriver: {
+            start_process: false
+          }
+        });
+
+        assert.deepStrictEqual(configData.test_settings['saucelabs.chrome'], {
+          extends: 'saucelabs',
+          desiredCapabilities: {
+            browserName: 'chrome',
+            screenResolution: '1280x1024',
+            browserVersion: 'latest',
+            javascriptEnabled: true,
+            acceptSslCerts: true,
+            timeZone: 'London',
+            chromeOptions: {
+              w3c: true
+            }
+          }
+        });
+
+        assert.deepStrictEqual(configData.test_settings['saucelabs.firefox'], {
+          extends: 'saucelabs',
+          desiredCapabilities: {
+            browserName: 'firefox',
+            screenResolution: '1280x1024',
+            browserVersion: 'latest',
+            javascriptEnabled: true,
+            acceptSslCerts: true,
+            timeZone: 'London'
+          }
+        });
+
         assert.deepStrictEqual(configData.test_settings['cucumber-js'], {
           src_folders: ['examples/cucumber-js/features/step_definitions'],
           test_runner: {
diff --git a/test/src/index/testSettings.js b/test/src/index/testSettings.js
index 917dcd89..84f13419 100644
--- a/test/src/index/testSettings.js
+++ b/test/src/index/testSettings.js
@@ -7,7 +7,7 @@ const eq = assert.strictEqual;
 
 describe('test Settings', function () {
   it('testSetOptions', function () {
-    let client = Nightwatch.createClient({
+    const client = Nightwatch.createClient({
       use_xpath: true,
       launch_url: '/home'
     });
@@ -21,7 +21,7 @@ describe('test Settings', function () {
 
   it('test set test_runner invalid value', function () {
     assert.throws(function() {
-      let client = Nightwatch.createClient({
+      const client = Nightwatch.createClient({
         test_runner: null
       });
     }, /Error: Invalid "test_runner" settings specified; received: null/);
@@ -29,7 +29,7 @@ describe('test Settings', function () {
 
   it('test set test_runner invalid value - function', function () {
     assert.throws(function() {
-      let client = Nightwatch.createClient({
+      const client = Nightwatch.createClient({
         test_runner: function() {}
       });
     }, /Error: Invalid "test_runner" settings specified; received: function\(\) \{\}/);
@@ -37,7 +37,7 @@ describe('test Settings', function () {
 
   it('test set test_runner string value', function () {
 
-    let client = Nightwatch.createClient({
+    const client = Nightwatch.createClient({
       test_runner: 'mocha'
     });
 
@@ -49,7 +49,7 @@ describe('test Settings', function () {
 
   it('testSetWebdriverOptionsDefaults', function () {
     HttpRequest.globalSettings = {};
-    let client = Nightwatch.createClientDefaults();
+    const client = Nightwatch.createClientDefaults();
 
     eq(client.options.webdriver.host, 'localhost');
     eq(client.options.webdriver.port, 4444);
@@ -63,7 +63,7 @@ describe('test Settings', function () {
   });
 
   it('testSetWebdriverOptions', function () {
-    let client = Nightwatch.createClient({
+    const client = Nightwatch.createClient({
       webdriver: {
         host: '127.0.0.1',
         ssl: false,
@@ -78,8 +78,8 @@ describe('test Settings', function () {
       }
     });
 
-    let HttpRequest = common.require('http/request.js');
-    let request = new HttpRequest({});
+    const HttpRequest = common.require('http/request.js');
+    const request = new HttpRequest({});
 
     eq(request.reqOptions.host, '127.0.0.1');
     eq(request.reqOptions.port, 10195);
@@ -124,7 +124,7 @@ describe('test Settings', function () {
     const Nightwatch = common.require('index.js');
     const Settings = common.require('settings/settings.js');
 
-    let settings = Settings.parse({
+    const settings = Settings.parse({
       selenium: {
         start_process: false
       },
@@ -134,8 +134,8 @@ describe('test Settings', function () {
 
     Nightwatch.client(settings, null);
 
-    let HttpRequest = common.require('http/request.js');
-    let request = new HttpRequest({});
+    const HttpRequest = common.require('http/request.js');
+    const request = new HttpRequest({});
 
     eq(request.reqOptions.port, 80);
   });
@@ -143,7 +143,7 @@ describe('test Settings', function () {
   it('test create Transport for remote with Firefox', function() {
     const Settings = common.require('settings/settings.js');
 
-    let settings = Settings.parse({
+    const settings = Settings.parse({
       selenium: {
         start_process: false
       }
@@ -162,7 +162,7 @@ describe('test Settings', function () {
   });
 
   it('testSetOptionsCredentials', function () {
-    let client = Nightwatch.createClient({
+    const client = Nightwatch.createClient({
       username: 'test-user',
       accessKey: 'test-access-key'
     });
@@ -174,7 +174,7 @@ describe('test Settings', function () {
   });
 
   it('testEnableFailFast', function () {
-    let client = Nightwatch.createClient({}, null, {
+    const client = Nightwatch.createClient({}, null, {
       'fail-fast': true
     });
 
@@ -182,7 +182,7 @@ describe('test Settings', function () {
   });
 
   it('testSetOptionsScreenshots – path empty', function () {
-    let client = Nightwatch.createClient({
+    const client = Nightwatch.createClient({
       screenshots: {
         enabled: true,
         path: ''
@@ -197,7 +197,7 @@ describe('test Settings', function () {
   });
 
   it('testSetOptionsScreenshots – path not empty', function () {
-    let client = Nightwatch.createClient({
+    const client = Nightwatch.createClient({
       screenshots: {
         enabled: true,
         path: '/tmp/'
@@ -207,7 +207,7 @@ describe('test Settings', function () {
 
     eq(client.api.options.log_screenshot_data, true);
     eq(client.options.screenshots.on_error, true);
-
+    
     if (process.platform === 'win32') {
       assert.ok(client.settings.screenshots.path.endsWith('\\tmp'));
     } else {
@@ -217,7 +217,7 @@ describe('test Settings', function () {
 
 
   it('testSetOptionsScreenshotsOnError', function () {
-    let client = Nightwatch.createClient({
+    const client = Nightwatch.createClient({
       screenshots: {
         enabled: true,
         on_error: true,
@@ -230,7 +230,7 @@ describe('test Settings', function () {
   });
 
   it('testSetOptionsScreenshotsThrows', function () {
-    let client = Nightwatch.createClient({
+    const client = Nightwatch.createClient({
       screenshots: true
     });
 
@@ -242,7 +242,7 @@ describe('test Settings', function () {
   });
 
   it('testEndSessionOnFail', function () {
-    let client = Nightwatch.createClient({
+    const client = Nightwatch.createClient({
       end_session_on_fail: true
     });
 
@@ -253,7 +253,7 @@ describe('test Settings', function () {
   });
 
   it('testSetRequestTimeoutOptions', function () {
-    let client = Nightwatch.createClient({
+    const client = Nightwatch.createClient({
       webdriver: {
         timeout_options: {
           timeout: 10000,
@@ -268,15 +268,15 @@ describe('test Settings', function () {
     });
 
 
-    let HttpRequest = common.require('http/request.js');
-    let request = new HttpRequest({});
+    const HttpRequest = common.require('http/request.js');
+    const request = new HttpRequest({});
 
     assert.strictEqual(request.httpOpts.timeout, 10000);
     assert.strictEqual(request.retryAttempts, 5);
   });
 
   it('Test initialize with parallel cli argument', function () {
-    let settings = Settings.parse({
+    const settings = Settings.parse({
       selenium_port: 10195,
       silent: false,
       output: false
diff --git a/test/src/runner/cucumber-integration/testCucumberWithBrowserstack.js b/test/src/runner/cucumber-integration/testCucumberWithBrowserstack.js
new file mode 100644
index 00000000..e99ac769
--- /dev/null
+++ b/test/src/runner/cucumber-integration/testCucumberWithBrowserstack.js
@@ -0,0 +1,100 @@
+const path = require('path');
+const assert = require('assert');
+const nock = require('nock');
+const common = require('../../../common.js');
+const {runTests} = common.require('index.js');
+
+xdescribe('Cucumber  Browserstack integration', function() {
+  
+  before(function(done) {
+    try {
+      nock.activate();
+    // eslint-disable-next-line no-empty
+    } catch (err) {
+    }
+   
+    nock('https://hub-cloud.browserstack.com')
+      .post('/wd/hub/session')
+      .reply(201, {
+        status: 0,
+        sessionId: '1352110219202',
+        value: {
+          browserName: 'chrome'
+        }
+      });
+
+    nock('https://api.browserstack.com')
+      .get('/automate/builds.json')
+      .reply(200, []);
+
+    nock('https://hub-cloud.browserstack.com')
+      .post('/wd/hub/session/1352110219202/url')
+      .reply(200, {
+        value: null
+      });
+
+    nock('https://hub-cloud.browserstack.com')
+      .post('/wd/hub/session/1352110219202/elements')
+      .reply(200, {
+        status: 0,
+        sessionId: '1352110219202',
+        value: [{
+          'ELEMENT': '0'
+        }]
+      });
+
+    nock('https://hub-cloud.browserstack.com')
+      .delete('/wd/hub/session/1352110219202')
+      .reply(200, {
+        value: []
+      });
+
+    done();
+  });
+
+  after(function(done) {
+    nock.restore();
+    done();
+  });
+
+
+  it('testCucumberSampleTests - [Passed]', function() {
+   
+    nock('https://api.browserstack.com')
+      .put('/automate/sessions/1352110219202.json', body => body.status === 'passed')
+      .reply(200);
+
+    const source = [path.join(__dirname, '../../../cucumber-integration-tests/sample_cucumber_tests/integration/testSample.js')];
+    
+    return runTests({
+      source,
+      tags: ['@pass'],
+      verbose: false,
+      config: path.join(__dirname, '../../../extra/cucumber-config.js'),
+      env: 'browserstack'
+    }, {}).then(failures => {
+      assert.strictEqual(failures, false, 'Cucumber has test failures. Run with verbose to investigate.');
+    });
+  });
+
+  it('testCucumberSampleTests with failures', function() {
+
+
+    nock('https://api.browserstack.com')
+      .put('/automate/sessions/1352110219202.json', body => body.status === 'failed')
+      .reply(200);
+    
+    const source = [path.join(__dirname, '../../../cucumber-integration-tests/sample_cucumber_tests/integration/testWithFailures.js')];
+    
+    return runTests({
+      source,
+      env: 'browserstack',
+      tags: ['@fail'],
+      verbose: false,
+      config: path.join(__dirname, '../../../extra/cucumber-config.js')
+    }, {}).then(failures => {
+      assert.strictEqual(failures, true, 'Cucumber tests should have failed. Run with verbose to investigate.');
+    });
+  });
+});
+