Repository: Codeception/CodeceptJS
Pull request #1729 (2019-06-21 15:39:35)

fix: Tests did not fail when an exception raised in the async bootstrap

codacy-bot: 
![Codacy](https://app.codacy.com/assets/images/favicon.png) Here is an overview of what got changed by this pull request:


```diff

Clones added
============
- lib/hooks.js  1
         
```


See the complete overview on [Codacy](https://app.codacy.com/app/DavertMik/CodeceptJS/pullRequest?prid=3753068&bid=13178803)
        

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 64edd832..9915d6b0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -16,7 +16,7 @@
 * [Puppeteer] Fixed `UnhandledPromiseRejectionWarning: "Execution context was destroyed...` by @adrielcodeco
 * [WebDriver] Keep browser window dimensions when starting a new session by @spiroid
 * Replace Ghekrin plceholders with values in files that combine a scenerio outline and table by @medtoure18.
-* Added Documentation to [locate elements in React Native](https://codecept.io/react-native-locators) apps. By @DimGun.
+* Added Documentation to [locate elements in React Native](https://codecept.io/mobile-react-native-locators) apps. By @DimGun.
 * Adding optional `path` parameter to `bdd:snippets` command to append snippets to a specific file. By @cthorsen31.
 * Added optional `output` parameter to `def` command by @LukoyanovE.
 * [Puppeteer] Added `grabDataFromPerformanceTiming` by @PeterNgTr.
diff --git a/docs/changelog.md b/docs/changelog.md
index 64edd832..9915d6b0 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -16,7 +16,7 @@
 * [Puppeteer] Fixed `UnhandledPromiseRejectionWarning: "Execution context was destroyed...` by @adrielcodeco
 * [WebDriver] Keep browser window dimensions when starting a new session by @spiroid
 * Replace Ghekrin plceholders with values in files that combine a scenerio outline and table by @medtoure18.
-* Added Documentation to [locate elements in React Native](https://codecept.io/react-native-locators) apps. By @DimGun.
+* Added Documentation to [locate elements in React Native](https://codecept.io/mobile-react-native-locators) apps. By @DimGun.
 * Adding optional `path` parameter to `bdd:snippets` command to append snippets to a specific file. By @cthorsen31.
 * Added optional `output` parameter to `def` command by @LukoyanovE.
 * [Puppeteer] Added `grabDataFromPerformanceTiming` by @PeterNgTr.
diff --git a/lib/codecept.js b/lib/codecept.js
index 08972f39..71201315 100644
--- a/lib/codecept.js
+++ b/lib/codecept.js
@@ -161,17 +161,20 @@ class Codecept {
       }
       mocha.files = mocha.files.filter(t => fsPath.basename(t, '.js') === test || t === test);
     }
-    event.emit(event.all.before, this);
-    mocha.run(() => {
-      const done = () => {
-        event.emit(event.all.result, this);
-        event.emit(event.all.after, this);
-      };
+    const done = () => {
+      event.emit(event.all.result, this);
+      event.emit(event.all.after, this);
+    };
+
+    try {
+      event.emit(event.all.before, this);
+      mocha.run(() => this.teardown(done));
+    } catch (e) {
       this.teardown(done);
-    });
+      throw new Error(e);
+    }
   }
 
-
   static version() {
     return JSON.parse(readFileSync(`${__dirname}/../package.json`, 'utf8')).version;
   }
diff --git a/lib/hooks.js b/lib/hooks.js
index 66d51af5..354da999 100644
--- a/lib/hooks.js
+++ b/lib/hooks.js
@@ -1,6 +1,7 @@
 const getParamNames = require('./utils').getParamNames;
 const fsPath = require('path');
 const fileExists = require('./utils').fileExists;
+const output = require('./output');
 
 module.exports = function (hook, done, stage) {
   stage = stage || 'bootstrap';
@@ -39,6 +40,9 @@ function loadCustomHook(module) {
 
 function callSync(callable, done) {
   if (isAsync(callable)) {
+    callAsync(callable, done);
+    if (!hasArguments(callable)) done();
+  } else if (hasArguments(callable)) {
     callable(done);
   } else {
     callable();
@@ -46,7 +50,20 @@ function callSync(callable, done) {
   }
 }
 
-function isAsync(fn) {
+function callAsync(callable, done = undefined) {
+  const called = done ? callable(done) : callable();
+  called.catch((err) => {
+    output.print('');
+    output.error(err.message);
+    output.print('');
+    output.print(output.colors.grey(err.stack.replace(err.message, '')));
+    process.exit(1);
+  });
+}
+
+const isAsync = fn => fn.constructor.name === 'AsyncFunction';
+
+function hasArguments(fn) {
   const params = getParamNames(fn);
   return params && params.length;
 }


Test patch:
diff --git a/test/data/sandbox/configs/bootstrap/failed_test.js b/test/data/sandbox/configs/bootstrap/failed_test.js
new file mode 100644
index 00000000..fc45269f
--- /dev/null
+++ b/test/data/sandbox/configs/bootstrap/failed_test.js
@@ -0,0 +1,7 @@
+Feature('Filesystem').tag('main');
+
+Scenario('check current dir', (I) => {
+  I.amInPath('.');
+  I.say('hello world');
+  I.seeFile('unknown.js');
+}).tag('slow').tag('@important');
diff --git a/test/data/sandbox/configs/bootstrap/fs_test.js b/test/data/sandbox/configs/bootstrap/fs_test.js
new file mode 100644
index 00000000..ac8b3dc3
--- /dev/null
+++ b/test/data/sandbox/configs/bootstrap/fs_test.js
@@ -0,0 +1,7 @@
+Feature('Filesystem').tag('main');
+
+Scenario('check current dir', (I) => {
+  I.amInPath('.');
+  I.say('hello world');
+  I.seeFile('fs_test.js');
+}).tag('slow').tag('@important');
diff --git a/test/data/sandbox/configs/bootstrap/invalid_require_test.js b/test/data/sandbox/configs/bootstrap/invalid_require_test.js
new file mode 100644
index 00000000..131bb1ba
--- /dev/null
+++ b/test/data/sandbox/configs/bootstrap/invalid_require_test.js
@@ -0,0 +1,9 @@
+require('invalidRequire');
+
+Feature('Filesystem');
+
+Scenario('check current dir', (I) => {
+  I.amInPath('.');
+  I.say('hello world');
+  I.seeFile('codecept.json');
+});
diff --git a/test/data/sandbox/codecept.bootstrap.obj.json b/test/data/sandbox/configs/bootstrap/obj.json
similarity index 59%
rename from test/data/sandbox/codecept.bootstrap.obj.json
rename to test/data/sandbox/configs/bootstrap/obj.json
index bde7f03d..7d20efd3 100644
--- a/test/data/sandbox/codecept.bootstrap.obj.json
+++ b/test/data/sandbox/configs/bootstrap/obj.json
@@ -1,13 +1,13 @@
 {
-  "tests": "./*_test.js",
+  "tests": "./fs_test.js",
   "timeout": 10000,
   "output": "./output",
   "helpers": {
     "FileSystem": {}
   },
   "include": {},
-  "bootstrap": "hooks.js",
-  "teardown": "hooks.js",
+  "bootstrap": "../../hooks.js",
+  "teardown": "../../hooks.js",
   "mocha": {},
   "name": "sandbox"
 }
\ No newline at end of file
diff --git a/test/data/sandbox/codecept.bootstrap.sync.json b/test/data/sandbox/configs/bootstrap/sync.json
similarity index 66%
rename from test/data/sandbox/codecept.bootstrap.sync.json
rename to test/data/sandbox/configs/bootstrap/sync.json
index 70c34bc0..dcc85ff8 100644
--- a/test/data/sandbox/codecept.bootstrap.sync.json
+++ b/test/data/sandbox/configs/bootstrap/sync.json
@@ -1,12 +1,12 @@
 {
-  "tests": "./*_test.js",
+  "tests": "./fs_test.js",
   "timeout": 10000,
   "output": "./output",
   "helpers": {
     "FileSystem": {}
   },
   "include": {},
-  "bootstrap": "bootstrap.sync.js",
+  "bootstrap": "../../bootstrap.sync.js",
   "mocha": {},
   "name": "sandbox"
 }
\ No newline at end of file
diff --git a/test/data/sandbox/configs/bootstrap/with.args.async.func.js b/test/data/sandbox/configs/bootstrap/with.args.async.func.js
new file mode 100644
index 00000000..4e759ce2
--- /dev/null
+++ b/test/data/sandbox/configs/bootstrap/with.args.async.func.js
@@ -0,0 +1,15 @@
+exports.config = {
+  tests: './fs_test.js',
+  timeout: 10000,
+  output: './output',
+  helpers: {
+    FileSystem: {},
+  },
+  include: {},
+  bootstrap: async (done) => {
+    console.log('I am bootstrap');
+    done();
+  },
+  mocha: {},
+  name: 'sandbox',
+};
diff --git a/test/data/sandbox/configs/bootstrap/with.args.bootstrap.teardown.js b/test/data/sandbox/configs/bootstrap/with.args.bootstrap.teardown.js
new file mode 100644
index 00000000..d6494a28
--- /dev/null
+++ b/test/data/sandbox/configs/bootstrap/with.args.bootstrap.teardown.js
@@ -0,0 +1,19 @@
+exports.config = {
+  tests: './invalid_require_test.js',
+  timeout: 10000,
+  output: './output',
+  helpers: {
+    FileSystem: {},
+  },
+  include: {},
+  bootstrap: async (done) => {
+    console.log('I am bootstrap');
+    done();
+  },
+  teardown: (done) => {
+    console.log('I am teardown');
+    done();
+  },
+  mocha: {},
+  name: 'sandbox',
+};
diff --git a/test/data/sandbox/configs/bootstrap/with.args.failed.bootstrap.async.func.js b/test/data/sandbox/configs/bootstrap/with.args.failed.bootstrap.async.func.js
new file mode 100644
index 00000000..a11417b4
--- /dev/null
+++ b/test/data/sandbox/configs/bootstrap/with.args.failed.bootstrap.async.func.js
@@ -0,0 +1,15 @@
+exports.config = {
+  tests: './fs_test.js',
+  timeout: 10000,
+  output: './output',
+  helpers: {
+    FileSystem: {},
+  },
+  include: {},
+  bootstrap: async (done) => {
+    console.log('I am bootstrap');
+    throw new Error('Error from async bootstrap');
+  },
+  mocha: {},
+  name: 'sandbox',
+};
diff --git a/test/data/sandbox/configs/bootstrap/with.args.failed.bootstrap.teardown.js b/test/data/sandbox/configs/bootstrap/with.args.failed.bootstrap.teardown.js
new file mode 100644
index 00000000..50dd6321
--- /dev/null
+++ b/test/data/sandbox/configs/bootstrap/with.args.failed.bootstrap.teardown.js
@@ -0,0 +1,19 @@
+exports.config = {
+  tests: './fs_test.js',
+  timeout: 10000,
+  output: './output',
+  helpers: {
+    FileSystem: {},
+  },
+  include: {},
+  bootstrap: async (done) => {
+    console.log('I am bootstrap');
+    throw new Error('Error from async bootstrap');
+  },
+  teardown: (done) => {
+    console.log('I am teardown');
+    done();
+  },
+  mocha: {},
+  name: 'sandbox',
+};
diff --git a/test/data/sandbox/configs/bootstrap/without.args.async.func.js b/test/data/sandbox/configs/bootstrap/without.args.async.func.js
new file mode 100644
index 00000000..94cbf50e
--- /dev/null
+++ b/test/data/sandbox/configs/bootstrap/without.args.async.func.js
@@ -0,0 +1,14 @@
+exports.config = {
+  tests: './fs_test.js',
+  timeout: 10000,
+  output: './output',
+  helpers: {
+    FileSystem: {},
+  },
+  include: {},
+  bootstrap: async () => {
+    console.log('I am bootstrap');
+  },
+  mocha: {},
+  name: 'sandbox',
+};
diff --git a/test/data/sandbox/configs/bootstrap/without.args.bootstrap.teardown.js b/test/data/sandbox/configs/bootstrap/without.args.bootstrap.teardown.js
new file mode 100644
index 00000000..86de5da3
--- /dev/null
+++ b/test/data/sandbox/configs/bootstrap/without.args.bootstrap.teardown.js
@@ -0,0 +1,17 @@
+exports.config = {
+  tests: './invalid_require.test.js',
+  timeout: 10000,
+  output: './output',
+  helpers: {
+    FileSystem: {},
+  },
+  include: {},
+  bootstrap: async () => {
+    console.log('I am bootstrap');
+  },
+  teardown: async () => {
+    console.log('I am teardown');
+  },
+  mocha: {},
+  name: 'sandbox',
+};
diff --git a/test/data/sandbox/configs/bootstrap/without.args.failed.bootstrap.async.func.js b/test/data/sandbox/configs/bootstrap/without.args.failed.bootstrap.async.func.js
new file mode 100644
index 00000000..e4b4b607
--- /dev/null
+++ b/test/data/sandbox/configs/bootstrap/without.args.failed.bootstrap.async.func.js
@@ -0,0 +1,15 @@
+exports.config = {
+  tests: './fs_test.js',
+  timeout: 10000,
+  output: './output',
+  helpers: {
+    FileSystem: {},
+  },
+  include: {},
+  bootstrap: async () => {
+    console.log('I am bootstrap');
+    throw new Error('Error from async bootstrap');
+  },
+  mocha: {},
+  name: 'sandbox',
+};
diff --git a/test/data/sandbox/configs/bootstrap/without.args.failed.bootstrap.teardown.js b/test/data/sandbox/configs/bootstrap/without.args.failed.bootstrap.teardown.js
new file mode 100644
index 00000000..15d417ff
--- /dev/null
+++ b/test/data/sandbox/configs/bootstrap/without.args.failed.bootstrap.teardown.js
@@ -0,0 +1,18 @@
+exports.config = {
+  tests: './fs_test.js',
+  timeout: 10000,
+  output: './output',
+  helpers: {
+    FileSystem: {},
+  },
+  include: {},
+  bootstrap: async () => {
+    console.log('I am bootstrap');
+    throw new Error('Error from async bootstrap');
+  },
+  teardown: async () => {
+    console.log('I am teardown');
+  },
+  mocha: {},
+  name: 'sandbox',
+};
diff --git a/test/runner/bootstrap_test.js b/test/runner/bootstrap_test.js
new file mode 100644
index 00000000..00b2b109
--- /dev/null
+++ b/test/runner/bootstrap_test.js
@@ -0,0 +1,203 @@
+const assert = require('assert');
+const path = require('path');
+const exec = require('child_process').exec;
+
+const runner = path.join(__dirname, '/../../bin/codecept.js');
+const codecept_dir = path.join(__dirname, '/../data/sandbox/configs/bootstrap');
+const codecept_run = `${runner} run`;
+const codecept_run_config = (config, grep) => `${codecept_run} --config ${codecept_dir}/${config} ${grep ? `--grep ${grep}` : ''}`;
+const config_run_override = (config, override) => `${codecept_run} --config ${codecept_dir}/${config} --override '${JSON.stringify(override)}'`;
+
+
+describe('CodeceptJS Bootstrap and Teardown', () => {
+  // success
+  it('should run bootstrap', (done) => {
+    exec(codecept_run_config('sync.json', '@important'), (err, stdout, stderr) => {
+      stdout.should.include('Filesystem'); // feature
+      stdout.should.include('I am bootstrap');
+      assert(!err);
+      done();
+    });
+  });
+
+  it('should run teardown', (done) => {
+    exec(config_run_override('../../', { teardown: 'bootstrap.sync.js' }), (err, stdout, stderr) => {
+      stdout.should.include('Filesystem'); // feature
+      stdout.should.include('I am bootstrap');
+      assert(!err);
+      done();
+    });
+  });
+
+  it('should run async bootstrap', (done) => {
+    exec(config_run_override('../../', { bootstrap: 'bootstrap.async.js' }), (err, stdout, stderr) => {
+      stdout.should.include('Ready: 0');
+      stdout.should.include('Go: 1');
+      stdout.should.include('Filesystem'); // feature
+      assert(!err);
+      done();
+    });
+  });
+
+  it('should run bootstrap/teardown as object', (done) => {
+    exec(codecept_run_config('obj.json'), (err, stdout, stderr) => {
+      stdout.should.include('Filesystem'); // feature
+      stdout.should.include('I am bootstrap');
+      stdout.should.include('I am teardown');
+      assert(!err);
+      done();
+    });
+  });
+
+  it('should run async bootstrap function without args', (done) => {
+    exec(codecept_run_config('without.args.async.func.js'), (err, stdout, stderr) => {
+      stdout.should.include('Filesystem'); // feature
+      stdout.should.include('I am bootstrap');
+      assert(!err);
+      done();
+    });
+  });
+
+  it('should run async bootstrap function with args', (done) => {
+    exec(codecept_run_config('with.args.async.func.js'), (err, stdout, stderr) => {
+      stdout.should.include('Filesystem'); // feature
+      stdout.should.include('I am bootstrap');
+      assert(!err);
+      done();
+    });
+  });
+
+  // failed test
+  it('should fail with code 1 when test failed and async bootstrap function without args', (done) => {
+    exec(config_run_override('without.args.async.func.js', { tests: './failed_test.js' }), (err, stdout, stderr) => {
+      assert(err);
+      assert.equal(err.code, 1);
+      stdout.should.include('Filesystem'); // feature
+      stdout.should.include('I am bootstrap');
+      stdout.should.include('✖ check current dir @slow @important');
+      done();
+    });
+  });
+
+  it('should fail with code 1 when test failed and async bootstrap function with args', (done) => {
+    exec(config_run_override('with.args.async.func.js', { tests: './failed_test.js' }), (err, stdout, stderr) => {
+      assert(err);
+      assert.equal(err.code, 1);
+      stdout.should.include('Filesystem'); // feature
+      stdout.should.include('I am bootstrap');
+      stdout.should.include('✖ check current dir @slow @important');
+      done();
+    });
+  });
+
+  // failed bootstrap
+  it('should fail with code 1 when async bootstrap function without args failed', (done) => {
+    exec(codecept_run_config('without.args.failed.bootstrap.async.func.js'), (err, stdout, stderr) => {
+      assert.equal(err.code, 1);
+      stdout.should.include('Error from async bootstrap');
+      stdout.should.not.include('✔ check current dir @slow @important in 2ms');
+      assert(err);
+      done();
+    });
+  });
+
+  it('should fail with code 1 when async bootstrap function with args failed', (done) => {
+    exec(codecept_run_config('with.args.failed.bootstrap.async.func.js'), (err, stdout, stderr) => {
+      assert.equal(err.code, 1);
+      stdout.should.include('Error from async bootstrap');
+      stdout.should.not.include('✔ check current dir @slow @important in 2ms');
+      assert(err);
+      done();
+    });
+  });
+
+  // failed in test file
+  it('should fail with code 1 when raise exceptin in the test file and async bootstrap function with args', (done) => {
+    exec(config_run_override('with.args.async.func.js', { tests: './invalid_require_test.js' }), (err, stdout, stderr) => {
+      assert(err);
+      assert.equal(err.code, 1);
+      stdout.should.include('Cannot find module \'invalidRequire\'');
+      stdout.should.not.include('✔ check current dir @slow @important in 2ms');
+      done();
+    });
+  });
+
+  it('should fail with code 1 when raise exceptin in the test file and async bootstrap function without args', (done) => {
+    exec(config_run_override('without.args.async.func.js', { tests: './invalid_require_test.js' }), (err, stdout, stderr) => {
+      assert(err);
+      assert.equal(err.code, 1);
+      stdout.should.include('Cannot find module \'invalidRequire\'');
+      stdout.should.not.include('✔ check current dir @slow @important in 2ms');
+      done();
+    });
+  });
+
+  // with teardown
+  it('should run async bootstrap/teardown with args', (done) => {
+    exec(config_run_override('with.args.bootstrap.teardown.js', { tests: './fs_test.js' }), (err, stdout, stderr) => {
+      stdout.should.include('Filesystem'); // feature
+      stdout.should.include('I am bootstrap');
+      stdout.should.include('I am teardown');
+      assert(!err);
+      done();
+    });
+  });
+
+  it('should run async bootstrap/teardown without args', (done) => {
+    exec(config_run_override('without.args.bootstrap.teardown.js', { tests: './fs_test.js' }), (err, stdout, stderr) => {
+      stdout.should.include('Filesystem'); // feature
+      stdout.should.include('I am bootstrap');
+      stdout.should.include('I am teardown');
+      assert(!err);
+      done();
+    });
+  });
+
+  // with teaedown - failed tests
+  it('should fail with code 1 when test failed and async bootstrap/teardown function with args', (done) => {
+    exec(config_run_override('with.args.bootstrap.teardown.js', { tests: './failed_test.js' }), (err, stdout, stderr) => {
+      assert(err);
+      assert.equal(err.code, 1);
+      stdout.should.include('Filesystem'); // feature
+      stdout.should.include('I am bootstrap');
+      stdout.should.include('✖ check current dir @slow @important');
+      stdout.should.include('I am teardown');
+      done();
+    });
+  });
+
+  it('should fail with code 1 when test failed and async bootstrap/teardown function without args', (done) => {
+    exec(config_run_override('without.args.bootstrap.teardown.js', { tests: './failed_test.js' }), (err, stdout, stderr) => {
+      assert(err);
+      assert.equal(err.code, 1);
+      stdout.should.include('Filesystem'); // feature
+      stdout.should.include('I am bootstrap');
+      stdout.should.include('✖ check current dir @slow @important');
+      stdout.should.include('I am teardown');
+      done();
+    });
+  });
+
+  // with teardown and fail bootstrap - teardown not call
+  it('should fail with code 1 when async bootstrap with args failed and not call teardown', (done) => {
+    exec(codecept_run_config('with.args.failed.bootstrap.teardown.js'), (err, stdout, stderr) => {
+      assert(err);
+      assert.equal(err.code, 1);
+      stdout.should.include('Error from async bootstrap');
+      stdout.should.not.include('✔ check current dir @slow @important in 2ms');
+      stdout.should.not.include('I am teardown');
+      done();
+    });
+  });
+
+  it('should fail with code 1 when async bootstrap without args failed and not call teardown', (done) => {
+    exec(codecept_run_config('without.args.failed.bootstrap.teardown.js'), (err, stdout, stderr) => {
+      assert(err);
+      assert.equal(err.code, 1);
+      stdout.should.include('Error from async bootstrap');
+      stdout.should.not.include('✔ check current dir @slow @important in 2ms');
+      stdout.should.not.include('I am teardown');
+      done();
+    });
+  });
+});
diff --git a/test/runner/codecept_test.js b/test/runner/codecept_test.js
index a305bae0..b2deef17 100644
--- a/test/runner/codecept_test.js
+++ b/test/runner/codecept_test.js
@@ -168,34 +168,6 @@ describe('CodeceptJS Runner', () => {
     });
   });
 
-  it('should run bootstrap', (done) => {
-    exec(codecept_run_config('codecept.bootstrap.sync.json'), (err, stdout, stderr) => {
-      stdout.should.include('Filesystem'); // feature
-      stdout.should.include('I am bootstrap');
-      assert(!err);
-      done();
-    });
-  });
-
-  it('should run teardown', (done) => {
-    exec(config_run_override({ teardown: 'bootstrap.sync.js' }), (err, stdout, stderr) => {
-      stdout.should.include('Filesystem'); // feature
-      stdout.should.include('I am bootstrap');
-      assert(!err);
-      done();
-    });
-  });
-
-  it('should run async bootstrap', (done) => {
-    exec(config_run_override({ bootstrap: 'bootstrap.async.js' }), (err, stdout, stderr) => {
-      stdout.should.include('Ready: 0');
-      stdout.should.include('Go: 1');
-      stdout.should.include('Filesystem'); // feature
-      assert(!err);
-      done();
-    });
-  });
-
   it('should run hooks', (done) => {
     exec(codecept_run_config('codecept.hooks.js'), (err, stdout, stderr) => {
       stdout.should.include('Filesystem'); // feature
@@ -275,16 +247,6 @@ describe('CodeceptJS Runner', () => {
     });
   });
 
-  it('should run bootstrap/teardown as object', (done) => {
-    exec(codecept_run_config('codecept.bootstrap.obj.json'), (err, stdout, stderr) => {
-      stdout.should.include('Filesystem'); // feature
-      stdout.should.include('I am bootstrap');
-      stdout.should.include('I am teardown');
-      assert(!err);
-      done();
-    });
-  });
-
   it('should run dynamic config', (done) => {
     exec(codecept_run_config('config.js'), (err, stdout, stderr) => {
       stdout.should.include('Filesystem'); // feature