Repository: nightwatchjs/nightwatch (js)

Issue:
Retain HTML report across test runs
## Describe the bug
<!--
A clear and concise description of what the bug is, how to reproduce it, and what the expected behaviour is.

Please make sure to format the code samples, verbose output, configuration and other inline debug information (use the Preview functionality). Check out the [GitHub Markdown guide](https://guides.github.com/features/mastering-markdown/).
-->
Currently, the HTML reporter does not maintain test results across test runs instead it replaces the report from the previous test run.

## Proposed Solution
Instead of replacing the HTML Report, there should be a flag to retain HTML reports across test runs.


## Your Environment
<!--- Include the relevant details your environment -->

| Executable | Version |
| ---: | :--- |
| `nightwatch --version` | 2.3 |





Pull request:
retain html report
## Changes
- Add a flag to retain HTML report across test runs
- Default behaviour is not changed it generates different HTML reports if `retain_report` is set to true

## Impacts
- fixes #3307 

Patch:
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
deleted file mode 100644
index 335d6d46..00000000
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ /dev/null
@@ -1,94 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ""
-labels: ''
-assignees: ''
-
----
-
-## Describe the bug
-<!--
-A clear and concise description of what the bug is, how to reproduce it, and what the expected behaviour is.
-
-Please make sure to format the code samples, verbose output, configuration and other inline debug information (use the Preview functionality). Check out the [GitHub Markdown guide](https://guides.github.com/features/mastering-markdown/).
--->
-
-## Sample test
-<!-- 
-Include a sample test which reproduces the problem you're experiencing. If possible, the test should be against a public url.
-
-Please add the test and other info inline, not as attachments or screenshots.
--->
-
-<details><summary>sampleTest.js</summary><p>
-<!-- browsers demand the next line be empty -->
-
-```js
-// Please add the sample test here
-
-module.exports = {
-  sampleTest: function(browser) {
-
-  }
-}
-```
-</p></details>
-
-**Run with command**
-<!-- Include the command used to run the test, if applicable. -->
-
-```sh
-$ nightwatch test/sampleTest.js --your-other-arguments-here
-```
-
-### Verbose output
-<details><summary>debug.log</summary><p>
-<!-- browsers demand the next line be empty -->
-
-```txt
-<!-- Include the verbose output, if possible (run nightwatch with `--verbose` argument) -->
-```
-</p></details>
-
-
-## Configuration
-<details><summary>nightwatch.json</summary><p>
-<!-- browsers demand the next line be empty -->
-
-<!-- Please paste your nightwatch.json or nightwatch.conf.js here; make sure to leave out any sensitive details -->
-
-```js
-{
-  "your": { "config": "here" }
-}
-```
-</p></details>
-
-## Your Environment
-<!--- Include the relevant details your environment -->
-
-| Executable | Version |
-| ---: | :--- |
-| `nightwatch --version` | VERSION |
-| `npm --version`  | VERSION |
-| `yarn --version` | VERSION |
-| `node --version` | VERSION |
-
-| Browser driver | Version |
-| --- | --- |
-| NAME | VERSION |
-<!-- For example:
-| chromedriver | 76 |
-| geckodriver | v0.25.0 |
-| selenium-server | 3.141 |
--->
-
-| OS | Version |
-| --- | --- |
-| NAME | VERSION |
-<!-- For example:
-| macOS Sierra | 10.12.3 |
-| Windows 10 | 1607 |
-| Ubuntu | 16.10 |
--->
diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml
new file mode 100644
index 00000000..e9070c13
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.yml
@@ -0,0 +1,164 @@
+name: "Bug Report"
+description: "File a bug report to help us improve Nightwatch."
+body:
+  - type: "markdown"
+    attributes:
+      value: |
+        Thanks in advance for your contribution. Before submitting a new issue, try searching for a similar one here: https://github.com/nightwatchjs/nightwatch/search?type=Issues.
+  - type: "input"
+    id: "description"
+    attributes:
+      label: "Description of the bug/issue"
+      description: "A brief description of the issue, how to reproduce it, and the expected behavior."
+      placeholder: |
+        When I ____, I expected ____ to happen but ____ happened instead.
+    validations:
+      required: true
+  - type: "textarea"
+    id: "steps"
+    attributes:
+      label: "Steps to reproduce"
+      description: |
+        Explain how to cause the issue in the provided reproduction.
+      value: |
+        1. Go to '...'
+        2. Click on '...'
+        3. Scroll down to '...'
+        4. See error
+  - type: textarea
+    id: "sample-test"
+    attributes:
+      label: "Sample test"
+      description:
+        Include a sample test that reproduces the problem you're experiencing. If possible, the test should be against a public URL.
+
+        Please add the test and other info inline, not as attachments or screenshots.
+      placeholder: |
+        ```js
+        // Please add the sample test here
+
+        module.exports = {
+          sampleTest: function() {
+            browser
+              .navigateTo('<URL here>')
+              .click('<selector>')
+              .end();
+          }
+        }
+        ```
+  - type: input
+    id: "run-with-command"
+    attributes:
+      label: "Command to run"
+      description: "Include the command used to run the test."
+      placeholder: |
+        nightwatch test/sampleTest.js --your-other-arguments-here
+  - type: textarea
+    id: "verbose-output"
+    attributes:
+      label: "Verbose Output"
+      description: |
+        "Extended nightwatch command logging during the session. 
+        Please use `nightwatch --verbose` and paste the output here"
+      placeholder: |
+        ```sh
+        [Sample Test] Test Suite
+        ────────────────────────────────────────────────────────
+        ⠋ Starting ChromeDriver on port 9515...
+          Request POST /session
+          {
+            desiredCapabilities: {
+              browserName: 'chrome',
+              'goog:chromeOptions': { w3c: true, args: [] },
+              name: 'Sample Test'
+            },
+            capabilities: {
+              alwaysMatch: {
+                browserName: 'chrome',
+                'goog:chromeOptions': { w3c: true, args: [] }
+              }
+            }
+        ```
+  - type: textarea
+    id: "configuration"
+    attributes:
+      label: "Nightwatch Configuration"
+      description: "Add your nightwatch.json or nightwatch.conf.js here; Make sure to leave out any sensitive details."
+      placeholder: |
+        //
+        // Refer to the online docs for more details:
+        // https://nightwatchjs.org/gettingstarted/configuration/
+        //
+        //  _   _  _         _      _                     _          _
+        // | \ | |(_)       | |    | |                   | |        | |
+        // |  \| | _   __ _ | |__  | |_ __      __  __ _ | |_   ___ | |__
+        // | . ` || | / _` || '_ \ | __|\ \ /\ / / / _` || __| / __|| '_ \
+        // | |\  || || (_| || | | || |_  \ V  V / | (_| || |_ | (__ | | | |
+        // \_| \_/|_| \__, ||_| |_| \__|  \_/\_/   \__,_| \__| \___||_| |_|
+        //             __/ |
+        //            |___/
+        //
+        module.exports = {
+          src_folders: [],
+          page_objects_path: [],
+          custom_commands_path: [],
+          custom_assertions_path: '',
+          plugins: [],
+          globals_path: '',
+          webdriver: {},
+          test_settings: {}
+        }
+    validations:
+      required: true
+  - type: "markdown"
+    attributes:
+      value: |
+        ## Your Environment
+        Include the relevant details related to your environment.
+  - type: "input"
+    id: "nightwatch-version"
+    attributes:
+      label: "Nightwatch.js Version"
+      description: "The version of Nightwatch you use."
+      placeholder: "2.3.0"
+    validations:
+      required: true
+  - type: "input"
+    id: "npm-version"
+    attributes:
+      label: "NPM Version"
+      description: "The version of NPM you use."
+      placeholder: "8.12.1"
+    validations:
+      required: true
+  - type: "input"
+    id: "node-version"
+    attributes:
+      label: "Node Version"
+      description: "The version of Node you use."
+      placeholder: "14.0.0"
+  - type: "input"
+    id: "yarn-version"
+    attributes:
+      label: "Yarn Version"
+      description: "The version of Yarn you use."
+      placeholder: "1.0.1"
+  - type: "input"
+    id: "browser"
+    attributes:
+      label: "Browser"
+      description: "The browser(s) this issue occurred with."
+      placeholder: "Firefox 102.0.1"
+  - type: "input"
+    id: "operating-system"
+    attributes:
+      label: "Operating System"
+      description: "The operating system(s) this issue occurred with."
+      placeholder: "MacOs Monterey 12.5"
+  - type: "textarea"
+    id: "additional-information"
+    attributes:
+      label: "Additional Information"
+      description: |
+        Use this section to provide any additional information you might have
+        like screenshots, notes, or links to ideas.
diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
new file mode 100644
index 00000000..df1cfbef
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/config.yml
@@ -0,0 +1,11 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Ask a question
+    url: https://github.com/nightwatchjs/nightwatch/discussions
+    about: Ask questions and discuss topics with other community members
+  - name: Documentation Request
+    url: https://github.com/nightwatchjs/nightwatch-docs/issues/new
+    about: Request for documentation to be added/altered
+  - name: Chat with other community members. Ask support questions, discuss news and feature requests.
+    url: https://discord.gg/x5RuXc6Z
+    about: The official Nightwatchjs Discord community
diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml
new file mode 100644
index 00000000..4d877c88
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature-request.yml
@@ -0,0 +1,49 @@
+name: "Feature Request"
+description: "Request a feature or enhancement for Nightwatch"
+labels: ["enhancement"]
+body:
+  - type: "markdown"
+    attributes:
+      value: |
+        Thanks in advance for your contribution. Before requesting a new feature, try searching for a similar issue here: https://github.com/nightwatchjs/nightwatch/search?type=Issues.
+  - type: "textarea"
+    id: "description"
+    attributes:
+      label: "Description"
+      description: "Please describe your request in one or two sentences."
+    validations:
+      required: true
+  - type: "textarea"
+    id: "justification"
+    attributes:
+      label: "Problem Statement/Justification"
+      description: |
+        Please provide valid reason(s) why this should be added to Nightwatch
+
+        If this feature is related to a problem you've noticed, mention it as
+        well.
+    validations:
+      required: true
+  - type: "textarea"
+    id: "proposed-solution"
+    attributes:
+      label: "Proposed Solution or API"
+      description: |
+        Please provide code snippets, gists, or links to the ideal
+        design or API.
+    validations:
+      required: true
+  - type: "textarea"
+    id: "alternatives"
+    attributes:
+      label: "Alternatives"
+      description: |
+        What alternative solutions have you considered before making this
+        request?
+  - type: "textarea"
+    id: "additional-information"
+    attributes:
+      label: "Additional Information"
+      description: |
+        What resources (links, screenshots, etc.) do you have to assist this
+        effort?
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
deleted file mode 100644
index 672afc62..00000000
--- a/.github/ISSUE_TEMPLATE/feature_request.md
+++ /dev/null
@@ -1,23 +0,0 @@
----
-name: Feature request
-about: Suggest an enhancement for Nightwatch
-title: "[Enhancement]"
-labels: ''
-assignees: ''
-
----
-
-**Is your feature request related to a problem? Please describe.**
-<!-- A clear and concise description of what the problem is. -->
-
-**Describe the solution you'd like**
-<!-- Explain the use case in detail. -->
-
-**Describe how the solution will work**
-<!--And include a sample test case and/or usage, if applicable. -->
-
-**Describe alternatives you've considered**
-<!-- A clear and concise description of any alternative solutions or features you've considered, if there are any available.-->
-
-**Additional context**
-<!-- Add any other context or screenshots about the feature request here.-->
diff --git a/.github/label-commenter-config.yml b/.github/label-commenter-config.yml
new file mode 100644
index 00000000..d2f3e615
--- /dev/null
+++ b/.github/label-commenter-config.yml
@@ -0,0 +1,39 @@
+  # Configuration for Label Commenter - https://github.com/peaceiris/actions-label-commenter
+labels:
+  - name: needs-triaging
+    labeled:
+      issue:
+        body: |
+          @{{ issue.user.login }}, thank you for creating this issue. We will troubleshoot it as soon as we can.
+
+          ---
+          <details>
+            <summary>Info for maintainers</summary>
+            <div>
+              <br>
+              <p>
+                Triage this issue by using labels.
+              </p>
+              <p>
+                If information is missing, add a helpful comment and add the <code>I-issue-template</code> label.
+              </p>
+              <p>
+                Thank you!
+              </p>
+            </div>
+          </details>
+  - name: I-issue-template
+    labeled:
+      issue:
+        body: |
+          Hi, @{{ issue.user.login }}.
+          Please follow the issue template, we need more information to reproduce the issue.
+
+          Either a complete code snippet (if more than one file is needed, provide a GitHub repo and instructions to run the code), the specific versions used, or a more detailed description to help us understand the issue.
+
+          Reply to this issuee when all information is provided, thank you.
+  - name: enhancement
+    labeled:
+      issue:
+        body: |
+          Thank you for setting this as an enhancement. One of the product folk will triage this again to help see when we can fit this in an upcoming sprint.
diff --git a/.github/workflows/label-commenter.yml b/.github/workflows/label-commenter.yml
new file mode 100644
index 00000000..5c709475
--- /dev/null
+++ b/.github/workflows/label-commenter.yml
@@ -0,0 +1,18 @@
+# Configuration for Label Commenter - https://github.com/peaceiris/actions-label-commenter
+name: Label Commenter
+
+on:
+  issues:
+    types: [ labeled ]
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  comment:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Label Commenter
+        uses: peaceiris/actions-label-commenter@v1
diff --git a/lib/reporter/global-reporter.js b/lib/reporter/global-reporter.js
index bfc838c0..2cc6d2e5 100644
--- a/lib/reporter/global-reporter.js
+++ b/lib/reporter/global-reporter.js
@@ -185,7 +185,7 @@ module.exports = class GlobalReporter {
   }
 
   async writeReport(reporter, globalResults) {
-    const {globals, output_folder, start_session} = this.settings;
+    const {globals, output_folder, start_session, folder_format} = this.settings;
     const needsCallback = reporter.write.length === 3;
     const options = {
       filename_prefix: this.settings.report_prefix,
@@ -193,7 +193,8 @@ module.exports = class GlobalReporter {
       globals,
       start_session,
       reporter,
-      openReport: this.openReport
+      openReport: this.openReport,
+      folder_format
     };
 
     if (needsCallback) {
diff --git a/lib/reporter/reporters/html.js b/lib/reporter/reporters/html.js
index 701b97be..6fe9b024 100644
--- a/lib/reporter/reporters/html.js
+++ b/lib/reporter/reporters/html.js
@@ -62,6 +62,20 @@ class HtmlReporter extends BaseReporter {
       });
   }
 
+  getFolderPrefix() {
+    let folderPrefix = '';
+    const {folder_format} = this.options; 
+    if (folder_format) {
+      if (typeof folder_format === 'function') {
+        folderPrefix = folder_format(this.results);
+      } else if (typeof folder_format === 'string') {
+        folderPrefix = folder_format;
+      }
+    }
+
+    return folderPrefix;
+  }
+
   async writeReport(data) {
     const results = this.results;
     const {modules}  = results;
@@ -80,7 +94,8 @@ class HtmlReporter extends BaseReporter {
       return prev;
     }, {});
 
-    const destFolder = path.join(output_folder, 'nightwatch-html-report');
+    const destFolder = path.join(output_folder, this.getFolderPrefix(), 'nightwatch-html-report');
+
     await Utils.createFolder(destFolder);
 
     const filename = path.join(destFolder, 'index.html');
diff --git a/lib/settings/defaults.js b/lib/settings/defaults.js
index 8c596cbb..aa17f10b 100644
--- a/lib/settings/defaults.js
+++ b/lib/settings/defaults.js
@@ -108,8 +108,13 @@ module.exports = {
   // this can be useful when persisting data between test suites is needed, such as a cookie or session information
   persist_globals: false,
 
-  // The location where the JUnit XML report files will be saved. Set this to false if you want to disable XML reporting
-  output_folder: 'tests_output',
+  reporter_options: {
+    // The location where the JUnit XML and HTML report files will be saved. Set this to false if you want to disable XML reporting
+    output_folder: 'tests_output',
+
+    //The folder formatting to use while saving HTML report
+    folder_format: null
+  },
 
   // A string or array of folders (excluding subfolders) where the tests are located.
   src_folders: null,
diff --git a/lib/settings/settings.js b/lib/settings/settings.js
index 516b5acf..5ed7f3b8 100644
--- a/lib/settings/settings.js
+++ b/lib/settings/settings.js
@@ -275,6 +275,7 @@ class Settings {
     this.setUnitTestsMode();
     this.setParallelMode();
     this.setTestRunner();
+    this.setReporterOptions();
 
     if (typeof this.settings.src_folders == 'string') {
       this.settings.src_folders = [this.settings.src_folders];
@@ -287,6 +288,10 @@ class Settings {
     return this;
   }
 
+  setReporterOptions() {
+    defaultsDeep(this.settings, this.settings.reporter_options);
+  }
+
   setParallelMode() {
     const Concurrency = require('../runner/concurrency');
 


Test patch:
diff --git a/lib/runner/test-runners/mocha.js b/lib/runner/test-runners/mocha.js
index ecede70c..d601b02e 100644
--- a/lib/runner/test-runners/mocha.js
+++ b/lib/runner/test-runners/mocha.js
@@ -134,7 +134,7 @@ class MochaRunner {
     }
 
     if (this.mochaOpts.reporter === 'mochawesome' && !this.mochaOpts.reporterOptions.reportDir) {
-      this.mochaOpts.reporterOptions.reportDir = 'html-report';
+      this.mochaOpts.reporterOptions.reportDir =  settings.reportDir || 'html-report';
     }
 
     const MochaNightwatch = MochaRunner.MochaNightwatch({settings, argv, addtOpts});
diff --git a/test/lib/utils.js b/test/lib/utils.js
index 2e5aa589..58378922 100644
--- a/test/lib/utils.js
+++ b/test/lib/utils.js
@@ -1,5 +1,6 @@
 var Nightwatch = require('./nightwatch.js');
 const fs = require('fs');
+const glob = require('glob');
 
 module.exports = {
 
@@ -41,14 +42,19 @@ module.exports = {
     return process.platform === 'win32' ? '\r\n' : '\n';
   },
 
-  readFilePromise(fileName) {
+  readFilePromise(pattern) {
     return new Promise(function(resolve, reject) {
-      fs.readFile(fileName, function(err, result) {
-        if (err) {
-          return reject(err);
+      glob(pattern, (error, matches)=> {
+        if (error) {
+          return reject(error);
         }
-
-        resolve(result);
+        fs.readFile(matches[0], function(err, result) {
+          if (err) {
+            return reject(err);
+          }
+          
+          return resolve(result);
+        });
       });
     });
   },
diff --git a/test/src/cli/testCliRunner.js b/test/src/cli/testCliRunner.js
index a2eb3490..50c7a4b6 100644
--- a/test/src/cli/testCliRunner.js
+++ b/test/src/cli/testCliRunner.js
@@ -56,6 +56,12 @@ describe('Test CLI Runner', function() {
       }
     });
 
+    mockery.registerMock('./reporter_options.json', {
+      reporter_options: {
+        output_folder: 'output'
+      }
+    });
+
     mockery.registerMock('./empty.json', {
       src_folders: 'tests'
     });
@@ -399,6 +405,35 @@ describe('Test CLI Runner', function() {
     assert.strictEqual(runner.settings.output_folder, false);
   });
 
+  it('testSetOutputFolder using reporterOptions', function(done) {
+    mockery.registerMock('fs', {
+      statSync: function(module) {
+        if (module === './settings.json' || module === './nightwatch.conf.js') {
+          throw new Error('Does not exist');
+        }
+
+        return {
+          isFile: function() {
+            return true;
+          }
+        };
+      },
+      constants,
+      rmdirSync
+    });
+
+    const CliRunner = common.require('runner/cli/cli.js');
+    const runner = new CliRunner({
+      config: './reporter_options.json',
+      env: 'default'
+    }).setup();
+
+    assert.strictEqual(runner.test_settings.output_folder, 'output');
+
+    done();
+  });
+  
+
   it('testReadSettingsDeprecated', function(done) {
     mockery.registerMock('fs', {
       statSync: function(module) {
@@ -436,6 +471,8 @@ describe('Test CLI Runner', function() {
     done();
   });
 
+ 
+
   it('testCustomSettingsFileAndEnvironment', function() {
     mockery.registerMock('fs', {
       statSync: function(module) {
diff --git a/test/src/cli/testCliRunnerMocha.js b/test/src/cli/testCliRunnerMocha.js
index 0f8aeb98..cd3a8664 100644
--- a/test/src/cli/testCliRunnerMocha.js
+++ b/test/src/cli/testCliRunnerMocha.js
@@ -168,6 +168,59 @@ describe('test CLI Runner Mocha', function() {
       assert.deepStrictEqual(testFiles, ['test1.js', 'test2.js']);
     });
   });
+
+  it('testRunWithMochaReporterOptions', function() {
+    const testFiles = [];
+
+    const mochaOptions = {
+      timeout: 20000, 
+      reporterOptions: {
+        reportDir: 'html-dir'
+      },
+      reporter: 'mochawesome'
+    };
+
+    if (isCi) {
+      mochaOptions.color = false;
+    }
+
+    mockery.registerMock('mochawesome', {});
+
+    mockery.registerMock('./withmochaReportOptions.json', {
+      src_folders: ['tests'],
+      output_folder: false,
+      test_settings: {
+        'default': {
+          silent: true
+        }
+      },
+      reporter_options: {
+        reportDir: 'html-dir'
+      },
+      test_runner: {
+        type: 'mocha',
+        options: {
+          reporter: 'mochawesome'
+        }
+      }
+    });
+
+    createMockedMocha(function(options) {
+      assert.deepStrictEqual(options, mochaOptions);
+    }, testFiles);
+
+    const CliRunner = common.require('runner/cli/cli.js');
+    const runner = new CliRunner({
+      config: './withmochaReportOptions.json',
+      env: 'default',
+      reporter: 'junit'
+    }).setup();
+
+    return runner.runTests().then(function() {
+      assert.deepStrictEqual(testFiles, ['test1.js', 'test2.js']);
+    });
+  });
+
 });
 
 
diff --git a/test/src/runner/testRunnerHtmlOutput.js b/test/src/runner/testRunnerHtmlOutput.js
index ac5f0a0f..c03c7d3a 100644
--- a/test/src/runner/testRunnerHtmlOutput.js
+++ b/test/src/runner/testRunnerHtmlOutput.js
@@ -116,4 +116,43 @@ describe('testRunnerHTMLOutput', function() {
       });
      
   });
+  
+  it('test html report folder with a folder format function', function () {
+
+    const testsPath = [
+      path.join(__dirname, '../../sampletests/withfailures')
+    ];
+
+    MockServer.addMock({
+      url: '/wd/hub/session/1352110219202/screenshot',
+      method: 'GET',
+      response: JSON.stringify({
+        sessionId: '1352110219202',
+        status: 0,
+        value: '<faketag>fakedata'
+      })
+    }, true);
+
+
+    return runTests({source: testsPath, reporter: 'html'}, settings({
+      output_folder: outputPath,
+      globals: {
+        waitForConditionPollInterval: 20,
+        waitForConditionTimeout: 50,
+        retryAssertionTimeout: 50,
+        reporter: function () {
+        }
+      },
+      reporter_options: {
+        folder_format: function() {
+          return Date.now().toString();
+        }
+      }
+    }))
+      .then(_ => {
+        return readFilePromise(`${outputPath}${path.sep}*${path.sep}nightwatch-html-report${path.sep}index.html`);
+      }).
+      then(_ => {
+      });
+  });
 });