Repository: asyncapi/generator (js)

Issue:
Support remote AsyncAPI files
After adding support for remote templates (#261), it makes total sense now to support remote AsyncAPI files. This should be useful especially on CI systems.

Pull request:
feat: fetching asyncapi file from server in cli and library

<!--   Thank you for your contribution. Before you submit the pull request:
1. Follow contributing guidelines and make sure PR title follows Conventional Commits specification -> https://github.com/asyncapi/generator/blob/master/CONTRIBUTING.md
2. Test your changes and attach their results to the pull request.
3. Update the relevant documentation.
-->

**Description**

- new function exposed -> `generateFromURL`
- support in CLI so we can do `ag https://raw.githubusercontent.com/asyncapi/asyncapi/master/examples/2.0.0/streetlights.yml @asyncapi/html-template`

validation if path is filesystem path or url is very basic here, not based on any sophisticated regex but just using `url.parse`. Let me know if you think it is bad approach. I don't see a use case that someone uses this feature as a library and on a server passes relative path to fetch the file from the same server where the generator is running, I think it is always about fetching from different server (hostname)


**Related issue(s)**
Resolves #266 

Patch:
diff --git a/README.md b/README.md
index fbb0260..168db83 100644
--- a/README.md
+++ b/README.md
@@ -83,6 +83,11 @@ Options:
 ag asyncapi.yaml @asyncapi/html-template
 ```
 
+**Generating from a URL:**
+```bash
+ag https://raw.githubusercontent.com/asyncapi/asyncapi/master/examples/2.0.0/streetlights.yml @asyncapi/html-template
+```
+
 **Specify where to put the result:**
 ```bash
 ag asyncapi.yaml @asyncapi/html-template -o ./docs
diff --git a/cli.js b/cli.js
index 8e3dbc2..6b17051 100755
--- a/cli.js
+++ b/cli.js
@@ -7,14 +7,14 @@ const xfs = require('fs.extra');
 const packageInfo = require('./package.json');
 const Generator = require('./lib/generator');
 const Watcher = require('./lib/watcher');
-const { isLocalTemplate } = require('./lib/utils');
+const { isLocalTemplate, isFilePath } = require('./lib/utils');
 
 const red = text => `\x1b[31m${text}\x1b[0m`;
 const magenta = text => `\x1b[35m${text}\x1b[0m`;
 const yellow = text => `\x1b[33m${text}\x1b[0m`;
 const green = text => `\x1b[32m${text}\x1b[0m`;
 
-let asyncapiFile;
+let asyncapiDocPath;
 let template;
 const params = {};
 const noOverwriteGlobs = [];
@@ -49,8 +49,8 @@ const showErrorAndExit = err => {
 program
   .version(packageInfo.version)
   .arguments('<asyncapi> <template>')
-  .action((asyncAPIPath, tmpl) => {
-    asyncapiFile = path.resolve(asyncAPIPath);
+  .action((path, tmpl) => {
+    asyncapiDocPath = path;
     template = tmpl;
   })
   .option('-d, --disable-hook <hookType>', 'disable a specific hook type', disableHooksParser)
@@ -63,10 +63,11 @@ program
   .option('--watch-template', 'watches the template directory and the AsyncAPI document, and re-generate the files when changes occur. Ignores the output directory. This flag should be used only for template development.')
   .parse(process.argv);
 
-if (!asyncapiFile) {
-  console.error(red('> Path to AsyncAPI file not provided.'));
+if (!asyncapiDocPath) {
+  console.error(red('> Path or URL to AsyncAPI file not provided.'));
   program.help(); // This exits the process
 }
+const isAsyncapiDocLocal = isFilePath(asyncapiDocPath);
 
 xfs.mkdirp(program.output, async err => {
   if (err) return showErrorAndExit(err);
@@ -78,19 +79,25 @@ xfs.mkdirp(program.output, async err => {
 
   // If we want to watch for changes do that
   if (program.watchTemplate) {
+    let watcher;
     const watchDir = path.resolve(template);
+    const outputPath = path.resolve(watchDir, program.output);
     // Template name is needed as it is not always a part of the cli commad
     // There is a use case that you run generator from a root of the template with `./` path
     const templateName = require(path.resolve(watchDir,'package.json')).name;
 
-    console.log(`[WATCHER] Watching for changes in the template directory ${magenta(watchDir)} and in the AsyncAPI file ${magenta(asyncapiFile)}`);
-
+    if (isAsyncapiDocLocal) {
+      console.log(`[WATCHER] Watching for changes in the template directory ${magenta(watchDir)} and in the AsyncAPI file ${magenta(asyncapiDocPath)}`);
+      watcher = new Watcher([asyncapiDocPath, watchDir], outputPath);
+    } else {
+      console.log(`[WATCHER] Watching for changes in the template directory ${magenta(watchDir)}`);
+      watcher = new Watcher(watchDir, outputPath);
+    }
     // Must check template in its installation path in generator to use isLocalTemplate function
     if (!await isLocalTemplate(path.resolve(Generator.DEFAULT_TEMPLATES_DIR, templateName))) {
       console.warn(`WARNING: ${template} is a remote template. Changes may be lost on subsequent installations.`);
     }
-    const outputPath = path.resolve(watchDir, program.output);
-    const watcher = new Watcher([asyncapiFile, watchDir], outputPath);
+    
     watcher.watch(async (changedFiles) => {
       console.clear();
       console.log('[WATCHER] Change detected');
@@ -139,7 +146,11 @@ function generate(targetDir) {
         debug: program.debug
       });
 
-      await generator.generateFromFile(asyncapiFile);
+      if (isAsyncapiDocLocal) {
+        await generator.generateFromFile(path.resolve(asyncapiDocPath));
+      } else {
+        await generator.generateFromURL(asyncapiDocPath);
+      }
       console.log(green('\n\nDone! ✨'));
       console.log(`${yellow('Check out your shiny new generated files at ') + magenta(program.output) + yellow('.')}\n`);
       resolve();
diff --git a/docs/api.md b/docs/api.md
index 7881308..d8885b2 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -16,9 +16,11 @@
         * [.debug](#Generator+debug) : <code>Boolean</code>
         * [.install](#Generator+install) : <code>Boolean</code>
         * [.templateConfig](#Generator+templateConfig) : <code>Object</code>
+        * [.hooks](#Generator+hooks) : <code>Object</code>
         * [.templateParams](#Generator+templateParams) : <code>Object</code>
         * [.generate(asyncapiDocument)](#Generator+generate) ⇒ <code>Promise</code>
         * [.generateFromString(asyncapiString, [parserOptions])](#Generator+generateFromString) ⇒ <code>Promise</code>
+        * [.generateFromURL(asyncapiURL)](#Generator+generateFromURL) ⇒ <code>Promise</code>
         * [.generateFromFile(asyncapiFile)](#Generator+generateFromFile) ⇒ <code>Promise</code>
         * [.installTemplate([force])](#Generator+installTemplate)
     * _static_
@@ -117,6 +119,12 @@ Install the template and its dependencies, even when the template has already be
 ### generator.templateConfig : <code>Object</code>
 The template configuration.
 
+**Kind**: instance property of [<code>Generator</code>](#Generator)  
+<a name="Generator+hooks"></a>
+
+### generator.hooks : <code>Object</code>
+Hooks object with hooks functionst grouped by the hook type.
+
 **Kind**: instance property of [<code>Generator</code>](#Generator)  
 <a name="Generator+templateParams"></a>
 
@@ -198,6 +206,35 @@ try {
   console.error(e);
 }
 ```
+<a name="Generator+generateFromURL"></a>
+
+### generator.generateFromURL(asyncapiURL) ⇒ <code>Promise</code>
+Generates files from a given template and AsyncAPI file stored on external server.
+
+**Kind**: instance method of [<code>Generator</code>](#Generator)  
+
+| Param | Type | Description |
+| --- | --- | --- |
+| asyncapiURL | <code>String</code> | Link to server and AsyncAPI file. |
+
+**Example**  
+```js
+generator
+  .generateFromURL('https://example.com/asyncapi.yaml')
+  .then(() => {
+    console.log('Done!');
+  })
+  .catch(console.error);
+```
+**Example** *(Using async/await)*  
+```js
+try {
+  await generator.generateFromURL('https://example.com/asyncapi.yaml');
+  console.log('Done!');
+} catch (e) {
+  console.error(e);
+}
+```
 <a name="Generator+generateFromFile"></a>
 
 ### generator.generateFromFile(asyncapiFile) ⇒ <code>Promise</code>
diff --git a/docs/authoring.md b/docs/authoring.md
index 438916b..a40c2a4 100644
--- a/docs/authoring.md
+++ b/docs/authoring.md
@@ -183,6 +183,7 @@ The `.tp-config.json` file contains a JSON object that may have the following in
 |`supportedProtocols`| [String] | A list with all the protocols this template supports.
 |`parameters`| Object[String, Object] | An object with all the parameters that can be passed when generating the template. When using the command line, it's done by indicating `--param name=value` or `-p name=value`.
 |`parameters[param].description`| String | A user-friendly description about the parameter.
+|`parameters[param].default`| Any | Default value of the parameter if not specified. Shouldn't be used for mandatory `required=true` parameters.
 |`parameters[param].required`| Boolean | Whether the parameter is required or not.
 |`conditionalFiles`| Object[String, Object] | An object containing all the file paths that should be conditionally rendered. Each key represents a file path and each value must be an object with the keys `subject` and `validation`.
 |`conditionalFiles[filePath].subject`| String | The `subject` is a [JMESPath](http://jmespath.org/) query to grab the value you want to apply the condition to. It queries an object with the whole AsyncAPI document and, when specified, the given server. The object looks like this: `{ asyncapi: { ... }, server: { ... } }`.
@@ -201,6 +202,11 @@ The `.tp-config.json` file contains a JSON object that may have the following in
     "server": {
       "description": "The server you want to use in the code.",
       "required": true
+    },
+    "dummyParameter": {
+      "description": "Example of parameter with default value.",
+      "default": "just a string",
+      "required": false
     }
   },
   "conditionalFiles": {
diff --git a/lib/__mocks__/utils.js b/lib/__mocks__/utils.js
index 0ad6b7e..d203b9c 100644
--- a/lib/__mocks__/utils.js
+++ b/lib/__mocks__/utils.js
@@ -5,6 +5,11 @@ utils.readFile = jest.fn(async (filePath) => {
   return utils.__files[filePath];
 });
 
+utils.__contentOfFetchedFile = '';
+utils.fetchSpec = jest.fn(async (fileUrl) => {
+  return utils.__contentOfFetchedFile;
+});
+
 utils.__isFileSystemPathValue = false;
 utils.isFileSystemPath = jest.fn(() => utils.__isFileSystemPathValue);
 
diff --git a/lib/generator.js b/lib/generator.js
index d44dca7..c9a4447 100644
--- a/lib/generator.js
+++ b/lib/generator.js
@@ -24,7 +24,8 @@ const {
   readDir,
   writeFile,
   copyFile,
-  exists
+  exists,
+  fetchSpec
 } = require('./utils');
 const { registerFilters } = require('./filtersRegistry');
 const { registerHooks } = require('./hooksRegistry');
@@ -109,6 +110,7 @@ class Generator {
     this.install = install;
     /** @type {Object} The template configuration. */
     this.templateConfig = {};
+    /** @type {Object} Hooks object with hooks functionst grouped by the hook type. */
     this.hooks = {};
 
     // Load template configuration
@@ -233,6 +235,34 @@ class Generator {
     return this.generate(this.asyncapi);
   }
 
+  /**
+   * Generates files from a given template and AsyncAPI file stored on external server.
+   *
+   * @example
+   * generator
+   *   .generateFromURL('https://example.com/asyncapi.yaml')
+   *   .then(() => {
+   *     console.log('Done!');
+   *   })
+   *   .catch(console.error);
+   *
+   * @example <caption>Using async/await</caption>
+   * try {
+   *   await generator.generateFromURL('https://example.com/asyncapi.yaml');
+   *   console.log('Done!');
+   * } catch (e) {
+   *   console.error(e);
+   * }
+   *
+   * @param  {String} asyncapiURL Link to AsyncAPI file
+   * @return {Promise}
+   */
+  async generateFromURL(asyncapiURL) {
+    const doc = await fetchSpec(asyncapiURL);
+
+    return this.generateFromString(doc);
+  }
+
   /**
    * Generates files from a given template and AsyncAPI file.
    *
@@ -642,7 +672,26 @@ class Generator {
     } catch (e) {
       this.templateConfig = {};
     }
-    await this.validateTemplateConfig();
+    await this.loadDefaultValues();
+  }
+
+  /**
+   * Loads default values of parameters from template config. If value was already set as parameter it will not be
+   * overrided.
+   * @private
+   */
+  async loadDefaultValues() {
+    const parameters = this.templateConfig.parameters;
+    const defaultValues = Object.keys(parameters || {}).filter(key => parameters[key].default);
+
+    defaultValues.filter(dv => this.templateParams[dv] === undefined).forEach(dv =>
+      Object.defineProperty(this.templateParams, dv, {
+        enumerable: true,
+        get() {
+          return parameters[dv].default;
+        }
+      })
+    );
   }
 
   /**
@@ -660,17 +709,7 @@ class Generator {
       throw new Error(`This template is not compatible with the current version of the generator (${packageJson.version}). This template is compatible with the following version range: ${generator}.`);
     }
 
-    const requiredParams = [];
-    Object.keys(parameters || {}).forEach(key => {
-      if (parameters[key].required === true) requiredParams.push(key);
-    });
-
-    if (Array.isArray(requiredParams)) {
-      const missingParams = requiredParams.filter(rp => this.templateParams[rp] === undefined);
-      if (missingParams.length) {
-        throw new Error(`This template requires the following missing params: ${missingParams}.`);
-      }
-    }
+    this.verifyParameters(parameters);
 
     if (typeof conditionalFiles === 'object') {
       const fileNames = Object.keys(conditionalFiles) || [];
@@ -694,6 +733,25 @@ class Generator {
     }
   }
 
+  /**
+   * Checks that all required parameters are set and all specified by user parameters are present in template.
+   *
+   * @private
+   * @param  {Array} [parameters] parameters known from template configuration.
+   */
+  verifyParameters(parameters) {
+    const missingParams = Object.keys(parameters || {})
+      .filter(key => parameters[key].required === true && this.templateParams[key] === undefined);
+    if (missingParams.length) {
+      throw new Error(`This template requires the following missing params: ${missingParams}.`);
+    }
+
+    const wrongParams = Object.keys(this.templateParams || {}).filter(key => parameters[key] === undefined);
+    if (wrongParams.length) {
+      console.warn(`Warning: This template doesn't have the following params: ${wrongParams}.`);
+    }
+  }
+
   /**
    * Launches all the hooks registered at a given hook point/name.
    *
diff --git a/lib/utils.js b/lib/utils.js
index 332ed59..98bb060 100644
--- a/lib/utils.js
+++ b/lib/utils.js
@@ -1,6 +1,8 @@
 const fs = require('fs');
 const util = require('util');
 const path = require('path');
+const fetch = require('node-fetch');
+const url = require('url');
 const utils = module.exports;
 
 utils.lstat = util.promisify(fs.lstat);
@@ -80,4 +82,28 @@ utils.getLocalTemplateDetails = async (templatePath) => {
     link: linkTarget,
     resolvedLink: path.resolve(path.dirname(templatePath), linkTarget),
   };
+};
+
+/**
+ * Fetches an AsyncAPI document from the given URL and return its content as string
+ * 
+ * @param {String} url URL where the AsyncAPI document is located.
+ * @returns Promise<String>} Content of fetched file.
+ */
+utils.fetchSpec = (url) => {
+  return new Promise((resolve, reject) => {
+    fetch(url)
+      .then(res => resolve(res.text()))
+      .catch(reject);
+  });
+};
+
+/**
+ *  Checks if given string is URL and if not, we assume it is file path
+ * 
+ * @param {String} str Information representing file path or url
+ * @returns {Boolean}
+ */
+utils.isFilePath = (str) => {
+  return !url.parse(str).hostname;
 };
\ No newline at end of file
diff --git a/package-lock.json b/package-lock.json
index 5e26f49..969e45a 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1,6 +1,6 @@
 {
   "name": "@asyncapi/generator",
-  "version": "0.46.1",
+  "version": "0.48.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
diff --git a/package.json b/package.json
index df9e801..dc1bd68 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@asyncapi/generator",
-  "version": "0.46.1",
+  "version": "0.48.0",
   "description": "The AsyncAPI generator. It can generate documentation, code, anything!",
   "main": "./lib/generator.js",
   "bin": {
@@ -55,6 +55,7 @@
     "js-yaml": "^3.13.1",
     "markdown-it": "^8.4.1",
     "minimatch": "^3.0.4",
+    "node-fetch": "^2.6.0",
     "npmi": "^4.0.0",
     "nunjucks": "^3.2.0",
     "semver": "^7.3.2",


Test patch:
diff --git a/test/generator.test.js b/test/generator.test.js
index fd30263..fe0cec3 100644
--- a/test/generator.test.js
+++ b/test/generator.test.js
@@ -271,7 +271,7 @@ describe('Generator', () => {
       const generateFromStringMock = jest.fn().mockResolvedValue();
       const gen = new Generator('testTemplate', __dirname);
       gen.generateFromString = generateFromStringMock;
-      await gen.generateFromFile('fake-asyncapi.yml');
+      await gen.generateFromFile(filePath);
       expect(utils.readFile).toHaveBeenCalled();
       expect(utils.readFile.mock.calls[0][0]).toBe(filePath);
       expect(utils.readFile.mock.calls[0][1]).toStrictEqual({ encoding: 'utf8' });
@@ -280,6 +280,22 @@ describe('Generator', () => {
     });
   });
 
+  describe('#generateFromURL', () => {
+    it('calls fetch and generateFromString with the right params', async () => {
+      const utils = require('../lib/utils');
+      const asyncapiURL = 'http://example.com/fake-asyncapi.yml';
+      utils.__contentOfFetchedFile = 'fake text';
+      
+      const generateFromStringMock = jest.fn().mockResolvedValue();
+      const gen = new Generator('testTemplate', __dirname);
+      gen.generateFromString = generateFromStringMock;
+      await gen.generateFromURL(asyncapiURL);
+      expect(utils.fetchSpec).toHaveBeenCalled();
+      expect(utils.fetchSpec.mock.calls[0][0]).toBe(asyncapiURL);
+      expect(generateFromStringMock.mock.calls[0][0]).toBe('fake text');
+    });
+  });
+
   describe('#installTemplate', () => {
     let npmiMock;
     let utils;
@@ -383,4 +399,127 @@ describe('Generator', () => {
       expect(content).toBe(utils.__files[filePath]);
     });
   });
+
+  describe('#loadDefaultValues', () => {
+    it('default value of parameter is set', async () => {
+      const gen = new Generator('testTemplate', __dirname, {
+        templateParams: {
+          test: true
+        }
+      });
+      gen.templateConfig  = {
+        parameters: {
+          paramWithDefault: {
+            description: 'Parameter with default value',
+            default: 'default',
+            required: false
+          },
+          paramWithoutDefault: {
+            description: 'Parameter without default value',
+            required: false
+          },
+          test: {
+            description: 'test',
+            required: false
+          }
+        }
+      };
+
+      await gen.loadDefaultValues();
+
+      expect(gen.templateParams).toStrictEqual({
+        test: true,
+        paramWithDefault: 'default'
+      });
+    });
+
+    it('default value of parameter is not override user value', async () => {
+      const gen = new Generator('testTemplate', __dirname, {
+        templateParams: {
+          test: true
+        }
+      });
+      gen.templateConfig = {
+        parameters: {
+          test: {
+            description: 'Parameter with default value',
+            default: false,
+            required: false
+          }
+        }
+      };
+
+      await gen.loadDefaultValues();
+
+      expect(gen.templateParams).toStrictEqual({
+        test: true
+      });
+    });
+
+    it('no default values', async () => {
+      const gen = new Generator('testTemplate', __dirname, {
+        templateParams: {
+          test: true
+        }
+      });
+      gen.templateConfig = {
+        parameters: {
+          test: {
+            description: 'Parameter with default value',
+            required: false
+          },
+          anotherParam: {
+            description: 'Yeat another param',
+            required: false
+          }
+        }
+      };
+
+      await gen.loadDefaultValues();
+
+      expect(gen.templateParams).toStrictEqual({
+        test: true
+      });
+    });
+  });
+
+  describe('#verifyParameters', () => {
+    it('required parameter is missed', () => {
+      const gen = new Generator('testTemplate', __dirname, {
+        templateParams: {
+          test: true
+        }
+      });
+      const params = {
+        requiredParam: {
+          required: true
+        },
+        test: {
+          required: false
+        }
+      };
+
+      expect(() => {
+        gen.verifyParameters(params);
+      }).toThrow(/requiredParam/);
+    });
+    it('wrong parameter is passed', () => {
+      const gen = new Generator('testTemplate', __dirname, {
+        templateParams: {
+          notExistInConf: true
+        }
+      });
+      const params = {
+        existInConf: {
+          required: false
+        }
+      };
+      console.warn = jest.fn();
+
+      gen.verifyParameters(params);
+
+      expect(console.warn).toBeCalledWith(expect.stringContaining('notExistInConf'));
+      expect(console.warn).toBeCalledWith(expect.not.stringContaining('existInConf'));
+    });
+  });
 });