Repository: 11ty/eleventy (js)

Issue:
Hooks to allow plugins to customize data file formats (think: yaml, toml, etc)
Similar to #117 but for data files instead of template files.

Related #265.

Pull request:
Gloryofrobots data hooks via #635
See #635 for original comments. Fixes #630.

Patch:
diff --git a/package.json b/package.json
index f6b7fbd7..457dd2dd 100755
--- a/package.json
+++ b/package.json
@@ -81,7 +81,8 @@
     "toml": "^3.0.0",
     "viperhtml": "^2.17.0",
     "vue": "^2.6.10",
-    "vue-server-renderer": "^2.6.10"
+    "vue-server-renderer": "^2.6.10",
+    "js-yaml": "^3.13.1"
   },
   "dependencies": {
     "@11ty/dependency-tree": "^1.0.0",
@@ -119,4 +120,4 @@
   },
   "pre-commit": "lint-staged",
   "pre-push": "test"
-}
\ No newline at end of file
+}
diff --git a/src/TemplateData.js b/src/TemplateData.js
index 8aa5802d..83862e14 100644
--- a/src/TemplateData.js
+++ b/src/TemplateData.js
@@ -111,10 +111,19 @@ class TemplateData {
 
   async getTemplateDataFileGlob() {
     let dir = await this.getInputDir();
-    return TemplatePath.addLeadingDotSlashArray([
+    let paths = [
       `${dir}/**/*.json`, // covers .11tydata.json too
       `${dir}/**/*${this.config.jsDataFileSuffix}.js`
-    ]);
+    ];
+
+    if (this.hasUserDataExtensions()) {
+      let userPaths = this.getUserDataExtensions().map(
+        extension => `${dir}/**/*.${extension}` // covers .11tydata.{extension} too
+      );
+      paths = userPaths.concat(paths);
+    }
+
+    return TemplatePath.addLeadingDotSlashArray(paths);
   }
 
   async getTemplateJavaScriptDataFileGlob() {
@@ -126,7 +135,15 @@ class TemplateData {
 
   async getGlobalDataGlob() {
     let dir = await this.getInputDir();
-    return [this._getGlobalDataGlobByExtension(dir, "(json|js)")];
+    let userExtensions = "";
+    // creating glob string for user extensions
+    if (this.hasUserDataExtensions()) {
+      userExtensions = this.getUserDataExtensions().join("|") + "|";
+    }
+
+    return [
+      this._getGlobalDataGlobByExtension(dir, "(" + userExtensions + "json|js)")
+    ];
   }
 
   getWatchPathCache() {
@@ -157,6 +174,7 @@ class TemplateData {
     let files = TemplatePath.addLeadingDotSlashArray(
       await this.getGlobalDataFiles()
     );
+
     let dataFileConflicts = {};
 
     for (let j = 0, k = files.length; j < k; j++) {
@@ -218,7 +236,31 @@ class TemplateData {
     return localData;
   }
 
-  async _getLocalJsonString(path) {
+  getUserDataExtensions() {
+    if (!this.config.dataExtensions) {
+      return [];
+    }
+
+    // returning extensions in reverse order to create proper extension order
+    // later added formats will override first ones
+    return Array.from(this.config.dataExtensions.keys()).reverse();
+  }
+
+  getUserDataParser(extension) {
+    return this.config.dataExtensions.get(extension);
+  }
+
+  isUserDataExtension(extension) {
+    return (
+      this.config.dataExtensions && this.config.dataExtensions.has(extension)
+    );
+  }
+
+  hasUserDataExtensions() {
+    return this.config.dataExtensions && this.config.dataExtensions.size > 0;
+  }
+
+  async _loadFileContents(path) {
     let rawInput;
     try {
       rawInput = await fs.readFile(path, "utf-8");
@@ -228,56 +270,90 @@ class TemplateData {
     return rawInput;
   }
 
+  async _parseDataFile(path, rawImports, ignoreProcessing, parser) {
+    let rawInput = await this._loadFileContents(path);
+    let engineName = this.dataTemplateEngine;
+
+    if (!rawInput) {
+      return {};
+    }
+
+    if (ignoreProcessing || engineName === false) {
+      try {
+        return parser(rawInput);
+      } catch (e) {
+        throw new TemplateDataParseError(
+          `Having trouble parsing data file ${path}`,
+          e
+        );
+      }
+    } else {
+      let fn = await new TemplateRender(engineName).getCompiledTemplate(
+        rawInput
+      );
+
+      try {
+        // pass in rawImports, don’t pass in global data, that’s what we’re parsing
+        let raw = await fn(rawImports);
+        return parser(raw);
+      } catch (e) {
+        throw new TemplateDataParseError(
+          `Having trouble parsing data file ${path}`,
+          e
+        );
+      }
+    }
+  }
+
   async getDataValue(path, rawImports, ignoreProcessing) {
-    if (ignoreProcessing || TemplatePath.getExtension(path) === "js") {
+    let extension = TemplatePath.getExtension(path);
+
+    // ignoreProcessing = false for global data files
+    // ignoreProcessing = true for local data files
+    if (
+      extension === "js" ||
+      (extension === "json" && (ignoreProcessing || !this.dataTemplateEngine))
+    ) {
+      // JS data file or require’d JSON (no preprocessing needed)
       let localPath = TemplatePath.absolutePath(path);
-      if (await fs.pathExists(localPath)) {
-        let dataBench = bench.get(`\`${path}\``);
-        dataBench.before();
-        deleteRequireCache(localPath);
-        let returnValue = require(localPath);
-        if (typeof returnValue === "function") {
-          returnValue = await returnValue();
-        }
-
-        dataBench.after();
-        return returnValue;
-      } else {
+      if (!(await fs.pathExists(localPath))) {
         return {};
       }
-    } else {
-      let rawInput = await this._getLocalJsonString(path);
-      let engineName = this.dataTemplateEngine;
-
-      if (rawInput) {
-        if (ignoreProcessing || engineName === false) {
-          try {
-            return JSON.parse(rawInput);
-          } catch (e) {
-            throw new TemplateDataParseError(
-              `Having trouble parsing data file ${path}`,
-              e
-            );
-          }
-        } else {
-          let fn = await new TemplateRender(engineName).getCompiledTemplate(
-            rawInput
-          );
 
-          try {
-            // pass in rawImports, don’t pass in global data, that’s what we’re parsing
-            return JSON.parse(await fn(rawImports));
-          } catch (e) {
-            throw new TemplateDataParseError(
-              `Having trouble parsing data file ${path}`,
-              e
-            );
-          }
-        }
+      let dataBench = bench.get(`\`${path}\``);
+      dataBench.before();
+      deleteRequireCache(localPath);
+
+      let returnValue = require(localPath);
+      if (typeof returnValue === "function") {
+        returnValue = await returnValue();
       }
+
+      dataBench.after();
+      return returnValue;
+    } else if (this.isUserDataExtension(extension)) {
+      // Other extensions
+      var parser = this.getUserDataParser(extension);
+      return this._parseDataFile(path, rawImports, ignoreProcessing, parser);
+    } else if (extension === "json") {
+      // File to string, parse with JSON (preprocess)
+      return this._parseDataFile(
+        path,
+        rawImports,
+        ignoreProcessing,
+        JSON.parse
+      );
+    } else {
+      throw new TemplateDataParseError(
+        `Could not find an appropriate data parser for ${path}. Do you need to add a plugin to your config file?`
+      );
     }
+  }
 
-    return {};
+  _pushExtensionsToPaths(paths, curpath, extensions) {
+    for (let extension of extensions) {
+      paths.push(curpath + "." + extension);
+    }
   }
 
   async getLocalDataPaths(templatePath) {
@@ -286,19 +362,34 @@ class TemplateData {
     let inputDir = TemplatePath.addLeadingDotSlash(
       TemplatePath.normalize(this.inputDir)
     );
+
     debugDev("getLocalDataPaths(%o)", templatePath);
     debugDev("parsed.dir: %o", parsed.dir);
 
+    let userExtensions = this.getUserDataExtensions();
+
     if (parsed.dir) {
       let fileNameNoExt = EleventyExtensionMap.removeTemplateExtension(
         parsed.base
       );
+
       let filePathNoExt = parsed.dir + "/" + fileNameNoExt;
       let dataSuffix = this.config.jsDataFileSuffix;
       debug("Using %o to find data files.", dataSuffix);
+
+      // data suffix
       paths.push(filePathNoExt + dataSuffix + ".js");
       paths.push(filePathNoExt + dataSuffix + ".json");
+      // inject user extensions
+      this._pushExtensionsToPaths(
+        paths,
+        filePathNoExt + dataSuffix,
+        userExtensions
+      );
+
+      // top level
       paths.push(filePathNoExt + ".json");
+      this._pushExtensionsToPaths(paths, filePathNoExt, userExtensions);
 
       let allDirs = TemplatePath.getAllDirs(parsed.dir);
       debugDev("allDirs: %o", allDirs);
@@ -307,15 +398,33 @@ class TemplateData {
         let dirPathNoExt = dir + "/" + lastDir;
 
         if (!inputDir) {
+          // data suffix
           paths.push(dirPathNoExt + dataSuffix + ".js");
           paths.push(dirPathNoExt + dataSuffix + ".json");
+          this._pushExtensionsToPaths(
+            paths,
+            dirPathNoExt + dataSuffix,
+            userExtensions
+          );
+
+          // top level
           paths.push(dirPathNoExt + ".json");
+          this._pushExtensionsToPaths(paths, dirPathNoExt, userExtensions);
         } else {
           debugDev("dirStr: %o; inputDir: %o", dir, inputDir);
           if (dir.indexOf(inputDir) === 0 && dir !== inputDir) {
+            // data suffix
             paths.push(dirPathNoExt + dataSuffix + ".js");
             paths.push(dirPathNoExt + dataSuffix + ".json");
+            this._pushExtensionsToPaths(
+              paths,
+              dirPathNoExt + dataSuffix,
+              userExtensions
+            );
+
+            // top level
             paths.push(dirPathNoExt + ".json");
+            this._pushExtensionsToPaths(paths, dirPathNoExt, userExtensions);
           }
         }
       }
diff --git a/src/UserConfig.js b/src/UserConfig.js
index 76df00d9..fc82a927 100644
--- a/src/UserConfig.js
+++ b/src/UserConfig.js
@@ -56,6 +56,9 @@ class UserConfig {
     // this.templateExtensionAliases = {};
     this.watchJavaScriptDependencies = true;
     this.browserSyncConfig = {};
+
+    // using Map to preserve insertion order
+    this.dataExtensions = new Map();
   }
 
   versionCheck(expected) {
@@ -603,13 +606,18 @@ class UserConfig {
       // templateExtensionAliases: this.templateExtensionAliases,
       watchJavaScriptDependencies: this.watchJavaScriptDependencies,
       browserSyncConfig: this.browserSyncConfig,
-      frontMatterParsingOptions: this.frontMatterParsingOptions
+      frontMatterParsingOptions: this.frontMatterParsingOptions,
+      dataExtensions: this.dataExtensions
     };
   }
 
   // addExtension(fileExtension, userClass) {
   //   this.userExtensionMap[ fileExtension ] = userClass;
   // }
+
+  addDataExtension(formatExtension, formatParser) {
+    this.dataExtensions.set(formatExtension, formatParser);
+  }
 }
 
 module.exports = UserConfig;


Test patch:
diff --git a/test/TemplateDataTest.js b/test/TemplateDataTest.js
index 5892367f..9350c062 100644
--- a/test/TemplateDataTest.js
+++ b/test/TemplateDataTest.js
@@ -52,6 +52,7 @@ test("Add local data", async t => {
   t.is(withLocalData.localdatakey1, "localdatavalue1");
 
   // from the js file
+  // this checks priority/overrides
   t.is(withLocalData.localdatakeyfromjs, "howdydoody");
   t.is(withLocalData.localdatakeyfromjs2, "howdy2");
 });
diff --git a/test/UserDataExtensionsTest.js b/test/UserDataExtensionsTest.js
new file mode 100644
index 00000000..fa5cc1e6
--- /dev/null
+++ b/test/UserDataExtensionsTest.js
@@ -0,0 +1,92 @@
+import test from "ava";
+import TemplateData from "../src/TemplateData";
+let yaml = require("js-yaml");
+
+function injectDataExtensions(dataObj) {
+  dataObj.config.dataExtensions = new Map([
+    ["yaml", s => yaml.safeLoad(s)],
+    ["nosj", JSON.parse]
+  ]);
+}
+
+test("Local data", async t => {
+  let dataObj = new TemplateData("./test/stubs-630/");
+  injectDataExtensions(dataObj);
+
+  let data = await dataObj.getData();
+
+  // YAML GLOBAL DATA
+  t.is(data.globalData2.datakey1, "datavalue2");
+  t.is(data.globalData2.datakey2, "@11ty/eleventy--yaml");
+
+  // NOSJ (JSON) GLOBAL DATA
+  t.is(data.globalData3.datakey1, "datavalue3");
+  t.is(data.globalData3.datakey2, "@11ty/eleventy--nosj");
+
+  let withLocalData = await dataObj.getLocalData(
+    "./test/stubs-630/component-yaml/component.njk"
+  );
+  // console.log("localdata", withLocalData);
+
+  t.is(withLocalData.yamlKey1, "yaml1");
+  t.is(withLocalData.yamlKey2, "yaml2");
+  t.is(withLocalData.yamlKey3, "yaml3");
+  t.is(withLocalData.nosjKey1, "nosj1");
+  t.is(withLocalData.jsonKey1, "json1");
+  t.is(withLocalData.jsonKey2, "json2");
+  t.is(withLocalData.jsKey1, "js1");
+});
+
+test("Local files", async t => {
+  let dataObj = new TemplateData("./test/stubs-630/");
+  injectDataExtensions(dataObj);
+  let files = await dataObj.getLocalDataPaths(
+    "./test/stubs-630/component-yaml/component.njk"
+  );
+  t.deepEqual(files, [
+    "./test/stubs-630/component-yaml/component-yaml.yaml",
+    "./test/stubs-630/component-yaml/component-yaml.nosj",
+    "./test/stubs-630/component-yaml/component-yaml.json",
+    "./test/stubs-630/component-yaml/component-yaml.11tydata.yaml",
+    "./test/stubs-630/component-yaml/component-yaml.11tydata.nosj",
+    "./test/stubs-630/component-yaml/component-yaml.11tydata.json",
+    "./test/stubs-630/component-yaml/component-yaml.11tydata.js",
+    "./test/stubs-630/component-yaml/component.yaml",
+    "./test/stubs-630/component-yaml/component.nosj",
+    "./test/stubs-630/component-yaml/component.json",
+    "./test/stubs-630/component-yaml/component.11tydata.yaml",
+    "./test/stubs-630/component-yaml/component.11tydata.nosj",
+    "./test/stubs-630/component-yaml/component.11tydata.json",
+    "./test/stubs-630/component-yaml/component.11tydata.js"
+  ]);
+});
+
+test("Global data", async t => {
+  let dataObj = new TemplateData("./test/stubs-630/");
+
+  injectDataExtensions(dataObj);
+
+  t.deepEqual(await dataObj.getGlobalDataGlob(), [
+    "./test/stubs-630/_data/**/*.(nosj|yaml|json|js)"
+  ]);
+
+  let dataFilePaths = await dataObj.getGlobalDataFiles();
+  let data = await dataObj.getData();
+
+  // JS GLOBAL DATA
+  t.is(data.globalData0.datakey1, "datavalue0");
+
+  // JSON GLOBAL DATA
+  t.is(data.globalData1.datakey1, "datavalue1");
+  t.is(data.globalData1.datakey2, "@11ty/eleventy--json");
+
+  // YAML GLOBAL DATA
+  t.is(data.globalData2.datakey1, "datavalue2");
+  t.is(data.globalData2.datakey2, "@11ty/eleventy--yaml");
+
+  // NOSJ (JSON) GLOBAL DATA
+  t.is(data.globalData3.datakey1, "datavalue3");
+  t.is(data.globalData3.datakey2, "@11ty/eleventy--nosj");
+
+  t.is(data.subdir.globalDataSubdir.keyyaml, "yaml");
+});
diff --git a/test/stubs-630/_data/globalData0.js b/test/stubs-630/_data/globalData0.js
new file mode 100644
index 00000000..4f79aca5
--- /dev/null
+++ b/test/stubs-630/_data/globalData0.js
@@ -0,0 +1,3 @@
+module.exports = {
+  datakey1: "datavalue0"
+};
diff --git a/test/stubs-630/_data/globalData1.json b/test/stubs-630/_data/globalData1.json
new file mode 100644
index 00000000..5f02feff
--- /dev/null
+++ b/test/stubs-630/_data/globalData1.json
@@ -0,0 +1,4 @@
+{
+  "datakey1": "datavalue1",
+  "datakey2": "{{pkg.name}}--json"
+}
diff --git a/test/stubs-630/_data/globalData2.yaml b/test/stubs-630/_data/globalData2.yaml
new file mode 100644
index 00000000..9a852bfa
--- /dev/null
+++ b/test/stubs-630/_data/globalData2.yaml
@@ -0,0 +1,2 @@
+datakey1: datavalue2
+datakey2: "{{pkg.name}}--yaml"
diff --git a/test/stubs-630/_data/globalData3.nosj b/test/stubs-630/_data/globalData3.nosj
new file mode 100644
index 00000000..3f01a07e
--- /dev/null
+++ b/test/stubs-630/_data/globalData3.nosj
@@ -0,0 +1,4 @@
+{
+  "datakey1": "datavalue3",
+  "datakey2": "{{pkg.name}}--nosj"
+}
diff --git a/test/stubs-630/_data/subdir/globalDataSubdir.yaml b/test/stubs-630/_data/subdir/globalDataSubdir.yaml
new file mode 100644
index 00000000..7e610b57
--- /dev/null
+++ b/test/stubs-630/_data/subdir/globalDataSubdir.yaml
@@ -0,0 +1,1 @@
+keyyaml: "yaml"
diff --git a/test/stubs-630/component-yaml/component.11tydata.js b/test/stubs-630/component-yaml/component.11tydata.js
new file mode 100644
index 00000000..f8e5b987
--- /dev/null
+++ b/test/stubs-630/component-yaml/component.11tydata.js
@@ -0,0 +1,3 @@
+module.exports = {
+  jsKey1: "js1"
+};
diff --git a/test/stubs-630/component-yaml/component.11tydata.json b/test/stubs-630/component-yaml/component.11tydata.json
new file mode 100644
index 00000000..0af98b49
--- /dev/null
+++ b/test/stubs-630/component-yaml/component.11tydata.json
@@ -0,0 +1,3 @@
+{
+    "jsonKey1": "json1"
+}
diff --git a/test/stubs-630/component-yaml/component.11tydata.nosj b/test/stubs-630/component-yaml/component.11tydata.nosj
new file mode 100644
index 00000000..76bc8b93
--- /dev/null
+++ b/test/stubs-630/component-yaml/component.11tydata.nosj
@@ -0,0 +1,3 @@
+{
+  "nosjKey1": "nosj1"
+}
diff --git a/test/stubs-630/component-yaml/component.11tydata.yaml b/test/stubs-630/component-yaml/component.11tydata.yaml
new file mode 100644
index 00000000..b61c3866
--- /dev/null
+++ b/test/stubs-630/component-yaml/component.11tydata.yaml
@@ -0,0 +1,5 @@
+yamlKey2: "yaml2"
+yamlKey3: "yaml3"
+jsonKey1: "overriden"
+jsKey1: "overriden"
+nosjKey1: "overriden"
diff --git a/test/stubs-630/component-yaml/component.json b/test/stubs-630/component-yaml/component.json
new file mode 100644
index 00000000..21e49a8c
--- /dev/null
+++ b/test/stubs-630/component-yaml/component.json
@@ -0,0 +1,5 @@
+{
+    "jsonKey2": "json2",
+    "jsKey1": "overriden",
+    "yamlKey3": "overriden"
+}
diff --git a/test/stubs-630/component-yaml/component.njk b/test/stubs-630/component-yaml/component.njk
new file mode 100644
index 00000000..4772f745
--- /dev/null
+++ b/test/stubs-630/component-yaml/component.njk
@@ -0,0 +1,1 @@
+{{localkeyOverride}}
\ No newline at end of file
diff --git a/test/stubs-630/component-yaml/component.yaml b/test/stubs-630/component-yaml/component.yaml
new file mode 100644
index 00000000..f65aadd9
--- /dev/null
+++ b/test/stubs-630/component-yaml/component.yaml
@@ -0,0 +1,5 @@
+yamlKey1: "yaml1"
+yamlKey2: "overriden"
+jsonKey1: "overriden"
+jsonKey2: "overriden"
+jsKey1: "overriden"