Repository: dynamoosejs/dynamoose
Pull request #731 (2020-03-16 03:10:38)

saveUnknown Wildcard


Patch:
diff --git a/BREAKING_CHANGES.md b/BREAKING_CHANGES.md
index 8eaeb339..c59b8237 100644
--- a/BREAKING_CHANGES.md
+++ b/BREAKING_CHANGES.md
@@ -20,3 +20,4 @@
 - `expires.returnExpiredItems` has been renamed to `expires.items.returnExpired`
 - `Model.transaction.conditionCheck` has been renamed to `Model.transaction.condition`
 - `Model.transaction.condition` options parameter now gets appended to the object returned. This means you can no longer use the helpers that Dynamoose provided to make conditions. Instead, pass in the DynamoDB API level conditions you wish to use
+- In the past the `saveUnknown` option for attribute names would handle all nested properties. Now you must use `*` to indicate one level of wildcard or `**` to indicate infinate levels of wildcard. So if you have an object property (`address`) and want to parse one level of values (no sub objects) you can use `address.*`, or `address.**` to all for infinate levels of values (including sub objects)
diff --git a/docs/api/Schema.md b/docs/api/Schema.md
index 0e14be86..fe7313f8 100644
--- a/docs/api/Schema.md
+++ b/docs/api/Schema.md
@@ -6,7 +6,7 @@ You can use this method to create a schema. The `schema` parameter is an object
 
 The `options` parameter is an optional object with the following options:
 
-- `saveUnknown` array | boolean (default: false) - This setting lets you specify if the schema should allow properties not defined in the schema. If you pass `true` in for this option all unknown properties will be allowed. If you pass in an array of strings, only properties that are included in that array will be allowed. If you retrieve items from DynamoDB with `saveUnknown` enabled, all custom Dynamoose types will be returned as the underlying DynamoDB type (ex. Dates will be returned as a Number representing number of milliseconds since Jan 1 1970)
+- `saveUnknown` array | boolean (default: false) - This setting lets you specify if the schema should allow properties not defined in the schema. If you pass `true` in for this option all unknown properties will be allowed. If you pass in an array of strings, only properties that are included in that array will be allowed. If you pass in an array of strings, you can use `*` to indicate a wildcard nested property one level deep, or `**` to indicate a wildcard nested property infinite levels deep. If you retrieve items from DynamoDB with `saveUnknown` enabled, all custom Dynamoose types will be returned as the underlying DynamoDB type (ex. Dates will be returned as a Number representing number of milliseconds since Jan 1 1970)
 - `timestamps` boolean | object (default: false) - This setting lets you indicate to Dynamoose that you would like it to handle storing timestamps in your documents for both creation and most recent update times. If you pass in an object for this setting you must specify two keys `createdAt` & `updatedAt`, each with a value of a string being the name of the attribute for each timestamp. If you pass in `null` for either of those keys that specific timestamp won't be added to the schema. If you set this option to `true` it will use the default attribute names of `createdAt` & `updatedAt`.
 
 ```js
diff --git a/lib/Document.js b/lib/Document.js
index 59835a56..03908c51 100644
--- a/lib/Document.js
+++ b/lib/Document.js
@@ -114,7 +114,10 @@ function DocumentCarrier(model) {
 			const genericKey = key.replace(/\d+/gu, "0"); // This is a key replacing all list numbers with 0 to standardize things like checking if it exists in the schema
 			const keyParts = key.split(".");
 			const parentKey = [...keyParts].splice(0, keyParts.length - 1).join(".");
-			const parentKeyIsArray = parentKey && model.schema.getAttributeType(parentKey) === "L";
+			let parentKeyIsArray;
+			try {
+				parentKeyIsArray = parentKey && model.schema.getAttributeType(parentKey) === "L";
+			} catch (e) {} // eslint-disable-line no-empty
 			const existsInSchema = model.schema.attributes().includes(genericKey) || parentKeyIsArray;
 			const isCustomType = existsInSchema && model.schema.getAttributeTypeDetails(key).customType;
 			const valueType = typeof value;
@@ -127,7 +130,7 @@ function DocumentCarrier(model) {
 				typeMatches = true;
 			}
 			const saveUnknownSetting = model.schema.getSettingValue("saveUnknown");
-			const saveUnknownAllowed = Array.isArray(saveUnknownSetting) ? saveUnknownSetting.includes(key) : saveUnknownSetting;
+			const saveUnknownAllowed = utils.dynamoose.saveunknown_attribute_allowed_check(saveUnknownSetting, key);
 			if ((existsInSchema && (typeMatches || (settings.customTypesDynamo && customValueType === expectedType) || (value === dynamooseUndefined))) || (settings.saveUnknown && saveUnknownAllowed)) {
 				if (!existsInSchema && settings.saveUnknown && saveUnknownAllowed) {
 					const foundAttributeType = utils.attribute_types.find((type) => type.isOfType && type.isOfType(customValue || value, settings.type, {"saveUnknown": true}));
diff --git a/lib/utils/attribute_types.js b/lib/utils/attribute_types.js
index 1fc3baa0..bd52e372 100644
--- a/lib/utils/attribute_types.js
+++ b/lib/utils/attribute_types.js
@@ -13,7 +13,7 @@ module.exports = [
 		}
 	}, "toDynamo": (val) => ({"wrapperName": "Set", "type": "Binary", "values": [...val]}), "fromDynamo": (val) => new Set(val.values)}, // Binary Set
 	{"dynamodbType": "L", "isOfType": (val) => Array.isArray(val), "nestedType": true, "defaultNestedType": () => []}, // List
-	{"dynamodbType": "M", "javascriptType": "object", "nestedType": true, "defaultNestedType": () => ({})}, // Map
+	{"dynamodbType": "M", "isOfType": (val) => Boolean(val) && val.constructor === Object, "javascriptType": "object", "nestedType": true, "defaultNestedType": () => ({})}, // Map
 	{"dynamodbType": "N", "javascriptType": "number"}, // Number
 	{"dynamodbType": "NS", "isSet": true, "isOfType": (val, type, settings = {}) => {
 		if (type === "toDynamo") {
diff --git a/lib/utils/dynamoose/index.js b/lib/utils/dynamoose/index.js
new file mode 100644
index 00000000..fc8f4d9d
--- /dev/null
+++ b/lib/utils/dynamoose/index.js
@@ -0,0 +1,3 @@
+module.exports = {
+	"saveunknown_attribute_allowed_check": require("./saveunknown_attribute_allowed_check")
+};
diff --git a/lib/utils/dynamoose/saveunknown_attribute_allowed_check.js b/lib/utils/dynamoose/saveunknown_attribute_allowed_check.js
new file mode 100644
index 00000000..5b7419cf
--- /dev/null
+++ b/lib/utils/dynamoose/saveunknown_attribute_allowed_check.js
@@ -0,0 +1,20 @@
+module.exports = (saveUnknown, checkKey) => {
+	if (Array.isArray(saveUnknown)) {
+		return Boolean(saveUnknown.find((key) => {
+			const keyParts = key.split(".");
+			let index = 0;
+			return checkKey.split(".").every((checkKeyPart) => {
+				const keyPart = keyParts[index];
+
+				if (keyPart === "**" || !isNaN(parseInt(checkKeyPart))) {
+					return true;
+				} else {
+					index++;
+					return keyPart === "*" || checkKeyPart === keyPart;
+				}
+			});
+		}));
+	} else {
+		return saveUnknown;
+	}
+};
diff --git a/lib/utils/index.js b/lib/utils/index.js
index f3ac0dbd..b5b14f7d 100644
--- a/lib/utils/index.js
+++ b/lib/utils/index.js
@@ -7,5 +7,6 @@ module.exports = {
 	"set_immediate_promise": require("./set_immediate_promise"),
 	"unique_array_elements": require("./unique_array_elements"),
 	"dynamodb": require("./dynamodb"),
-	"object": require("./object")
+	"object": require("./object"),
+	"dynamoose": require("./dynamoose")
 };


Test patch:
diff --git a/test/Document.js b/test/Document.js
index 73f50260..15a2be3f 100644
--- a/test/Document.js
+++ b/test/Document.js
@@ -390,6 +390,50 @@ describe("Document", () => {
 					}]);
 				});
 
+				it("Should save correct object with object property and saveUnknown set to one level nested", async () => {
+					putItemFunction = () => Promise.resolve();
+					User = new Model("User", new Schema({"id": Number, "address": Object}, {"saveUnknown": ["address.*"]}), {"create": false, "waitForActive": false});
+					user = new User({"id": 1, "address": {"country": "world", "zip": 12345, "metadata": {"name": "Home"}}});
+					await callType.func(user).bind(user)();
+					expect(putParams).to.eql([{
+						"Item": {"id": {"N": "1"}, "address": {"M": {"country": {"S": "world"}, "zip": {"N": "12345"}, "metadata": {"M": {}}}}},
+						"TableName": "User"
+					}]);
+				});
+
+				it("Should save correct object with object property and saveUnknown set to all level nested", async () => {
+					putItemFunction = () => Promise.resolve();
+					User = new Model("User", new Schema({"id": Number, "address": Object}, {"saveUnknown": ["address.**"]}), {"create": false, "waitForActive": false});
+					user = new User({"id": 1, "address": {"country": "world", "zip": 12345, "metadata": {"name": "Home"}}});
+					await callType.func(user).bind(user)();
+					expect(putParams).to.eql([{
+						"Item": {"id": {"N": "1"}, "address": {"M": {"country": {"S": "world"}, "zip": {"N": "12345"}, "metadata": {"M": {"name": {"S": "Home"}}}}}},
+						"TableName": "User"
+					}]);
+				});
+
+				it("Should save correct object with object property and saveUnknown set to one level nested with parent as array", async () => {
+					putItemFunction = () => Promise.resolve();
+					User = new Model("User", new Schema({"id": Number, "address": Object}, {"saveUnknown": ["addresses.*"]}), {"create": false, "waitForActive": false});
+					user = new User({"id": 1, "addresses": [{"country": "world", "zip": 12345, "metadata": [{"name": "Home"}]}]});
+					await callType.func(user).bind(user)();
+					expect(putParams).to.eql([{
+						"Item": {"id": {"N": "1"}, "addresses": {"L": [{"M": {"country": {"S": "world"}, "zip": {"N": "12345"}, "metadata": {"L": [{"M": {}}]}}}]}},
+						"TableName": "User"
+					}]);
+				});
+
+				it("Should save correct object with object property and saveUnknown set to all level nested with parent as array", async () => {
+					putItemFunction = () => Promise.resolve();
+					User = new Model("User", new Schema({"id": Number, "address": Object}, {"saveUnknown": ["addresses.**"]}), {"create": false, "waitForActive": false});
+					user = new User({"id": 1, "addresses": [{"country": "world", "zip": 12345, "metadata": [{"name": "Home"}]}]});
+					await callType.func(user).bind(user)();
+					expect(putParams).to.eql([{
+						"Item": {"id": {"N": "1"}, "addresses": {"L": [{"M": {"country": {"S": "world"}, "zip": {"N": "12345"}, "metadata": {"L": [{"M": {"name": {"S": "Home"}}}]}}}]}},
+						"TableName": "User"
+					}]);
+				});
+
 				it("Should save correct object with array", async () => {
 					putItemFunction = () => Promise.resolve();
 					User = new Model("User", {"id": Number, "friends": {"type": Array, "schema": [String]}}, {"create": false, "waitForActive": false});
diff --git a/test/Schema.js b/test/Schema.js
index 5bf9ef9c..4047c7c8 100644
--- a/test/Schema.js
+++ b/test/Schema.js
@@ -57,6 +57,12 @@ describe("Schema", () => {
 		expect(() => new Schema({"id": String, "friends": {"type": Array, "schema": [{"type": Object, "schema": {"1": [String], "data": {"type": Array, "schema": [Buffer, String]}}}]}})).to.throw("Attributes names must not be numbers.");
 	});
 
+	it.skip("Should throw error if attribute names contains star", () => {
+		expect(() => new Schema({"*": String})).to.throw("Attributes names must not include stars.");
+		expect(() => new Schema({"id": String, "friends": {"type": Array, "schema": [{"type": Object, "schema": {"*": String, "data": {"type": Array, "schema": [Buffer, String]}}}]}})).to.throw("Attributes names must not include stars.");
+		expect(() => new Schema({"id": String, "friends": {"type": Array, "schema": [{"type": Object, "schema": {"*": [String], "data": {"type": Array, "schema": [Buffer, String]}}}]}})).to.throw("Attributes names must not include stars.");
+	});
+
 	it("Should not throw error if valid schema passed in", () => {
 		expect(() => new Schema({"id": Number, "friends": [String]})).to.not.throw();
 	});
diff --git a/test/utils/dynamoose/saveunknown_attribute_allowed_check.js b/test/utils/dynamoose/saveunknown_attribute_allowed_check.js
new file mode 100644
index 00000000..0471b9e6
--- /dev/null
+++ b/test/utils/dynamoose/saveunknown_attribute_allowed_check.js
@@ -0,0 +1,90 @@
+const {expect} = require("chai");
+const utils = require("../../../lib/utils");
+
+describe("utils.dynamoose.saveunknown_attribute_allowed_check", () => {
+	it("Should be a function", () => {
+		expect(utils.dynamoose.saveunknown_attribute_allowed_check).to.be.a("function");
+	});
+
+	const tests = [
+		{
+			"input": [true, "random"],
+			"output": true
+		},
+		{
+			"input": [false, "random"],
+			"output": false
+		},
+		{
+			"input": [["random"], "random"],
+			"output": true
+		},
+		{
+			"input": [["other"], "random"],
+			"output": false
+		},
+		{
+			"input": [["random"], "random.test"],
+			"output": false
+		},
+		{
+			"input": [["random.test"], "random"],
+			"output": true
+		},
+		{
+			"input": [["random.*"], "random"],
+			"output": true
+		},
+		{
+			"input": [["random.*"], "random.test"],
+			"output": true
+		},
+		{
+			"input": [["random.*"], "random.test.random"],
+			"output": false
+		},
+		{
+			"input": [["random.*"], "random.0.random"],
+			"output": true
+		},
+		{
+			"input": [["random.*"], "random.1.random"],
+			"output": true
+		},
+		{
+			"input": [["random.*.hello"], "random.test.hello"],
+			"output": true
+		},
+		{
+			"input": [["random.*.hello"], "random.test.random"],
+			"output": false
+		},
+		{
+			"input": [["random.*.hello"], "random.test.hello.test"],
+			"output": false
+		},
+		{
+			"input": [["random.**"], "random.test.random"],
+			"output": true
+		},
+		{
+			"input": [["**"], "random.test.random"],
+			"output": true
+		},
+		// TODO: add support for this later
+		// {
+		// 	"input": [["random.**.hello"], "random.test.random"],
+		// 	"output": false
+		// },
+		// {
+		// 	"input": [["random.**.hello"], "random.test.random.hello"],
+		// 	"output": true
+		// }
+	];
+
+	tests.forEach((test) => {
+		it(`Should return ${test.output} for ${test.input}`, () => {
+			expect(utils.dynamoose.saveunknown_attribute_allowed_check(...test.input)).to.eql(test.output);
+		});
+	});
+});