Repository: washingtonpost/ans-schema (js)

Issue:
Proposal: improve support for tags, add categories
**Problem**
ANS Tags do not support wordpress categories

**Proposed Solution**
Add to utils/tag.json:
 - slug = tag or category slug, e.g. 'my-category-slug'
 - text = tag or category name ~~(currently listed in spec as "tag")~~
 - description = tag or category description
 - additional_properties = (same as elsewhere, used for non-standard metadata)

**Example**
```tags: [
{
_id: "DGBTWSJS2BHNTPWLVVUYPM6T6E",
  text: "My Category Name",
  additional_properties: {wordpress_type: "category"},
  slug:  "my-category-name",
  description: "My superfab category"
},
{
_id: "AMKDOSHIJ5FYVJS5UASQZ2G2CY",
text: "My Tag Name",
  additional_properties: {wordpress_type: "tag"},
slug: "my-tag-name",
description: "My unparalleled tag"
}
]
```

**Question**
Currently, tags have two properties, on the api output: _id and "tag".  Am I misreading the spec when it appears that the properties should be _id and "text"?

Pull request description:
Tags
Fixes #87 

Patch:
diff --git a/README.md b/README.md
index 0d3e281..a5f2eae 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@ JSON schema definition and supporting example/validation code for The Washington
 ANS ("Arc Native Specification") is the collection of schema documents that comprise the Washington Post's definition of "content", in so far as content is passed back and forth between systems in the Arc ecosystem of applications.
 
 ## Schema files
-ANS Schema files are defined with the [JSON Schema specification](https://spacetelescope.github.io/understanding-json-schema/index.html).  Schemas are defined in individual files under the [src/main/resrouces/schema/ans/_version_/](src/main/resources/schema/ans/0.5.6/) directory.
+ANS Schema files are defined with the [JSON Schema specification](https://spacetelescope.github.io/understanding-json-schema/index.html).  Schemas are defined in individual files under the [src/main/resrouces/schema/ans/_version_/](src/main/resources/schema/ans/0.5.8/) directory.
 
 Small, self-contained, attributes such as "had copyright" or "had unique id" are modeled as **traits**, which are combined together or applied directly to more complicated schema objects for the purpose of re-usability.  The "content.json" schema, for example, composes the traits "credit", "locale", "location", and "copyright".
 
@@ -17,26 +17,26 @@ One can use a variety of third party tools to validate their content against the
 ## Examples
 A few examples are provided of content documents that validate against the Washington Post ANS schema:
 
-1. [An Example Story](tests/fixtures/schema/0.5.6/story-fixture-tiny-house.json)
-2. [An Example Video](tests/fixtures/schema/0.5.6/video-fixture-nationals.json)
-3. [An Example Image](tests/fixtures/schema/0.5.6/image-fixture-good.json)
+1. [An Example Story](tests/fixtures/schema/0.5.8/story-fixture-tiny-house.json)
+2. [An Example Video](tests/fixtures/schema/0.5.8/video-fixture-nationals.json)
+3. [An Example Image](tests/fixtures/schema/0.5.8/image-fixture-good.json)
 
 
 ## Validating Locally ##
 
-This project contains a node library that can be used to validate ANS documents locally.  It is the same validator used in the ANS Service: https://ans.arc.nile.works/validate/0.5.6
+This project contains a node library that can be used to validate ANS documents locally.  It is the same validator used in the ANS Service: https://ans.arc.nile.works/validate/0.5.8
 
 You can run the validator on the command line from the project directory to validate an ANS document:
 
 ```
 npm install
-npm run-script ans -- --ansdata='{"type":"story", "version":"0.5.5"}' --version=0.5.6 validate
+npm run-script ans -- --ansdata='{"type":"story", "version":"0.5.5"}' --version=0.5.8 validate
 
 [
   {
     "keyword": "enum",
     "dataPath": ".version",
-    "schemaPath": "https://raw.githubusercontent.com/washingtonpost/ans-schema/master/src/main/resources/schema/ans/0.5.6/traits/trait_version.json/enum",
+    "schemaPath": "https://raw.githubusercontent.com/washingtonpost/ans-schema/master/src/main/resources/schema/ans/0.5.8/traits/trait_version.json/enum",
     "params": {},
     "message": "should be equal to one of values"
   }
@@ -46,7 +46,7 @@ npm run-script ans -- --ansdata='{"type":"story", "version":"0.5.5"}' --version=
 A file can also be used as input:
 
 ```
-npm run-script ans -- --ansfile=test.json --version=0.5.6 validate
+npm run-script ans -- --ansfile=test.json --version=0.5.8 validate
 
 ```
 
@@ -55,8 +55,8 @@ You can also load the validator as a library in your node project:
 ```
 var ans = require('ans-schema');
 
-ans.getValidatorForVersion('0.5.6', function(err, validator) {
-  var errors = validator.validate({"type":"story", "version":"0.5.6"});
+ans.getValidatorForVersion('0.5.8', function(err, validator) {
+  var errors = validator.validate({"type":"story", "version":"0.5.8"});
 
   if (errors.length > 0) {
     console.log("There were some errors: " + JSON.stringify(errors));
@@ -77,7 +77,7 @@ npm run-script ans -- --ansdata='{"type":"story", "version":"0.5.0", "content_el
 
 {
   "type": "story",
-  "version": "0.5.6",
+  "version": "0.5.8",
   "content_elements": [
     {
       "type": "text",
@@ -93,7 +93,7 @@ npm run-script ans -- --ansdata='{"type":"story", "version":"0.5.0", "content_el
         "additional_properties": {
           "type": "section"
         },
-        "version": "0.5.6",
+        "version": "0.5.8",
         "type": "site",
         "name": "(unnamed)",
         "primary": false
@@ -108,19 +108,19 @@ npm run-script ans -- --ansdata='{"type":"story", "version":"0.5.0", "content_el
 Fixes an invalid document composed of valid sub-documents of differing ANS versions. Essentially this converts a very specific kind of invalid document to a valid one by selective upverting.
 
 ```
-npm run-script ans -- --ansdata='{"type":"story", "version":"0.5.6", "content_elements":[{"type":"gallery", "version":"0.5.3", "content_elements":[ {"type":"image", "version":"0.5.3", "url":"http://foo.com/img.jpg"}]}]}' sync
+npm run-script ans -- --ansdata='{"type":"story", "version":"0.5.8", "content_elements":[{"type":"gallery", "version":"0.5.3", "content_elements":[ {"type":"image", "version":"0.5.3", "url":"http://foo.com/img.jpg"}]}]}' sync
 
 {
   "type": "story",
-  "version": "0.5.6",
+  "version": "0.5.8",
   "content_elements": [
     {
       "type": "gallery",
-      "version": "0.5.6",
+      "version": "0.5.8",
       "content_elements": [
         {
           "type": "image",
-          "version": "0.5.6",
+          "version": "0.5.8",
           "url": "http://foo.com/img.jpg"
         }
       ]
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 015c326..35b70b3 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -1,5 +1,26 @@
 # ANS Release Notes
 
+### 1.2.7 2017/01/09 ###
+
+* Adds `description`, `slug`, `additional_properties` properties to tags
+* Disables non-spec properties on tags
+
+### 1.2.6 2017/01/05 ###
+
+* Creates ANS 0.5.8
+  * Adds `alignment` property
+  * Adds `element_group` content element
+
+### 1.2.5 2016/11/30 ###
+
+* Adds `primary_site` to taxonomy
+
+### 1.2.4 2016/11/11 ###
+
+* Adds `licensable` to images
+* Adds `channels` to top-level content items (standardizes on "channels" instead of "channel")
+* Fixes `description` in images
+
 ### 1.2.3 2016/08/31 ###
 
 * Adds url operation, redirect object, new fields to image_operation to ANS 0.5.7
diff --git a/lib/transform_utils.js b/lib/transform_utils.js
index 49b8108..1244af1 100644
--- a/lib/transform_utils.js
+++ b/lib/transform_utils.js
@@ -2,7 +2,8 @@
 
 var _ = require('lodash');
 
-var top_level_types = ["story", "video", "image", "gallery", "revision"];
+var top_level_types = ["story", "video", "image", "gallery", "results", "audio", "redirect"];
+var recursion_keys = [ 'promo_items', 'related_content', 'content_elements' ];
 
 // Given a version number, generates a function that sets input.version to that version number
 var version_incrementer = function(new_version) {
@@ -56,5 +57,6 @@ var accumulatorFor = function(object) {
 module.exports = {
   'top_level_types': top_level_types,
   'version_incrementer': version_incrementer,
-  'accumulatorFor': accumulatorFor
+  'accumulatorFor': accumulatorFor,
+  'recursion_keys': recursion_keys
 };
diff --git a/lib/upvert/0.5.7.js b/lib/upvert/0.5.7.js
index c4d195a..e37a6f2 100644
--- a/lib/upvert/0.5.7.js
+++ b/lib/upvert/0.5.7.js
@@ -6,15 +6,84 @@ var transform_utils = require('../transform_utils');
 
 var version_incrementer = transform_utils.version_incrementer;
 var top_level_types = transform_utils.top_level_types;
+var recursion_keys = transform_utils.recursion_keys;
 var accumulatorFor = transform_utils.accumulatorFor;
 
-var upvert = function(input) {
-  var output = version_incrementer("0.5.8")(input);
+// Tags in 0.5.7 were mostly undefined object. In 0.5.8 they have specified properties. Fields which
+// match the new properties should be preserved, those that do not should be moved into additional_properties.
+var convertTag = function convertTag(old_tag) {
+
+  var additional_properties = _.has(old_tag, 'additional_properties') && _.isObject(old_tag.additional_properties) ?
+      _.cloneDeep(old_tag.additional_properties) : {};
+
+  var tag = _.transform(old_tag, function(result, value, key, object) {
+    // Specced Tag fields for 0.5.8
+    if (_.includes(['_id', 'text', 'description', 'slug'], key)) {
+      result[key] = value;
+    }
+    else {
+      // Example: .foo should become .additional_properties.foo
+      // In the unlikely even that .additional_properties.foo already existed on the old object,
+      // then this will overwrite it (but it will show up as .additional_properties.additional_properties.foo)
+      additional_properties[key] = value;
+      // 'tag' was in use in some places in lieu of text, so copy over if possible
+      if (key === 'tag' && !_.has(object, 'text')) {
+        result['text'] = value;
+      }
+    }
+  }, {});
+
+  if (_.size(additional_properties) > 0) {
+    tag.additional_properties = additional_properties;
+  }
+
+  return tag;
+};
+
+// Convert a top-level ANS document and recur at known points
+var convertDocument = function convertDocument(doc) {
+  return _.transform(doc, function(result, value, key, object) {
+    if (key === 'taxonomy') {
+      var taxonomy = {};
+      _.assign(taxonomy, value, { 'tags': _.map(value.tags, convertTag) });
+      result[key] = taxonomy;
+    }
+    else if (_.includes(recursion_keys, key)) {
+      result[key] = convert(value);
+    }
+  }, doc);
+};
 
-  // TODO: Add upvert logic here
-  // If change is non-breaking, you can leave this as-is.
+// Generic recur-and-convert down object tree
+var convert = function convert(ans) {
+  if (!_.isObject(ans)) {
+    return ans;
+  }
+
+  if (_.isArray(ans)) {
+    return _.map(ans, convert);
+  }
+
+  if (_.has(ans, 'type') &&
+      _.includes(top_level_types, ans.type)) {
+    return convertDocument(ans);
+  }
+
+  return _.transform(ans, function(result, value, key, object) {
+    if (key === 'additional_properties' || key === 'referent_properties') {
+      result[key] = value;
+    }
+    else {
+      result[key] = convert(value);
+    }
+  }, accumulatorFor(ans));
+};
+
+
+var upvert = function(input) {
+  var output = version_incrementer('0.5.8')(input);
 
-  return output;
+  return convertDocument(output);
 };
 
 module.exports = upvert;
diff --git a/package.json b/package.json
index 4723241..2d2ac31 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "ans-schema",
   "description": "The Washington Post's Arc Native Specification",
-  "version": "1.2.6",
+  "version": "1.2.7",
   "homepage": "https://github.com/washingtonpost/ans-schema",
   "repository": {
     "type": "git",
diff --git a/src/main/resources/schema/ans/0.5.8/utils/tag.json b/src/main/resources/schema/ans/0.5.8/utils/tag.json
index 2be63d1..83ac0f9 100644
--- a/src/main/resources/schema/ans/0.5.8/utils/tag.json
+++ b/src/main/resources/schema/ans/0.5.8/utils/tag.json
@@ -4,13 +4,25 @@
   "title": "Tag",
   "description": "Models a keyword used in describing a piece of content.",
   "type": "object",
+  "additionalProperties": false,
   "properties": {
     "_id": {
       "$ref": "https://raw.githubusercontent.com/washingtonpost/ans-schema/master/src/main/resources/schema/ans/0.5.8/traits/trait_id.json"
     },
     "text": {
       "type": "string",
-      "description": "The text of the tag"
+      "description": "The text of the tag as displayed to users."
+    },
+    "description": {
+      "type": "string",
+      "description": "A more detailed description of the tag."
+    },
+    "additional_properties": {
+      "$ref": "https://raw.githubusercontent.com/washingtonpost/ans-schema/master/src/main/resources/schema/ans/0.5.8/traits/trait_additional_properties.json"
+    },
+    "slug": {
+      "$ref": "https://raw.githubusercontent.com/washingtonpost/ans-schema/master/src/main/resources/schema/ans/0.5.8/traits/trait_slug.json"
     }
-  }
+  },
+  "required": [ "text" ]
 }


Test patch:
diff --git a/tests/fixtures/schema/0.5.8/story-fixture-good.json b/tests/fixtures/schema/0.5.8/story-fixture-good.json
index 7137c5d..2335020 100644
--- a/tests/fixtures/schema/0.5.8/story-fixture-good.json
+++ b/tests/fixtures/schema/0.5.8/story-fixture-good.json
@@ -105,7 +105,12 @@
     "tags": [
       {
         "_id": "fred",
-        "text": "Fred"
+        "text": "Fred",
+        "slug": "fred",
+        "description": "For all things Fred",
+        "additional_properties": {
+          "foo": "bar"
+        }
       },
       {
         "_id": "writing",
diff --git a/tests/fixtures/schema/0.5.8/taxonomy-fixture-good.json b/tests/fixtures/schema/0.5.8/taxonomy-fixture-good.json
index a3a9ff5..0d1b598 100644
--- a/tests/fixtures/schema/0.5.8/taxonomy-fixture-good.json
+++ b/tests/fixtures/schema/0.5.8/taxonomy-fixture-good.json
@@ -1,12 +1,23 @@
 {
-    "keywords": [
-        {
-            "keyword": "foo"
-        },
-        {
-            "keyword": "bar",
-            "score": 0.333,
-            "frequency": 5
-        }
-    ]
+  "keywords": [
+    {
+      "keyword": "foo"
+    },
+    {
+      "keyword": "bar",
+      "score": 0.333,
+      "frequency": 5
+    }
+  ],
+  "tags": [
+    {
+      "_id": "YRAXSARSPFFC5O577BS5XIJ5M4",
+      "text": "apples and bananas",
+      "slug": "apples-and-bananas",
+      "description": "Apples & Bananas",
+      "additional_properties": {
+        "something": "else"
+      }
+    }
+  ]
 }
diff --git a/tests/fixtures/transforms/0.5.7/story-fixture-good-tags.json b/tests/fixtures/transforms/0.5.7/story-fixture-good-tags.json
new file mode 100644
index 0000000..8725e0b
--- /dev/null
+++ b/tests/fixtures/transforms/0.5.7/story-fixture-good-tags.json
@@ -0,0 +1,33 @@
+{
+  "type": "story",
+  "version": "0.5.7",
+  "taxonomy": {
+    "tags": [
+      {
+        "_id": "ABC",
+        "text": "Foo"
+      },
+      {
+        "text": "Bar",
+        "tag": "wrong"
+      },
+      {
+        "_id": "DEF",
+        "tag": "Baz"
+      },
+      {
+        "_id": "GHI",
+        "text": "Car",
+        "description": "Not automobiles",
+        "slug": "car",
+        "prop1": "foo",
+        "prop2": "bar",
+        "additional_properties": {
+          "one": "two",
+          "prop1": "baz",
+          "slug": "carrr--"
+        }
+      }
+    ]
+  }
+}
diff --git a/tests/fixtures/transforms/0.5.7/story-fixture-good.json b/tests/fixtures/transforms/0.5.7/story-fixture-good.json
index 779261d..6044355 100644
--- a/tests/fixtures/transforms/0.5.7/story-fixture-good.json
+++ b/tests/fixtures/transforms/0.5.7/story-fixture-good.json
@@ -120,6 +120,19 @@
         },
         "primary": false
       }
+    ],
+    "tags": [
+      {
+        "_id": "tag1",
+        "text": "tag one",
+        "description": "The first tag",
+        "slug": "tag-one",
+        "foo": "bar"
+      },
+      {
+        "_id": "tag2",
+        "tag": "tag two"
+      }
     ]
   },
   "publish_date": "2015-06-24T09:49:00.10Z",
@@ -257,4 +270,4 @@
       }
     }
   ]
-}
\ No newline at end of file
+}
diff --git a/tests/transform-tests.js b/tests/transform-tests.js
index 0cf2019..08958c5 100644
--- a/tests/transform-tests.js
+++ b/tests/transform-tests.js
@@ -485,9 +485,66 @@ describe("Transformations: ", function() {
     });
 
     describe("0.5.7 to 0.5.8", function() {
+      describe("Tags", function() {
+        it("should leave valid tags alone", function() {
+          var result = transforms.upvert(fixtures['0.5.7']['story-fixture-good-tags'], '0.5.8');
+          result.taxonomy.tags[0]._id.should.eql("ABC");
+          result.taxonomy.tags[0].text.should.eql("Foo");
+          result.taxonomy.tags[0].should.not.have.property("additional_properties");
+          result.taxonomy.tags[0].should.not.have.property("slug");
+          result.taxonomy.tags[0].should.not.have.property("description");
+          result.taxonomy.tags[0].should.not.have.property("tag");
 
-    });
+          result.taxonomy.tags[1].text.should.eql("Bar");
+        });
+
+        it("should promote 'tag' to 'text' where the former exists and the latter does not", function() {
+          var result = transforms.upvert(fixtures['0.5.7']['story-fixture-good-tags'], '0.5.8');
+          result.taxonomy.tags[2].text.should.eql("Baz");
+        });
+
+        it("should not promote 'tag' to 'text' where the latter exists", function() {
+          var result = transforms.upvert(fixtures['0.5.7']['story-fixture-good-tags'], '0.5.8');
+          result.taxonomy.tags[1].text.should.eql("Bar");
+        });
+
+        it("should leave in place 0.5.8 fields 'description' and 'slug' if they exist", function() {
+          var result = transforms.upvert(fixtures['0.5.7']['story-fixture-good-tags'], '0.5.8');
+          result.taxonomy.tags[3].slug.should.eql("car");
+          result.taxonomy.tags[3].description.should.eql("Not automobiles");
+        });
+
+        it("should add non-spec properties to additional properties", function() {
+          var result = transforms.upvert(fixtures['0.5.7']['story-fixture-good-tags'], '0.5.8');
+          result.taxonomy.tags[3].additional_properties.prop1.should.eql("foo");
+          result.taxonomy.tags[3].additional_properties.prop2.should.eql("bar");
+        });
 
+        it("should preserve non-colliding additional propeties when possible", function() {
+          var result = transforms.upvert(fixtures['0.5.7']['story-fixture-good-tags'], '0.5.8');
+          result.taxonomy.tags[3].additional_properties.one.should.eql("two");
+          result.taxonomy.tags[3].additional_properties.slug.should.eql("carrr--");
+        });
+
+        it("should copy original additional_properties to sub-object", function() {
+          var result = transforms.upvert(fixtures['0.5.7']['story-fixture-good-tags'], '0.5.8');
+          result.taxonomy.tags[3].additional_properties.should.have.property("additional_properties");
+          result.taxonomy.tags[3].additional_properties.additional_properties.one.should.eql("two");
+          result.taxonomy.tags[3].additional_properties.additional_properties.prop1.should.eql("baz");
+          result.taxonomy.tags[3].additional_properties.additional_properties.slug.should.eql("carrr--");
+
+          result.taxonomy.tags[1].additional_properties.tag.should.eql("wrong");
+          result.taxonomy.tags[2].additional_properties.tag.should.eql("Baz");
+        });
+
+        it("should not create second additional_properties if first did not exist", function() {
+          var result = transforms.upvert(fixtures['0.5.7']['story-fixture-good-tags'], '0.5.8');
+          result.taxonomy.tags[1].additional_properties.should.not.have.property("additional_properties");
+          result.taxonomy.tags[2].additional_properties.should.not.have.property("additional_properties");
+
+        });
+      });
+    });
   });
 
   describe("Synchronizer", function() {