Repository: tobie/specref (js)

Issue:
Browserspec script is removing entries
The browser spec script is removing entries (currently `TC39-RESIZABLEARRAYBUFFER`) which (rightfully) breaks the build.

Script needs editing to keep those references around and potentially mark them as obsolete.



Pull request:
Resume browser-specs source, preserving entries
This update leverages the new `obsoletedBy` and `formerNames` properties reported by browser-specs to:
- track spec transitions over time,
- make sure that entries in `browser-specs.json` are preserved,
- and avoid the creation of ghost entries.

In particular, on top of creating `aliasOf` relationships when needed, the script now sets `isSuperseded` and `obsoletedBy` properties - or `isRetired` - for discontinued specs.

Next time the script runs, among other minor updates, it will flag `TC39-RESIZABLEARRAYBUFFER` as superseded and obsoleted by `ECMASCRIPT`, and flag `CLOSE-WATCHER` as superseded and obsoleted by `HTML`.

This should fix #752.

Patch:
diff --git a/refs/ietf.json b/refs/ietf.json
index bd67c7d8a..f1cc1f9c9 100644
--- a/refs/ietf.json
+++ b/refs/ietf.json
@@ -3676,6 +3676,7 @@
         "seeAlso": [
             "RFC0265"
         ],
+        "hasErrata": "http://www.rfc-editor.org/errata_search.php?rfc=264",
         "publisher": "IETF",
         "source": "https://www.rfc-editor.org/in-notes/rfc-index.xml"
     },
@@ -26018,7 +26019,6 @@
         ],
         "rawDate": "1995-08",
         "status": "Informational",
-        "hasErrata": "http://www.rfc-editor.org/errata_search.php?rfc=1823",
         "publisher": "IETF",
         "source": "https://www.rfc-editor.org/in-notes/rfc-index.xml"
     },
@@ -69303,6 +69303,7 @@
         ],
         "rawDate": "2006-06",
         "status": "Proposed Standard",
+        "hasErrata": "http://www.rfc-editor.org/errata_search.php?rfc=4561",
         "publisher": "IETF",
         "source": "https://www.rfc-editor.org/in-notes/rfc-index.xml"
     },
@@ -124288,6 +124289,7 @@
         ],
         "rawDate": "2017-03",
         "status": "Informational",
+        "hasErrata": "http://www.rfc-editor.org/errata_search.php?rfc=8087",
         "publisher": "IETF",
         "source": "https://www.rfc-editor.org/in-notes/rfc-index.xml"
     },
@@ -133082,6 +133084,7 @@
         ],
         "rawDate": "2019-09",
         "status": "Proposed Standard",
+        "hasErrata": "http://www.rfc-editor.org/errata_search.php?rfc=8641",
         "publisher": "IETF",
         "source": "https://www.rfc-editor.org/in-notes/rfc-index.xml"
     },
diff --git a/refs/w3c.json b/refs/w3c.json
index 7f56cdb26..46f0dd79b 100644
--- a/refs/w3c.json
+++ b/refs/w3c.json
@@ -11507,7 +11507,7 @@
         ],
         "href": "https://www.w3.org/TR/WGSL/",
         "title": "WebGPU Shading Language",
-        "rawDate": "2023-08-01",
+        "rawDate": "2023-08-02",
         "status": "WD",
         "publisher": "W3C",
         "edDraft": "https://gpuweb.github.io/gpuweb/wgsl/",
@@ -15938,6 +15938,22 @@
                     "https://www.w3.org/2020/gpu/"
                 ],
                 "source": "https://www.w3.org/2002/01/tr-automation/tr.rdf"
+            },
+            "20230802": {
+                "authors": [
+                    "Alan Baker",
+                    "Mehmet Oguz Derin",
+                    "David Neto"
+                ],
+                "href": "https://www.w3.org/TR/2023/WD-WGSL-20230802/",
+                "title": "WebGPU Shading Language",
+                "rawDate": "2023-08-02",
+                "status": "WD",
+                "publisher": "W3C",
+                "deliveredBy": [
+                    "https://www.w3.org/2020/gpu/"
+                ],
+                "source": "https://www.w3.org/2002/01/tr-automation/tr.rdf"
             }
         },
         "repository": "https://github.com/gpuweb/gpuweb"
@@ -126999,7 +127015,7 @@
         ],
         "href": "https://www.w3.org/TR/vc-data-model-2.0/",
         "title": "Verifiable Credentials Data Model v2.0",
-        "rawDate": "2023-07-30",
+        "rawDate": "2023-08-02",
         "status": "WD",
         "publisher": "W3C",
         "edDraft": "https://w3c.github.io/vc-data-model/",
@@ -128029,6 +128045,24 @@
                     "https://www.w3.org/2017/vc/"
                 ],
                 "source": "https://www.w3.org/2002/01/tr-automation/tr.rdf"
+            },
+            "20230802": {
+                "authors": [
+                    "Manu Sporny",
+                    "Orie Steele",
+                    "Michael Jones",
+                    "Gabe Cohen",
+                    "Oliver Terbu"
+                ],
+                "href": "https://www.w3.org/TR/2023/WD-vc-data-model-2.0-20230802/",
+                "title": "Verifiable Credentials Data Model v2.0",
+                "rawDate": "2023-08-02",
+                "status": "WD",
+                "publisher": "W3C",
+                "deliveredBy": [
+                    "https://www.w3.org/2017/vc/"
+                ],
+                "source": "https://www.w3.org/2002/01/tr-automation/tr.rdf"
             }
         },
         "repository": "https://github.com/w3c/vc-data-model"
@@ -128704,8 +128738,8 @@
             "Andres Uribe"
         ],
         "href": "https://www.w3.org/TR/vc-json-schema/",
-        "title": "Verifiable Credentials JSON Schema Specification 2023",
-        "rawDate": "2023-08-01",
+        "title": "Verifiable Credentials JSON Schema Specification",
+        "rawDate": "2023-08-02",
         "status": "WD",
         "publisher": "W3C",
         "edDraft": "https://w3c.github.io/vc-json-schema/",
@@ -128905,6 +128939,22 @@
                     "https://www.w3.org/2017/vc/"
                 ],
                 "source": "https://www.w3.org/2002/01/tr-automation/tr.rdf"
+            },
+            "20230802": {
+                "authors": [
+                    "Gabe Cohen",
+                    "Orie Steele",
+                    "Andres Uribe"
+                ],
+                "href": "https://www.w3.org/TR/2023/WD-vc-json-schema-20230802/",
+                "title": "Verifiable Credentials JSON Schema Specification",
+                "rawDate": "2023-08-02",
+                "status": "WD",
+                "publisher": "W3C",
+                "deliveredBy": [
+                    "https://www.w3.org/2017/vc/"
+                ],
+                "source": "https://www.w3.org/2002/01/tr-automation/tr.rdf"
             }
         },
         "repository": "https://github.com/w3c/vc-json-schema"
diff --git a/refs/wg21.json b/refs/wg21.json
index 73bdef040..6714c1c7c 100644
--- a/refs/wg21.json
+++ b/refs/wg21.json
@@ -32853,7 +32853,7 @@
         "source": "https://wg21.link/index.json"
     },
     "D2572R1": {
-        "title": "std::format() fill character allowances",
+        "title": "P2572R1: std::format() fill character allowances",
         "href": "https://wg21.link/d2572r1",
         "publisher": "WG21",
         "source": "https://wg21.link/index.json"
@@ -32877,7 +32877,7 @@
         "source": "https://wg21.link/index.json"
     },
     "D2652R2": {
-        "title": "D2552R2 - Fix `allocator_traits::allocate_at_least`",
+        "title": "Disallow User Specialization of allocator_traits (Bug fix from R1)",
         "href": "https://wg21.link/d2652r2",
         "publisher": "WG21",
         "source": "https://wg21.link/index.json"
@@ -75081,7 +75081,7 @@
     "EDIT6431": {
         "title": "[intro.races] p13 - add missing serial comma between items",
         "href": "https://wg21.link/edit6431",
-        "status": "open",
+        "status": "closed",
         "publisher": "WG21",
         "source": "https://wg21.link/index.json"
     },
@@ -75099,6 +75099,13 @@
         "publisher": "WG21",
         "source": "https://wg21.link/index.json"
     },
+    "EDIT6434": {
+        "title": "[intro.execution] \"Sequenced before\" should be a strict partial order",
+        "href": "https://wg21.link/edit6434",
+        "status": "open",
+        "publisher": "WG21",
+        "source": "https://wg21.link/index.json"
+    },
     "EDIT644": {
         "title": "[strings] Formatting and whitespace harmonization",
         "href": "https://wg21.link/edit644",
@@ -232178,6 +232185,9 @@
     "WG21-EDIT6433": {
         "aliasOf": "EDIT6433"
     },
+    "WG21-EDIT6434": {
+        "aliasOf": "EDIT6434"
+    },
     "WG21-EDIT644": {
         "aliasOf": "EDIT644"
     },
diff --git a/scripts/browser-specs.js b/scripts/browser-specs.js
index 6c48b5db0..24c56f006 100644
--- a/scripts/browser-specs.js
+++ b/scripts/browser-specs.js
@@ -30,10 +30,27 @@ function convertSpec(spec) {
     if (spec.nightly.repository) {
         ref.repository = spec.nightly.repository;
     }
+    if (spec.standing === 'discontinued') {
+        ref.isRetired = true;
+    }
+    if (spec.obsoletedBy) {
+        ref.obsoletedBy = spec.obsoletedBy;
+    }
     return ref;
 }
 
 
+function isInSpecref(id) {
+    if (updatedList[id]) {
+        return true;
+    }
+    var ref = refs[id];
+    if (ref && ref.source !== SOURCE) {
+        return true;
+    }
+    return false;
+}
+
 console.log("Updating", PUBLISHER, "references...");
 console.log("Fetching", SOURCE + "...");
 request({
@@ -48,59 +65,112 @@ request({
     }
 
     console.log("Parsing", SOURCE + "...");
-    JSON.parse(body)
+    const browserSpecs = JSON.parse(body);
+    browserSpecs
         .filter(function(spec) {
-            // Only keep specs from browser-specs for which the info came from
-            // the spec itself (other specs should already be in Specref)
+            // Only keep specs from browser-specs that meet the following
+            // criteria:
+            // 1. The info comes from the spec itself. Other specs should
+            // already be in Specref.
+            // 2. The name ID is available, or lives in browser-specs.json.
+            // Given the way that browser-specs is maintained, a name collision
+            // means that the spec already started to appear in another source
+            // but this transition has not yet been picked up by browser-specs.
+            // That's good, no need to throw an error, let's give priority to
+            // the other source.
             const uppercaseId = spec.shortname.toUpperCase();
-            return spec.source === 'spec';
+            return spec.source === 'spec' && !isInSpecref(uppercaseId);
         })
         .forEach(function(spec) {
-            ref = convertSpec(spec);
-            
-            var id = spec.shortname;
-            var uppercaseId = id.toUpperCase();
-
-            // Spec may already exist in Specref under another name
-            var rv = bibref.reverseLookup([ref.href])[ref.href];
-            if (rv) {
+            // Add an entry to browser-specs.json for the spec, unless Specref
+            // already has an entry somewhere else for a spec with the same URL.
+            const uppercaseId = spec.shortname.toUpperCase();
+            const rv = bibref.reverseLookup([spec.url])[spec.url];
+            if (rv && !current[rv.id]) {
                 if (current[uppercaseId] &&
-                        rv.id.toUpperCase() != uppercaseId &&
-                        // avoid inadvertently catching drafts.
-                        bibref.normalizeUrl(rv.href) == bibref.normalizeUrl(ref.href)) {
-                    // Make the existing entry in browser-specs.json an alias
-                    // of the other one
+                        rv.id.toUpperCase() !== uppercaseId) {
+                    // There was an entry for this spec in browser-specs.json
+                    // but Specref has an entry for a spec with the same URL
+                    // somewhere else. Let's give that other entry priority and
+                    // mark the browser-specs entry as an alias of it.
                     updatedList[uppercaseId] = { aliasOf: rv.id };
                     return;
                 }
                 else if (!current[uppercaseId]) {
-                    // Entry did not exist in browser-specs.json, let's not
-                    // create one
+                    // There is an entry for this spec in Specref somewhere else
+                    // and we had not yet created a browser-specs.json entry.
+                    // Let's not.
                     return;
                 }
             }
 
-            if (!(uppercaseId in refs)) {
-                updatedList[uppercaseId] = ref;
-            } else {
-                var existingRef = refs[uppercaseId];
-                while (existingRef.aliasOf) { existingRef = refs[existingRef.aliasOf]; }
-                if (!("source" in existingRef) || existingRef.source == SOURCE ||
-                        bibref.normalizeUrl(ref.href) == bibref.normalizeUrl(existingRef.href)) {
-                    updatedList[uppercaseId] = ref;
-                } else {
-                    throw new Error("cannot override " + uppercaseId);
+            // No entry in Specref, let's add one
+            updatedList[uppercaseId] = convertSpec(spec);
+        });
+
+    // For continuity, add back aliases that we may have created earlier on.
+    // Also add back entries that used to exist but that are now known under
+    // another name.
+    const browserSpecsWithFormerNames = browserSpecs.filter(s => s.formerNames);
+    Object.keys(current)
+        .filter(function(id) { return !updatedList[id]; })
+        .forEach(function(id) {
+            var ref = current[id];
+            var browserSpec = browserSpecsWithFormerNames.find(s =>
+                s.formerNames.find(n => n === id || n.toUpperCase() === id)
+            );
+            if (browserSpec) {
+                const uppercaseId = browserSpec.shortname.toUpperCase();
+                if (updatedList[uppercaseId]) {
+                    updatedList[id] = { aliasOf: uppercaseId };
                 }
+                else {
+                    let rv = bibref.reverseLookup([uppercaseId])[uppercaseId];
+                    if (!rv) {
+                        rv = bibref.reverseLookup([browserSpec.url])[browserSpec.url];
+                    }
+                    if (rv) {
+                        updatedList[id] = { aliasOf: rv.id };
+                    }
+                }
+            }
+            else if (ref.aliasOf && isInSpecref(ref.aliasOf)) {
+                updatedList[id] = ref;
+            }
+            else {
+                throw new Error("Missing aliasOf target in browser-specs.json: " + ref.aliasOf + " needed for " + id);
             }
         });
-    // For continuity, add back aliases that we may have created earlier on,
-    // unless the targeted entry no longer exists
-    Object.keys(current).forEach(function(id) {
-        var ref = current[id];
-        if (!updatedList[id] && ref.aliasOf && refs[ref.aliasOf]) {
-            updatedList[id] = ref;
-        }
-    });
+
+    // Make sure that "obsoletedBy" properties target actual specs in Specref
+    Object.keys(updatedList)
+        .filter(function(id) { return !!updatedList[id].obsoletedBy; })
+        .forEach(function(id) {
+            var ref = updatedList[id];
+            ref.obsoletedBy = ref.obsoletedBy
+                .map(n => {
+                    if (isInSpecref(n)) {
+                        return n;
+                    }
+                    else if (isInSpecref(n.toUpperCase())) {
+                        return n.toUpperCase();
+                    }
+                    else {
+                        return null;
+                    }
+                })
+                .filter(n => !!n);
+            if (ref.obsoletedBy.length > 0) {
+                ref.isSuperseded = true;
+                if (ref.isRetired) {
+                    delete ref.isRetired;
+                }
+            }
+            else {
+                delete ref.obsoletedBy;
+            }
+        });
+
     updatedList = helper.sortRefs(updatedList);
     console.log("updating existing refs.")
     helper.writeBiblio(FILENAME, updatedList);
diff --git a/scripts/run-all b/scripts/run-all
index 4e91fcaa5..c31c08615 100755
--- a/scripts/run-all
+++ b/scripts/run-all
@@ -2,7 +2,7 @@
 
 node "./scripts/list-refs.js" &&
 node "./scripts/run.js" &&
-# node "./scripts/browser-specs.js" &&
+node "./scripts/browser-specs.js" &&
 node "./scripts/ietf.js" &&
 node "./scripts/w3c.js" &&
 node "./scripts/csswg.js" &&


Test patch:
diff --git a/test/ref-list.json b/test/ref-list.json
index 409f354d4..6e33986eb 100644
--- a/test/ref-list.json
+++ b/test/ref-list.json
@@ -16416,6 +16416,7 @@
     "EDIT6431",
     "EDIT6432",
     "EDIT6433",
+    "EDIT6434",
     "EDIT644",
     "EDIT645",
     "EDIT646",
@@ -86656,6 +86657,8 @@
     "VC-DATA-MODEL-2.0-20230728",
     "vc-data-model-2.0-20230730",
     "VC-DATA-MODEL-2.0-20230730",
+    "vc-data-model-2.0-20230802",
+    "VC-DATA-MODEL-2.0-20230802",
     "vc-data-model-20170803",
     "VC-DATA-MODEL-20170803",
     "vc-data-model-20190208",
@@ -86782,6 +86785,8 @@
     "VC-JSON-SCHEMA-20230731",
     "vc-json-schema-20230801",
     "VC-JSON-SCHEMA-20230801",
+    "vc-json-schema-20230802",
+    "VC-JSON-SCHEMA-20230802",
     "vc-jws-2020",
     "VC-JWS-2020",
     "vc-jws-2020-20221208",
@@ -101430,6 +101435,7 @@
     "WG21-EDIT6431",
     "WG21-EDIT6432",
     "WG21-EDIT6433",
+    "WG21-EDIT6434",
     "WG21-EDIT644",
     "WG21-EDIT645",
     "WG21-EDIT646",
@@ -114970,6 +114976,7 @@
     "WGSL-20230726",
     "WGSL-20230731",
     "WGSL-20230801",
+    "WGSL-20230802",
     "WHATWG-BOOKS",
     "WHATWG-COMPAT",
     "WHATWG-CONSOLE",