Repository: mdn/kuma

Issue:
Support unified git-style diff for revise results
The revise tool produces a "results" directory with one subdir for each page that was modified, and "ref.html" and "rev.html" files under that. We can diff ref and rev, but have to diff each page individually, which is cumbersome when a lot of files have been changed.

It would be great if "revise" offered a way to present a unified diff for all the pages that have changed.



Discussion:

Resolving patch:
diff --git a/kuma/wiki/views/revision.py b/kuma/wiki/views/revision.py
--- a/kuma/wiki/views/revision.py
+++ b/kuma/wiki/views/revision.py
@@ -14,7 +14,7 @@
 from django.views.decorators.cache import never_cache
 from django.views.decorators.clickjacking import xframe_options_sameorigin
 from django.views.decorators.csrf import csrf_exempt
-from django.views.decorators.http import etag, require_GET, require_POST
+from django.views.decorators.http import last_modified, require_GET, require_POST
 from ratelimit.decorators import ratelimit
 from rest_framework.authentication import TokenAuthentication
 from rest_framework.decorators import api_view, authentication_classes
@@ -27,6 +27,7 @@
 )
 from kuma.core.utils import smart_int
 
+from .utils import get_last_modified_header
 from .. import kumascript
 from ..decorators import prevent_indexing, process_document_path
 from ..events import EditDocumentEvent
@@ -212,10 +213,10 @@ def revision_api(request, document_slug, document_locale):
     """
     GET a document's raw HTML with select macros rendered or removed, or
     POST new raw HTML to a document. POST's are allowed only for clients
-    using a valid token passed via the "Authorization" header. The "ETag"
-    header is returned, and conditional requests handled for both GET and
-    POST requests, but conditional request handling is only intended for
-    POST requests to avoid collisions.
+    using a valid token passed via the "Authorization" header. The "Last-
+    Modified" header is returned, and conditional requests handled for both
+    GET and POST requests, but conditional request handling is only intended
+    for POST requests to avoid collisions.
     """
     doc = get_object_or_404(Document, slug=document_slug, locale=document_locale)
 
@@ -260,17 +261,17 @@ def revision_api(request, document_slug, document_locale):
     return do_revision_api_get(request, doc, mode, select_macros)
 
 
-def get_etag(request, doc, *args):
+def get_last_modified(request, doc, *args):
     """
-    Returns the id of the document's current revision as a string.
+    Returns the datetime of the creation of the document's current revision.
     """
-    return str(doc.current_revision.id)
+    return doc.current_revision.created
 
 
-@etag(get_etag)
+@last_modified(get_last_modified)
 def do_revision_api_get(request, doc, mode, select_macros):
     """
-    Handles the GET, including adding the ETag header, assuming that
+    Handles the GET, including adding the Last-Modified header, assuming that
     validation has already been performed.
     """
     if mode and select_macros:
@@ -285,7 +286,7 @@ def do_revision_api_get(request, doc, mode, select_macros):
     return response
 
 
-@etag(get_etag)
+@last_modified(get_last_modified)
 def do_revision_api_post(request, doc, data):
     """
     Handles the POST, including conditional POST's, given the document and
@@ -301,8 +302,8 @@ def do_revision_api_post(request, doc, data):
     response = HttpResponse(doc.html, status=201)
     rev_url = f"{doc.get_absolute_url()}$revision/{doc.current_revision.id}"
     response["Location"] = request.build_absolute_uri(rev_url)
-    # Set the "ETag" header or else the "etag" decorator will set it according
-    # to the document's previous revision, i.e. the current revision prior to
-    # the "revise" method call above.
-    response["ETag"] = f'"{str(doc.current_revision.id)}"'
+    # Set the "Last-Modified" header or else the "last_modified" decorator will
+    # set it according to the document's previous revision, i.e. the current
+    # revision prior to the "revise" method call above.
+    response["Last-Modified"] = get_last_modified_header(doc.current_revision.created)
     return response
diff --git a/kuma/wiki/views/utils.py b/kuma/wiki/views/utils.py
--- a/kuma/wiki/views/utils.py
+++ b/kuma/wiki/views/utils.py
@@ -1,4 +1,7 @@
 import hashlib
+from calendar import timegm
+
+from django.utils.http import http_date
 
 
 def split_slug(slug):
@@ -50,3 +53,7 @@ def document_form_initial(document):
 
 def calculate_etag(content):
     return hashlib.md5(content.encode()).hexdigest()
+
+
+def get_last_modified_header(dt):
+    return http_date(timegm(dt.utctimetuple()))
diff --git a/scripts/revise/revise.py b/scripts/revise/revise.py
--- a/scripts/revise/revise.py
+++ b/scripts/revise/revise.py
@@ -14,6 +14,7 @@
 from functools import partial, wraps
 from http import HTTPStatus
 from pathlib import Path
+from subprocess import run
 from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
 
 import click
@@ -102,6 +103,20 @@ def commit(dir):
     handle_commit(dir)
 
 
+@revise.command()
+@click.argument("dir", required=True)
+@click.option("--no-pager", is_flag=True)
+def diff(dir, no_pager):
+    dir = Path(dir)
+    if not dir.exists():
+        raise ReviseException(f"{dir} does not exist")
+    pager = "--no-pager" if no_pager else "--paginate"
+    cmd = f"git {pager} diff {{rev}} {{ref}}"
+    for rev in dir.rglob(f"rev.html"):
+        ref = rev.with_name("ref.html")
+        run(cmd.format(rev=rev, ref=ref).split())
+
+
 def handle_commit(dir):
     token = get_token()
     dir = Path(dir)
@@ -119,13 +134,13 @@ def do_commit(dir, token, log):
         metadata = get_metadata(rev.with_name("metadata.yaml"), log)
         if not metadata:
             continue
-        etag = metadata["etag"]
+        last_modified = metadata["last_modified"]
         parts = urlsplit(metadata["url"])
         doc = parts.path
         url = urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
         headers = {
-            "If-Match": etag,
             "Authorization": f"Token {token}",
+            "If-Unmodified-Since": last_modified,
         }
         try:
             with rev.open() as f:
@@ -179,7 +194,7 @@ def do_edit(cmd, config_path, output_dir, log):
         rev_url = f"{ref_url}?{qs}"
 
         try:
-            ref_text, rev_text, etag = get_ref_and_rev(ref_url, rev_url)
+            ref_text, rev_text, last_modified = get_ref_and_rev(ref_url, rev_url)
         except requests.HTTPError as e:
             log(f"error: while getting data for {doc} ({str(e)})", fg="red")
             continue
@@ -207,7 +222,7 @@ def do_edit(cmd, config_path, output_dir, log):
 
         log(f"- {metadata_path}")
         with metadata_path.open("w") as f:
-            yaml.dump(dict(url=rev_url, etag=etag), f)
+            yaml.dump(dict(url=rev_url, last_modified=last_modified), f)
 
 
 def get_ref_and_rev(ref_url, rev_url):
@@ -215,12 +230,12 @@ def get_ref_and_rev(ref_url, rev_url):
     while not done:
         ref_resp = get(ref_url)
         ref_resp.raise_for_status()
-        ref_etag = ref_resp.headers.get("etag")
+        ref_last_modified = ref_resp.headers.get("last-modified")
         rev_resp = get(rev_url)
         rev_resp.raise_for_status()
-        rev_etag = rev_resp.headers.get("etag")
-        done = ref_etag == rev_etag
-    return (ref_resp.text, rev_resp.text, rev_etag)
+        rev_last_modified = rev_resp.headers.get("last-modified")
+        done = ref_last_modified == rev_last_modified
+    return (ref_resp.text, rev_resp.text, rev_last_modified)
 
 
 def get_fresh_output_dir(output_dir):
@@ -251,6 +266,13 @@ def get_config(path):
     for s in ("render", "remove"):
         if (s in macros) and not macros[s]:
             raise ReviseException(f'the "{s}" block within {path} is empty')
+    # Let's ensure we use the wiki site (mainly so POST's don't redirect).
+    site = config["site"]
+    status_url = urljoin(site, "_kuma_status.json")
+    resp = get(status_url)
+    if resp.status_code != requests.codes.ok:
+        raise ReviseException(f"{site} doesn't seem to be an MDN site")
+    config["site"] = resp.json()["settings"]["WIKI_SITE_URL"]
     return config
 
 
@@ -261,7 +283,7 @@ def get_metadata(path, log):
     except (OSError, yaml.YAMLError) as e:
         log(f"error: {str(e)}", fg="red")
         return None
-    for k in ("etag", "url"):
+    for k in ("last_modified", "url"):
         if not metadata.get(k):
             log(f'error: {path} has no "{k}" key or its value is empty', fg="red")
             return None


Test patch:
diff --git a/kuma/wiki/tests/test_views_revision.py b/kuma/wiki/tests/test_views_revision.py
--- a/kuma/wiki/tests/test_views_revision.py
+++ b/kuma/wiki/tests/test_views_revision.py
@@ -12,6 +12,7 @@
 from kuma.core.utils import urlparams
 
 from ..models import Document, Revision
+from ..views.utils import get_last_modified_header
 
 
 @pytest.fixture
@@ -246,7 +247,9 @@ def test_revision_api_get(doc_with_macros, client, constance_config, qs, expecte
     assert_no_cache_header(response)
     assert response["X-Frame-Options"] == "deny"
     assert response["X-Robots-Tag"] == "noindex"
-    assert response["ETag"] == f'"{str(doc_with_macros.current_revision.id)}"'
+    assert response["Last-Modified"] == get_last_modified_header(
+        doc_with_macros.current_revision.created
+    )
 
 
 @pytest.mark.parametrize("case", ("json", "form-data", "form-urlencoded"))
@@ -285,7 +288,7 @@ def test_revision_api_post(
         )
     )
     assert response.content.decode() == "yada"
-    assert response["ETag"] == f'"{str(new_rev.id)}"'
+    assert response["Last-Modified"] == get_last_modified_header(new_rev.created)
     assert_no_cache_header(response)
 
 
@@ -301,12 +304,12 @@ def test_revision_api_conditional_post(
         "wiki.revision_api", args=[doc_with_macros.slug], locale=doc_with_macros.locale
     )
 
-    # First let's get some revised content and the ETag header.
+    # First let's get some revised content and the Last-Modified header.
     response = client.get(url + "?mode=remove&macros=m1", HTTP_HOST=settings.WIKI_HOST)
     assert response.status_code == 200
     revised_content = response.content.decode()
     assert revised_content == '{{M2("z")}}'
-    etag_from_get = response["ETag"]
+    last_modified_from_get = response["Last-Modified"]
 
     # Let's POST the revised content, but with the condition that no one else
     # has created a new revision for the document since we performed our GET.
@@ -314,7 +317,7 @@ def test_revision_api_conditional_post(
         url,
         data=dict(content=revised_content),
         HTTP_HOST=settings.WIKI_HOST,
-        HTTP_IF_MATCH=etag_from_get,
+        HTTP_IF_UNMODIFIED_SINCE=last_modified_from_get,
         HTTP_AUTHORIZATION=f"Token {wiki_user_2_token.key}",
     )
     doc_with_macros.refresh_from_db()
@@ -331,9 +334,9 @@ def test_revision_api_conditional_post(
     )
     assert response.content.decode() == revised_content
     assert_no_cache_header(response)
-    assert response["ETag"] == f'"{str(new_rev.id)}"'
+    assert response["Last-Modified"] == get_last_modified_header(new_rev.created)
 
-    # Now let's pretend we're someone else, holding the same "etag_from_get"
+    # This time we're someone else, holding the same "last_modified_from_get"
     # value, who also wants to conditionally revise the document, but since
     # it has changed, the condition will not be satisfied and the POST will
     # fail.
@@ -341,7 +344,7 @@ def test_revision_api_conditional_post(
         url,
         data=dict(content="yada"),
         HTTP_HOST=settings.WIKI_HOST,
-        HTTP_IF_MATCH=etag_from_get,
+        HTTP_IF_UNMODIFIED_SINCE=last_modified_from_get,
         HTTP_AUTHORIZATION=f"Token {wiki_user_2_token.key}",
     )
     assert response.status_code == 412