Repository: zulip/zulip

Issue:
Lint for duplicate IDs in our HTML
We should have a lint rule that checks for duplicate IDs in our HTML templates.  

This would avoid problems like ad513bab3342bbab7089eae20720b0dd9de6ed53.


Discussion:
Is this looking for duplicate ids within a single template or across all templates?
The code in `tools/lib/html_branches.py` should be able to mostly solve this problem.
@timabbott I can take this up, please guide me to get started on this.
One can start by reading and understanding the script that @showell mentioned, testing it with some print statements, etc., until you understand how it works.  
@mahim97 Once you get oriented to this code, check in on Zulip with @tommyip, who also expressed some interest in this issue (not sure how much):  https://chat.zulip.org/#narrow/stream/tools/topic/duplicate.20HTML.20IDs
@mahim97 Please go ahead and do it! I was just poking around to see if there is anything I could help.

Resolving patch:
diff --git a/tools/lib/html_branches.py b/tools/lib/html_branches.py
--- a/tools/lib/html_branches.py
+++ b/tools/lib/html_branches.py
@@ -4,6 +4,7 @@
 from typing import Optional
 
 import re
+from collections import defaultdict
 
 from .template_parser import (
     tokenize,
@@ -109,11 +110,38 @@ def get_tag_info(token):
         m = re.search(regex, s)
         if m:
             for g in m.groups():
-                lst += g.split()
+                lst += split_for_id_and_class(g)
 
     return TagInfo(tag=tag, classes=classes, ids=ids, token=token)
 
 
+def split_for_id_and_class(element):
+    # type: (str) -> List[str]
+    # Here we split a given string which is expected to contain id or class
+    # attributes from HTML tags. This also takes care of template variables
+    # in string during splitting process. For eg. 'red black {{ a|b|c }}'
+    # is split as ['red', 'black', '{{ a|b|c }}']
+    outside_braces = True # type: bool
+    lst = []
+    s = ''
+
+    for ch in element:
+        if ch == '{':
+            outside_braces = False
+        if ch == '}':
+            outside_braces = True
+        if ch == ' ' and outside_braces:
+            if not s == '':
+                lst.append(s)
+            s = ''
+        else:
+            s += ch
+    if not s == '':
+        lst.append(s)
+
+    return lst
+
+
 def html_branches(text, fn=None):
     # type: (str, str) -> List[HtmlTreeBranch]
 
@@ -163,3 +191,21 @@ def html_tag_tree(text):
             stack.pop()
 
     return top_level
+
+
+def build_id_dict(templates):
+    # type: (List[str]) -> (Dict[str,List[str]])
+
+    template_id_dict = defaultdict(list) # type: (Dict[str,List[str]])
+
+    for fn in templates:
+        text = open(fn).read()
+        list_tags = tokenize(text)
+
+        for tag in list_tags:
+            info = get_tag_info(tag)
+
+            for ids in info.ids:
+                template_id_dict[ids].append("Line " + str(info.token.line) + ":" + fn)
+
+    return template_id_dict


Test patch:
diff --git a/tools/tests/test_html_branches.py b/tools/tests/test_html_branches.py
--- a/tools/tests/test_html_branches.py
+++ b/tools/tests/test_html_branches.py
@@ -2,6 +2,7 @@
 from __future__ import print_function
 
 import unittest
+import os
 
 import tools.lib.template_parser
 
@@ -9,6 +10,8 @@
     get_tag_info,
     html_branches,
     html_tag_tree,
+    build_id_dict,
+    split_for_id_and_class,
 )
 
 
@@ -100,3 +103,37 @@ def test_html_branches(self):
         self.assertEqual(branches[0].staircase_text(), '\n    html\n        head\n            title\n')
         self.assertEqual(branches[1].staircase_text(), '\n    html\n        body\n            p\n                br\n')
         self.assertEqual(branches[2].staircase_text(), '\n    html\n        body\n            p\n')
+
+    def test_build_id_dict(self):
+        # type: () -> None
+        TEST_TEMPLATES_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_template_data")
+        templates = ["test_template1.html", "test_template2.html"]
+        templates = [os.path.join(TEST_TEMPLATES_DIR, fn) for fn in templates]
+
+        template_id_dict = build_id_dict(templates)
+
+        self.assertEqual(list(template_id_dict.keys()), ['hello_{{ message }}', 'intro', 'below_navbar'])
+        self.assertEqual(template_id_dict['hello_{{ message }}'], [
+                         'Line 12:/srv/zulip/tools/tests/test_template_data/test_template1.html',
+                         'Line 12:/srv/zulip/tools/tests/test_template_data/test_template2.html'])
+        self.assertEqual(template_id_dict['intro'], [
+                         'Line 10:/srv/zulip/tools/tests/test_template_data/test_template1.html',
+                         'Line 11:/srv/zulip/tools/tests/test_template_data/test_template1.html',
+                         'Line 11:/srv/zulip/tools/tests/test_template_data/test_template2.html'])
+        self.assertEqual(template_id_dict['below_navbar'], ['Line 10:/srv/zulip/tools/tests/test_template_data/test_template2.html'])
+
+    def test_split_for_id_and_class(self):
+        # type: () -> None
+        id1 = "{{ red|blue }}"
+        id2 = "search_box_{{ page }}"
+
+        class1 = "chat_box message"
+        class2 = "stream_{{ topic }}"
+        class3 = "foo {{ a|b|c }} bar"
+
+        self.assertEqual(split_for_id_and_class(id1), ['{{ red|blue }}'])
+        self.assertEqual(split_for_id_and_class(id2), ['search_box_{{ page }}'])
+
+        self.assertEqual(split_for_id_and_class(class1), ['chat_box', 'message'])
+        self.assertEqual(split_for_id_and_class(class2), ['stream_{{ topic }}'])
+        self.assertEqual(split_for_id_and_class(class3), ['foo', '{{ a|b|c }}', 'bar'])
diff --git a/tools/tests/test_template_data/test_template1.html b/tools/tests/test_template_data/test_template1.html
new file mode 100644
--- /dev/null
+++ b/tools/tests/test_template_data/test_template1.html
@@ -0,0 +1,17 @@
+<!-- test -->
+      <!DOCTYPE html>
+      <html>
+      <head>
+          <title>Test</title>
+          <meta charset="utf-8" />
+          <link rel="stylesheet" href="style.css" />
+      </head>
+      <body>
+        <div id="intro">
+          <p id="intro">
+            <span id="hello_{{ message }}">Hello World!!</span>
+            This is a test file for checking correct working of duplicate id detection module.
+          </p>
+        </div>
+      </body>
+      </html>
diff --git a/tools/tests/test_template_data/test_template2.html b/tools/tests/test_template_data/test_template2.html
new file mode 100644
--- /dev/null
+++ b/tools/tests/test_template_data/test_template2.html
@@ -0,0 +1,17 @@
+<!-- test -->
+      <!DOCTYPE html>
+      <html>
+      <head>
+          <title>Test</title>
+          <meta charset="utf-8" />
+          <link rel="stylesheet" href="style.css" />
+      </head>
+      <body>
+        <div id="below_navbar">
+          <p id="intro">
+            <span id="hello_{{ message }}">Hello World!!</span>
+            This is a test file for checking correct working of duplicate id detection module.
+          </p>
+        </div>
+      </body>
+      </html>