Repository: evennia/evennia

Issue:
[BUG] (webclient conversion of blank spaces unmanageable in EvForm)
#### Describe the bug
The webclient does not show blank spaces the same way that telnet does, due to the conversion to html. An EvForm that looks perfect from mudlet might look strangely jogged around in the webclient. This is a problem majorly with EvTable and EvForm because in other situations, it can be managed by using different symbols for blank spaces. But when it comes to more dynamic content, it is difficult to predict what management might be necessary.

#### To Reproduce
Steps to reproduce the behavior:
1.  Use this form and command: https://pastebin.com/YiHLXEA0
2.  Connect through the webclient and run the command.
3.  Connect through mudlet and run the command.
4. See error

#### Expected behavior
Blank spaces and blank rows inside of EvTables should be properly represented in EvForm format in order not to run into problems with the html conversion for the webclient.

#### Environment, Evennia version, OS etc
         Evennia version: 0.9.5 (rev 76af934fd)
         OS: nt
         Python: 3.7.9
         Twisted: 20.3.0
         Django: 2.2.5

#### Additional context
The html conversion for the webclient could be fixed to not have this problem: it seems that `|_` works better than tabs or blank spaces.



Discussion:
Possibly related to #2677 ?
I was poking around at this and while I'm not sure _how_ this would be happening, it seems as though the text2html parsing is applying to the template separately from the table as a whole, resulting in some double-spaces not being caught.

Specifically, on the line `left_table.add_row(" ", "Here is a blank space.")` and the two lines below, I noticed that the actual text in the element is 11 spaces (not including the left-border padding) while the displayed text is only 10 spaces, indicating that the first space in that element is staying ` ` instead of being converted to `&nbsp;`

I did a non-solution test where I changed the double-space replacement to replace all spaces with nbsp and confirmed it has the correct visible spacing in that situation. (It also breaks all of the html tags, so please don't try this as a fix 🤣 )

p.s. your example table isn't set to wrap lines, so the long line is being truncated with `[...]` indicating overflow instead

p.p.s. ...In further mysterious developments, I undid all of my changes to `utils/text2html.py` and restarted the server to go back to the base game state, and the spacing is somehow no longer broken when I use the `formtest` command. I have no idea why it would be behaving differently.

Resolving patch:
diff --git a/evennia/utils/text2html.py b/evennia/utils/text2html.py
--- a/evennia/utils/text2html.py
+++ b/evennia/utils/text2html.py
@@ -79,11 +79,11 @@ class TextToHTMLparser(object):
     # create stop markers
     fgstop = "(?:\033\[1m|\033\[22m){0,1}\033\[3[0-8].*?m|\033\[0m|$"
     bgstop = "(?:\033\[1m|\033\[22m){0,1}\033\[4[0-8].*?m|\033\[0m|$"
-    bgfgstop = bgstop[:-2] + r"(\s*)" + fgstop
+    bgfgstop = bgstop[:-2] + fgstop
 
     fgstart = "((?:\033\[1m|\033\[22m){0,1}\033\[3[0-8].*?m)"
     bgstart = "((?:\033\[1m|\033\[22m){0,1}\033\[4[0-8].*?m)"
-    bgfgstart = bgstart + r"(\s*)" + "((?:\033\[1m|\033\[22m){0,1}\033\[[3-4][0-8].*?m){0,1}"
+    bgfgstart = bgstart + r"((?:\033\[1m|\033\[22m){0,1}\033\[[3-4][0-8].*?m){0,1}"
 
     # extract color markers, tagging the start marker and the text marked
     re_fgs = re.compile(fgstart + "(.*?)(?=" + fgstop + ")")
@@ -97,12 +97,9 @@ class TextToHTMLparser(object):
     re_blink = re.compile("(?:%s)(.*?)(?=%s|%s)" % (blink.replace("[", r"\["), fgstop, bgstop))
     re_inverse = re.compile("(?:%s)(.*?)(?=%s|%s)" % (inverse.replace("[", r"\["), fgstop, bgstop))
     re_string = re.compile(
-        r"(?P<htmlchars>[<&>])|(?P<tab>[\t]+)|(?P<space> +)|"
-        r"(?P<spacestart>^ )|(?P<lineend>\r\n|\r|\n)",
+        r"(?P<htmlchars>[<&>])|(?P<tab>[\t]+)|(?P<lineend>\r\n|\r|\n)",
         re.S | re.M | re.I,
     )
-    re_dblspace = re.compile(r" {2,}", re.M)
-    re_invisiblespace = re.compile(r"( <.*?>)( )")
     re_url = re.compile(
         r'(?<!=")((?:ftp|www|https?)\W+(?:(?!\.(?:\s|$)|&\w+;)[^"\',;$*^\\(){}<>\[\]\s])+)(\.(?:\s|$)|&\w+;|)'
     )
@@ -111,20 +108,16 @@ class TextToHTMLparser(object):
 
     def _sub_bgfg(self, colormatch):
         # print("colormatch.groups()", colormatch.groups())
-        bgcode, prespace, fgcode, text, postspace = colormatch.groups()
+        bgcode, fgcode, text = colormatch.groups()
         if not fgcode:
-            ret = r"""<span class="%s">%s%s%s</span>""" % (
+            ret = r"""<span class="%s">%s</span>""" % (
                 self.bg_colormap.get(bgcode, self.fg_colormap.get(bgcode, "err")),
-                prespace and "&nbsp;" * len(prespace) or "",
-                postspace and "&nbsp;" * len(postspace) or "",
                 text,
             )
         else:
-            ret = r"""<span class="%s"><span class="%s">%s%s%s</span></span>""" % (
+            ret = r"""<span class="%s"><span class="%s">%s</span></span>""" % (
                 self.bg_colormap.get(bgcode, self.fg_colormap.get(bgcode, "err")),
                 self.fg_colormap.get(fgcode, self.bg_colormap.get(fgcode, "err")),
-                prespace and "&nbsp;" * len(prespace) or "",
-                postspace and "&nbsp;" * len(postspace) or "",
                 text,
             )
         return ret
@@ -265,20 +258,6 @@ def convert_urls(self, text):
         # change pages (and losing our webclient session).
         return self.re_url.sub(r'<a href="\1" target="_blank">\1</a>\2', text)
 
-    def re_double_space(self, text):
-        """
-        HTML will swallow any normal space after the first, so if any slipped
-        through we must make sure to replace them with " &nbsp;"
-        """
-        return self.re_dblspace.sub(self.sub_dblspace, text)
-
-    def re_invisible_space(self, text):
-        """
-        If two spaces are separated by an invisble html element, they act as a
-        hidden double-space and the last of them should be replaced by &nbsp;
-        """
-        return self.re_invisiblespace.sub(self.sub_invisiblespace, text)
-
     def sub_mxp_links(self, match):
         """
         Helper method to be passed to re.sub,
@@ -332,28 +311,10 @@ def sub_text(self, match):
         elif cdict["lineend"]:
             return "<br>"
         elif cdict["tab"]:
-            text = cdict["tab"].replace("\t", " " + "&nbsp;" * (self.tabstop - 1))
-            return text
-        elif cdict["space"] or cdict["spacestart"]:
-            text = cdict["space"]
-            text = " " if len(text) == 1 else " " + text[1:].replace(" ", "&nbsp;")
+            text = cdict["tab"].replace("\t", " " * (self.tabstop))
             return text
         return None
 
-    def sub_dblspace(self, match):
-        "clean up double-spaces"
-        return " " + "&nbsp;" * (len(match.group()) - 1)
-
-    def sub_invisiblespace(self, match):
-        "clean up invisible spaces"
-        return match.group(1) + "&nbsp;"
-
-    def handle_single_first_space(self, text):
-        "Don't swallow an initial lone space"
-        if text.startswith(" "):
-            return "&nbsp;" + text[1:]
-        return text
-
     def parse(self, text, strip_ansi=False):
         """
         Main access function, converts a text containing ANSI codes
@@ -383,9 +344,6 @@ def parse(self, text, strip_ansi=False):
         result = self.convert_linebreaks(result)
         result = self.remove_backspaces(result)
         result = self.convert_urls(result)
-        result = self.re_double_space(result)
-        result = self.re_invisible_space(result)
-        result = self.handle_single_first_space(result)
         # clean out eventual ansi that was missed
         ## result = parse_ansi(result, strip_ansi=True)
 


Test patch:
diff --git a/evennia/utils/tests/test_text2html.py b/evennia/utils/tests/test_text2html.py
--- a/evennia/utils/tests/test_text2html.py
+++ b/evennia/utils/tests/test_text2html.py
@@ -120,13 +120,6 @@ def test_convert_urls(self):
         )
         # TODO: doesn't URL encode correctly
 
-    def test_re_double_space(self):
-        parser = text2html.HTML_PARSER
-        self.assertEqual("foo", parser.re_double_space("foo"))
-        self.assertEqual(
-            "a &nbsp;red &nbsp;&nbsp;&nbsp;foo", parser.re_double_space("a  red    foo")
-        )
-
     def test_sub_mxp_links(self):
         parser = text2html.HTML_PARSER
         mocked_match = mock.Mock()
@@ -156,7 +149,7 @@ def test_sub_text(self):
             "tab": "\t",
             "space": "",
         }
-        self.assertEqual(" &nbsp;", parser.sub_text(mocked_match))
+        self.assertEqual("  ", parser.sub_text(mocked_match))
 
         mocked_match.groupdict.return_value = {
             "htmlchars": "",
@@ -165,7 +158,7 @@ def test_sub_text(self):
             "space": " ",
             "spacestart": " ",
         }
-        self.assertEqual(" &nbsp; &nbsp;", parser.sub_text(mocked_match))
+        self.assertEqual("    ", parser.sub_text(mocked_match))
 
         mocked_match.groupdict.return_value = {
             "htmlchars": "",
@@ -181,24 +174,13 @@ def test_parse_tab_to_html(self):
         parser = text2html.HTML_PARSER
         parser.tabstop = 4
         # single tab
-        self.assertEqual(parser.parse("foo|>foo"), "foo &nbsp;&nbsp;&nbsp;foo")
+        self.assertEqual(parser.parse("foo|>foo"), "foo    foo")
 
         # space and tab
-        self.assertEqual(parser.parse("foo |>foo"), "foo &nbsp;&nbsp;&nbsp;&nbsp;foo")
+        self.assertEqual(parser.parse("foo |>foo"), "foo     foo")
 
         # space, tab, space
-        self.assertEqual(parser.parse("foo |> foo"), "foo &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;foo")
-
-    def test_parse_space_to_html(self):
-        """test space parsing - a single space should be kept, two or more
-        should get &nbsp;"""
-        parser = text2html.HTML_PARSER
-        # single space
-        self.assertEqual(parser.parse("foo foo"), "foo foo")
-        # double space
-        self.assertEqual(parser.parse("foo  foo"), "foo &nbsp;foo")
-        # triple space
-        self.assertEqual(parser.parse("foo   foo"), "foo &nbsp;&nbsp;foo")
+        self.assertEqual(parser.parse("foo |> foo"), "foo      foo")
 
     def test_parse_html(self):
         self.assertEqual("foo", text2html.parse_html("foo"))