Repository: tlienart/Franklin.jl (julia)

Issue:
Footnote becomes H2
Happens if you have one and only one empty line after a footnote before a `---` divider:
```
julia> println(Franklin.fd2html("""
       hello[^a]

       [^a]: world

       ---
       """))
<p>hello<sup id="fnref:a"><a href="#fndef:a" class="fnref">[1]</a></sup>
<h2><table class="fndef" id="fndef:a">
    <tr>
        <td class="fndef-backref"><a href="#fnref:a">[1]</a></td>
        <td class="fndef-content">world</td>
    </tr>
</table>
</h2>
```

Pull request:
closes #432 this ended up being a bit more annoying than I thought. 

Anyway long story short the PR moves a few other things like regexes, nothing to worry about. Main change is to look for `---`, `***`, `___` (and more) to find horizontal rules that are well separated e.g. `\n\n----`; that's not all cases (most are still handled by Julia's Markdown parser) but it helps avoid issues such as #432. 

The issue is that there was an ambiguity with lines like

```
<----------------
Here is a well separated comment
------------->
```

Basically the parser would try to capture the last line as a  horizontal rule, then drop it, and move to `>` which wouldn't be recognised as part of a `-->`. 

To avoid ambiguities, comment indicators now  **must be separated** so:

```
<-- This is allowed -->
```

```
<--_also allowed_-->
```

```
<---not allowed---->
```

This requires a change in FranklinTemplates for the tests to pass (patch release 0.5.7)

**Note**: these comments are not part of Common Mark so I feel reasonably free to add this constraint.

Patch:
diff --git a/src/converter/latex/hyperrefs.jl b/src/converter/latex/hyperrefs.jl
index 4aa7837b1..f9a1bb815 100644
--- a/src/converter/latex/hyperrefs.jl
+++ b/src/converter/latex/hyperrefs.jl
@@ -82,7 +82,7 @@ function lx_biblabel(lxc::LxCom, _)::String
 end
 
 function lx_toc(::LxCom, _)
-    minlevel = LOCAL_VARS["mintoclevel"].first
-    maxlevel = LOCAL_VARS["maxtoclevel"].first
+    minlevel = locvar("mintoclevel")
+    maxlevel = locvar("maxtoclevel")
     return "{{toc $minlevel $maxlevel}}"
 end
diff --git a/src/converter/markdown/blocks.jl b/src/converter/markdown/blocks.jl
index 19444a2fa..3442c3ac6 100644
--- a/src/converter/markdown/blocks.jl
+++ b/src/converter/markdown/blocks.jl
@@ -23,6 +23,7 @@ function convert_block(β::AbstractBlock, lxdefs::Vector{LxDef})::AS
     βn == :LINK_DEF        && return ""
 
     βn == :DOUBLE_BRACE    && return β.ss # let HTML converter deal with it
+    βn == :HORIZONTAL_RULE && return "<hr />"
 
     # Math block --> needs to call further processing to resolve possible latex
     βn ∈ MATH_BLOCKS_NAMES && return convert_math_block(β, lxdefs)
diff --git a/src/converter/markdown/md.jl b/src/converter/markdown/md.jl
index 1af91f128..12dca764b 100644
--- a/src/converter/markdown/md.jl
+++ b/src/converter/markdown/md.jl
@@ -43,6 +43,8 @@ function convert_md(mds::AS,
     validate_footnotes!(tokens)
     # ignore header tokens that are not at the start of a line
     validate_headers!(tokens)
+    # capture some hrule (see issue #432)
+    hrules = find_hrules!(tokens)
     #> 1b. Find indented blocks (ONLY if not recursive to avoid ambiguities!)
     if !isrecursive
         find_indented_blocks!(tokens, mds)
@@ -89,7 +91,7 @@ function convert_md(mds::AS,
 
     #> 3[ex]. find double brace blocks, note we do it on pre_ocb tokens
     # as the step `find_all_ocblocks` possibly found and deactivated {...}.
-    dbb = find_double_brace_blocks(toks_pre_ocb)
+    dbb     = find_double_brace_blocks(toks_pre_ocb)
 
     # ------------------------------------------------------------------------
     #> 4. Page variable definition (mddefs), also if in config, update lxdefs
@@ -116,12 +118,12 @@ function convert_md(mds::AS,
     #
     # filter out the fnrefs that are left (still active)
     # and add them to the blocks to insert
-    fnrefs = filter!(τ -> τ.name == :FOOTNOTE_REF, tokens)
+    fnrefs = filter(τ -> τ.name == :FOOTNOTE_REF, tokens)
 
     #> 1. Merge all the blocks that will need further processing before
     # insertion
     b2insert = merge_blocks(lxcoms, deactivate_divs(blocks),
-                            sp_chars, fnrefs, dbb)
+                            sp_chars, fnrefs, dbb, hrules)
 
     #> 2. Form intermediate markdown + html
     inter_md, mblocks = form_inter_md(mds, b2insert, lxdefs)
diff --git a/src/manager/rss_generator.jl b/src/manager/rss_generator.jl
index 04eda58b3..90152c1a3 100644
--- a/src/manager/rss_generator.jl
+++ b/src/manager/rss_generator.jl
@@ -100,9 +100,9 @@ function rss_generator()::Nothing
     isempty(RSS_DICT) && return nothing
 
     # are the basic defs there? otherwise warn and break
-    rss_title = GLOBAL_VARS["website_title"] |> first
-    rss_descr = GLOBAL_VARS["website_descr"] |> first
-    rss_link  = GLOBAL_VARS["website_url"]   |> first
+    rss_title = globvar("website_title")
+    rss_descr = globvar("website_descr")
+    rss_link  = globvar("website_url")
 
     if any(isempty, (rss_title, rss_descr, rss_link))
         @warn """I found RSS items but the RSS feed is not properly described:
diff --git a/src/parser/markdown/tokens.jl b/src/parser/markdown/tokens.jl
index 6d4e0eafb..321ebc1bb 100644
--- a/src/parser/markdown/tokens.jl
+++ b/src/parser/markdown/tokens.jl
@@ -30,23 +30,24 @@ possibilities to consider in which case the order is important: the first case
 that works will be taken.
 """
 const MD_TOKENS = LittleDict{Char, Vector{TokenFinder}}(
-    '<'  => [ isexactly("<!--") => :COMMENT_OPEN,     # <!-- ...
+    '<'  => [ isexactly("<!--")  => :COMMENT_OPEN,     # <!-- ...
              ],
-    '-'  => [ isexactly("-->")  => :COMMENT_CLOSE,    #      ... -->
+    '-'  => [ isexactly("-->")   => :COMMENT_CLOSE,    #  ... -->
+              incrlook(is_hr1)   => :HORIZONTAL_RULE,  # ---+
              ],
-    '~'  => [ isexactly("~~~")  => :ESCAPE,           # ~~~  ... ~~~
+    '~'  => [ isexactly("~~~")   => :ESCAPE,           # ~~~  ... ~~~
              ],
-    '['  => [ incrlook(is_footnote) => :FOOTNOTE_REF,    # [^...](:)? defs will be separated after
+    '['  => [ incrlook(is_footnote) => :FOOTNOTE_REF, # [^...](:)? defs will be separated after
              ],
     ']'  => [ isexactly("]: ") => :LINK_DEF,
              ],
     '\\' => [ # -- special characters, see `find_special_chars` in ocblocks
-              isexactly("\\\\")       => :CHAR_LINEBREAK, # --> <br/>
-              isexactly("\\", (' ',)) => :CHAR_BACKSPACE, # --> &#92;
-              isexactly("\\*")        => :CHAR_ASTERISK,  # --> &#42;
-              isexactly("\\_")        => :CHAR_UNDERSCORE,# --> &#95;
-              isexactly("\\`")        => :CHAR_BACKTICK,  # --> &#96;
-              isexactly("\\@")        => :CHAR_ATSIGN,    # --> &#64;
+              isexactly("\\\\")       => :CHAR_LINEBREAK,   # --> <br/>
+              isexactly("\\", (' ',)) => :CHAR_BACKSPACE,   # --> &#92;
+              isexactly("\\*")        => :CHAR_ASTERISK,    # --> &#42;
+              isexactly("\\_")        => :CHAR_UNDERSCORE,  # --> &#95;
+              isexactly("\\`")        => :CHAR_BACKTICK,    # --> &#96;
+              isexactly("\\@")        => :CHAR_ATSIGN,      # --> &#64;
               # -- maths
               isexactly("\\{")        => :INACTIVE,         # See note [^1]
               isexactly("\\}")        => :INACTIVE,         # See note [^1]
@@ -61,11 +62,11 @@ const MD_TOKENS = LittleDict{Char, Vector{TokenFinder}}(
               isexactly("\\end{eqnarray}")   => :MATH_EQA_CLOSE,
               # -- latex
               isexactly("\\newcommand")      => :LX_NEWCOMMAND,
-              incrlook((_, c) -> α(c))       => :LX_COMMAND,     # \command⎵*
+              incrlook((_, c) -> α(c))       => :LX_COMMAND,  # \command⎵*
              ],
-    '@'  => [ isexactly("@def", (' ',))   => :MD_DEF_OPEN,  # @def var = ...
-              isexactly("@@", SPACE_CHAR) => :DIV_CLOSE,    # @@⎵*
-              incrlook(is_div_open)       => :DIV_OPEN,     # @@dname
+    '@'  => [ isexactly("@def", (' ',))   => :MD_DEF_OPEN,    # @def var = ...
+              isexactly("@@", SPACE_CHAR) => :DIV_CLOSE,      # @@⎵*
+              incrlook(is_div_open)       => :DIV_OPEN,       # @@dname
              ],
     '#'  => [ isexactly("#",      (' ',)) => :H1_OPEN, # see note [^2]
               isexactly("##",     (' ',)) => :H2_OPEN,
@@ -77,10 +78,11 @@ const MD_TOKENS = LittleDict{Char, Vector{TokenFinder}}(
     '&'  => [ incrlook(is_html_entity) => :CHAR_HTML_ENTITY,
              ],
     '$'  => [ isexactly("\$", ('$',), false) => :MATH_A,  # $⎵*
-              isexactly("\$\$") => :MATH_B,              # $$⎵*
+              isexactly("\$\$")              => :MATH_B,  # $$⎵*
              ],
-    '_'  => [ isexactly("_\$>_") => :MATH_I_OPEN,   # internal use when resolving a latex command
-              isexactly("_\$<_") => :MATH_I_CLOSE,  # within mathenv (e.g. \R <> \mathbb R)
+    '_'  => [ isexactly("_\$>_") => :MATH_I_OPEN,  # internal use when resolving a latex command
+              isexactly("_\$<_") => :MATH_I_CLOSE, # within mathenv (e.g. \R <> \mathbb R)
+              incrlook(is_hr2)   => :HORIZONTAL_RULE,
              ],
     '`'  => [ isexactly("`", ('`',), false)  => :CODE_SINGLE, # `⎵
               isexactly("``",('`',), false)  => :CODE_DOUBLE, # ``⎵*
@@ -89,6 +91,8 @@ const MD_TOKENS = LittleDict{Char, Vector{TokenFinder}}(
               is_language()                  => :CODE_LANG,   # ```lang*
               is_language2()                 => :CODE_LANG2,  # `````lang*
              ],
+    '*'  => [ incrlook(is_hr3)   => :HORIZONTAL_RULE,
+             ]
     ) # end dict
 #= NOTE
 [1] capturing \{ here will force the head to move after it thereby not
diff --git a/src/parser/markdown/validate.jl b/src/parser/markdown/validate.jl
index 5dde4a5bf..0b179402c 100644
--- a/src/parser/markdown/validate.jl
+++ b/src/parser/markdown/validate.jl
@@ -33,7 +33,7 @@ $SIGNATURES
 Verify that a given string corresponds to a well formed html entity.
 """
 function validate_html_entity(ss::AS)
-    !isnothing(match(r"&(?:[a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});", ss))
+    !isnothing(match(HTML_ENT_PAT, ss))
 end
 
 """
@@ -60,7 +60,6 @@ function validate_headers!(tokens::Vector{Token})::Nothing
     return
 end
 
-
 """
 $(SIGNATURES)
 
@@ -174,3 +173,36 @@ function find_double_brace_blocks(tokens)
     end
     return dbb
 end
+
+"""
+$SIGNATURES
+
+Check that a `---` or `***` or `___` starts at the beginning of a line and is preceded by an empty line, if that's not the case, discard it.
+"""
+function find_hrules!(tokens::Vector{Token})
+    keep = Int[]
+    rm   = Int[]
+    for (i, τ) in enumerate(tokens)
+        τ.name == :HORIZONTAL_RULE || continue
+        # check if it has the right format
+        if !(startswith(τ.ss, "---") ||
+             startswith(τ.ss, "___") ||
+             startswith(τ.ss, "***")) || length(unique(τ.ss)) > 1
+            push!(rm, i)
+            continue
+        end
+        # check if it's at the start of the string  or
+        # if it's preceded by an empty line, in which case mark it as
+        # horizontal rule, leave all other cases to be dealt with by Julia's
+        # Markdown parser.
+        s, k = str(τ), from(τ)
+        if !(k == 1 || s[prevind(s, k, 2):prevind(s, k)] == "\n\n")
+            push!(rm, i)
+            continue
+        end
+        push!(keep, i)
+    end
+    hrules = tokens[keep]
+    deleteat!(tokens, sort(vcat(rm, keep)))
+    return hrules
+end
diff --git a/src/parser/tokens.jl b/src/parser/tokens.jl
index 3c688ff1a..ae0d75abc 100644
--- a/src/parser/tokens.jl
+++ b/src/parser/tokens.jl
@@ -50,6 +50,7 @@ end
 HTML_SPCH(ss) = HTML_SPCH(ss, "")
 
 
+
 """
 $(TYPEDEF)
 
@@ -284,6 +285,28 @@ function is_footnote(i::Int, c::Char)
     i > 2  && return αη(c, (']', ':'))
 end
 
+"""
+$SIGNATURES
+
+Check if it looks like `---+`.
+"""
+is_hr1(::Int, c::Char) = c == '-'
+
+"""
+$SIGNATURES
+
+Check if it looks like `___+`.
+"""
+is_hr2(::Int, c::Char) = c == '_'
+
+"""
+$SIGNATURES
+
+Check if it looks like `***+`.
+"""
+is_hr3(::Int, c::Char) = c == '*'
+
+
 """
 TokenFinder
 
diff --git a/src/regexes.jl b/src/regexes.jl
index 6473b8805..4173adc5b 100644
--- a/src/regexes.jl
+++ b/src/regexes.jl
@@ -50,6 +50,11 @@ const ESC_LINK_PAT = r"(&#33;)?&#91;(.*?)&#93;(?!:)(?:&#91;(.*?)&#93;)?"
 
 const FN_DEF_PAT = r"^\[\^[\p{L}0-9_]+\](:)?$"
 
+#= =====================================================
+HTML entity pattern
+===================================================== =#
+const HTML_ENT_PAT = r"&(?:[a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});"
+
 #= =====================================================
 HBLOCK patterns, see html blocks
 NOTE: the &#123 is { and 125 is }, this is because


Test patch:
diff --git a/test/parser/markdown-extra.jl b/test/parser/markdown-extra.jl
index b5df0f68a..7dfca39d1 100644
--- a/test/parser/markdown-extra.jl
+++ b/test/parser/markdown-extra.jl
@@ -136,3 +136,45 @@ end
     @test isapproxstr(s, """
         hello \\(\\rho=\\frac{e^{-\\beta \\mathcal{E}_{s}}} {\\mathcal{Z}} \\)""")
 end
+
+# issue 432 and consequences
+@testset "Hz rule" begin
+    # issue 432
+    s = raw"""
+        hello[^a]
+
+        [^a]: world
+
+        ---
+        """ |> fd2html_td
+    @test isapproxstr(s, """
+        <p>hello<sup id="fnref:a"><a href="#fndef:a" class="fnref">[1]</a></sup>
+        <table class="fndef" id="fndef:a">
+            <tr>
+                <td class="fndef-backref"><a href="#fnref:a">[1]</a></td>
+                <td class="fndef-content">world</td>
+            </tr>
+        </table>
+        <hr /></p>
+        """)
+    s = raw"""
+        A
+        ---
+        """ |> fd2html_td
+    @test isapproxstr(s, """
+        <h2>A</h2>""")
+    s = raw"""
+        A
+        ----
+        ****
+        """ |> fd2html_td
+    @test isapproxstr(s, "<h2>A</h2>\n<hr />")
+
+    # cases where nothing should happen
+    s = raw"""
+        A
+        ---*
+        ***_
+        """ |> fd2html_td
+    @test isapproxstr(s, "<p>A –-* ***_</p>") # note double -- is transformed -
+end
diff --git a/test/regexes.jl b/test/regexes.jl
index 8020fc34b..6b914fbc4 100644
--- a/test/regexes.jl
+++ b/test/regexes.jl
@@ -71,6 +71,26 @@ end
     end
 end
 
+### HTML ENTITY
+@testset "html-ent" begin
+    for s in (
+        "&tab;",
+     	"&newline;",
+        "&excl;",
+        "&quot;",
+        "&#x00009;",
+        "&#x00027;",
+        "&#39;",
+        "&#58;",
+        "&#123;",
+        "&ccirc;",
+        "&#x00107;",
+        "&#263;"
+        )
+        @test !isnothing(match(F.HTML_ENT_PAT, s))
+    end
+end
+
 ### HBLOCK REGEXES
 
 @testset "hb-if" begin