Repository: tlienart/JuDoc.jl (julia)

Issue:
Parsing issue with \note
```
\newcommand{\note}[1]{@@note @@title ⚠ Note@@ @@content #1 @@ @@}
```

```
\note{This assumes that you're working on a user folder with final URL looking like 
`username.github.io`. See [this example](https://github.com/tlienart2/tlienart2.github.io) for 
instance.}
```

![Screen Shot 2020-04-05 at 17 20 58](https://user-images.githubusercontent.com/10897531/78502529-dbb7a480-7761-11ea-81d0-1aff67c6738a.png)

Pull request description:
Non breaking patch 

### 444

See the comment in https://github.com/tlienart/Franklin.jl/issues/444#issuecomment-611937199, the PR essentially adds `deactivate_inner_blocks!` which does the work that used to be done when finding ocblocks. It's refactored a bit and tested more.

Auxiliary changes to tests to adjust to this (basically now blocks are sorted so that changed a few things in the tests)

### 447

adds a paragraph in the readme

Closes #444 closes #447

Patch:
diff --git a/README.md b/README.md
index bf775b09e..8f3c8113a 100644
--- a/README.md
+++ b/README.md
@@ -80,6 +80,16 @@ julia> serve()
 Modify the files in `MyNewSite/src` and see the changes being live-rendered in your browser.
 Head to [the docs](https://franklinjl.org) for more information.
 
+### Heads up!
+
+While Franklin broadly supports standard Markdown there are a few things that may trip you which are either due to Franklin or due to Julia's Markdown library, here are key ones you should keep in mind:
+
+* when writing a list, the content of the list item **must** be on a single line (no line break)
+* (as of `v0.6.15`) you can write comments with `<!-- comments -->` the comment markers `<!--` and `-->` **must** be separated by a character that is not a `-` to work properly so `<!--A-->` is ok but `<!---A--->` is not, best is to just systematically use a whitespace: `<!-- A -->`.
+* (as of `v0.7`) code blocks should be delimited with backticks `` ` `` you *can* also use indented blocks to delimit code blocks but you now have to **opt in** explicitly on pages that would use them by using `@def indented_code = true`, if you want to use that everywhere, write that in the `config.md`. Note that indented blocks are **ambiguous** with some of the other things that Franklin provides (div blocks, latex commands) and so if you use them, you are responsible for avoiding ambiguities (effectively that means _not using indentation for anything else than code_)
+
+
+
 ## Associated repositories
 
 * [LiveServer.jl](https://github.com/asprionj/LiveServer.jl) a package coded with [Jonas Asprion](https://github.com/asprionj) to render and watch the content of a local folder in the browser.
diff --git a/src/converter/markdown/md.jl b/src/converter/markdown/md.jl
index 8de8e70a7..41fe7915e 100644
--- a/src/converter/markdown/md.jl
+++ b/src/converter/markdown/md.jl
@@ -68,6 +68,7 @@ function convert_md(mds::AS,
     #>> a'. find the rest
     blocks2, tokens = find_all_ocblocks(tokens, vcat(MD_OCB2, MD_OCB_MATH))
     append!(blocks, blocks2)
+    deactivate_inner_blocks!(blocks)
     #>> b. merge CODE_BLOCK_IND which are separated by emptyness
     merge_indented_blocks!(blocks, mds)
     #>> b'. only keep indented code blocks which are not contained in larger
@@ -91,7 +92,7 @@ function convert_md(mds::AS,
 
     #> 3[ex]. find double brace blocks, note we do it on pre_ocb tokens
     # as the step `find_all_ocblocks` possibly found and deactivated {...}.
-    dbb     = find_double_brace_blocks(toks_pre_ocb)
+    dbb = find_double_brace_blocks(toks_pre_ocb)
 
     # ------------------------------------------------------------------------
     #> 4. Page variable definition (mddefs), also if in config, update lxdefs
diff --git a/src/parser/markdown/tokens.jl b/src/parser/markdown/tokens.jl
index 321ebc1bb..bb4d34f3c 100644
--- a/src/parser/markdown/tokens.jl
+++ b/src/parser/markdown/tokens.jl
@@ -242,7 +242,8 @@ const MATH_BLOCKS_NAMES = [e.name for e ∈ MD_OCB_MATH]
 """
 MD_OCB_NO_INNER
 
-List of names of blocks which will deactivate any block contained within them.
+List of names of blocks which will deactivate any block contained within them
+as their content will be reprocessed later on.
 See [`find_all_ocblocks`](@ref).
 """
-const MD_OCB_NO_INNER = vcat(MD_OCB_ESC, MATH_BLOCKS_NAMES, :LXB)
+const MD_OCB_NO_INNER = vcat(MD_OCB_ESC, MATH_BLOCKS_NAMES, :LXB, MD_HEADER)
diff --git a/src/parser/ocblocks.jl b/src/parser/ocblocks.jl
index 31a90a0a4..21fcee046 100644
--- a/src/parser/ocblocks.jl
+++ b/src/parser/ocblocks.jl
@@ -75,7 +75,8 @@ $(SIGNATURES)
 Convenience function to find all ocblocks e.g. such as `MD_OCBLOCKS`. Returns a
 vector of vectors of ocblocks.
 """
-function find_all_ocblocks(tokens::Vector{Token}, ocplist::Vector{OCProto}; inmath=false)
+function find_all_ocblocks(tokens::Vector{Token}, ocplist::Vector{OCProto};
+                           inmath=false)
 
     ocbs_all = Vector{OCBlock}()
     for ocp ∈ ocplist
@@ -85,36 +86,49 @@ function find_all_ocblocks(tokens::Vector{Token}, ocplist::Vector{OCProto}; inma
         ocbs, tokens = find_ocblocks(tokens, ocp; inmath=inmath)
         append!(ocbs_all, ocbs)
     end
-    # it may happen that a block is contained in a larger block
-    # there are two situations where we want to do something about it:
-    # Case 1 (solved here) where a block is inside a larger escape block
-    # Case 2 (solved in check_and_merge_indented_blocks!) when an indented
-    #   code block is contained inside a larger block
-    #
+    return ocbs_all, tokens
+end
+
+"""
+$SIGNATURES
+
+Deactivate blocks inside other blocks when the outer block is bound to be
+reprocessed. There's effectively two cases:
+1. (solved here) when a block is inside a larger 'escape' block.
+2. (solved in `check_and_merge_indented_blocks!`) when an indented code block
+    is contained inside a larger block.
+"""
+function deactivate_inner_blocks!(blocks::Vector{OCBlock})
+    # see #444 it's important to ensure the blocks are sorted and they now
+    # may not be given that we're finding them in 2 passes.
+    sort!(blocks, by=(β->from(β)))
     # CASE 1: block inside a larger escape block.
     #   this can happen if there is a code block in an escape block
     #   (see e.g. #151) or if there's indentation in a math block.
-    #   To fix this, we browse the escape blocks in backwards order and check
-    #   if there is any other block within it.
-    i = length(ocbs_all)
-    active = ones(Bool, i)
-    all_heads = from.(ocbs_all)
-    while i > 1
-        cur_ocb = ocbs_all[i]
-        if active[i] && cur_ocb.name ∈ MD_OCB_NO_INNER
-            # find all blocks within the span of this block, deactivate all of them
-            cur_head = all_heads[i]
-            cur_tail = to(cur_ocb)
-            mask = filter(j -> active[j] && cur_head < all_heads[j] < cur_tail, 1:i-1)
+    i       = 1
+    nb      = length(blocks)
+    active  = ones(Bool, nb)
+    heads   = from.(blocks)
+    while i < nb
+        cblock = blocks[i]
+        if active[i] && cblock.name ∈ MD_OCB_NO_INNER
+            # find all blocks within the span of this block, deactivate them
+            chead = heads[i]
+            ctail = to(cblock)
+            # look at all blocks starting after the current one that may
+            # be within its span (note that, at worst, we have a few thousand
+            # blocks here so that it doesn't need to be super optimised...)
+            mask = filter(j -> active[j] && chead < heads[j] < ctail, i+1:nb)
             active[mask] .= false
         end
-        i -= 1
+        i += 1
     end
-    deleteat!(ocbs_all, map(!, active))
-    return ocbs_all, tokens
+    deleteat!(blocks, map(!, active))
+    return nothing
 end
 
 
+
 """
 $(SIGNATURES)
 


Test patch:
diff --git a/test/converter/markdown3.jl b/test/converter/markdown3.jl
index 9a4d9fb5c..128f85df7 100644
--- a/test/converter/markdown3.jl
+++ b/test/converter/markdown3.jl
@@ -101,9 +101,9 @@ end
     steps = explore_md_steps(st)
     blocks, tokens = steps[:ocblocks]
     @test blocks[1].name == :CODE_INLINE
-    @test F.content(blocks[1]) == "double ` double"
+    @test F.content(blocks[1]) == "single"
+    @test F.content(blocks[2]) == "double ` double"
     @test blocks[2].name == :CODE_INLINE
-    @test F.content(blocks[2]) == "single"
 
     st = raw"""A `single` and ``double ` double`` and ``` triple ``` B"""
     steps = explore_md_steps(st)
@@ -116,23 +116,23 @@ end
     @test tokens[6].name == :CODE_TRIPLE
     @test tokens[7].name == :CODE_TRIPLE
     blocks, tokens = steps[:ocblocks]
-    @test blocks[1].name == :CODE_BLOCK
-    @test F.content(blocks[1]) == " triple "
+    @test blocks[3].name == :CODE_BLOCK
+    @test F.content(blocks[3]) == " triple "
     @test blocks[2].name == :CODE_INLINE
-    @test blocks[3].name == :CODE_INLINE
+    @test blocks[1].name == :CODE_INLINE
 
     st = raw"""A `single` and ``double ` double`` and ``` triple `` triple```
                and ```julia 1+1``` and `single again` done"""
     steps = explore_md_steps(st)
     blocks, _ = steps[:ocblocks]
-    @test blocks[1].name == :CODE_BLOCK_LANG
-    @test F.content(blocks[1]) == " 1+1"
-    @test blocks[2].name == :CODE_BLOCK
-    @test F.content(blocks[2]) == " triple `` triple"
-    @test blocks[3].name == :CODE_INLINE
-    @test F.content(blocks[3]) == "double ` double"
-    @test blocks[4].name == :CODE_INLINE
-    @test F.content(blocks[4]) == "single"
+    @test blocks[4].name == :CODE_BLOCK_LANG
+    @test F.content(blocks[4]) == " 1+1"
+    @test blocks[3].name == :CODE_BLOCK
+    @test F.content(blocks[3]) == " triple `` triple"
+    @test blocks[2].name == :CODE_INLINE
+    @test F.content(blocks[2]) == "double ` double"
+    @test blocks[1].name == :CODE_INLINE
+    @test F.content(blocks[1]) == "single"
 end
 
 @testset "\\ and \`" begin # see issue 203
diff --git a/test/global/ordering.jl b/test/global/ordering.jl
index 4f99ffb69..82cfe9f81 100644
--- a/test/global/ordering.jl
+++ b/test/global/ordering.jl
@@ -60,8 +60,8 @@ end
     steps = st |> explore_md_steps
     blocks, = steps[:ocblocks]
     @test length(blocks) == 2
-    @test blocks[1].name == :COMMENT
-    @test blocks[2].name == :MATH_EQA
+    @test blocks[1].name == :MATH_EQA
+    @test blocks[2].name == :COMMENT
 
     @test isapproxstr(st |> seval, raw"""
             <p>A
diff --git a/test/parser/latex++.jl b/test/parser/latex++.jl
new file mode 100644
index 000000000..fec61d182
--- /dev/null
+++ b/test/parser/latex++.jl
@@ -0,0 +1,22 @@
+# Let's test latex commands to death... especially in light of #444
+
+@testset "lx++1" begin
+    # 444
+    s = "\\newcommand{\\note}[1]{#1} \\note{A `B` C} D" |> fd2html_td
+    @test isapproxstr(s, """
+        A <code>B</code> C D
+        """)
+    s = raw"""
+        \newcommand{\note}[1]{@@note #1 @@}
+        \note{A}
+        \note{A `B` C}
+        \note{A @@cc B @@ D}
+        \note{A @@cc B `D` E @@ F}
+        """ |> fd2html_td
+    @test isapproxstr(s, """
+        <div class="note">A</div>
+        <div class="note">A <code>B</code> C</div>
+        <div class="note">A <div class="cc">B</div> D</div>
+        <div class="note">A <div class="cc">B <code>D</code> E</div> F</div>
+        """)
+end
diff --git a/test/parser/markdown+latex.jl b/test/parser/markdown+latex.jl
index 266c099e8..0f384b611 100644
--- a/test/parser/markdown+latex.jl
+++ b/test/parser/markdown+latex.jl
@@ -24,27 +24,26 @@ end
 
     steps = explore_md_steps(st)
     blocks, tokens = steps[:ocblocks]
-    braces = filter(β -> β.name == :LXB, blocks)
-
-    # escape block
-    β = blocks[2]
-    @test β.name == :ESCAPE
-    @test β.ss == "~~~\nescape block\n~~~"
 
     # inline code block
     β = blocks[1]
     @test β.name == :CODE_INLINE
     @test β.ss == "`code`"
 
-    # brace block
-    β = braces[1]
-    @test β.name == :LXB
-    @test β.ss == "{target}"
-
     # div block
-    β = blocks[4]
+    β = blocks[2]
     @test β.name == :DIV
     @test β.ss == "@@dname block @@"
+
+    # escape block
+    β = blocks[3]
+    @test β.name == :ESCAPE
+    @test β.ss == "~~~\nescape block\n~~~"
+
+    # escape block
+    β = blocks[4]
+    @test β.name == :LXB
+    @test β.ss == "{target}"
 end
 
 
@@ -96,8 +95,8 @@ end
     @test lxdefs[3].narg == 0
     @test lxdefs[3].def  == "\\mathbb R"
 
-    @test blocks[2].name == :ESCAPE
-    @test blocks[1].name == :CODE_BLOCK_LANG
+    @test blocks[1].name == :ESCAPE
+    @test blocks[2].name == :CODE_BLOCK_LANG
 
     lxcoms, tokens = F.find_lxcoms(tokens, lxdefs, braces)
 
@@ -215,12 +214,12 @@ end
 
     lxdefs, tokens, braces, blocks, lxcoms = explore_md_steps(st)[:latex]
 
-    @test blocks[1].name == :COMMENT
-    @test F.content(blocks[1]) == " comment "
+    @test blocks[1].name == :MD_DEF
+    @test F.content(blocks[1]) == " title = \"Convex Optimisation I\""
+    @test blocks[2].name == :COMMENT
+    @test F.content(blocks[2]) == " comment "
     @test blocks[3].name == :H2
     @test F.content(blocks[3]) == " blah <!-- ✅ 19/9/999 -->"
-    @test blocks[2].name == :MD_DEF
-    @test F.content(blocks[2]) == " title = \"Convex Optimisation I\""
 
     @test lxcoms[1].ss == "\\com{A}"
     @test lxcoms[2].ss == "\\com{B}"
diff --git a/test/runtests.jl b/test/runtests.jl
index fa9f88b14..5d26305d9 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -34,6 +34,7 @@ include("parser/2-blocks.jl")
 include("parser/markdown+latex.jl")
 include("parser/markdown-extra.jl")
 include("parser/footnotes+links.jl")
+include("parser/latex++.jl")
 println("🍺")
 
 # EVAL
diff --git a/test/test_utils.jl b/test/test_utils.jl
index 49174e532..d59d8f07d 100644
--- a/test/test_utils.jl
+++ b/test/test_utils.jl
@@ -59,15 +59,21 @@ function explore_md_steps(mds)
     steps = OrderedDict{Symbol,NamedTuple}()
 
     # tokenize
-    tokens  = F.find_tokens(mds, F.MD_TOKENS, F.MD_1C_TOKENS)
-    fn_refs = F.validate_footnotes!(tokens)
+    tokens = F.find_tokens(mds, F.MD_TOKENS, F.MD_1C_TOKENS)
+    F.validate_footnotes!(tokens)
     F.validate_headers!(tokens)
+    hrules = F.find_hrules!(tokens)
     F.find_indented_blocks!(tokens, mds)
     steps[:tokenization] =  (tokens=tokens,)
-    steps[:fn_validation] = (fn_refs=fn_refs,)
 
     # ocblocks
-    blocks, tokens = F.find_all_ocblocks(tokens, F.MD_OCB_ALL)
+    blocks, tokens = F.find_all_ocblocks(tokens, F.MD_OCB)
+    toks_pre_ocb   = deepcopy(tokens)
+
+    blocks2, tokens = F.find_all_ocblocks(tokens, vcat(F.MD_OCB2, F.MD_OCB_MATH))
+    append!(blocks, blocks2)
+    F.deactivate_inner_blocks!(blocks)
+
     F.merge_indented_blocks!(blocks, mds)
     F.filter_indented_blocks!(blocks)
     steps[:ocblocks] = (blocks=blocks, tokens=tokens)
@@ -87,10 +93,16 @@ function explore_md_steps(mds)
     steps[:latex] = (lxdefs=lxdefs, tokens=tokens, braces=braces,
                      blocks=blocks, lxcoms=lxcoms)
 
+    dbb = F.find_double_brace_blocks(toks_pre_ocb)
+
     sp_chars = F.find_special_chars(tokens)
     steps[:spchars] = (spchars=sp_chars,)
 
-    b2insert = F.merge_blocks(lxcoms, F.deactivate_divs(blocks), sp_chars)
+    fnrefs = filter(τ -> τ.name == :FOOTNOTE_REF, tokens)
+    steps[:fnrefs] = (fnrefs=fnrefs,)
+
+    b2insert = F.merge_blocks(lxcoms, F.deactivate_divs(blocks),
+                              sp_chars, fnrefs, dbb, hrules)
     steps[:b2insert] = (b2insert=b2insert,)
 
     inter_md, mblocks = F.form_inter_md(mds, b2insert, lxdefs)