Repository: tlienart/JuDoc.jl
Pull request #312 (2019-12-18 17:27:57)

Improvements to parser
* remove the hacky ` * J.EOS` trick to mark the end of the page string which was unnecessary and amounted  to copying big strings around too much
* cleaner solution to #310 to make sure that indented lines are grouped and preceded by a blank line, this reduces risks of clashes
* more tests and general improvements for the parser.

**Note**:
- does not affect users unless you had been using the non-markdown-kosher indented-lines without adding a blank line before the code, this is now not allowed.
- eventually I'll go through the Common  Mark test cases [here](https://spec.commonmark.org/0.29/spec.json) we can't expect to pass everything as we don't cover everything (either us or Julia Markdown) but it'd be good to check what and see what can be covered without too much  work.


Patch:
diff --git a/src/JuDoc.jl b/src/JuDoc.jl
index edf70ab3..72ae47d4 100644
--- a/src/JuDoc.jl
+++ b/src/JuDoc.jl
@@ -81,6 +81,7 @@ include("build.jl") # check if user has Node/minify
 
 # PARSING
 include("parser/tokens.jl")
+include("parser/indent.jl")
 include("parser/ocblocks.jl")
 # > latex
 include("parser/lx_tokens.jl")
diff --git a/src/converter/html.jl b/src/converter/html.jl
index 273a482b..fcf3814e 100644
--- a/src/converter/html.jl
+++ b/src/converter/html.jl
@@ -51,7 +51,7 @@ function jd2html(st::AbstractString; internal::Bool=false, dir::String="")::Stri
         def_GLOBAL_LXDEFS!()
         def_GLOBAL_PAGE_VARS!()
     end
-    m, v = convert_md(st * EOS, collect(values(GLOBAL_LXDEFS)); isinternal=internal)
+    m, v = convert_md(st, collect(values(GLOBAL_LXDEFS)); isinternal=internal)
     h = convert_html(m, v)
     return h
 end
diff --git a/src/converter/lx.jl b/src/converter/lx.jl
index ba945b09..7cdb8413 100644
--- a/src/converter/lx.jl
+++ b/src/converter/lx.jl
@@ -40,7 +40,7 @@ function resolve_lxcom(lxc::LxCom, lxdefs::Vector{LxDef}; inmath::Bool=false)::S
             # \command#1 --> \command blah and no error.
             partial = replace(partial, "#$argnum" => " " * content_)
         end
-        partial = ifelse(inmath, mathenv(partial), partial) * EOS
+        partial = ifelse(inmath, mathenv(partial), partial)
     end
     # reprocess (we don't care about jd_vars=nothing)
     plug, _ = convert_md(partial, lxdefs, isrecursive=true, isconfig=false, has_mddefs=false)
@@ -312,7 +312,7 @@ function resolve_lx_input_plainoutput(rpath::AS, reproc::Bool=false; code::Bool=
     content = read(out_file, String)
     isempty(content) && return ""
     reproc || return html_code(content)
-    return content * EOS
+    return content
 end
 
 
@@ -330,7 +330,7 @@ function resolve_lx_input_textfile(rpath::AS)::String
     isfile(fpath) || throw(ErrorException("I found a \\textinput but no relevant file."))
     content = read(fpath, String)
     isempty(content) && return ""
-    return content * EOS
+    return content
 end
 
 
diff --git a/src/converter/lx_simple.jl b/src/converter/lx_simple.jl
index 8adcbd6e..374c724b 100644
--- a/src/converter/lx_simple.jl
+++ b/src/converter/lx_simple.jl
@@ -67,7 +67,7 @@ function resolve_lx_literate(lxc::LxCom)::String
     end
     # if haschanged=true and not full pass then this will be handled cell by cell
     # comparing with cell files following `eval_and_resolve_code`
-    return read(opath, String) * EOS
+    return read(opath, String)
 end
 
 
diff --git a/src/converter/md.jl b/src/converter/md.jl
index 6716f652..811b10a1 100644
--- a/src/converter/md.jl
+++ b/src/converter/md.jl
@@ -16,7 +16,7 @@ well as a dictionary of page variables.
 * `isconfig=false`:    a bool indicating whether the file to convert is the configuration file
 * `has_mddefs=true`:   a bool indicating whether to look for definitions of page variables
 """
-function convert_md(mds::String, pre_lxdefs::Vector{LxDef}=Vector{LxDef}();
+function convert_md(mds::AS, pre_lxdefs::Vector{LxDef}=Vector{LxDef}();
                     isrecursive::Bool=false, isinternal::Bool=false,
                     isconfig::Bool=false, has_mddefs::Bool=true
                     )::Tuple{String,Union{Nothing,PageVars}}
@@ -29,6 +29,9 @@ function convert_md(mds::String, pre_lxdefs::Vector{LxDef}=Vector{LxDef}();
         def_PAGE_LINK_DEFS!()   # page-specific link definition candidates [..]: (...)
     end
 
+    # if it's a substring, force it to a string
+    mds = String(mds)
+
     #
     # Parsing of the markdown string
     # (to find latex command, latex definitions, math envs etc.)
@@ -44,6 +47,7 @@ function convert_md(mds::String, pre_lxdefs::Vector{LxDef}=Vector{LxDef}();
     #> 1b. Find indented blocks (ONLY if not recursive to avoid ambiguities!)
     if !isrecursive
         find_indented_blocks!(tokens, mds)
+        filter_lr_indent!(tokens, mds)
     end
 
     # -----------------------------------------------------------------------------------
@@ -132,7 +136,9 @@ yet been defined.
 * `lxdefs`: existing latex definitions prior to the math block
 * `offset`: where the mathblock is with respect to the parent string
 """
-function convert_md_math(ms::String, lxdefs::Vector{LxDef}=Vector{LxDef}(), offset::Int=0)::String
+function convert_md_math(ms::AS, lxdefs::Vector{LxDef}=Vector{LxDef}(), offset::Int=0)::String
+    # if a substring is given, copy it as string
+    ms = String(ms)
     #
     # Parsing of the markdown string
     # (to find latex command, latex definitions, math envs etc.)
@@ -155,7 +161,7 @@ function convert_md_math(ms::String, lxdefs::Vector{LxDef}=Vector{LxDef}(), offs
 
     htmls = IOBuffer()
 
-    strlen   = prevind(ms, lastindex(ms))
+    strlen   = lastindex(ms)
     len_lxc  = length(lxcoms)
     next_lxc = iszero(len_lxc) ? BIG_INT : from(lxcoms[1])
 
@@ -172,7 +178,7 @@ function convert_md_math(ms::String, lxdefs::Vector{LxDef}=Vector{LxDef}(), offs
         next_lxc = from_ifsmaller(lxcoms, lxc_idx, len_lxc)
     end
     # add anything after the last command
-    (head ≤ strlen) && write(htmls, chop(ms, head=prevind(ms, head), tail=1))
+    (head ≤ strlen) && write(htmls, chop(ms, head=prevind(ms, head), tail=0))
     return String(take!(htmls))
 end
 
@@ -203,8 +209,7 @@ that a piece will need to be plugged in there later.
 """
 function form_inter_md(mds::AS, blocks::Vector{<:AbstractBlock},
                       lxdefs::Vector{LxDef})::Tuple{String, Vector{AbstractBlock}}
-    # final character is the EOS character
-    strlen  = prevind(mds, lastindex(mds))
+    strlen  = lastindex(mds)
     intermd = IOBuffer()
     # keep track of the matching blocks for each insert
     mblocks = Vector{AbstractBlock}()
diff --git a/src/converter/md_blocks.jl b/src/converter/md_blocks.jl
index 7c672130..9eb2d6f6 100644
--- a/src/converter/md_blocks.jl
+++ b/src/converter/md_blocks.jl
@@ -25,8 +25,7 @@ function convert_block(β::AbstractBlock, lxcontext::LxContext)::AS
     # Div block --> need to process the block as a sub-element
     if βn == :DIV
         raw_ct = strip(content(β))
-        ct, _ = convert_md(raw_ct * EOS, lxcontext.lxdefs;
-                           isrecursive=true, has_mddefs=false)
+        ct, _ = convert_md(raw_ct, lxcontext.lxdefs; isrecursive=true, has_mddefs=false)
         name = chop(otok(β).ss, head=2, tail=0)
         return html_div(name, ct)
     end
@@ -92,7 +91,7 @@ function convert_math_block(β::OCBlock, lxdefs::Vector{LxDef})::String
         end
     end
     # assemble the katex decorators, the resolved content etc
-    write(htmls, pm[3], convert_md_math(inner * EOS, lxdefs, from(β)), pm[4])
+    write(htmls, pm[3], convert_md_math(inner, lxdefs, from(β)), pm[4])
     return String(take!(htmls))
 end
 
@@ -104,7 +103,7 @@ Helper function for the case of a header block (H1, ..., H6).
 """
 function convert_header(β::OCBlock, lxdefs::Vector{LxDef})::String
     hk       = lowercase(string(β.name)) # h1, h2, ...
-    title, _ = convert_md(content(β) * EOS, lxdefs;
+    title, _ = convert_md(content(β), lxdefs;
                           isrecursive=true, has_mddefs=false)
     rstitle  = refstring(title)
     # check if the header has appeared before and if so suggest
@@ -346,7 +345,7 @@ function convert_footnote_def(β::OCBlock, lxdefs::Vector{LxDef})::String
         return ""
     end
     # need to process the content which could contain stuff
-    ct, _ = convert_md(content(β) * EOS, lxdefs;
+    ct, _ = convert_md(content(β), lxdefs;
                        isrecursive=true, has_mddefs=false)
     """
     <table class="fndef" id="fndef:$id">
diff --git a/src/manager/file_utils.jl b/src/manager/file_utils.jl
index 66c1bd78..54636475 100644
--- a/src/manager/file_utils.jl
+++ b/src/manager/file_utils.jl
@@ -9,7 +9,7 @@ function process_config()::Nothing
     # read the config.md file if it is present
     config_path = joinpath(PATHS[:src], "config.md")
     if isfile(config_path)
-        convert_md(read(config_path, String) * EOS; isconfig=true)
+        convert_md(read(config_path, String); isconfig=true)
     else
         @warn "I didn't find a config file. Ignoring."
     end
@@ -36,7 +36,7 @@ function write_page(root::String, file::String, head::String,
      # of paths
     JD_ENV[:CUR_PATH] = fpath[lastindex(PATHS[:src])+length(PATH_SEP)+1:end]
 
-    (content, jd_vars) = convert_md(read(fpath, String) * EOS, collect(values(GLOBAL_LXDEFS)))
+    (content, jd_vars) = convert_md(read(fpath, String), collect(values(GLOBAL_LXDEFS)))
 
     # Check for RSS elements
     if GLOBAL_PAGE_VARS["generate_rss"].first && JD_ENV[:FULL_PASS] &&
diff --git a/src/misc_utils.jl b/src/misc_utils.jl
index 5cd84b5e..24514322 100644
--- a/src/misc_utils.jl
+++ b/src/misc_utils.jl
@@ -66,7 +66,7 @@ julia> ss = SubString("hello", 2:4); JuDoc.from(ss)
 ```
 """
 from(ss::SubString)::Int = nextind(str(ss), ss.offset)
-
+from(s::String) = 1
 
 """
 $(SIGNATURES)
@@ -80,6 +80,7 @@ julia> ss = SubString("hello", 2:4); JuDoc.to(ss)
 ```
 """
 to(ss::SubString)::Int = ss.offset + ss.ncodeunits
+to(s::String) = lastindex(s)
 
 """
 $(SIGNATURES)
diff --git a/src/parser/indent.jl b/src/parser/indent.jl
new file mode 100644
index 00000000..a160eeeb
--- /dev/null
+++ b/src/parser/indent.jl
@@ -0,0 +1,184 @@
+"""
+$(SIGNATURES)
+
+In initial phase, discard all `:LR_INDENT` that don't meet the requirements for an
+indented code block. I.e.: where the first  line is not preceded by a blank line and
+then subsequently followed by indented lines.
+"""
+function filter_lr_indent!(vt::Vector{Token}, s::String)::Nothing
+    tind_idx = [i for i in eachindex(vt) if vt[i].name == :LR_INDENT]
+    isempty(tind_idx) && return nothing
+    disable = zeros(Bool, length(tind_idx))
+    open = false
+    i    = 1
+    while i <= length(tind_idx)
+        t = vt[tind_idx[i]]
+        if !open
+            # check if previous line is blank
+            prev_idx = prevind(s, from(t))
+            if prev_idx < 1 || s[prev_idx] != '\n'
+                disable[i] = true
+            else
+                open = true
+            end
+            i += 1
+        else
+            # if touching previous line, keep open and go to next
+            prev = vt[tind_idx[i-1]]
+            if prev.lno + 1 == t.lno
+                i += 1
+            else
+                # otherwise close, and continue (will go to !open)
+                open = false
+                continue
+            end
+        end
+    end
+    for i in eachindex(disable)
+        disable[i] || continue
+        vt[tind_idx[i]] = Token(:LINE_RETURN, subs(vt[tind_idx[i]].ss, 1))
+    end
+    return nothing
+end
+
+"""
+$(SIGNATURES)
+
+Find markers for indented lines (i.e. a line return followed by a tab or 4 spaces).
+"""
+function find_indented_blocks!(tokens::Vector{Token}, st::String)::Nothing
+    # index of the line return tokens
+    lr_idx = [j for j in eachindex(tokens) if tokens[j].name == :LINE_RETURN]
+    remove = Int[]
+    # go over all line return tokens; if they are followed by either four spaces
+    # or by a tab, then check if the line is empty or looks like a list, otherwise
+    # change the token for a LR_INDENT token which will be captured as part of code
+    # blocks.
+    for i in 1:length(lr_idx)-1
+        # capture start and finish of the line (from line return to line return)
+        start  = from(tokens[lr_idx[i]])   # first :LINE_RETURN
+        finish = from(tokens[lr_idx[i+1]]) # next  :LINE_RETURN
+        line   = subs(st, start, finish)
+        indent = ""
+        if startswith(line, "\n    ")
+            indent = "    "
+        elseif startswith(line, "\n\t")
+            indent = "\t"
+        else
+            continue
+        end
+        # is there something on that line? if so, does it start with a list indicator
+        # like `*`, `-`, `+` or [0-9](.|\)) ? in which case this takes precedence (commonmark)
+        # TODO: document clearly that with fenced code blocks there are far fewer cases for issues
+        code_line = subs(st, nextind(st, start+length(indent)), prevind(st, finish))
+        scl       = strip(code_line)
+        isempty(scl) && continue
+        # list takes precedence (this may cause clash but then just use fenced code blocks...)
+        looks_like_a_list = scl[1] ∈ ('*', '-', '+') ||
+                            (length(scl) ≥ 2 &&
+                                scl[1] ∈ ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9') &&
+                                scl[2] ∈ ('.', ')'))
+        looks_like_a_list && continue
+        # if here, it looks like a code line (and will be considered as such)
+        tokens[lr_idx[i]] = Token(:LR_INDENT, subs(st, start, start+length(indent)), i+1)
+    end
+    return nothing
+end
+
+
+"""
+$SIGNATURES
+
+When two indented code blocks follow each other and there's nothing in between (empty line(s)),
+merge them into a super block.
+"""
+function merge_indented_blocks!(blocks::Vector{OCBlock}, mds::AS)::Nothing
+    # indices of CODE_BLOCK_IND
+    idx = [i for i in eachindex(blocks) if blocks[i].name == :CODE_BLOCK_IND]
+    isempty(idx) && return
+    # check if they're separated by something or nothing
+    inter_space = [(subs(mds, to(blocks[idx[i]]), from(blocks[idx[i+1]])) |> strip |> length) > 0
+                    for i in 1:length(idx)-1]
+
+    curseq     = Int[] # to keep track of current list of blocks to merge
+    del_blocks = Int[] # to keep track of blocks that will be removed afterwards
+
+    # if there's no inter_space, add to the list, if there is, close and merge
+    for i in eachindex(inter_space)
+        if inter_space[i] && !isempty(curseq)
+            # close and merge all in curseq and empty curseq
+            form_super_block!(blocks, idx, curseq, del_blocks)
+        elseif !inter_space[i]
+            push!(curseq, i)
+        end
+    end
+    !isempty(curseq) && form_super_block!(blocks, idx, curseq, del_blocks)
+    # remove the blocks that have been merged
+    deleteat!(blocks, del_blocks)
+    return
+end
+
+
+"""
+$SIGNATURES
+
+Helper function to [`merge_indented_blocks`](@ref).
+"""
+function form_super_block!(blocks::Vector{OCBlock}, idx::Vector{Int},
+                           curseq::Vector{Int}, del_blocks::Vector{Int})::Nothing
+    push!(curseq, curseq[end]+1)
+    first_block = blocks[idx[curseq[1]]]
+    last_block  = blocks[idx[curseq[end]]]
+    # replace the first block with the super block
+    blocks[idx[curseq[1]]] = OCBlock(:CODE_BLOCK_IND, (otok(first_block) => ctok(last_block)))
+    # append all blocks but the first to the delete list
+    append!(del_blocks, curseq[2:end])
+    empty!(curseq)
+    return
+end
+
+
+"""
+$SIGNATURES
+
+Discard any indented block that is within a larger block to avoid ambiguities (see #285).
+"""
+function filter_indented_blocks!(blocks::Vector{OCBlock})::Nothing\
+    # retrieve the indices of the indented blocks
+    idx       = [i for i in eachindex(blocks) if blocks[i].name == :CODE_BLOCK_IND]
+    isempty(idx) && return nothing
+    # keep track of the ones that are active
+    active    = ones(Bool, length(idx))
+    # retrieve their span
+    indblocks = blocks[idx]
+    froms     = indblocks .|> from
+    tos       = indblocks .|> to
+    # retrieve the max/min span for faster processing
+    minfrom   = froms |> minimum
+    maxto     = tos |> maximum
+    update    = false
+    # go over all blocks and check if they contain an indented block
+    for block in blocks
+        # discard if the block is before or after all indented blocks
+        from_, to_ = from(block), to(block)
+        (to_ < minfrom || from_ > maxto) && continue
+        # otherwise check and deactivate if it's contained
+        for k in eachindex(active)
+            active[k] || continue
+            indblock = blocks[idx[k]]
+            if from_ < from(indblock) && to_ > to(indblock)
+                active[k] = false
+                update = true
+            end
+        end
+        if update
+            froms[.!active] .= typemax(Int)
+            tos[.!active]   .= 0
+            minfrom          = froms |> minimum
+            maxto            = tos |> maximum
+            update           = false
+        end
+    end
+    deleteat!(blocks, idx[.!active])
+    return nothing
+end
diff --git a/src/parser/md_tokens.jl b/src/parser/md_tokens.jl
index dfc4ca5a..1df5f480 100644
--- a/src/parser/md_tokens.jl
+++ b/src/parser/md_tokens.jl
@@ -62,9 +62,9 @@ const MD_TOKENS = LittleDict{Char, Vector{TokenFinder}}(
               isexactly("\\newcommand")      => :LX_NEWCOMMAND,
               incrlook((_, c) -> α(c))       => :LX_COMMAND,     # \command⎵*
              ],
-    '@'  => [ isexactly("@def", (' ',)) => :MD_DEF_OPEN,  # @def var = ...
-              isexactly("@@", SPACER)   => :DIV_CLOSE,    # @@⎵*
-              incrlook(is_div_open)     => :DIV_OPEN, # @@dname
+    '@'  => [ isexactly("@def", (' ',))   => :MD_DEF_OPEN,  # @def var = ...
+              isexactly("@@", SPACE_CHAR) => :DIV_CLOSE,    # @@⎵*
+              incrlook(is_div_open)       => :DIV_OPEN,     # @@dname
              ],
     '#'  => [ isexactly("#",      (' ',)) => :H1_OPEN, # see note [^2]
               isexactly("##",     (' ',)) => :H2_OPEN,
@@ -81,12 +81,12 @@ const MD_TOKENS = LittleDict{Char, Vector{TokenFinder}}(
     '_'  => [ isexactly("_\$>_") => :MATH_I_OPEN,   # internal use when resolving a latex command
               isexactly("_\$<_") => :MATH_I_CLOSE,  # within mathenv (e.g. \R <> \mathbb R)
              ],
-    '`'  => [ isexactly("`", ('`',), false) => :CODE_SINGLE, # `⎵
-              isexactly("``",('`',), false) => :CODE_DOUBLE, # ``⎵*
-              isexactly("```", SPACER)      => :CODE_TRIPLE, # ```⎵*
-              isexactly("`````", SPACER)    => :CODE_PENTA,  # `````⎵*
-              is_language()                 => :CODE_LANG,   # ```lang*
-              is_language2()                => :CODE_LANG2,  # `````lang*
+    '`'  => [ isexactly("`", ('`',), false)  => :CODE_SINGLE, # `⎵
+              isexactly("``",('`',), false)  => :CODE_DOUBLE, # ``⎵*
+              isexactly("```", SPACE_CHAR)   => :CODE_TRIPLE, # ```⎵*
+              isexactly("`````", SPACE_CHAR) => :CODE_PENTA,  # `````⎵*
+              is_language()                  => :CODE_LANG,   # ```lang*
+              is_language2()                 => :CODE_LANG2,  # `````lang*
              ],
     ) # end dict
 #= NOTE
diff --git a/src/parser/ocblocks.jl b/src/parser/ocblocks.jl
index 385f9da7..c0d8bdeb 100644
--- a/src/parser/ocblocks.jl
+++ b/src/parser/ocblocks.jl
@@ -125,150 +125,6 @@ function merge_blocks(lvb::Vector{<:AbstractBlock}...)
     return blocks
 end
 
-
-"""
-$(SIGNATURES)
-
-Find markers for indented lines (i.e. a line return followed by a tab or 4 spaces).
-"""
-function find_indented_blocks!(tokens::Vector{Token}, st::String)::Nothing
-    # index of the line return tokens
-    lr_idx = [j for j in eachindex(tokens) if tokens[j].name == :LINE_RETURN]
-    remove = Int[]
-    # go over all line return tokens; if they are followed by either four spaces
-    # or by a tab, then check if the line is empty or looks like a list, otherwise
-    # change the token for a LR_INDENT token which will be captured as part of code
-    # blocks.
-    for i in 1:length(lr_idx)-1
-        # capture start and finish of the line (from line return to line return)
-        start  = from(tokens[lr_idx[i]])   # first :LINE_RETURN
-        finish = from(tokens[lr_idx[i+1]]) # next  :LINE_RETURN
-        line   = subs(st, start, finish)
-        indent = ""
-        if startswith(line, "\n    ")
-            indent = "    "
-        elseif startswith(line, "\n\t")
-            indent = "\t"
-        else
-            continue
-        end
-        # is there something on that line? if so, does it start with a list indicator
-        # like `*`, `-`, `+` or [0-9](.|\)) ? in which case this takes precedence (commonmark)
-        # TODO: document clearly that with fenced code blocks there are far fewer cases for issues
-        code_line = subs(st, nextind(st, start+length(indent)), prevind(st, finish))
-        scl       = strip(code_line)
-        isempty(scl) && continue
-        # list takes precedence (this may cause clash but then just use fenced code blocks...)
-        looks_like_a_list = scl[1] ∈ ('*', '-', '+') ||
-                            (length(scl) ≥ 2 &&
-                                scl[1] ∈ ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9') &&
-                                scl[2] ∈ ('.', ')'))
-        looks_like_a_list && continue
-        # if here, it looks like a code line (and will be considered as such)
-        tokens[lr_idx[i]] = Token(:LR_INDENT, subs(st, start, start+length(indent)))
-    end
-    return nothing
-end
-
-
-"""
-$SIGNATURES
-
-When two indented code blocks follow each other and there's nothing in between (empty line(s)),
-merge them into a super block.
-"""
-function merge_indented_blocks!(blocks::Vector{OCBlock}, mds::String)::Nothing
-    # indices of CODE_BLOCK_IND
-    idx = [i for i in eachindex(blocks) if blocks[i].name == :CODE_BLOCK_IND]
-    isempty(idx) && return
-    # check if they're separated by something or nothing
-    inter_space = [(subs(mds, to(blocks[idx[i]]), from(blocks[idx[i+1]])) |> strip |> length) > 0
-                    for i in 1:length(idx)-1]
-
-    curseq     = Int[] # to keep track of current list of blocks to merge
-    del_blocks = Int[] # to keep track of blocks that will be removed afterwards
-
-    # if there's no inter_space, add to the list, if there is, close and merge
-    for i in eachindex(inter_space)
-        if inter_space[i] && !isempty(curseq)
-            # close and merge all in curseq and empty curseq
-            form_super_block!(blocks, idx, curseq, del_blocks)
-        elseif !inter_space[i]
-            push!(curseq, i)
-        end
-    end
-    !isempty(curseq) && form_super_block!(blocks, idx, curseq, del_blocks)
-    # remove the blocks that have been merged
-    deleteat!(blocks, del_blocks)
-    return
-end
-
-
-"""
-$SIGNATURES
-
-Helper function to [`merge_indented_blocks`](@ref).
-"""
-function form_super_block!(blocks::Vector{OCBlock}, idx::Vector{Int},
-                           curseq::Vector{Int}, del_blocks::Vector{Int})::Nothing
-    push!(curseq, curseq[end]+1)
-    first_block = blocks[idx[curseq[1]]]
-    last_block  = blocks[idx[curseq[end]]]
-    # replace the first block with the super block
-    blocks[idx[curseq[1]]] = OCBlock(:CODE_BLOCK_IND, (otok(first_block) => ctok(last_block)))
-    # append all blocks but the first to the delete list
-    append!(del_blocks, curseq[2:end])
-    empty!(curseq)
-    return
-end
-
-
-"""
-$SIGNATURES
-
-Discard any indented block that is within a larger block to avoid ambiguities (see #285).
-"""
-function filter_indented_blocks!(blocks::Vector{OCBlock})::Nothing\
-    # retrieve the indices of the indented blocks
-    idx       = [i for i in eachindex(blocks) if blocks[i].name == :CODE_BLOCK_IND]
-    isempty(idx) && return nothing
-    # keep track of the ones that are active
-    active    = ones(Bool, length(idx))
-    # retrieve their span
-    indblocks = blocks[idx]
-    froms     = indblocks .|> from
-    tos       = indblocks .|> to
-    # retrieve the max/min span for faster processing
-    minfrom   = froms |> minimum
-    maxto     = tos |> maximum
-    update    = false
-    # go over all blocks and check if they contain an indented block
-    for block in blocks
-        # discard if the block is before or after all indented blocks
-        from_, to_ = from(block), to(block)
-        (to_ < minfrom || from_ > maxto) && continue
-        # otherwise check and deactivate if it's contained
-        for k in eachindex(active)
-            active[k] || continue
-            indblock = blocks[idx[k]]
-            if from_ < from(indblock) && to_ > to(indblock)
-                active[k] = false
-                update = true
-            end
-        end
-        if update
-            froms[.!active] .= typemax(Int)
-            tos[.!active]   .= 0
-            minfrom          = froms |> minimum
-            maxto            = tos |> maximum
-            update           = false
-        end
-    end
-    deleteat!(blocks, idx[.!active])
-    return nothing
-end
-
-
 """
 $SIGNATURES
 
diff --git a/src/parser/tokens.jl b/src/parser/tokens.jl
index fa783a61..1f0f8b43 100644
--- a/src/parser/tokens.jl
+++ b/src/parser/tokens.jl
@@ -26,7 +26,9 @@ math expression.
 struct Token <: AbstractBlock
     name::Symbol
     ss::SubString
+    lno::Int  # for LRINDENT it's useful to store line number
 end
+Token(n, s) = Token(n, s, 0)
 
 to(β::Token) = ifelse(β.name == :EOS, from(β), to(β.ss))
 
@@ -107,7 +109,12 @@ content of a `{...}` block would be `...`.
 function content(ocb::OCBlock)::SubString
     s = str(ocb.ss) # this does not allocate
     cfrom = nextind(s, to(otok(ocb)))
-    cto = prevind(s, from(ctok(ocb)))
+    c = ctok(ocb)
+    if c.name == :EOS
+        cto = from(c)
+    else
+        cto = prevind(s, from(c))
+    end
     return subs(s, cfrom, cto)
 end
 
@@ -116,24 +123,28 @@ end
 FUNCTIONS / CONSTANTS THAT HELP DEFINE TOKENS
 =#
 
-
 """
 EOS
 
 Convenience symbol to mark the end of the string to parse (helps with corner cases where a token
 ends a document without being followed by a space).
 """
-const EOS = '⌑'
-
+const EOS = '\0'
 
 """
-SPACER
+SPACE_CHARS
 
 Convenience list of characters that would correspond to a `\\s` regex. see also
 https://github.com/JuliaLang/julia/blob/master/base/strings/unicode.jl.
 """
-const SPACER = (' ', '\n', '\t', '\v', '\f', '\r', '\u85', '\ua0', EOS)
+const SPACE_CHAR = (' ', '\n', '\t', '\v', '\f', '\r', '\u85', '\ua0', EOS)
 
+"""
+SPACER
+
+Convenience list of characters corresponding to digits.
+"""
+const NUM_CHAR   = ('1', '2', '3', '4', '5', '6', '7', '8', '9', '0')
 
 """
 $(SIGNATURES)
@@ -159,32 +170,40 @@ julia> s
 ```
 """
 function isexactly(refstring::AS, follow::NTuple{K,Char} where K = (),
-                   isfollowed=true)::Tuple{Int,Bool,Function,Nothing}
-    # number of steps from the start character
+                   isfollowed=true)::Tuple{Int,Bool,Function,Bool}
+    # number of code units from the start character
     steps = prevind(refstring, lastindex(refstring))
     # no offset (don't check next character)
-    isempty(follow) && return (steps, false, s -> (s == refstring), nothing)
-    # include next char for verification (--> offset of 1)
-    steps = nextind(refstring, steps)
+    isempty(follow) && return (steps, false, (s,_) -> (s == refstring), false)
+    # will include next char for verification (--> offset of 1)
+    steps  = lastindex(refstring)
+    nochop = !isfollowed || EOS ∈ follow
     # verification function; we want either (false false or true true))
-    λ(s) = (chop(s) == refstring) && !xor(isfollowed, s[end] ∈ follow)
-    return (steps, true, λ, nothing)
-end
+    # NOTE, chop because we will take the next char with it
+
+    λ(s, at_eos) = at_eos ?
+                    # if at_eos  then we don't get an extra char, compare as is
+                    # if isfollowed, then check EOS in follow
+                    s == refstring && !isfollowed || EOS ∈ follow :
+                    # if not, get extra char, compare  with
+                    chop(s) == refstring && !xor(isfollowed, s[end] ∈ follow)
 
+    return (steps, true, λ, nochop)
+end
 
 """
 $(SIGNATURES)
 
 Check whether `c` is a letter or is in a vector of character `ac`.
 """
-α(c::Char, ac::NTuple{K,Char}=()) where {K} = isletter(c) || (c ∈ ac)
+α(c::Char, ac::NTuple{K,Char}=()) where K = isletter(c) || (c ∈ ac)
 
 """
 $(SIGNATURES)
 
 Check whether `c` is alpha numeric or in vector of character `ac`
 """
-αη(c::Char, ac::NTuple{K,Char}=()) where {K} = α(c, tuple(ac..., ("$i"[1] for i in 0:9)...))
+αη(c::Char, ac::NTuple{K,Char}=()) where K = α(c, tuple(ac..., NUM_CHAR...))
 
 """
 $(SIGNATURES)
@@ -264,7 +283,7 @@ TokenFinder
 Convenience type to define tokens. The Tuple comes from the output of functions such as
 [`isexactly`](@ref).
 """
-const TokenFinder = Pair{Tuple{Int,Bool,Function,Union{Nothing,Function}},Symbol}
+const TokenFinder = Pair{Tuple{Int,Bool,Function,Union{Bool,Nothing,Function}},Symbol}
 
 
 """
@@ -275,33 +294,30 @@ that match specific tokens. The list of tokens found is returned.
 
 **Arguments**
 
-* `str`:          the initial text
-* `tokens_dict`:  dictionary of possible tokens (multiple characters long)
-* `stokens_dict`: dictionaro of possible tokens (single character)
+* `str`:   the initial text
+* `dictn`: dictionary of possible tokens (multiple characters long)
+* `dict1`: dictionaro of possible tokens (single character)
 """
 function find_tokens(str::AS,
-                     tokens_dict::AbstractDict{Char,Vector{TokenFinder}},
-                     stokens_dict::AbstractDict{Char,Symbol})::Vector{Token}
+                     dictn::AbstractDict{Char,Vector{TokenFinder}},
+                     dict1::AbstractDict{Char,Symbol})::Vector{Token}
+    isempty(str) && return Token[]
     # storage to keep track of the tokens found
-    tokens = Vector{Token}()
-
-    # head_idx will travel over the valid characters from first to final one
-    # excluding it (the EOS character).
-    head_idx, EOS_idx = 1, lastindex(str)
-
-    while head_idx < EOS_idx
-        # read the character and check if corresponds to start of token
-        head = str[head_idx]
-
+    tokens    = Vector{Token}()
+    head_idx  = 1                # valid string index
+    end_idx   = lastindex(str)   # outer bound
+    while head_idx <= end_idx
+        head_char = str[head_idx]
         # 1. is it one of the single-char token?
-        if haskey(stokens_dict, head)
-            push!(tokens, Token(stokens_dict[head], subs(str, head_idx)))
+        if haskey(dict1, head_char)
+            tok = Token(dict1[head_char], subs(str, head_idx))
+            push!(tokens, tok)
 
         # 2. is it one of the multi-char token?
-        elseif haskey(tokens_dict, head)
-            for ((steps, offset, λ, ν), case) ∈ tokens_dict[head]
+    elseif haskey(dictn, head_char)
+            for ((steps, offset, λ, ν), case) ∈ dictn[head_char]
                 #=
-                ↪ steps = length of the lookahead, 0 if incremental
+                ↪ steps = length of the lookahead, 0 if incremental (greedy)
                 ↪ offset = if we need to check one character 'too much'
                 (e.g. this is the case if we want to check that something
                 is followed by a space)
@@ -311,6 +327,8 @@ function find_tokens(str::AS,
                     * for an incremental lookahead, it returns true if chars
                     given meet a condition, chars after head_idx are fed while
                     the condition holds.
+                ↪ ν = the (optional) validator function in the case of a greedy
+                    lookahead to check whether the sequence is valid
 
                 Either way, we push to the 'memory' the exact span (without the
                 potential offset) of the token and a symbol indicating what it
@@ -318,39 +336,54 @@ function find_tokens(str::AS,
                 it's pushed by 1 again after the if-else-end to start again).
                 =#
                 if steps > 0 # exact match of a given fixed pattern
-                    endchar_idx = nextind(str, head_idx, steps)
-                    endchar_idx > EOS_idx && continue
-                    stack = subs(str, head_idx, endchar_idx)
-                    if λ(stack)
-                        # offset==True --> looked at 1 extra char (lookahead)
-                        head_idx = prevind(str, endchar_idx, offset)
-                        push!(tokens, Token(case, chop(stack, tail=offset)))
+                    tail_idx = nextind(str, head_idx, steps)
+                    # is there space for the fixed pattern?
+                    at_eos = false
+                    if ν && tail_idx == nextind(str, end_idx)
+                        tail_idx = end_idx
+                        at_eos = true
+                    end
+                    tail_idx > end_idx && continue
+                    # consider the substring and verify whether it matches
+                    cand_seq = subs(str, head_idx, tail_idx)
+                    if λ(cand_seq, at_eos)
+                        # if offset==True --> looked at 1 extra char (lookahead)
+                        back_one = offset & !at_eos
+                        head_idx = prevind(str, tail_idx, back_one)
+                        tok = Token(case, chop(cand_seq, tail=back_one))
+                        push!(tokens, tok)
                         # token identified, no need to check other cases.
                         break
                     end
-                else # rule-based match, greedy catch until fail
-                    stack, shift = head, 1
-                    nextchar_idx = nextind(str, head_idx)
-                    while λ(shift, str[nextchar_idx])
-                        stack = subs(str, head_idx, nextchar_idx)
-                        shift += 1
-                        nextchar_idx = nextind(str, nextchar_idx)
+                else # rule-based match: greedy catch until fail
+                    nchars     = 1
+                    tail_idx   = head_idx
+                    probe_idx  = nextind(str, head_idx)
+                    probe_idx > end_idx && continue
+                    probe_char = str[probe_idx]
+                    while λ(nchars, probe_char)
+                        tail_idx   = probe_idx
+                        probe_idx  = nextind(str, probe_idx)
+                        probe_idx > end_idx && break
+                        probe_char = str[probe_idx]
+                        nchars   += 1
                     end
-                    endchar_idx = prevind(str, nextchar_idx)
-                    if endchar_idx > head_idx
-                        # if the validator is unhappy, don't move the head and
-                        # consider other rules
-                        isnothing(ν) || ν(stack) || continue
-                        # otherwise move ahead after the match
-                        push!(tokens, Token(case, stack))
-                        head_idx = endchar_idx
+                    if tail_idx > head_idx
+                        cand_seq = subs(str, head_idx, tail_idx)
+                        # check if the validator is happy otherwise skip
+                        isnothing(ν) || ν(cand_seq) || continue
+                        # if it's happy push the token & move after the match
+                        tok = Token(case, cand_seq)
+                        push!(tokens, tok)
+                        head_idx = tail_idx
                     end
                 end
             end
         end
-        # dictionaries have been checked etc, moving on to the next valid char
-        head_idx = nextind(str, head_idx)
+        head_idx  = nextind(str, head_idx)
     end
-    push!(tokens, Token(:EOS, subs(str, EOS_idx, EOS_idx)))
+    # finally push the end token
+    eos = Token(:EOS, subs(str, end_idx))
+    push!(tokens, eos)
     return tokens
 end


Test patch:
diff --git a/test/converter/eval.jl b/test/converter/eval.jl
index 6ab5f483..0a6f7cf1 100644
--- a/test/converter/eval.jl
+++ b/test/converter/eval.jl
@@ -14,7 +14,7 @@ J.JD_ENV[:CUR_PATH] = "index.md"
         then:
         \input{output}{./code/exca1}
         done.
-        """ * J.EOS |> seval
+        """ |> seval
 
     h2 = raw"""
         Simple code:
@@ -25,7 +25,7 @@ J.JD_ENV[:CUR_PATH] = "index.md"
         then:
         \input{output}{excb1}
         done.
-        """ * J.EOS |> seval
+        """ |> seval
 
     @test h == h2
 
@@ -60,7 +60,7 @@ end
         print(a**2)
         ```
         done.
-        """ * J.EOS |> seval
+        """ |> seval
 
     @test isapproxstr(h, raw"""
             <p>Simple code:
@@ -79,7 +79,7 @@ end
         then:
         \input{output}{/assets/scripts/test2}
         done.
-        """ * J.EOS |> seval
+        """ |> seval
 
     spath = joinpath(J.PATHS[:assets], "scripts", "test2.jl")
     @test isfile(spath)
@@ -110,7 +110,7 @@ end
         then:
         \input{output}{./code/abc2}
         done.
-        """ * J.EOS |> seval
+        """ |> seval
 
     spath = joinpath(J.PATHS[:assets], "pages", "pg1", "code", "abc2.jl")
     @test isfile(spath)
@@ -139,7 +139,7 @@ end
         then:
         \input{output}{scripts/test1}
         done.
-        """ * J.EOS |> seval
+        """ |> seval
     # dot(a, a) == 54
     @test occursin("""then: <pre><code class="plaintext">54</code></pre> done.""", h)
 end
@@ -154,7 +154,7 @@ end
         then:
         \input{plot}{tv2}
         done.
-        """ * J.EOS |> seval
+        """ |> seval
     @test occursin("then: <img src=\"/assets/index/code/output/tv2.png\" alt=\"\"> done.", h)
 end
 
@@ -167,7 +167,7 @@ end
         then:
         \input{output}{scripts/test1}
         done.
-        """ * J.EOS |> seval
+        """ |> seval
     # errors silently
     @test occursin("then: <pre><code class=\"plaintext\">There was an error running the code:\nDomainError", h)
 end
@@ -179,7 +179,7 @@ end
         sqrt(-1)
         ```
         done.
-        """ * J.EOS
+        """
 
     @test (@test_logs (:warn, "Eval of non-julia code blocks is not yet supported.") h |> seval) == "<p>Simple code: <pre><code class=\"language-python\">sqrt(-1)</code></pre> done.</p>\n"
 end
@@ -198,7 +198,7 @@ end
         ```
         done.
         \output{scripts/test186}
-        """ * J.EOS |> seval
+        """ |> seval
     @test isapproxstr(h, raw"""
             <p>Simple code: <pre><code class="language-julia">fn = "tempf.jl"
             write(fn, "a = 1+1")
diff --git a/test/converter/hyperref.jl b/test/converter/hyperref.jl
index ace67067..30e7afde 100644
--- a/test/converter/hyperref.jl
+++ b/test/converter/hyperref.jl
@@ -11,7 +11,7 @@
 
        * \biblabel{amari98b}{Amari and Douglas., 1998} **Amari** and **Douglas**: *Why Natural Gradient*, 1998.
        * \biblabel{bardenet17}{Bardenet et al., 2017} **Bardenet**, **Doucet** and **Holmes**: *On Markov Chain Monte Carlo Methods for Tall Data*, 2017.
-    """ * J.EOS;
+    """;
 
     J.def_GLOBAL_PAGE_VARS!()
     J.def_GLOBAL_LXDEFS!()
@@ -58,7 +58,7 @@ end
        B
        C \eqref{eq 1}.
        and *B $E$*.
-    """ * J.EOS
+    """
     h = st |> seval
     @test occursin(raw"""<a id="eq_1"></a>\[ x = x \]""", h)
     @test occursin(raw"""<span class="eqref">(<a href="/index.html#eq_1">1</a>)</span>.""", h)
@@ -77,7 +77,7 @@ end
         but further
         \eqa{ 1 &=& 1 \label{beyond hope}}
         and finally a \eqref{eq:a trivial one} and maybe \eqref{beyond hope}.
-        """ * J.EOS
+        """
     m, _ = J.convert_md(st, collect(values(J.GLOBAL_LXDEFS)))
 
     @test J.PAGE_EQREFS[J.PAGE_EQREFS_COUNTER] == 3
diff --git a/test/converter/lx_input.jl b/test/converter/lx_input.jl
index 2d7bc4e0..47b4a715 100644
--- a/test/converter/lx_input.jl
+++ b/test/converter/lx_input.jl
@@ -43,7 +43,7 @@ end
         Finally img:
         \input{plot:a}{s1.jl}
         done.
-        """ * J.EOS;
+        """;
 
     J.def_GLOBAL_PAGE_VARS!()
     J.def_GLOBAL_LXDEFS!()
@@ -63,6 +63,6 @@ end
     st = raw"""
         Some string
         \textinput{ccc/asset1}
-        """ * J.EOS;
+        """;
     @test isapproxstr(st |> conv, "<p>Some string blah <strong>blih</strong></p>")
 end
diff --git a/test/converter/markdown.jl b/test/converter/markdown.jl
index b9cf2bb0..79f3439b 100644
--- a/test/converter/markdown.jl
+++ b/test/converter/markdown.jl
@@ -6,7 +6,7 @@
         * \com and \comb{blah}
         * $f$ is a function
         * a last element
-        """ * J.EOS
+        """
 
     steps = explore_md_steps(st)
     lxdefs, tokens, braces, blocks, lxcoms = steps[:latex]
@@ -42,7 +42,7 @@ end
         """
 
     inter_md, = explore_md_steps(st)[:inter_md]
-    @test inter_md == " ##JDINSERT## \nfinally ⊙⊙𝛴⊙ and\n ##JDINSERT## \ndone"
+    @test inter_md == " ##JDINSERT## \nfinally ⊙⊙𝛴⊙ and\n ##JDINSERT## \ndone\n"
 end
 
 
@@ -50,7 +50,7 @@ end
     st = raw"""
         a\newcommand{\eqa}[1]{\begin{eqnarray}#1\end{eqnarray}}b@@d .@@
         \eqa{\sin^2(x)+\cos^2(x) &=& 1}
-        """ * J.EOS
+        """
 
     steps = explore_md_steps(st)
     lxdefs, tokens, braces, blocks, lxcoms = steps[:latex]
@@ -63,7 +63,6 @@ end
 
     @test J.convert_block(blocks2insert[1], lxcontext) == "<div class=\"d\">.</div>"
     @test isapproxstr(J.convert_block(blocks2insert[2], lxcontext), "\\[\\begin{array}{c} \\sin^2(x)+\\cos^2(x) &=& 1\\end{array}\\]")
-
     hstring = J.convert_inter_html(inter_html, blocks2insert, lxcontext)
     @test isapproxstr(hstring, raw"""
                         <p>
@@ -83,7 +82,7 @@ end
         ~~~
         \newcommand{\comb}[ 1]{\mathrm{#1}} text C1 $\comb{b}$ text C2
         \newcommand{\comc}[ 2]{part1:#1 and part2:#2} then \comc{AA}{BB}.
-        """ * J.EOS
+        """
 
     steps = explore_md_steps(st)
     lxdefs, tokens, braces, blocks, lxcoms = steps[:latex]
diff --git a/test/converter/markdown2.jl b/test/converter/markdown2.jl
index 15e47ab6..01a2153f 100644
--- a/test/converter/markdown2.jl
+++ b/test/converter/markdown2.jl
@@ -6,22 +6,22 @@ function inter(st::String)
 end
 
 @testset "issue163" begin
-    st = raw"""A _B `C` D_ E""" * J.EOS
+    st = raw"""A _B `C` D_ E"""
     imd, ih = inter(st)
     @test imd == "A _B  ##JDINSERT##  D_ E"
     @test ih == "<p>A <em>B  ##JDINSERT##  D</em> E</p>\n"
 
-    st = raw"""A _`B` C D_ E""" * J.EOS
+    st = raw"""A _`B` C D_ E"""
     imd, ih = inter(st)
     @test imd == "A _ ##JDINSERT##  C D_ E"
     @test ih == "<p>A <em>##JDINSERT##  C D</em> E</p>\n"
 
-    st = raw"""A _B C `D`_ E""" * J.EOS
+    st = raw"""A _B C `D`_ E"""
     imd, ih = inter(st)
     @test imd == "A _B C  ##JDINSERT## _ E"
     @test ih == "<p>A <em>B C  ##JDINSERT##</em> E</p>\n"
 
-    st = raw"""A _`B` C `D`_ E""" * J.EOS
+    st = raw"""A _`B` C `D`_ E"""
     imd, ih = inter(st)
     @test imd == "A _ ##JDINSERT##  C  ##JDINSERT## _ E"
     @test ih == "<p>A <em>##JDINSERT##  C  ##JDINSERT##</em> E</p>\n"
@@ -37,7 +37,7 @@ end
         ### Goodbye!
         ## Done
         done.
-        """ * J.EOS |> seval
+        """ |> seval
     @test isapproxstr(h, raw"""
         <div class="jd-toc">
           <ol>
diff --git a/test/converter/markdown3.jl b/test/converter/markdown3.jl
index 0dc278b4..36cdb999 100644
--- a/test/converter/markdown3.jl
+++ b/test/converter/markdown3.jl
@@ -9,7 +9,7 @@
         A \ b
         ```
         done
-        """ * J.EOS
+        """
 
     steps = explore_md_steps(st)
     tokens, = steps[:tokenization]
@@ -50,7 +50,7 @@ end
         A \ b
         ```
         done
-        """ * J.EOS
+        """
     steps = explore_md_steps(st)
     tokens, = steps[:tokenization]
     @test tokens[7].name == :CHAR_LINEBREAK
@@ -64,32 +64,32 @@ end
 end
 
 @testset "Backtick" begin # see issue #205
-    st = raw"""Blah \` etc""" * J.EOS
+    st = raw"""Blah \` etc"""
     @test isapproxstr(st |> seval, "<p>Blah &#96; etc</p>")
 end
 
 @testset "HTMLEnts" begin # see issue #206
-    st = raw"""Blah &pi; etc""" * J.EOS
+    st = raw"""Blah &pi; etc"""
     @test isapproxstr(st |> seval, "<p>Blah &pi; etc</p>")
     # but ill-formed ones (either deliberately or not) will be parsed
-    st = raw"""AT&T""" * J.EOS
+    st = raw"""AT&T"""
     @test isapproxstr(st |> seval, "<p>AT&amp;T</p>")
 end
 
 @testset "DoubleTicks" begin # see issue #204
-    st = raw"""A `single` B""" * J.EOS
+    st = raw"""A `single` B"""
     steps = explore_md_steps(st)
     tokens = steps[:tokenization].tokens
     @test tokens[1].name == :CODE_SINGLE
     @test tokens[2].name == :CODE_SINGLE
 
-    st = raw"""A ``double`` B""" * J.EOS
+    st = raw"""A ``double`` B"""
     steps = explore_md_steps(st)
     tokens = steps[:tokenization].tokens
     @test tokens[1].name == :CODE_DOUBLE
     @test tokens[2].name == :CODE_DOUBLE
 
-    st = raw"""A `single` and ``double`` B""" * J.EOS
+    st = raw"""A `single` and ``double`` B"""
     steps = explore_md_steps(st)
     tokens = steps[:tokenization].tokens
     @test tokens[1].name == :CODE_SINGLE
@@ -97,7 +97,7 @@ end
     @test tokens[3].name == :CODE_DOUBLE
     @test tokens[4].name == :CODE_DOUBLE
 
-    st = raw"""A `single` and ``double ` double`` B""" * J.EOS
+    st = raw"""A `single` and ``double ` double`` B"""
     steps = explore_md_steps(st)
     blocks, tokens = steps[:ocblocks]
     @test blocks[1].name == :CODE_INLINE
@@ -105,7 +105,7 @@ end
     @test blocks[2].name == :CODE_INLINE
     @test J.content(blocks[2]) == "single"
 
-    st = raw"""A `single` and ``double ` double`` and ``` triple ``` B""" * J.EOS
+    st = raw"""A `single` and ``double ` double`` and ``` triple ``` B"""
     steps = explore_md_steps(st)
     tokens = steps[:tokenization].tokens
     @test tokens[1].name == :CODE_SINGLE
@@ -122,7 +122,7 @@ end
     @test blocks[3].name == :CODE_INLINE
 
     st = raw"""A `single` and ``double ` double`` and ``` triple `` triple```
-               and ```julia 1+1``` and `single again` done""" * J.EOS
+               and ```julia 1+1``` and `single again` done"""
     steps = explore_md_steps(st)
     blocks, _ = steps[:ocblocks]
     @test blocks[1].name == :CODE_BLOCK_LANG
@@ -136,7 +136,7 @@ end
 end
 
 @testset "\\ and \`" begin # see issue 203
-    st = raw"""The `"Hello\n"` after the `readall` command is a returned value, whereas the `Hello` after the `run` command is printed output.""" * J.EOS
+    st = raw"""The `"Hello\n"` after the `readall` command is a returned value, whereas the `Hello` after the `run` command is printed output."""
     st |> seval
     @test isapproxstr(st |> seval, raw"""
                         <p>The <code>&quot;Hello\n&quot;</code> after
@@ -152,7 +152,7 @@ end
             1. you can introduce definitions using `\newcommand`
             1. you can use hyper-references with `\eqref`, `\cite`, ...
             1. you can show nice maths (via KaTeX)
-            """ * J.EOS
+            """
 
     @test isapproxstr(st |> seval, raw"""
                     <p>Essentially three things are imitated from LaTeX</p>
@@ -177,7 +177,7 @@ end
         [Python]: https://www.python.org/
         [1]: http://slashdot.org/
         end
-        """ * J.EOS
+        """
     @test isapproxstr(st |> seval, """
                         <p>
                         A <a href="https://julialang.org/">link</a> and
@@ -193,7 +193,7 @@ end
         blah
         [link]: https://julialang.org/
         [id]: ./path/to/img.png
-        """ * J.EOS
+        """
 
     @test isapproxstr(st |> seval, """
                       <p>
@@ -212,7 +212,7 @@ end
         [link]: https://julialang.org/
         [id]: ./path/to/img.png
         [not]: https://www.mozilla.org/
-        """ * J.EOS
+        """
 
     @test isapproxstr(st |> seval, """
                       <p>
@@ -226,6 +226,7 @@ end
 @testset "IndCode" begin # issue 207
     st = raw"""
         A
+
             a = 1+1
             if a > 1
                 @show a
@@ -233,7 +234,7 @@ end
             b = 2
             @show a+b
         end
-        """ * J.EOS
+        """
     @test isapproxstr(st |> seval, raw"""
         <p>
             A
@@ -248,13 +249,14 @@ end
 
     st = raw"""
         A `single` and ```python blah``` and
+
             a = 1+1
         then
         * blah
             + blih
             + bloh
         end
-        """ * J.EOS
+        """
     @test isapproxstr(st |> seval, raw"""
                         <p>
                         A <code>single</code> and
@@ -294,7 +296,7 @@ end
             end
 
         C
-        """ * J.EOS
+        """
     isapproxstr(st |> seval, raw"""
                             <p>A <pre><code class="language-julia">function foo()
 
@@ -316,7 +318,7 @@ end
 @testset "More ``" begin
     st = raw"""
          A ``blah``.
-         """ * J.EOS
+         """
     @test isapproxstr(st |> seval, """<p>A <code>blah</code>.</p>""")
 end
 
diff --git a/test/global/cases1.jl b/test/global/cases1.jl
index 2787bb79..44ec727a 100644
--- a/test/global/cases1.jl
+++ b/test/global/cases1.jl
@@ -3,7 +3,7 @@
         \newcommand{ \coma }[ 1]{hello #1}
         \newcommand{ \comb} [2 ]{\coma{#1}, goodbye #1, #2!}
         Then \comb{auth1}{auth2}.
-        """ * J.EOS
+        """
     @test isapproxstr(st |> conv,
             """<p>
             Then hello auth1, goodbye auth1, auth2$(Markdown.htmlesc("!")).
@@ -18,7 +18,7 @@ end
         \newcommand{\R}{\mathbb R}
         Then something like
         \eqa{ \E{f(X)} \in \R &\text{if}& f:\R\maptso\R }
-        """ * J.EOS
+        """
     @test isapproxstr(st |> conv,
             """<p>
             Then something like
@@ -33,7 +33,7 @@ end
         \com{A}
         \newcommand{\com}[1]{◲!#1◲}
         \com{A}
-        """ * J.EOS
+        """
     steps = st |> explore_md_steps
     @test steps[:inter_md].inter_md == "\n ##JDINSERT## \n\n ##JDINSERT## \n"
     @test st |> conv == "⭒A⭒\n◲A◲"
@@ -45,7 +45,7 @@ end
         Etc and `~~~` but hey.
         @@dd but `x` and `y`? @@
         done
-        """ * JuDoc.EOS
+        """
     @test isapproxstr(st |> conv,
             """<p>
             Etc and <code>~~~</code> but hey.
@@ -64,7 +64,7 @@ end
         end
         ```
         done
-        """ * JuDoc.EOS
+        """
     @test isapproxstr(st |> conv,
             """<p>
             Some code
@@ -83,7 +83,7 @@ end
         $$
             \min_{x\in \R^n} \quad f(x)+i_C(x).
         $$
-        """ * J.EOS
+        """
     @test isapproxstr(st |> conv,
             """\\[ \\min_{x\\in \\mathbb R^n} \\quad f(x)+i_C(x). \\]""")
 end
@@ -93,7 +93,7 @@ end
     st = raw"""
         \newcommand{\com}[1]{⭒!#1⭒}
         abc\com{A}\com{B}def.
-        """ * J.EOS
+        """
     @test st |> conv == "<p>abc⭒A⭒⭒B⭒def.</p>\n"
 
     st = raw"""
@@ -103,7 +103,7 @@ end
         \com{A}\com{B}
 
         def.
-        """ * J.EOS
+        """
     @test st |> conv == "<p>abc</p>\n⭒A⭒⭒B⭒\n<p>def.</p>\n"
 
     st = raw"""
@@ -113,7 +113,7 @@ end
         \com{A}
 
         def.
-        """ * J.EOS
+        """
     @test st |> conv == "<p>abc</p>\n⭒A⭒\n<p>def.</p>\n"
 
     st = raw"""
@@ -121,7 +121,7 @@ end
         blah \com{a}
         * \com{aaa} tt
         * ss \com{bbb}
-        """ * J.EOS
+        """
     @test isapproxstr(st |> conv,
             """<p>blah †a†
             <ul>
@@ -139,7 +139,7 @@ end
            $\scal{\mu, \nu} = \E[X]$
            exhibit B
            $\E[X] = \scal{\mu, \nu}$
-           end.""" * J.EOS
+           end."""
     @test isapproxstr(st |> conv,
             """<p>exhibit A
             \\(\\left\\langle \\mu, \\nu\\right\\rangle = \\mathbb E[X]\\)
@@ -158,7 +158,7 @@ end
         {{ if var1 }} targ1 {{ else if var2 }} targ2 {{ else }} targ3 {{ end }}
         ~~~
         done
-        """ * J.EOS
+        """
     @test stv(true, true)   |> conv == "<p>start \n targ1 \n done</p>\n"
     @test stv(false, true)  |> conv == "<p>start \n targ2 \n done</p>\n"
     @test stv(false, false) |> conv == "<p>start \n targ3 \n done</p>\n"
@@ -170,7 +170,7 @@ end
         | A | B |
         | :---: | :---: |
         | C | D |
-        """ * J.EOS
+        """
 
     if VERSION >= v"1.4.0-"
         @test isapproxstr(st |> conv,
@@ -190,7 +190,7 @@ end
         @@emptycore
         @@
         @@
-        """ * J.EOS
+        """
     st |> conv == "<div class=\"emptydiv\"><div class=\"emptycore\"></div>\n</div>\n"
 end
 
@@ -200,7 +200,7 @@ end
         Blah blah
         ## some title
         and then an anchor here \label{anchor} done.
-        """ * J.EOS
+        """
     J.def_GLOBAL_LXDEFS!()
     r = st |> conv
     @test occursin("here <a id=\"anchor\"></a> done.", r)
diff --git a/test/global/cases2.jl b/test/global/cases2.jl
index 872e6321..34861468 100644
--- a/test/global/cases2.jl
+++ b/test/global/cases2.jl
@@ -74,7 +74,7 @@ end
         3 | Facebook Social Circles | 4,039 | 88,234
 
         C
-        """ * J.EOS
+        """
 
     if VERSION >= v"1.4.0-"
         @test isapproxstr(st |> seval, raw"""<p>A</p>
diff --git a/test/global/ordering.jl b/test/global/ordering.jl
index a934b6ff..4f99ffb6 100644
--- a/test/global/ordering.jl
+++ b/test/global/ordering.jl
@@ -9,7 +9,7 @@
             indent
         D -->
         B
-        """ * J.EOS
+        """
     steps = st |> explore_md_steps
     blocks, = steps[:ocblocks]
     @test length(blocks) == 1
@@ -27,7 +27,7 @@ end
             1 + 1 &=& 2
         \end{eqnarray}
         B
-        """ * J.EOS
+        """
     steps = st |> explore_md_steps
     blocks, = steps[:ocblocks]
     @test length(blocks) == 1
@@ -56,7 +56,7 @@ end
         [blah](hello)
         -->
         C
-        """ * J.EOS
+        """
     steps = st |> explore_md_steps
     blocks, = steps[:ocblocks]
     @test length(blocks) == 2
@@ -80,7 +80,7 @@ end
              B
         }
         C
-        """ * J.EOS
+        """
     steps = st |> explore_md_steps
     blocks, = steps[:ocblocks]
 
@@ -98,7 +98,7 @@ end
 @testset "Ordering-5" begin
     st = raw"""
         A [❗️_ongoing_ ] C
-        """ * J.EOS
+        """
     @test isapproxstr(st |> seval, raw"""
         <p>A &#91;❗️<em>ongoing</em> &#93; C</p>
         """)
@@ -110,7 +110,7 @@ end
                 >> url
             -->
         C
-        """ * J.EOS
+        """
     @test isapproxstr(st |> seval, raw"""
             <p>0</p>
             <ul>
diff --git a/test/jd_paths_vars.jl b/test/jd_paths_vars.jl
index 9842cc2d..d8f3ea29 100644
--- a/test/jd_paths_vars.jl
+++ b/test/jd_paths_vars.jl
@@ -56,7 +56,7 @@ end
         @def title = "blah" <!-- comment -->
         @def hasmath = false
         etc
-        """ * J.EOS
+        """
     (m, jdv) = J.convert_md(st)
     @test jdv["title"].first == "blah"
     @test jdv["hasmath"].first == false
diff --git a/test/misc.jl b/test/misc.jl
index a413bdcd..df66f0d5 100644
--- a/test/misc.jl
+++ b/test/misc.jl
@@ -33,23 +33,23 @@ end
     steps, b, λ = J.isexactly("<!--")
     @test steps == length("<!--") - 1 # minus start char
     @test b == false
-    @test λ("<!--") == true
-    @test λ("<--") == false
+    @test λ("<!--",false) == true
+    @test λ("<--",false) == false
 
-    steps, b, λ = J.isexactly("\$", ('\$',))
+    steps, b, λ, = J.isexactly("\$", ('\$',))
     @test steps == 1
     @test b == true
-    @test λ("\$\$") == true
-    @test λ("\$a") == false
-    @test λ("a\$") == false
+    @test λ("\$\$",false) == true
+    @test λ("\$a",false) == false
+    @test λ("a\$",false) == false
 
     rs = "\$"
     steps, b, λ = J.isexactly(rs, ('\$',), false)
     @test steps == nextind(rs, prevind(rs, lastindex(rs)))
     @test b == true
-    @test λ("\$\$") == false
-    @test λ("\$a") == true
-    @test λ("a\$") == false
+    @test λ("\$\$",false) == false
+    @test λ("\$a",false) == true
+    @test λ("a\$",false) == false
 
     steps, b, λ = J.incrlook(isletter)
     @test steps == 0
diff --git a/test/parser/1-tokenize.jl b/test/parser/1-tokenize.jl
new file mode 100644
index 00000000..f806f457
--- /dev/null
+++ b/test/parser/1-tokenize.jl
@@ -0,0 +1,149 @@
+tok  = s -> J.find_tokens(s, J.MD_TOKENS, J.MD_1C_TOKENS)
+vfn  = s -> (t = tok(s); J.validate_footnotes!(t); t)
+vh   = s -> (t = vfn(s); J.validate_headers!(t); t)
+fib  = s -> (t = vh(s); J.find_indented_blocks!(t, s); t)
+fib2 = s -> (t = fib(s); J.filter_lr_indent!(t, s); t)
+
+islr(t) = t.name == :LINE_RETURN && t.ss == "\n"
+istok(t, n, s) = t.name == n && t.ss == s
+isind(t) = t.name == :LR_INDENT && t.ss == "\n    "
+
+##
+## FIND_TOKENS
+##
+
+@testset "P:1:find-tok" begin
+    t = raw"""
+    @def v = 5
+    @@da
+        @@db
+        @@
+    @@
+    $A$ and \[B\] and \com{hello} etc
+    """ |> tok
+    @test istok(t[1], :MD_DEF_OPEN, "@def")
+    @test islr(t[2])
+    @test istok(t[3], :DIV_OPEN,    "@@da")
+    @test islr(t[4])
+    @test istok(t[5], :DIV_OPEN, "@@db")
+    @test islr(t[6])
+    @test istok(t[7], :DIV_CLOSE, "@@")
+    @test islr(t[8])
+    @test istok(t[9], :DIV_CLOSE, "@@")
+    @test islr(t[10])
+    @test istok(t[11], :MATH_A, "\$")
+    @test istok(t[12], :MATH_A, "\$")
+    @test istok(t[13], :MATH_C_OPEN, "\\[")
+    @test istok(t[14], :MATH_C_CLOSE, "\\]")
+    @test istok(t[15], :LX_COMMAND, "\\com")
+    @test istok(t[16], :LXB_OPEN, "{")
+    @test istok(t[17], :LXB_CLOSE, "}")
+    @test islr(t[18])
+    @test istok(t[19], :EOS, "\n")
+end
+
+@testset "P:1:ctok" begin
+    # check that tokens at EOS close properly
+    # NOTE: this was avoided before by the addition of a special char
+    # to denote the end of the string but we don't do that anymore.
+    # see `isexactly` and `find_tokens` in the fixed pattern case.
+    t = "@@d ... @@" |> tok
+    @test istok(t[1], :DIV_OPEN, "@@d")
+    @test istok(t[2], :DIV_CLOSE, "@@")
+    @test istok(t[3], :EOS, "@")
+    t = "``` ... ```" |> tok
+    @test istok(t[1], :CODE_TRIPLE, "```")
+    @test istok(t[2], :CODE_TRIPLE, "```")
+    @test istok(t[3], :EOS, "`")
+    t = "<!--...-->" |> tok
+    @test istok(t[1], :COMMENT_OPEN, "<!--")
+    @test istok(t[2], :COMMENT_CLOSE, "-->")
+    t = "~~~...~~~" |> tok
+    @test istok(t[1], :ESCAPE, "~~~")
+    @test istok(t[2], :ESCAPE, "~~~")
+    t = "b `j`" |> tok
+    @test istok(t[1], :CODE_SINGLE, "`")
+    @test istok(t[2], :CODE_SINGLE, "`")
+    @test istok(t[3], :EOS, "`")
+end
+
+##
+## VALIDATE_FOOTNOTE!
+##
+
+@testset "P:1:val-fn" begin
+    t = raw"""
+    A [^B] and
+    [^B]: etc
+    """ |> vfn
+    @test istok(t[1], :FOOTNOTE_REF, "[^B]")
+    @test islr(t[2])
+    @test istok(t[3], :FOOTNOTE_DEF, "[^B]:")
+end
+
+##
+## VALIDATE_HEADERS!
+##
+
+@testset "P:1:val-hd" begin
+    t = raw"""
+    # A
+    ## B
+    and # C
+    """ |> vh
+    @test istok(t[1], :H1_OPEN, "#")
+    @test islr(t[2])
+    @test istok(t[3], :H2_OPEN, "##")
+    @test islr(t[4])
+    @test islr(t[5])
+    @test istok(t[6], :EOS, "\n")
+end
+
+##
+## FIND_INDENTED_BLOCKS!
+##
+
+@testset "P:1:fib" begin
+    s = raw"""
+    A
+
+        B1
+        B2
+        B3
+    C
+    @@da
+        @@db
+            E
+        @@
+    @@
+    E
+        F
+    G
+    """
+    t = s |> fib
+    @test islr(t[1])
+    @test isind(t[2]) && t[2].lno == 3
+    @test isind(t[3]) && t[3].lno == 4
+    @test isind(t[4]) && t[4].lno == 5 # B3
+    @test islr(t[5])
+    @test islr(t[6])
+    @test istok(t[7], :DIV_OPEN, "@@da")
+    @test isind(t[8]) && t[8].lno == 8
+    @test istok(t[9], :DIV_OPEN, "@@db")
+    @test isind(t[10]) && t[10].lno == 9  # in front of E
+    @test isind(t[11]) && t[11].lno == 10 # in front of @@
+    @test istok(t[12], :DIV_CLOSE, "@@")
+    @test islr(t[13])
+    @test istok(t[14], :DIV_CLOSE, "@@")
+    @test islr(t[15])
+    @test isind(t[16]) && t[16].lno == 13 # F
+    @test islr(t[17])
+    @test islr(t[18])
+    @test istok(t[19], :EOS, "\n")
+
+    t = s |> fib2
+    @test length(t) == 19
+
+    @test isind.([t[2], t[3], t[4]]) |> all
+    @test islr.([t[8], t[10], t[11], t[16]]) |> all
+end
diff --git a/test/parser/2-blocks.jl b/test/parser/2-blocks.jl
new file mode 100644
index 00000000..5ff91f71
--- /dev/null
+++ b/test/parser/2-blocks.jl
@@ -0,0 +1,135 @@
+tok  = s -> J.find_tokens(s, J.MD_TOKENS, J.MD_1C_TOKENS)
+vfn  = s -> (t = tok(s); J.validate_footnotes!(t); t)
+vh   = s -> (t = vfn(s); J.validate_headers!(t); t)
+fib  = s -> (t = vh(s); J.find_indented_blocks!(t, s); t)
+fib2 = s -> (t = fib(s); J.filter_lr_indent!(t, s); t)
+
+blk  = s -> (J.def_LOCAL_PAGE_VARS!(); J.find_all_ocblocks(fib2(s), J.MD_OCB_ALL))
+blk2 = s -> ((b, t) = blk(s); J.merge_indented_blocks!(b, s); b)
+blk3 = s -> (b = blk2(s); J.filter_indented_blocks!(b); b)
+
+isblk(b, n, s) = b.name == n && b.ss == s
+cont(b) = J.content(b)
+
+# Basics
+@testset "P:2:blk" begin
+    b, t = raw"""
+    A <!--
+
+        Hello
+    -->
+    Then ```julia 1+5``` and ~~~    ~~~.
+    """ |> blk
+    @test length(b) == 3
+    @test isblk(b[1], :COMMENT, "<!--\n\n    Hello\n-->")
+    @test isblk(b[2], :CODE_BLOCK_LANG, "```julia 1+5```")
+    @test cont(b[2]) == " 1+5"
+    @test isblk(b[3], :ESCAPE, "~~~    ~~~")
+end
+
+# with indentation
+@testset "P:2:blk-ind" begin
+    b, t = raw"""
+    A
+
+        B1
+        B2
+        B3
+    @@d1
+        B
+    @@
+    """ |> blk
+    @test length(b) == 2
+    @test isblk(b[1], :CODE_BLOCK_IND, "\n    B1\n    B2\n    B3\n")
+    @test isblk(b[2], :DIV, "@@d1\n    B\n@@")
+
+    b, t = raw"""
+    @@d1
+        @@d2
+            B
+        @@
+    @@
+    """ |> blk
+    @test length(b) == 1
+    @test isblk(b[1], :DIV, "@@d1\n    @@d2\n        B\n    @@\n@@")
+
+    b, t = raw"""
+    @@d1
+        @@d2
+            B
+        @@
+    @@
+    """ |> blk
+    @test length(b) == 1
+    @test isblk(b[1], :DIV, "@@d1\n    @@d2\n        B\n    @@\n@@")
+end
+
+# with indentation and grouping and filtering
+@testset "P:2:blk-indF" begin
+    b = raw"""
+    A
+
+        B1
+        B2
+
+        B3
+    C
+    """ |> blk3
+    @test isblk(b[1], :CODE_BLOCK_IND, "\n    B1\n    B2\n\n    B3\n")
+
+    b = raw"""
+    A
+
+        B
+        C
+            D
+        E
+
+    F
+
+        G
+
+
+
+        H
+    """ |> blk3
+    @test length(b) == 2
+    @test isblk(b[1], :CODE_BLOCK_IND, "\n    B\n    C\n        D\n    E\n")
+    @test isblk(b[2], :CODE_BLOCK_IND, "\n    G\n\n\n\n    H\n")
+
+    b = raw"""
+    @@d1
+
+        A
+        B
+        C
+    @@
+    """ |> blk3
+    @test length(b) == 1
+    @test isblk(b[1], :DIV, "@@d1\n\n    A\n    B\n    C\n@@")
+
+    b = raw"""
+    @@d1
+        @@d2
+            @@d3
+                B
+            @@
+        @@
+    @@
+    """ |> blk3
+    @test b[1].name == :DIV
+end
+
+@testset "P:2:blk-{}" begin
+    b = "{ABC}" |> blk3
+    @test J.content(b[1]) == "ABC"
+    b = "\\begin{eqnarray} \\sin^2(x)+\\cos^2(x) &=& 1\\end{eqnarray}" |> blk3
+    @test cont(b[1]) == " \\sin^2(x)+\\cos^2(x) &=& 1"
+    b = raw"""
+    a\newcommand{\eqa}[1]{\begin{eqnarray}#1\end{eqnarray}}b@@d .@@
+    \eqa{\sin^2(x)+\cos^2(x) &=& 1}
+    """ |> blk3
+    @test isblk(b[1], :LXB, raw"{\eqa}")
+    @test isblk(b[2], :LXB, raw"{\begin{eqnarray}#1\end{eqnarray}}")
+    @test isblk(b[3], :LXB, raw"{\sin^2(x)+\cos^2(x) &=& 1}")
+end
diff --git a/test/parser/footnotes.jl b/test/parser/footnotes.jl
index 0c585d6e..30eb632b 100644
--- a/test/parser/footnotes.jl
+++ b/test/parser/footnotes.jl
@@ -2,7 +2,7 @@
     J.JD_ENV[:CUR_PATH] = "index.md"
     st = """
         A[^1] B[^blah] C
-        """ * J.EOS
+        """
     @test isapproxstr(st |> seval, """
            <p>A<sup id="fnref:1"><a href="/index.html#fndef:1" class="fnref">[1]</a></sup>
               B<sup id="fnref:blah"><a href="/index.html#fndef:blah" class="fnref">[2]</a></sup>
@@ -13,7 +13,7 @@
         C
         [^1]: first footnote
         [^blah]: second footnote
-        """ * J.EOS
+        """
     @test isapproxstr(st |> seval, """
             <p>
                 A
diff --git a/test/parser/markdown+latex.jl b/test/parser/markdown+latex.jl
index d28692be..6b8637d5 100644
--- a/test/parser/markdown+latex.jl
+++ b/test/parser/markdown+latex.jl
@@ -1,14 +1,14 @@
 @testset "Find Tokens" begin
-    a = raw"""some markdown then `code` and @@dname block @@""" * J.EOS
+    a = raw"""some markdown then `code` and @@dname block @@"""
 
-    steps = explore_md_steps(a)
-    tokens, = steps[:tokenization]
+    tokens = J.find_tokens(a, J.MD_TOKENS, J.MD_1C_TOKENS)
 
     @test tokens[1].name == :CODE_SINGLE
     @test tokens[2].name == :CODE_SINGLE
     @test tokens[3].name == :DIV_OPEN
     @test tokens[3].ss == "@@dname"
     @test tokens[4].ss == "@@"
+    @test tokens[5].name == :EOS
 end
 
 
@@ -21,7 +21,7 @@ end
         escape block
         ~~~
         and done {target} done.
-        """ * J.EOS
+        """
 
     steps = explore_md_steps(st)
     blocks, tokens = steps[:ocblocks]
@@ -52,7 +52,7 @@ end
 @testset "Unicode lx" begin
     st = raw"""
     Call me “$x$”, not $🍕$.
-    """ * J.EOS
+    """
 
     steps = explore_md_steps(st)
     blocks, _ = steps[:ocblocks]
@@ -83,7 +83,7 @@ end
         ```latex
         \newcommand{\brol}{\mathbb B}
         ```
-        """ * J.EOS
+        """
 
     lxdefs, tokens, braces, blocks = explore_md_steps(st)[:latex]
 
@@ -112,7 +112,7 @@ end
     st = raw"""
         \newcommand{\com}{blah}
         \newcommand{\comb}[ 2]{hello #1 #2}
-        """ * J.EOS
+        """
 
     lxdefs, tokens, braces, blocks, lxcoms = explore_md_steps(st)[:latex]
 
@@ -128,12 +128,12 @@ end
     #
 
     # testing malformed newcommands
-    st = raw"""abc \newcommand abc""" * J.EOS
+    st = raw"""abc \newcommand abc"""
     tokens = J.find_tokens(st, J.MD_TOKENS, J.MD_1C_TOKENS)
     blocks, tokens = J.find_all_ocblocks(tokens, J.MD_OCB_ALL)
     # Ill formed newcommand (needs two {...})
     @test_throws J.LxDefError J.find_md_lxdefs(tokens, blocks)
-    st = raw"""abc \newcommand{abc} def""" * J.EOS
+    st = raw"""abc \newcommand{abc} def"""
     tokens = J.find_tokens(st, J.MD_TOKENS, J.MD_1C_TOKENS)
     blocks, tokens = J.find_all_ocblocks(tokens, J.MD_OCB_ALL)
     # Ill formed newcommand (needs two {...})
@@ -151,7 +151,7 @@ end
         ```
         etc \comb{blah} then maybe
         @@adiv inner part @@ final.
-        """ * J.EOS
+        """
 
     lxdefs, tokens, braces, blocks, lxcoms = explore_md_steps(st)[:latex]
 
@@ -173,7 +173,7 @@ end
     st = raw"""
         \newcommand{\comb}[1]{HH#1HH}
         etc \comb then.
-        """ * J.EOS
+        """
 
     tokens = J.find_tokens(st, J.MD_TOKENS, J.MD_1C_TOKENS)
     blocks, tokens = J.find_all_ocblocks(tokens, J.MD_OCB_ALL)
@@ -191,7 +191,7 @@ end
         ~~~
         \newcommand{\comb}[ 1]{\mathrm{#1}} text C1 $\comb{b}$ text C2
         \newcommand{\comc}[ 2]{part1:#1 and part2:#2} then \comc{AA}{BB}.
-        """ * J.EOS
+        """
 
     lxdefs, tokens, braces, blocks, lxcoms = explore_md_steps(st)[:latex]
 
@@ -212,7 +212,7 @@ end
         then some
         ## blah <!-- ✅ 19/9/999 -->
         end \com{B}.
-        """ * J.EOS
+        """
 
     lxdefs, tokens, braces, blocks, lxcoms = explore_md_steps(st)[:latex]
 
@@ -250,7 +250,7 @@ end
         5
         ###### t6
         6
-        """ * J.EOS
+        """
 
     tokens, blocks = explore_md_steps(st)[:filter]
 
@@ -276,7 +276,7 @@ end
         5
         ### t2
         6
-        """ * J.EOS |> seval
+        """ |> seval
     @test isapproxstr(h, """
         <h1 id="t1"><a href="/index.html#t1">t1</a></h1>
         1
@@ -299,7 +299,7 @@ end
         B
         ## example 2
         C
-        """ * J.EOS |> seval
+        """ |> seval
     @test  isapproxstr(h, """
         <h2 id="example"><a href="/index.html#example">example</a></h2>
         A
diff --git a/test/parser/markdown-extra.jl b/test/parser/markdown-extra.jl
index f8a9e4e0..7ffc1631 100644
--- a/test/parser/markdown-extra.jl
+++ b/test/parser/markdown-extra.jl
@@ -1,43 +1,43 @@
 @testset "Bold x*" begin # issue 223
-    h = raw"**x\***" * J.EOS |> seval
+    h = raw"**x\***" |> seval
     @test h == "<p><strong>x&#42;</strong></p>\n"
 
-    h = raw"_x\__" * J.EOS |> seval
+    h = raw"_x\__" |> seval
     @test h == "<p><em>x&#95;</em></p>\n"
 end
 
 @testset "Bold code" begin # issue 222
-    h = raw"""A **`master`** B.""" * J.EOS |> seval
+    h = raw"""A **`master`** B.""" |> seval
     @test h == "<p>A <strong><code>master</code></strong> B.</p>\n"
 end
 
 @testset "Tickssss" begin # issue 219
-    st = raw"""A `B` C""" * J.EOS
+    st = raw"""A `B` C"""
     tokens = J.find_tokens(st, J.MD_TOKENS, J.MD_1C_TOKENS)
     @test tokens[1].name == :CODE_SINGLE
     @test tokens[2].name == :CODE_SINGLE
 
-    st = raw"""A ``B`` C""" * J.EOS
+    st = raw"""A ``B`` C"""
     tokens = J.find_tokens(st, J.MD_TOKENS, J.MD_1C_TOKENS)
     @test tokens[1].name == :CODE_DOUBLE
     @test tokens[2].name == :CODE_DOUBLE
 
-    st = raw"""A ``` B ``` C""" * J.EOS
+    st = raw"""A ``` B ``` C"""
     tokens = J.find_tokens(st, J.MD_TOKENS, J.MD_1C_TOKENS)
     @test tokens[1].name == :CODE_TRIPLE
     @test tokens[2].name == :CODE_TRIPLE
 
-    st = raw"""A ````` B ````` C""" * J.EOS
+    st = raw"""A ````` B ````` C"""
     tokens = J.find_tokens(st, J.MD_TOKENS, J.MD_1C_TOKENS)
     @test tokens[1].name == :CODE_PENTA
     @test tokens[2].name == :CODE_PENTA
 
-    st = raw"""A ```b B ``` C""" * J.EOS
+    st = raw"""A ```b B ``` C"""
     tokens = J.find_tokens(st, J.MD_TOKENS, J.MD_1C_TOKENS)
     @test tokens[1].name == :CODE_LANG
     @test tokens[2].name == :CODE_TRIPLE
 
-    st = raw"""A `````b B ````` C""" * J.EOS
+    st = raw"""A `````b B ````` C"""
     tokens = J.find_tokens(st, J.MD_TOKENS, J.MD_1C_TOKENS)
     @test tokens[1].name == :CODE_LANG2
     @test tokens[2].name == :CODE_PENTA
@@ -84,6 +84,7 @@ end
     \hello
     """ |> jd2html_td
     @test isapproxstr(h, raw"""yaya  bar bar""")
+
     h = raw"""
     @@da
         @@db
diff --git a/test/runtests.jl b/test/runtests.jl
index 0a52dd44..284fd670 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -18,6 +18,8 @@ println("🍺")
 
 # PARSER folder
 println("PARSER/MD+LX")
+include("parser/1-tokenize.jl")
+include("parser/2-blocks.jl")
 include("parser/markdown+latex.jl")
 include("parser/markdown-extra.jl")
 include("parser/footnotes.jl")