Repository: tlienart/Franklin.jl
Pull request #574 (2020-08-03 08:29:21)

Emoji562
* markdown parser now includes a rule for `:...:` where if `...` is one of the base emoji code recognised by the Julia REPL then it will be directly transformed, e.g.:

```
julia> "abc :joy: etc" |> fd2html
"<p>abc 😂 etc</p>\n"
```

* minification pass now ignores the content of `_assets` (as it should have from the start)
* a new local variable `prerender` can be set to false to skip pre rendering for specific pages where it might mess with the rest of the javascript

related issues: #562 (emojis), #568 (minification) 

also patch release


Patch:
diff --git a/Project.toml b/Project.toml
index 202af42d..de31a780 100644
--- a/Project.toml
+++ b/Project.toml
@@ -1,7 +1,7 @@
 name = "Franklin"
 uuid = "713c75ef-9fc9-4b05-94a9-213340da978e"
 authors = ["Thibaut Lienart <tlienart@me.com>"]
-version = "0.9.1"
+version = "0.9.2"
 
 [deps]
 Crayons = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
@@ -17,6 +17,7 @@ Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
 NodeJS = "2bd173c7-0d6d-553b-b6af-13a54713934c"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 
 [compat]
diff --git a/src/Franklin.jl b/src/Franklin.jl
index 3525897e..66f8ffbc 100644
--- a/src/Franklin.jl
+++ b/src/Franklin.jl
@@ -19,6 +19,7 @@ import NodeJS
 import Literate
 import HTTP
 import Random
+import REPL.REPLCompletions: emoji_symbols
 
 export serve, publish, cleanpull, newsite, optimize, fd2html,
        literate_folder, verify_links, @OUTPUT, get_url
diff --git a/src/converter/html/blocks.jl b/src/converter/html/blocks.jl
index aa1fa5c2..6fd2a681 100644
--- a/src/converter/html/blocks.jl
+++ b/src/converter/html/blocks.jl
@@ -73,7 +73,7 @@ function process_html_cond(hs::AS, qblocks::Vector{AbstractBlock},
             end
         else
             # HIsPage//HIsNotPage
-            rpath = splitext(unixify(locvar("fd_rpath")))[1]
+            rpath = splitext(unixify(locvar(:fd_rpath)))[1]
             if FD_ENV[:STRUCTURE] < v"0.2"
                 # current path is relative to /src/ for instance
                 # /src/pages/blah.md -> pages/blah
@@ -82,8 +82,8 @@ function process_html_cond(hs::AS, qblocks::Vector{AbstractBlock},
                 rpath = replace(rpath, Regex("^pages") => "pub")
             end
             # are we currently on a tag page?
-            if !isempty(locvar("fd_tag"))
-                tag = locvar("fd_tag")
+            if !isempty(locvar(:fd_tag))
+                tag = locvar(:fd_tag)
                 if FD_ENV[:STRUCTURE] < v"0.2"
                     rpath = "/pub/tag/$tag/"
                 else
diff --git a/src/converter/html/html.jl b/src/converter/html/html.jl
index c1ec5f61..0f69763c 100644
--- a/src/converter/html/html.jl
+++ b/src/converter/html/html.jl
@@ -26,7 +26,7 @@ function convert_html(hs::AS; isoptim::Bool=false)::String
     # See issue #204, basically not all markdown links are processed  as
     # per common mark with the JuliaMarkdown, so this is a patch that kind
     # of does
-    if locvar("reflinks")
+    if locvar(:reflinks)
         fhs = find_and_fix_md_links(fhs)
     end
     isempty(fhs) && return ""
diff --git a/src/converter/latex/hyperrefs.jl b/src/converter/latex/hyperrefs.jl
index 82766913..d3c64e8d 100644
--- a/src/converter/latex/hyperrefs.jl
+++ b/src/converter/latex/hyperrefs.jl
@@ -82,7 +82,7 @@ function lx_biblabel(lxc::LxCom, _)::String
 end
 
 function lx_toc(::LxCom, _)
-    minlevel = locvar("mintoclevel")
-    maxlevel = locvar("maxtoclevel")
+    minlevel = locvar(:mintoclevel)
+    maxlevel = locvar(:maxtoclevel)
     return "{{toc $minlevel $maxlevel}}"
 end
diff --git a/src/converter/markdown/md.jl b/src/converter/markdown/md.jl
index e51c36b3..cd9f1ce0 100644
--- a/src/converter/markdown/md.jl
+++ b/src/converter/markdown/md.jl
@@ -43,12 +43,16 @@ function convert_md(mds::AbstractString,
     #> 1. Tokenize
     tokens = find_tokens(mds, MD_TOKENS, MD_1C_TOKENS)
     (:convert_md, "convert_md: '$([t.name for t in tokens])'") |> logger
+
     # distinguish fnref/fndef
     validate_footnotes!(tokens)
     # ignore header tokens that are not at the start of a line
     validate_headers!(tokens)
+    # validate emojis
+    validate_emojis!(tokens)
     # capture some hrule (see issue #432)
     hrules = find_hrules!(tokens)
+
     #> 1b. Find indented blocks (ONLY if not recursive to avoid ambiguities!)
     if !isrecursive
         find_indented_blocks!(tokens, mds)
@@ -122,8 +126,8 @@ function convert_md(mds::AbstractString,
     end
 
     # ------------------------------------------------------------------------
-    #> 5. Process special characters and html entities so that they can be
-    # injected as they are in the HTML later
+    #> 5. Process special characters, emojis and html entities so that they
+    # can be injected as they are in the HTML later
     sp_chars = find_special_chars(tokens)
 
     # ========================================================================
@@ -134,8 +138,8 @@ function convert_md(mds::AbstractString,
     # and add them to the blocks to insert
     fnrefs = filter(τ -> τ.name == :FOOTNOTE_REF, tokens)
 
-    # Discard indented blocks unless locvar("indented_code")
-    if !locvar("indented_code")
+    # Discard indented blocks unless locvar(:indented_code)
+    if !locvar(:indented_code)
         filter!(b -> b.name != :CODE_BLOCK_IND, blocks)
     end
 
@@ -157,10 +161,10 @@ function convert_md(mds::AbstractString,
     (:convert_md, "hstring: '$hstring'") |> logger
 
     # final var adjustment, infer title if not given
-    if isnothing(locvar("title")) && !isempty(PAGE_HEADERS)
+    if isnothing(locvar(:title)) && !isempty(PAGE_HEADERS)
         title = first(values(PAGE_HEADERS))[1]
         set_var!(LOCAL_VARS, "title", title)
-        ALL_PAGE_VARS[splitext(locvar("fd_rpath"))[1]]["title"] =
+        ALL_PAGE_VARS[splitext(locvar(:fd_rpath))[1]]["title"] =
             deepcopy(LOCAL_VARS["title"])
     end
 
diff --git a/src/converter/markdown/mddefs.jl b/src/converter/markdown/mddefs.jl
index 5815b490..34fb91fa 100644
--- a/src/converter/markdown/mddefs.jl
+++ b/src/converter/markdown/mddefs.jl
@@ -34,7 +34,7 @@ function process_mddefs(blocks::Vector{OCBlock}, isconfig::Bool,
     end
 
     # if in config file, update `GLOBAL_VARS` and return
-    rpath = splitext(locvar("fd_rpath"))[1]
+    rpath = splitext(locvar(:fd_rpath))[1]
     if isconfig
         set_vars!(GLOBAL_VARS, assignments)
         return nothing
@@ -42,7 +42,7 @@ function process_mddefs(blocks::Vector{OCBlock}, isconfig::Bool,
 
     # otherwise set local vars
     set_vars!(LOCAL_VARS, assignments)
-    rpath = splitext(locvar("fd_rpath"))[1]
+    rpath = splitext(locvar(:fd_rpath))[1]
 
     hasmath = locvar(:hasmath)
     hascode = locvar(:hascode)
@@ -64,7 +64,7 @@ function process_mddefs(blocks::Vector{OCBlock}, isconfig::Bool,
     (:process_mddefs, "assignments done & copy to ALL_PAGE_VARS") |> logger
 
     # TAGS
-    tags = Set(unique(locvar("tags")))
+    tags = Set(unique(locvar(:tags)))
     # Cases:
     # 0. there was no page tags before
     #   a. tags is empty --> do nothing
diff --git a/src/eval/codeblock.jl b/src/eval/codeblock.jl
index f9189256..ef53bd66 100644
--- a/src/eval/codeblock.jl
+++ b/src/eval/codeblock.jl
@@ -40,11 +40,11 @@ function should_eval(code::AS, rpath::AS)
     FD_ENV[:FORCE_REEVAL] && return true
 
     # 2. local setting forcing the current page to reeval everything
-    locvar("reeval") && return true
+    locvar(:reeval) && return true
 
     # 3. if space previously marked as stale, return true
     # note that on every page build, this is re-init as false.
-    locvar("fd_eval") && return true
+    locvar(:fd_eval) && return true
 
     # 4. if the code has changed reeval
     cp = form_codepaths(rpath)
@@ -93,7 +93,7 @@ function resolve_code_block(ss::SubString)::String
         # 3. here we have code that should be (re)evaluated
         # >> retrieve the modulename, the module may not exist
         # (& may not need to)
-        modname = modulename(locvar("fd_rpath"))
+        modname = modulename(locvar(:fd_rpath))
         # >> check if relevant module exists, otherwise create one
         mod = ismodule(modname) ?
                 getfield(Main, Symbol(modname)) :
@@ -118,7 +118,7 @@ function resolve_code_block(ss::SubString)::String
         set_var!(LOCAL_VARS, "fd_eval", true)
     end
     # >> finally return as html
-    if locvar("showall")
+    if locvar(:showall)
         return html_code(code, lang) *
                 reprocess("\\show{$rpath}", [GLOBAL_LXDEFS["\\show"]])
     end
diff --git a/src/eval/io.jl b/src/eval/io.jl
index bd359327..2ab4a91c 100644
--- a/src/eval/io.jl
+++ b/src/eval/io.jl
@@ -97,7 +97,7 @@ function parse_rpath(rpath::AS; canonical::Bool=false, code::Bool=false)::AS
         end
         # if the locvar ends with `index.md` but is not strictly
         # index.md then remove it
-        loc_rpath = locvar("fd_rpath")
+        loc_rpath = locvar(:fd_rpath)
         if endswith(loc_rpath, "index.md") && loc_rpath != "index.md"
             loc_rpath = replace(loc_rpath, r"index\.md$" => "")
         end
@@ -114,7 +114,7 @@ function parse_rpath(rpath::AS; canonical::Bool=false, code::Bool=false)::AS
                                      join_rpath(rpath[3:end]))
             else
                 full_path = joinpath(PATHS[:assets],
-                                     splitext(locvar("fd_rpath"))[1],
+                                     splitext(locvar(:fd_rpath))[1],
                                      join_rpath(rpath[3:end]))
             end
             return normpath(full_path)
diff --git a/src/manager/rss_generator.jl b/src/manager/rss_generator.jl
index df919017..4cbc2412 100644
--- a/src/manager/rss_generator.jl
+++ b/src/manager/rss_generator.jl
@@ -66,16 +66,16 @@ function add_rss_item()::RSSItem
 
     descr = fd2html(descr; internal=true) |> remove_html_ps
 
-    author    = locvar("rss_author")
-    category  = locvar("rss_category")
-    comments  = locvar("rss_comments")
-    enclosure = locvar("rss_enclosure")
+    author    = locvar(:rss_author)
+    category  = locvar(:rss_category)
+    comments  = locvar(:rss_comments)
+    enclosure = locvar(:rss_enclosure)
 
-    pubDate = locvar("rss_pubdate")
+    pubDate = locvar(:rss_pubdate)
     if pubDate == Date(1)
-        pubDate = locvar("date")
+        pubDate = locvar(:date)
         if !isa(pubDate, Date) || pubDate == Date(1)
-            pubDate = locvar("fd_mtime")
+            pubDate = locvar(:fd_mtime)
         end
     end
 
diff --git a/src/parser/markdown/tokens.jl b/src/parser/markdown/tokens.jl
index 3c5085b3..fa4d8e53 100644
--- a/src/parser/markdown/tokens.jl
+++ b/src/parser/markdown/tokens.jl
@@ -40,6 +40,8 @@ const MD_TOKENS = LittleDict{Char, Vector{TokenFinder}}(
              ],
     ']'  => [ isexactly("]: ") => :LINK_DEF,
              ],
+    ':'  => [ incrlook(is_emoji) => :CAND_EMOJI,
+             ],
     '\\' => [ # -- special characters, see `find_special_chars` in ocblocks
               isexactly("\\\\")       => :CHAR_LINEBREAK,   # --> <br/>
               isexactly("\\", (' ',)) => :CHAR_BACKSPACE,   # --> &#92;
diff --git a/src/parser/markdown/validate.jl b/src/parser/markdown/validate.jl
index 858365f8..3efa6454 100644
--- a/src/parser/markdown/validate.jl
+++ b/src/parser/markdown/validate.jl
@@ -32,9 +32,7 @@ $SIGNATURES
 
 Verify that a given string corresponds to a well formed html entity.
 """
-function validate_html_entity(ss::AS)
-    !isnothing(match(HTML_ENT_PAT, ss))
-end
+validate_html_entity(ss::AS) = !isnothing(match(HTML_ENT_PAT, ss))
 
 """
 $(SIGNATURES)
@@ -60,6 +58,41 @@ function validate_headers!(tokens::Vector{Token})::Nothing
     return
 end
 
+
+function validate_emojis!(tokens::Vector{Token})::Nothing
+    isempty(tokens) && return
+    s = str(tokens[1].ss) # doesn't allocate
+    rm = Int[]
+    for (i, τ) in enumerate(tokens)
+        τ.name == :CAND_EMOJI || continue
+        # check if the immediate next character is `:`
+        nextidx = nextind(s, to(τ))
+        if s[nextidx] == ':'
+            # check if the string describes a known emoji if not, remove,
+            # otherwise re-form the emoji to add the closing ':'.
+            key = "\\$(τ.ss):"
+            if key in keys(emoji_symbols)
+                tokens[i] = Token(:EMOJI, subs(s, from(τ), nextidx))
+            else
+                push!(rm, i)
+            end
+        else
+            push!(rm, i)
+        end
+    end
+    deleteat!(tokens, rm)
+    return
+end
+
+"""
+    emoji(token)
+
+Return the emoji corresponding to an Emoji token by querying `emoji_symbols`.
+This assumes that the token has name `:EMOJI` and so we know it's in the dict.
+"""
+emoji(τ::Token) = emoji_symbols["\\$(τ.ss)"]
+
+
 """
 $(SIGNATURES)
 
diff --git a/src/parser/ocblocks.jl b/src/parser/ocblocks.jl
index d4919db4..4fbebe2e 100644
--- a/src/parser/ocblocks.jl
+++ b/src/parser/ocblocks.jl
@@ -177,6 +177,7 @@ function find_special_chars(tokens::Vector{Token})
         τ.name == :CHAR_BACKSPACE   && push!(spch, HTML_SPCH(τ.ss, "&#92;"))
         τ.name == :CHAR_BACKTICK    && push!(spch, HTML_SPCH(τ.ss, "&#96;"))
         τ.name == :CHAR_LINEBREAK   && push!(spch, HTML_SPCH(τ.ss, "<br/>"))
+        τ.name == :EMOJI            && push!(spch, HTML_SPCH(τ.ss, emoji(τ)))
         τ.name == :CHAR_HTML_ENTITY &&
             validate_html_entity(τ.ss) && push!(spch, HTML_SPCH(τ.ss))
     end
diff --git a/src/parser/tokens.jl b/src/parser/tokens.jl
index c2c805c6..550dc092 100644
--- a/src/parser/tokens.jl
+++ b/src/parser/tokens.jl
@@ -287,6 +287,14 @@ is_html_entity(i::Int, c::Char) = αη(c, ('#',';'))
 """
 $(SIGNATURES)
 
+Check if it looks like an emoji indicator `:...` note that it does
+not take the final `:` this is checked and added in `validate_emoji!`.
+"""
+is_emoji(i::Int, c::Char) = αη(c, ('+','_','-'))
+
+"""
+$(SIGNATURES)
+
 Check if it looks like `\\[\\^[\\p{L}0-9]+\\]:`.
 """
 function is_footnote(i::Int, c::Char)
diff --git a/src/scripts/minify.py b/src/scripts/minify.py
index 8af565db..e2d7d88f 100644
--- a/src/scripts/minify.py
+++ b/src/scripts/minify.py
@@ -27,6 +27,10 @@ else:
     css_files  = []
     for root, dirs, files in os.walk("__site"):
         for fname in files:
+            path = os.path.join(root, fname)
+            # skip the assets folder which should be left untouched (#568)
+            if path.startswith(os.path.join("__site", "assets")):
+                continue
             if fname.endswith(".html"):
                 html_files.append(os.path.join(root, fname))
             if fname.endswith(".css"):
diff --git a/src/utils/html.jl b/src/utils/html.jl
index 2d49255f..83e6bbdf 100644
--- a/src/utils/html.jl
+++ b/src/utils/html.jl
@@ -138,7 +138,7 @@ end
 function _url_curpage()
     # go from /pages/.../something.md to /pub/.../something.html note that if
     # on windows then it would be \\ whence the PATH_SEP
-    rp = replace(locvar("fd_rpath"),
+    rp = replace(locvar(:fd_rpath),
             Regex("^pages$(escape_string(PATH_SEP))") => "pub$(PATH_SEP)")
     rp = unixify(rp)
     rp = splitext(rp)[1] * ".html"
@@ -148,7 +148,7 @@ end
 
 function _url_curpage2()
     # get the relative path to current page and split extension (.md)
-    fn = splitext(locvar("fd_rpath"))[1]
+    fn = splitext(locvar(:fd_rpath))[1]
     # if it's not `index` then add `index`:
     if splitdir(fn)[2] != "index"
         fn = joinpath(fn, "index")
diff --git a/src/utils/vars.jl b/src/utils/vars.jl
index 7b80c0af..b8a12f10 100644
--- a/src/utils/vars.jl
+++ b/src/utils/vars.jl
@@ -39,6 +39,7 @@ const GLOBAL_VARS_DEFAULT = [
     # keep track page=>tags and tag=>pages
     "fd_page_tags"     => Pair(nothing, (DTAG,  Nothing)),
     "fd_tag_pages"     => Pair(nothing, (DTAGI, Nothing)),
+    # -----------------------------------------------------
     # LEGACY
     "div_content" => Pair("", (String,)), # see build_page
     ]
@@ -70,6 +71,7 @@ const LOCAL_VARS_DEFAULT = [
     "reflinks"      => Pair(true,       (Bool,)),   # are there reflinks?
     "indented_code" => Pair(false,      (Bool,)),   # support indented code?
     "tags"          => Pair(String[],   (Vector{String},)),
+    "prerender"     => Pair(true,       (Bool,)),   # allow specific switch
     # -----------------
     # TABLE OF CONTENTS
     "mintoclevel" => Pair(1,  (Int,)), # set to 2 to ignore h1
@@ -193,7 +195,7 @@ function pagevar(rpath::AS, name::Union{Symbol,String})
                      joinpath(path(:folder), fpath)
         isfile(candpath) || return nothing
         # store curpath
-        bk_path = locvar("fd_rpath")
+        bk_path = locvar(:fd_rpath)
         bk_path_ = splitext(bk_path)[1]
 
         (:pagevar, "!haskey, bkpath: $bk_path, rpath: $rpath") |> logger
@@ -291,11 +293,11 @@ automatically construct them using the first three letters of the names in
 """
 function fd_date(d::DateTime)
     # aliases for locale data and format from local variables
-    format      = locvar("date_format")
-    months      = locvar("date_months")
-    shortmonths = locvar("date_shortmonths")
-    days        = locvar("date_days")
-    shortdays   = locvar("date_shortdays")
+    format      = locvar(:date_format)
+    months      = locvar(:date_months)
+    shortmonths = locvar(:date_shortmonths)
+    days        = locvar(:date_days)
+    shortdays   = locvar(:date_shortdays)
     # if vectors are empty, user has not defined custom locale,
     # defaults to english
     if all(isempty.((months, shortmonths, days, shortdays)))


Test patch:
diff --git a/test/parser/markdown-extra.jl b/test/parser/markdown-extra.jl
index e5b66558..cfde0889 100644
--- a/test/parser/markdown-extra.jl
+++ b/test/parser/markdown-extra.jl
@@ -188,3 +188,18 @@ end
         """ |> fd2html_td
     @test isapproxstr(s, raw"\[ φφφ1 abcdef \]")
 end
+
+@testset "emoji-basics" begin
+    s = "abc :+1: def" |> fd2html
+    @test s // "<p>abc 👍 def</p>"
+    s = "abc :+10: def" |> fd2html
+    @test s // "<p>abc :&#43;10: def</p>"
+    s = "abc :joy: def" |> fd2html
+    @test s // "<p>abc 😂 def</p>"
+    s = "abc :joy def:" |> fd2html
+    @test s // "<p>abc :joy def:</p>"
+    s = "abc : joy: def" |> fd2html
+    @test s // "<p>abc : joy: def</p>"
+    s = "abc :joi: def" |> fd2html
+    @test s // "<p>abc :joi: def</p>"
+end
diff --git a/test/test_utils.jl b/test/test_utils.jl
index 09296e80..aaf64295 100644
--- a/test/test_utils.jl
+++ b/test/test_utils.jl
@@ -9,7 +9,7 @@ chtml = t -> F.convert_html(t)
 conv  = st -> st |> cmd |> chtml
 
 set_curpath(path) =
-    (F.set_var!(F.LOCAL_VARS, "fd_rpath", path); F.locvar("fd_rpath"))
+    (F.set_var!(F.LOCAL_VARS, "fd_rpath", path); F.locvar(:fd_rpath))
 
 # convenience function that squeezes out all whitespaces and line returns out of a string
 # and checks if the resulting strings are equal. When expecting a specific string +- some
@@ -105,7 +105,7 @@ function explore_md_steps(mds)
     fnrefs = filter(τ -> τ.name == :FOOTNOTE_REF, tokens)
     steps[:fnrefs] = (fnrefs=fnrefs,)
 
-    if !F.locvar("indented_code")
+    if !F.locvar(:indented_code)
         filter!(b -> b.name != :CODE_BLOCK_IND, blocks)
     end