Repository: comonicon/Comonicon.jl (julia)

Issue:
Docstring parsing is truncated by the first `.` (period)

<img width="508" alt="Screen Shot 2021-11-02 at 23 06 51" src="https://user-images.githubusercontent.com/7741548/140012314-1c0ba1a5-53f6-49e5-9747-c5ecde89b368.png">

<img width="704" alt="Screen Shot 2021-11-02 at 23 05 19" src="https://user-images.githubusercontent.com/7741548/140012213-e61478ac-4f12-4055-9167-745fffecc9bb.png">

<img width="639" alt="Screen Shot 2021-11-02 at 23 05 32" src="https://user-images.githubusercontent.com/7741548/140012220-778b8af7-bf5a-470a-8b5f-36247c4eb993.png">

Pull request description:
fix #149

Patch:
diff --git a/src/ast/types.jl b/src/ast/types.jl
index 09baa1f9..49ee8f8c 100644
--- a/src/ast/types.jl
+++ b/src/ast/types.jl
@@ -3,18 +3,14 @@ const Maybe{T} = Union{Nothing,T}
 abstract type ComoniconExpr end
 
 Base.@kwdef struct Description <: ComoniconExpr
-    brief::Union{Nothing,String} = nothing
-    content::Union{Nothing,String} = nothing
+    brief::String = ""
+    content::String = ""
 end
 
+Base.convert(::Type{Description}, s::AbstractString) = Description(String(s))
 Base.convert(::Type{Description}, ::Nothing) = Description()
-Base.convert(::Type{Description}, x::String) = Description(x)
-Base.convert(::Type{Description}, x::AbstractString) = Description(String(x))
-
-Description(::Nothing) = Description(nothing, nothing)
-function Description(text::String)
-    return Description(brief(text), text)
-end
+Description(text::String) = Description(text, "")
+Description(::Nothing) = Description()
 
 Base.@kwdef struct Argument <: ComoniconExpr
     name::String
diff --git a/src/ast/utils.jl b/src/ast/utils.jl
index b57ca8b2..0785d824 100644
--- a/src/ast/utils.jl
+++ b/src/ast/utils.jl
@@ -61,21 +61,3 @@ function splitlines(s, width = 80)
     isempty(current_line) || push!(lines, strip(join(current_line)))
     return lines
 end
-
-"""
-    brief(text::String)
-
-Use the first sentence as the brief description.
-"""
-function brief(text::String)
-    index = findfirst(". ", text)
-    if index === nothing
-        index = findfirst(".", text)
-    end
-
-    if index === nothing
-        return text
-    else
-        return text[1:first(index)]
-    end
-end
diff --git a/src/frontend/cast.jl b/src/frontend/cast.jl
index e835c0e2..b5b9f9ae 100644
--- a/src/frontend/cast.jl
+++ b/src/frontend/cast.jl
@@ -477,7 +477,8 @@ end
 
 function cast(m::Module, name::String = default_name(m), line = LineNumberNode(0))
     isdefined(m, :CASTED_COMMANDS) || error("module $m does not contain any @cast commands")
-    NodeCommand(name, copy(m.CASTED_COMMANDS), split_docstring(m), line)
+    desc, intro = split_docstring(m)
+    NodeCommand(name, copy(m.CASTED_COMMANDS), Description(desc, intro), line)
 end
 
 function cast(
@@ -500,7 +501,7 @@ function cast(
         vararg,
         flags,
         options,
-        doc.desc,
+        Description(doc.desc, doc.intro),
         line,
     )
 end
diff --git a/src/frontend/markdown.jl b/src/frontend/markdown.jl
index 3b84609f..aba01a90 100644
--- a/src/frontend/markdown.jl
+++ b/src/frontend/markdown.jl
@@ -1,3 +1,9 @@
+const PARSED_SECTION = [
+    "Intro", "Introduction", # long description
+    "Arguments", "Args", # arguments
+    "Options", "Flags" # kwargs
+]
+
 function split_docstring(f::Function)
     doc = Base.Docs.doc(f)
     return split_docstring(doc)
@@ -5,17 +11,18 @@ end
 
 function split_docstring(m::Module)
     doc = Base.Docs.doc(m)
-    has_docstring(doc) || return
-    return read_description(doc)
+    has_docstring(doc) || return "", ""
+    return read_description(doc), read_intro(doc)
 end
 
 function split_docstring(doc::Markdown.MD)
     has_docstring(doc) || return JLMD()
     desc = read_description(doc)
+    intro = read_intro(doc)
     args = read_arguments(doc)
     flags = read_flags(doc)
     options = read_options(doc)
-    return JLMD(desc, args, options, flags)
+    return JLMD(desc, intro, args, options, flags)
 end
 
 function has_docstring(doc::Markdown.MD)
@@ -24,10 +31,19 @@ function has_docstring(doc::Markdown.MD)
     return !flag
 end
 
+function read_intro(md::Markdown.MD)
+    sec = read_section(md, ["Intro", "Introduction"])
+    isempty(sec) && return ""
+    length(sec) == 1 || error("section Intro/Introduction can only have one paragraph")
+    return strip(md_to_string(Markdown.MD(sec, md.meta)))
+end
+
 function read_arguments(md::Markdown.MD)
     args = Dict{String,String}()
     sec = read_section(md, ["Arguments", "Args"])
-    sec === nothing && return args
+    isempty(sec) && return args
+    length(sec) == 1 || error("section Arguments/Args can only have one paragraph")
+    sec = sec[]
 
     for each in sec.items
         name, doc = read_item(each[1])
@@ -39,7 +55,9 @@ end
 function read_flags(md::Markdown.MD)
     flags = Dict{String,JLMDFlag}()
     sec = read_section(md, "Flags")
-    sec === nothing && return flags
+    isempty(sec) && return flags
+    length(sec) == 1 || error("section Flags can only have one paragraph")
+    sec = sec[]
 
     for each in sec.items
         name, doc = read_item(each[1])
@@ -53,7 +71,9 @@ end
 function read_options(md::Markdown.MD)
     options = Dict{String,JLMDOption}()
     sec = read_section(md, "Options")
-    sec === nothing && return options
+    isempty(sec) && return options
+    length(sec) == 1 || error("section Options can only have one paragraph")
+    sec = sec[]
 
     for each in sec.items
         code, doc = read_item(each[1])
@@ -72,35 +92,39 @@ function read_description(md::Markdown.MD)
     end
 
     for line in lines
-        if line isa Markdown.Header{1} && line.text[1] in ["Arguments", "Args", "Options", "Flags"]
+        if line isa Markdown.Header{1} && line.text[1] in PARSED_SECTION
             break
         else
             push!(intro, line)
         end
     end
+    isempty(intro) && return ""
     return strip(md_to_string(Markdown.MD(intro, md.meta)))
 end
 
 function read_section(md::Markdown.MD, title::Vector{String})
     for each in title
         sec = read_section(md, each)
-        sec === nothing || return sec
+        isempty(sec) || return sec
     end
-    return
+    return []
 end
 
-function read_section(md::Markdown.MD, title::String)
+function read_section(md::Markdown.MD, title::String)::Vector{Any}
     ct = read_content(md)
     nlines = length(ct)
+    content = []
     for k in 1:nlines
         line = ct[k]
         if line isa Markdown.Header{1} && line.text[1] == title
-            if k + 1 <= nlines
-                return ct[k+1]
+            k + 1 ≤ nlines || return content # return on last line
+            for idx in k+1:nlines
+                ct[idx] isa Markdown.Header{1} && return content # another title
+                push!(content, ct[idx])
             end
         end
     end
-    return
+    return content
 end
 
 read_content(x) = x
@@ -145,8 +169,12 @@ function docstring(x)
     end
 end
 
-function md_to_string(md::Markdown.MD)
-    return sprint(md; context = :color => true) do io, x
+function md_to_string(md::Markdown.MD) # don't print newlines for one paragraph
+    return sprint(md; context = (
+                :color => true,
+                :displaysize => (typemax(Int), 1000)
+            )
+        ) do io, x
         show(io, MIME"text/plain"(), x)
     end
 end
diff --git a/src/frontend/types.jl b/src/frontend/types.jl
index d8f5d790..039ab258 100644
--- a/src/frontend/types.jl
+++ b/src/frontend/types.jl
@@ -29,6 +29,7 @@ end
 
 Base.@kwdef struct JLMD
     desc::String = ""
+    intro::String = ""
     arguments::Dict{String,String} = Dict{String,String}()
     options::Dict{String,JLMDOption} = Dict{String,JLMDOption}()
     flags::Dict{String,JLMDFlag} = Dict{String,JLMDFlag}()


Test patch:
diff --git a/test/ast/ast.jl b/test/ast/ast.jl
index d0c5396c..6c8d771c 100644
--- a/test/ast/ast.jl
+++ b/test/ast/ast.jl
@@ -2,9 +2,6 @@ using Comonicon.AST
 using Test
 using Faker
 
-desc = Description(Faker.text())
-@test endswith(desc.brief, ".") # brief should be a sentence
-
 arg = Argument(; name = "arg", type = Int)
 
 macro test_show(mime, ex)
@@ -24,13 +21,6 @@ macro test_show(mime, ex)
     return esc(ret)
 end
 
-@testset "convert(Description, ...)" begin
-    @test Description(nothing) == Description()
-    @test convert(Description, nothing) == Description()
-    @test convert(Description, "nothing") == Description("nothing")
-    @test convert(Description, split("abcd. efdasdas.", '.')[1]) == Description("abcd")
-end
-
 @test_show MIME"text/plain" begin
     "<arg>" in Argument(; name = "arg")
     "[arg...]" in Argument(; name = "arg", vararg = true)
diff --git a/test/codegen/julia/arg.jl b/test/codegen/julia/arg.jl
index d8011472..bcac9681 100644
--- a/test/codegen/julia/arg.jl
+++ b/test/codegen/julia/arg.jl
@@ -40,7 +40,7 @@ cmd = Entry(
         "main",
         Argument[],
         0,
-        Argument("xs", Any, true, false, nothing, Description(nothing, nothing), LineNumberNode(0)),
+        Argument("xs", Any, true, false, nothing, Description(), LineNumberNode(0)),
         Dict{String,Flag}(),
         Dict{String,Option}(),
         Description("", ""),
diff --git a/test/frontend/markdown.jl b/test/frontend/markdown.jl
index bf1be5bf..2ef1ea5d 100644
--- a/test/frontend/markdown.jl
+++ b/test/frontend/markdown.jl
@@ -20,6 +20,7 @@ using Comonicon:
     split_leaf_command,
     split_docstring,
     read_arguments,
+    read_intro,
     read_description,
     read_options,
     read_flags,
@@ -32,6 +33,12 @@ using Comonicon:
 
     description of the command.
 
+    # Intro
+
+    asdmwds dasklsam xasdklqm dasdm, qwdjiolkasjdsa
+    dasklmdas weqwlkjmdas kljnsadlksad qwlkdnasd
+    dasklmdlqwoi, dasdasklmd qw,asd. dasdjklnmldqw.
+
     # Args
 
     - `arg1`: argument 1.
@@ -41,6 +48,9 @@ using Comonicon:
     """)
 
     @test read_description(doc) == "description of the command."
+    @test read_intro(doc) == "asdmwds dasklsam xasdklqm dasdm, qwdjiolkasjdsa " *
+    "dasklmdas weqwlkjmdas kljnsadlksad qwlkdnasd " *
+    "dasklmdlqwoi, dasdasklmd qw,asd. dasdjklnmldqw."
 
     args = read_arguments(doc)
     @test args["arg1"] == "argument 1."