Repository: domluna/JuliaFormatter.jl
Pull request #177 (2020-01-26 17:22:32)

Remove superflous whitespace
Add `remove_extra_newlines` option which removes extra newlines.
For example:

```
a = 1






b = 2
```

would be rewritten as

```
a = 1

b = 2
```

If a NOTCODE node is found to not have any comments (only newlines)
in certain cases it is removed. Determined by `remove_empty_notcode`.

### Misc

Moves more FST related functions from pretty.jl to fst.jl


Patch:
diff --git a/README.md b/README.md
index d644f42a..8ddb078d 100644
--- a/README.md
+++ b/README.md
@@ -21,6 +21,12 @@ julia> using JuliaFormatter
 
 # Recursively formats all Julia files in the current directory
 julia> format(".")
+
+# Formats an individual file
+julia> format_file("foo.jl")
+
+# Formats a string (contents of a Julia file)
+julia> format_text(str)
 ```
 
 [Use With Github Actions](https://github.com/julia-actions/julia-format)
@@ -37,6 +43,8 @@ format_text(
     always_for_in = false,
     whitespace_typedefs::Bool = false,
     whitespace_ops_in_indices::Bool = false,
+    remove_extra_newlines::Bool = false,
+    style::AbstractStyle = DefaultStyle(),
 )
 
 format_file(
@@ -48,6 +56,8 @@ format_file(
     always_for_in = false,
     whitespace_typedefs::Bool = false,
     whitespace_ops_in_indices::Bool = false,
+    remove_extra_newlines::Bool = false,
+    style::AbstractStyle = DefaultStyle(),
 )
 
 format(
@@ -59,6 +69,8 @@ format(
     always_for_in = false,
     whitespace_typedefs::Bool = false,
     whitespace_ops_in_indices::Bool = false,
+    remove_extra_newlines::Bool = false,
+    style::AbstractStyle = DefaultStyle(),
 )
 ```
 
@@ -88,10 +100,28 @@ If `whitespace_typedefs` is true, whitespace is added for type definitions.
 Make this `true` if you prefer `Union{A <: B, C}` to `Union{A<:B,C}`.
 
 If `whitespace_ops_in_indices` is true, whitespace is added for binary operations
-in indices. Make this `true` if you prefer `arr[a + b]` to `arr[a+b]`. Additionally,
-if there's a colon `:` involved, parenthesis will be added to the LHS and RHS.
+in indices. Make this `true` if you prefer `arr[a + b]` to `arr[a+b]`. Additionally, if there's a colon `:` involved, parenthesis will be added to the LHS and RHS.
+
 Example: `arr[(i1 + i2):(i3 + i4)]` instead of `arr[i1+i2:i3+i4]`.
 
+If `remove_extra_newlines` is true superflous newlines will be removed. For example:
+
+```julia
+a = 1
+
+
+
+b = 2
+```
+
+is rewritten as
+
+```julia
+a = 1
+
+b = 2
+```
+
 ### Editor Plugins
 
 For integration with other editors:
diff --git a/docs/src/index.md b/docs/src/index.md
index f7c4cd51..5468a5ad 100644
--- a/docs/src/index.md
+++ b/docs/src/index.md
@@ -4,6 +4,8 @@
 
 Width-sensitive formatter for Julia code. Inspired by gofmt, refmt, and black. Built with [`CSTParser`](https://github.com/ZacLN/CSTParser.jl).
 
+![](https://user-images.githubusercontent.com/1813121/72941091-0b146300-3d68-11ea-9c95-75ec979caf6e.gif)
+
 ## Installation
 
 ```julia
@@ -17,6 +19,12 @@ julia> using JuliaFormatter
 
 # Recursively formats all Julia files in the current directory
 julia> format(".")
+
+# Formats an individual file
+julia> format_file("foo.jl")
+
+# Formats a string (contents of a Julia file)
+julia> format_text(str)
 ```
 
 ## Usage
@@ -31,6 +39,8 @@ format_text(
     always_for_in = false,
     whitespace_typedefs::Bool = false,
     whitespace_ops_in_indices::Bool = false,
+    remove_extra_newlines::Bool = false,
+    style::AbstractStyle = DefaultStyle(),
 )
 
 format_file(
@@ -42,6 +52,8 @@ format_file(
     always_for_in = false,
     whitespace_typedefs::Bool = false,
     whitespace_ops_in_indices::Bool = false,
+    remove_extra_newlines::Bool = false,
+    style::AbstractStyle = DefaultStyle(),
 )
 
 format(
@@ -53,6 +65,8 @@ format(
     always_for_in = false,
     whitespace_typedefs::Bool = false,
     whitespace_ops_in_indices::Bool = false,
+    remove_extra_newlines::Bool = false,
+    style::AbstractStyle = DefaultStyle(),
 )
 ```
 
@@ -85,3 +99,21 @@ If `whitespace_ops_in_indices` is true, whitespace is added for binary operation
 in indices. Make this `true` if you prefer `arr[a + b]` to `arr[a+b]`. Additionally,
 if there's a colon `:` involved, parenthesis will be added to the LHS and RHS.
 Example: `arr[(i1 + i2):(i3 + i4)]` instead of `arr[i1+i2:i3+i4]`.
+
+If `remove_extra_newlines` is true superflous newlines will be removed. For example:
+
+```julia
+a = 1
+
+
+
+b = 2
+```
+
+is rewritten as
+
+```julia
+a = 1
+
+b = 2
+```
diff --git a/docs/src/style.md b/docs/src/style.md
index 71b17726..71eac00f 100644
--- a/docs/src/style.md
+++ b/docs/src/style.md
@@ -34,7 +34,9 @@ Functions calls `foo(args...)`, tuples `(args...)`, arrays `[args...]`, braces `
 
 ```julia
 f(
+
 a,b
+
 ,c )
 
 ->
diff --git a/src/JuliaFormatter.jl b/src/JuliaFormatter.jl
index a13e28f4..b29b9175 100644
--- a/src/JuliaFormatter.jl
+++ b/src/JuliaFormatter.jl
@@ -3,7 +3,7 @@ module JuliaFormatter
 using CSTParser
 using Tokenize
 
-export format, format_text, format_file, format_dir
+export format, format_text, format_file
 
 is_str_or_cmd(t::Tokens.Kind) =
     t in (Tokens.CMD, Tokens.TRIPLE_CMD, Tokens.STRING, Tokens.TRIPLE_STRING)
@@ -158,6 +158,7 @@ Base.@kwdef struct Options
     always_for_in::Bool = false
     whitespace_typedefs::Bool = false
     whitespace_ops_in_indices::Bool = false
+    remove_extra_newlines::Bool = false
 end
 
 mutable struct State
@@ -211,6 +212,7 @@ include("styles/yas.jl")
         always_for_in::Bool = false,
         whitespace_typedefs::Bool = false,
         whitespace_ops_in_indices::Bool = false,
+        remove_extra_newlines::Bool = false,
         style::AbstractStyle = DefaultStyle(),
     )::String
 
@@ -232,9 +234,27 @@ If `whitespace_typedefs` is true, whitespace is added for type definitions.
 Make this `true` if you prefer `Union{A <: B, C}` to `Union{A<:B,C}`.
 
 If `whitespace_ops_in_indices` is true, whitespace is added for binary operations
-in indices. Make this `true` if you prefer `arr[a + b]` to `arr[a+b]`. Additionally,
-if there's a colon `:` involved, parenthesis will be added to the LHS and RHS.
+in indices. Make this `true` if you prefer `arr[a + b]` to `arr[a+b]`. Additionally, if there's a colon `:` involved, parenthesis will be added to the LHS and RHS.
+
 Example: `arr[(i1 + i2):(i3 + i4)]` instead of `arr[i1+i2:i3+i4]`.
+
+If `remove_extra_newlines` is true superflous newlines will be removed. For example:
+
+```julia
+a = 1
+
+
+
+b = 2
+```
+
+is rewritten as
+
+```julia
+a = 1
+
+b = 2
+```
 """
 function format_text(
     text::AbstractString;
@@ -243,6 +263,7 @@ function format_text(
     always_for_in::Bool = false,
     whitespace_typedefs::Bool = false,
     whitespace_ops_in_indices::Bool = false,
+    remove_extra_newlines::Bool = false,
     style::AbstractStyle = DefaultStyle(),
 )
     isempty(text) && return text
@@ -257,6 +278,7 @@ function format_text(
         always_for_in = always_for_in,
         whitespace_typedefs = whitespace_typedefs,
         whitespace_ops_in_indices = whitespace_ops_in_indices,
+        remove_extra_newlines = remove_extra_newlines,
     )
     s = State(Document(text), indent, margin, opts)
     t = pretty(style, x, s)
@@ -297,6 +319,8 @@ end
         always_for_in::Bool = false,
         whitespace_typedefs::Bool = false,
         whitespace_ops_in_indices::Bool = false,
+        remove_extra_newlines::Bool = false,
+        style::AbstractStyle = DefaultStyle(),
     )
 
 Formats the contents of `filename` assuming it's a Julia source file.
@@ -312,22 +336,7 @@ to `stdout`.
 
 ### Formatting Options
 
-`indent` - the number of spaces used for an indentation.
-
-`margin` - the maximum length of a line. Code exceeding this margin will be formatted
-across multiple lines.
-
-If `always_for_in` is true `=` is always replaced with `in` if part of a
-`for` loop condition.  For example, `for i = 1:10` will be transformed
-to `for i in 1:10`.
-
-If `whitespace_typedefs` is true, whitespace is added for type definitions.
-Make this `true` if you prefer `Union{A <: B, C}` to `Union{A<:B,C}`.
-
-If `whitespace_ops_in_indices` is true, whitespace is added for binary operations
-in indices. Make this `true` if you prefer `arr[a + b]` to `arr[a+b]`. Additionally,
-if there's a colon `:` involved, parenthesis will be added to the LHS and RHS.
-Example: `arr[(i1 + i2):(i3 + i4)]` instead of `arr[i1+i2:i3+i4]`.
+See `format_text` for description of formatting options.
 """
 function format_file(
     filename::AbstractString;
@@ -338,6 +347,8 @@ function format_file(
     always_for_in::Bool = false,
     whitespace_typedefs::Bool = false,
     whitespace_ops_in_indices::Bool = false,
+    remove_extra_newlines::Bool = false,
+    style::AbstractStyle = DefaultStyle(),
 )
     path, ext = splitext(filename)
     shebang_pattern = r"^#!\s*/.*\bjulia[0-9.-]*\b"
@@ -353,6 +364,8 @@ function format_file(
         always_for_in = always_for_in,
         whitespace_typedefs = whitespace_typedefs,
         whitespace_ops_in_indices = whitespace_ops_in_indices,
+        remove_extra_newlines = remove_extra_newlines,
+        style = style,
     )
     str = replace(str, r"\n*$" => "\n")
     overwrite ? write(filename, str) : write(path * "_fmt" * ext, str)
@@ -407,6 +420,8 @@ end
         overwrite::Bool = true,
         verbose::Bool = false,
         always_for_in::Bool = false,
+        remove_extra_newlines = false,
+        style = DefaultStyle(),
     )
 
 Recursively descend into files and directories, formatting and `.jl`
diff --git a/src/fst.jl b/src/fst.jl
index 6d5bba19..7d2445e6 100644
--- a/src/fst.jl
+++ b/src/fst.jl
@@ -224,9 +224,9 @@ function add_node!(t::FST, n::FST, s::State; join_lines = false, max_padding = -
             # If there are comments in between node elements
             # nesting is forced in an effort to preserve them.
 
-            if is_iterable(t)
+            if remove_empty_notcode(t)
                 nest = false
-                for l=notcode_startline:notcode_endline
+                for l = notcode_startline:notcode_endline
                     if hascomment(s.doc, l)
                         nest = true
                         break
@@ -341,3 +341,76 @@ end
     (fst.val == "{" || fst.val == "(" || fst.val == "[")
 @inline is_opener(cst::CSTParser.EXPR) =
     cst.kind === Tokens.LBRACE || cst.kind === Tokens.LPAREN || cst.kind === Tokens.LSQUARE
+
+@inline is_str(cst::CSTParser.EXPR) = is_str_or_cmd(cst.kind) || is_str_or_cmd(cst.typ)
+
+function is_iterable(x::Union{CSTParser.EXPR,FST})
+    x.typ === CSTParser.TupleH && return true
+    x.typ === CSTParser.Vect && return true
+    x.typ === CSTParser.Vcat && return true
+    x.typ === CSTParser.Braces && return true
+    x.typ === CSTParser.Call && return true
+    x.typ === CSTParser.Curly && return true
+    x.typ === CSTParser.Comprehension && return true
+    x.typ === CSTParser.MacroCall && return true
+    x.typ === CSTParser.InvisBrackets && return true
+    x.typ === CSTParser.Ref && return true
+    x.typ === CSTParser.TypedVcat && return true
+    return false
+end
+
+function is_block(cst::CSTParser.EXPR)
+    cst.typ === CSTParser.If && return true
+    cst.typ === CSTParser.Do && return true
+    cst.typ === CSTParser.Try && return true
+    cst.typ === CSTParser.Begin && return true
+    cst.typ === CSTParser.For && return true
+    cst.typ === CSTParser.While && return true
+    cst.typ === CSTParser.Let && return true
+    (cst.typ === CSTParser.Quote && cst[1].kind === Tokens.QUOTE) && return true
+    return false
+end
+
+function nest_block(cst::CSTParser.EXPR)
+    cst.typ === CSTParser.If && return true
+    cst.typ === CSTParser.Do && return true
+    cst.typ === CSTParser.Try && return true
+    cst.typ === CSTParser.For && return true
+    cst.typ === CSTParser.While && return true
+    cst.typ === CSTParser.Let && return true
+    return false
+end
+
+function remove_empty_notcode(fst::FST)
+    is_iterable(fst) && return true
+    fst.typ === CSTParser.BinaryOpCall && return true
+    fst.typ === CSTParser.ConditionalOpCall && return true
+    fst.typ === CSTParser.Comparison && return true
+    fst.typ === CSTParser.ChainOpCall && return true
+    return false
+end
+
+nest_assignment(cst::CSTParser.EXPR) = CSTParser.precedence(cst[2].kind) == 1
+
+function unnestable_arg(cst::CSTParser.EXPR)
+    is_iterable(cst) && return true
+    is_str(cst) && return true
+    cst.typ === CSTParser.LITERAL && return true
+    (cst.typ === CSTParser.BinaryOpCall && cst[2].kind === Tokens.DOT) && return true
+    return false
+end
+
+function nestable(::S, cst::CSTParser.EXPR) where {S<:AbstractStyle}
+    CSTParser.defines_function(cst) && cst[1].typ !== CSTParser.UnaryOpCall && return true
+    nest_assignment(cst) && return !is_str(cst[3])
+    true
+end
+
+function nest_rhs(cst::CSTParser.EXPR)::Bool
+    if CSTParser.defines_function(cst)
+        rhs = cst[3]
+        rhs.typ === CSTParser.Block && (rhs = rhs[1])
+        return nest_block(rhs)
+    end
+    false
+end
diff --git a/src/pretty.jl b/src/pretty.jl
index b3229813..8357d58a 100644
--- a/src/pretty.jl
+++ b/src/pretty.jl
@@ -1101,45 +1101,6 @@ end
 p_kw(style::S, cst::CSTParser.EXPR, s::State) where {S<:AbstractStyle} =
     p_kw(DefaultStyle(style), cst, s)
 
-@inline is_str(cst::CSTParser.EXPR) = is_str_or_cmd(cst.kind) || is_str_or_cmd(cst.typ)
-
-is_iterable(x::Union{CSTParser.EXPR,FST}) =
-    x.typ === CSTParser.TupleH || x.typ === CSTParser.Vect ||
-    x.typ === CSTParser.Vcat || x.typ === CSTParser.Braces || x.typ === CSTParser.Call ||
-    x.typ === CSTParser.Curly || x.typ === CSTParser.Comprehension ||
-    x.typ === CSTParser.MacroCall || x.typ === CSTParser.InvisBrackets ||
-    x.typ === CSTParser.Ref || x.typ === CSTParser.TypedVcat
-
-is_block(cst::CSTParser.EXPR) =
-    cst.typ === CSTParser.If ||
-    cst.typ === CSTParser.Do || cst.typ === CSTParser.Try || cst.typ === CSTParser.Begin ||
-    cst.typ === CSTParser.For || cst.typ === CSTParser.While || cst.typ === CSTParser.Let ||
-    (cst.typ === CSTParser.Quote && cst[1].kind === Tokens.QUOTE)
-
-nest_block(cst::CSTParser.EXPR) =
-    cst.typ === CSTParser.If || cst.typ === CSTParser.Do || cst.typ === CSTParser.Try ||
-    cst.typ === CSTParser.For || cst.typ === CSTParser.While || cst.typ === CSTParser.Let
-
-nest_assignment(cst::CSTParser.EXPR) = CSTParser.precedence(cst[2].kind) == 1
-
-unnestable_arg(cst::CSTParser.EXPR) =
-    is_iterable(cst) || is_str(cst) || cst.typ === CSTParser.LITERAL ||
-    (cst.typ === CSTParser.BinaryOpCall && cst[2].kind === Tokens.DOT)
-
-function nestable(::S, cst::CSTParser.EXPR) where {S<:AbstractStyle}
-    CSTParser.defines_function(cst) && cst[1].typ !== CSTParser.UnaryOpCall && return true
-    nest_assignment(cst) && return !is_str(cst[3])
-    true
-end
-
-function nest_rhs(cst::CSTParser.EXPR)::Bool
-    if CSTParser.defines_function(cst)
-        rhs = cst[3]
-        rhs.typ === CSTParser.Block && (rhs = rhs[1])
-        return nest_block(rhs)
-    end
-    false
-end
 
 # BinaryOpCall
 function p_binaryopcall(
diff --git a/src/print.jl b/src/print.jl
index 78e2db70..5993ebb7 100644
--- a/src/print.jl
+++ b/src/print.jl
@@ -119,19 +119,26 @@ end
 function print_notcode(io::IOBuffer, fst::FST, s::State; fmttag = false)
     s.on || return
     for l = fst.startline:fst.endline
-        ws = fst.indent
-        if fmttag
-            ws, v = get(s.doc.comments, l, (0, "\n"))
-        else
-            _, v = get(s.doc.comments, l, (0, "\n"))
+        ws, v = get(s.doc.comments, l, (0, "\n"))
+        !fmttag && (ws = fst.indent)
+
+        # If the current newline is followed by another newline
+        # don't print the current newline.
+        if s.opts.remove_extra_newlines
+            _, vn = get(s.doc.comments, l + 1, (0, "\n"))
+            vn == "\n" && v == "\n" && (v = "")
         end
+
         v == "" && continue
         v == "\n" && (ws = 0)
+
         if l == fst.endline && v[end] == '\n'
             v = v[1:prevind(v, end)]
         end
+
         ws > 0 && write(io, repeat(" ", ws))
         write(io, v)
+
         if l != fst.endline && v[end] != '\n'
             write(io, "\n")
         end
diff --git a/src/styles/yas.jl b/src/styles/yas.jl
index 02acdd2c..a9703bdc 100644
--- a/src/styles/yas.jl
+++ b/src/styles/yas.jl
@@ -43,7 +43,6 @@ end
 
 function p_call(ys::YASStyle, cst::CSTParser.EXPR, s::State)
     t = FST(cst, nspaces(s))
-
     for (i, a) in enumerate(cst)
         n = pretty(ys, a, s)
         if CSTParser.is_comma(a) && i < length(cst) && !is_punc(cst[i+1])


Test patch:
diff --git a/test/files/cppwrapper.jl b/test/files/cppwrapper.jl
index 194565c0..dc54efca 100644
--- a/test/files/cppwrapper.jl
+++ b/test/files/cppwrapper.jl
@@ -217,7 +217,6 @@ function TetgenIO(
     trifacemarkers = Cint[],
     edges = LineFace{Cint}[],
     edgemarkers = Cint[],
-
 ) where {T}
     TetgenIO(
         points,
diff --git a/test/runtests.jl b/test/runtests.jl
index 0d3298ee..7d7187a1 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -10,6 +10,7 @@ fmt1(
     always_for_in = false,
     whitespace_typedefs = false,
     whitespace_ops_in_indices = false,
+    remove_extra_newlines = false,
 ) = JuliaFormatter.format_text(
     s,
     indent = i,
@@ -17,6 +18,7 @@ fmt1(
     always_for_in = always_for_in,
     whitespace_typedefs = whitespace_typedefs,
     whitespace_ops_in_indices = whitespace_ops_in_indices,
+    remove_extra_newlines = remove_extra_newlines,
 )
 
 # Verifies formatting the formatted text
@@ -28,9 +30,26 @@ function fmt(
     always_for_in = false,
     whitespace_typedefs = false,
     whitespace_ops_in_indices = false,
+    remove_extra_newlines = false,
 )
-    s1 = fmt1(s, i, m, always_for_in, whitespace_typedefs, whitespace_ops_in_indices)
-    fmt1(s1, i, m, always_for_in, whitespace_typedefs, whitespace_ops_in_indices)
+    s1 = fmt1(
+        s,
+        i,
+        m,
+        always_for_in,
+        whitespace_typedefs,
+        whitespace_ops_in_indices,
+        remove_extra_newlines,
+    )
+    fmt1(
+        s1,
+        i,
+        m,
+        always_for_in,
+        whitespace_typedefs,
+        whitespace_ops_in_indices,
+        remove_extra_newlines,
+    )
 end
 fmt(s, i, m) = fmt(s; i = i, m = m)
 fmt1(s, i, m) = fmt1(s, i, m, false, false, false)
@@ -4201,4 +4220,167 @@ some_function(
         @test fmt(str_, whitespace_typedefs = true) == str
     end
 
+    @testset "remove excess newlines" begin
+        str_ = """
+        var = foo(a,
+
+        b,     c,
+
+
+
+
+
+        d)"""
+        str = "var = foo(a, b, c, d)"
+        @test fmt(str_) == str
+
+        str = """
+        var =
+            foo(
+                a,
+                b,
+                c,
+                d,
+            )"""
+        @test fmt(str_, 4, 1) == str
+
+        str_ = """
+        var = foo(a,
+
+        b,     c,
+
+
+        # comment !!!
+
+
+        d)"""
+        str = """
+        var = foo(
+            a,
+            b,
+            c,
+
+
+            # comment !!!
+
+
+            d,
+        )"""
+        @test fmt(str_) == str
+
+        str = """
+        var = foo(
+            a,
+            b,
+            c,
+
+            # comment !!!
+
+            d,
+        )"""
+        @test fmt(str_, remove_extra_newlines = true) == str
+
+        str_ = """
+        a = 10
+
+        # foo1
+        # ooo
+
+
+
+        # aooo
+
+
+        # aaaa
+        b = 20
+
+
+
+        # hello
+        """
+        str = """
+        a = 10
+
+        # foo1
+        # ooo
+
+        # aooo
+
+        # aaaa
+        b = 20
+
+        # hello
+        """
+        @test fmt(str, remove_extra_newlines = true) == str
+
+        str_ = """
+        var =
+
+            func(a,
+
+            b,
+
+            c)"""
+        str = """var = func(a, b, c)"""
+        @test fmt(str_) == str
+        @test fmt(str_, remove_extra_newlines = true) == str
+
+        str_ = """
+        var =
+
+            a &&
+
+
+        b &&  
+        c"""
+        str = """var = a && b && c"""
+        @test fmt(str_) == str
+        @test fmt(str_, remove_extra_newlines = true) == str
+
+        str_ = """
+        var =
+
+            a ?
+
+
+        b :          
+
+
+
+        c"""
+        str = """var = a ? b : c"""
+        @test fmt(str_) == str
+        @test fmt(str_, remove_extra_newlines = true) == str
+
+        str_ = """
+        var =
+
+            a +
+
+
+        b +          
+
+
+
+        c"""
+        str = """var = a + b + c"""
+        @test fmt(str_) == str
+        @test fmt(str_, remove_extra_newlines = true) == str
+
+        str_ = """
+        var =
+
+            a   ==
+
+
+        b   ==          
+
+
+
+        c"""
+        str = """var = a == b == c"""
+        @test fmt(str_) == str
+        @test fmt(str_, remove_extra_newlines = true) == str
+    end
+
 end