Repository: domluna/JuliaFormatter.jl (julia)

Issue:
Strange Error when formatting a file
Consider the file here

https://github.com/JuliaManifolds/Manopt.jl/blob/chambolle-pock/examples/ChambollePock/SPD_Signal_TV.jl

running on that file a format yields an error, that I do not understand (see below) but more surprisingly, commenting out line  21, i.e. changing it to

```julia
#export_any = export_orig || export_primal || export_table
```

makes formatter pass even with true, so a correctly formatted file.
Any idea why without commenting out the following error occurs?

Happens on JuliaFormatter v0.10.8 with Julia 1.5.2.

```julia
julia> format("examples/ChambollePock/SPD_Signal_TV.jl")
ERROR: MethodError: no method matching iterate(::Nothing)
Closest candidates are:
  iterate(::DataStructures.SparseIntSet, ::Any...) at~/.julia/packages/DataStructures/DLSxi/src/sparse_int_set.jl:147
  iterate(::Base.Iterators.ProductIterator{Tuple{}}) at iterators.jl:959
  iterate(::Base.Iterators.ProductIterator{Tuple{}}, ::Any) at iterators.jl:960
  ...
Stacktrace:
 [1] indexed_iterate(::Nothing, ::Int64) at ./tuple.jl:84
 [2] #p_literal#71 at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:414 [inlined]
 [3] #p_literal#76 at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:475 [inlined]
 [4] p_literal(::BlueStyle, ::CSTParser.EXPR, ::JuliaFormatter.State) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:475
 [5] pretty(::DefaultStyle, ::CSTParser.EXPR, ::JuliaFormatter.State; kwargs::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:13
 [6] pretty(::DefaultStyle, ::CSTParser.EXPR, ::JuliaFormatter.State) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:3
 [7] pretty(::BlueStyle, ::CSTParser.EXPR, ::JuliaFormatter.State; kwargs::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:144
 [8] pretty(::BlueStyle, ::CSTParser.EXPR, ::JuliaFormatter.State) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:144
 [9] p_call(::DefaultStyle, ::CSTParser.EXPR, ::JuliaFormatter.State) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:1588
 [10] p_call(::BlueStyle, ::CSTParser.EXPR, ::JuliaFormatter.State) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/blue/pretty.jl:96
 [11] pretty(::DefaultStyle, ::CSTParser.EXPR, ::JuliaFormatter.State; kwargs::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:71
 [12] pretty(::DefaultStyle, ::CSTParser.EXPR, ::JuliaFormatter.State) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:3
 [13] pretty(::BlueStyle, ::CSTParser.EXPR, ::JuliaFormatter.State; kwargs::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:144
 [14] pretty(::BlueStyle, ::CSTParser.EXPR, ::JuliaFormatter.State) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:144
 [15] p_binaryopcall(::BlueStyle, ::CSTParser.EXPR, ::JuliaFormatter.State; nonest::Bool, nospace::Bool) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/blue/pretty.jl:181
 [16] p_binaryopcall(::BlueStyle, ::CSTParser.EXPR, ::JuliaFormatter.State) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/blue/pretty.jl:110
 [17] pretty(::DefaultStyle, ::CSTParser.EXPR, ::JuliaFormatter.State; kwargs::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:83
 [18] pretty(::DefaultStyle, ::CSTParser.EXPR, ::JuliaFormatter.State) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:3
 [19] pretty(::BlueStyle, ::CSTParser.EXPR, ::JuliaFormatter.State; kwargs::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:144
 [20] pretty(::BlueStyle, ::CSTParser.EXPR, ::JuliaFormatter.State) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:144
 [21] p_fileh(::DefaultStyle, ::CSTParser.EXPR, ::JuliaFormatter.State) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:155
 [22] p_fileh(::BlueStyle, ::CSTParser.EXPR, ::JuliaFormatter.State) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:159
 [23] pretty(::DefaultStyle, ::CSTParser.EXPR, ::JuliaFormatter.State; kwargs::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:131
 [24] pretty(::DefaultStyle, ::CSTParser.EXPR, ::JuliaFormatter.State) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:3
 [25] pretty(::BlueStyle, ::CSTParser.EXPR, ::JuliaFormatter.State; kwargs::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:144
 [26] pretty at~/.julia/packages/JuliaFormatter/e1ear/src/styles/default/pretty.jl:144 [inlined]
 [27] format_text(::CSTParser.EXPR, ::BlueStyle, ::JuliaFormatter.State) at~/.julia/packages/JuliaFormatter/e1ear/src/JuliaFormatter.jl:328
 [28] format_text(::String, ::BlueStyle, ::JuliaFormatter.Options) at~/.julia/packages/JuliaFormatter/e1ear/src/JuliaFormatter.jl:324
 [29] #format_text#156 at~/.julia/packages/JuliaFormatter/e1ear/src/JuliaFormatter.jl:316 [inlined]
 [30] format_text at~/.julia/packages/JuliaFormatter/e1ear/src/JuliaFormatter.jl:314 [inlined]
 [31] #format_text#155 at~/.julia/packages/JuliaFormatter/e1ear/src/JuliaFormatter.jl:310 [inlined]
 [32] format_file(::String; overwrite::Bool, verbose::Bool, format_markdown::Bool, format_options::Base.Iterators.Pairs{Symbol,BlueStyle,Tuple{Symbol},NamedTuple{(:style,),Tuple{BlueStyle}}}) at~/.julia/packages/JuliaFormatter/e1ear/src/JuliaFormatter.jl:420
 [33] format(::Tuple{String}; options::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at~/.julia/packages/JuliaFormatter/e1ear/src/JuliaFormatter.jl:529
 [34] format at~/.julia/packages/JuliaFormatter/e1ear/src/JuliaFormatter.jl:505 [inlined]
 [35] #format#165 at~/.julia/packages/JuliaFormatter/e1ear/src/JuliaFormatter.jl:554 [inlined]
 [36] format(::String) at~/.julia/packages/JuliaFormatter/e1ear/src/JuliaFormatter.jl:554
 [37] top-level scope at REPL[2]:1
```

Pull request:
fixes #332

Patch:
diff --git a/Project.toml b/Project.toml
index 0a94fc74c..c9a480bc0 100644
--- a/Project.toml
+++ b/Project.toml
@@ -1,7 +1,7 @@
 name = "JuliaFormatter"
 uuid = "98e50ef6-434e-11e9-1051-2b60c6c9e899"
 authors = ["Dominique Luna <dluna132@gmail.com>"]
-version = "0.10.8"
+version = "0.10.9"
 
 [deps]
 CSTParser = "00ebfdb7-1f24-5e51-bd34-a7502290713f"
diff --git a/src/document.jl b/src/document.jl
index 40bdd8b08..843b3578e 100644
--- a/src/document.jl
+++ b/src/document.jl
@@ -149,6 +149,8 @@ function Document(text::AbstractString)
 
         if t.kind === Tokens.COMMENT
             goffset += (t.endbyte - t.startbyte + 1)
+        elseif t.kind === Tokens.WHITESPACE
+            goffset += (t.endbyte - t.startbyte + 1)
         else
             goffset += length(Tokenize.untokenize(t))
         end


Test patch:
diff --git a/test/document.jl b/test/document.jl
index 8f8730180..936b81b1e 100644
--- a/test/document.jl
+++ b/test/document.jl
@@ -1,12 +1,31 @@
 @testset "Document" begin
-    s = """
-    \"""
-    𝔽𝔽
+    @testset "count unicode literals in bytes" begin
+        s = """
+        \"""
+        𝔽𝔽
 
-    \"""
-    struct A end
-    """
-    d = JuliaFormatter.Document(s)
-    ranges = Dict(1 => 1:4, 2 => 5:7, 3 => 8:8, 4 => 9:12, 5 => 13:25, 6 => 26:25)
-    @test ranges == d.line_to_range
+        \"""
+        struct A end
+        """
+        d = JuliaFormatter.Document(s)
+        ranges = Dict(1 => 1:4, 2 => 5:7, 3 => 8:8, 4 => 9:12, 5 => 13:25, 6 => 26:25)
+        @test ranges == d.line_to_range
+    end
+
+    @testset "count unicode whitespace in bytes" begin
+        s0 = """a = b || c ;
+               f("A")"""
+        d = JuliaFormatter.Document(s0)
+        ranges = Dict(1 => 1:13, 2 => 14:19)
+        @test ranges == d.line_to_range
+
+        # this string has a nbsp after 'c'
+        # so it should have an additional byte because
+        # it's unicode
+        s = """a = b || c ;
+               f("A")"""
+        d = JuliaFormatter.Document(s)
+        ranges = Dict(1 => 1:14, 2 => 15:20)
+        @test ranges == d.line_to_range
+    end
 end
diff --git a/test/issues.jl b/test/issues.jl
index ac54c696a..4d6f2eeb1 100644
--- a/test/issues.jl
+++ b/test/issues.jl
@@ -621,4 +621,15 @@
         """
         @test format_text(str, align_assignment = true) == str
     end
+
+    @testset "issue 332" begin
+        # this string has a nbsp after 'c'
+        # so it should have an additional byte because
+        # it's unicode
+        str_ = """a = b || c ;
+               f("A")"""
+        str = """a = b || c;
+               f("A")"""
+        @test format_text(str_) == str
+    end
 end