Repository: editorconfig-checker/editorconfig-checker (go)

Issue:
max_line_length should support non ASCII characters
Currently it seems like `ä`, `ö` and `ü` are counted as two characters which makes the `max_line_length` rule not usable for not plain ASCII files (like UTF-8 so everything with emojis which maybe should also count as one character ;))

Pull request description:
Update max_line_length validator to work with UTF8, fixes #115
This fixes the `max_line_length` validator so it's charset-aware. For utf-8, it will now use the rune count in the string instead of the byte count, which fixes #115. Unfortunately, I couldn't figure out how to easily add support for UTF-16 because the "unicode/utf16" package doesn't provide a `RuneCountInString` method.

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c8e02a3..8049f3b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,7 @@
 ### Deprecated
 ### Removed
 ### Fixed
+* Fix `max_line_length` validation for UTF-8 [#115](https://github.com/editorconfig-checker/editorconfig-checker/issues/115)
 ### Security
 ### Misc
 
diff --git a/go.mod b/go.mod
index 5945773..c12f04a 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@ go 1.12
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/editorconfig/editorconfig-core-go/v2 v2.3.8
+	github.com/editorconfig/editorconfig-core-go/v2 v2.3.7
 	github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f // indirect
 	github.com/smartystreets/assertions v1.0.1 // indirect
 )
diff --git a/go.sum b/go.sum
index 2d8b560..c8c2dd4 100644
--- a/go.sum
+++ b/go.sum
@@ -26,7 +26,6 @@ github.com/editorconfig/editorconfig-core-go/v2 v2.3.6 h1:sJDkoFZCazT2lhtyUXKgzz
 github.com/editorconfig/editorconfig-core-go/v2 v2.3.6/go.mod h1:keiVCwWAotcoxvCkHCtJbGWoV7/oNgCmRjB5+LvuEE0=
 github.com/editorconfig/editorconfig-core-go/v2 v2.3.7 h1:e88U5ztaklGv7X0gHIgR/cCJmHkLyVAS8aOIoEPKJP0=
 github.com/editorconfig/editorconfig-core-go/v2 v2.3.7/go.mod h1:NifC+uYhAGV/U2AxhZa3bNy4EdFMHz9mVU02vbGSMWQ=
-github.com/editorconfig/editorconfig-core-go/v2 v2.3.8/go.mod h1:z7TIMh40583cev3v8ei7V1RRPKeHQbttoa4Vm5/5u7g=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -82,6 +81,5 @@ gopkg.in/ini.v1 v1.60.1 h1:P5y5shSkb0CFe44qEeMBgn8JLow09MP17jlJHanke5g=
 gopkg.in/ini.v1 v1.60.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.60.2 h1:7i8mqModL63zqi8nQn8Q3+0zvSCZy1AxhBgthKfi4WU=
 gopkg.in/ini.v1 v1.60.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/pkg/validation/validation.go b/pkg/validation/validation.go
index 5846317..c49a3ec 100644
--- a/pkg/validation/validation.go
+++ b/pkg/validation/validation.go
@@ -139,7 +139,9 @@ func ValidateMaxLineLength(fileInformation files.FileInformation, config config.
 		return error.ValidationError{}
 	}
 
-	if currentError := validators.MaxLineLength(fileInformation.Line, maxLineLength); !config.Disable.MaxLineLength && currentError != nil {
+	charSet := fileInformation.Editorconfig.Raw["charset"]
+
+	if currentError := validators.MaxLineLength(fileInformation.Line, maxLineLength, charSet); !config.Disable.MaxLineLength && currentError != nil {
 		config.Logger.Verbose(fmt.Sprintf("Max line length error found in %s on %d", fileInformation.FilePath, fileInformation.LineNumber))
 		return error.ValidationError{LineNumber: -1, Message: currentError}
 	}
diff --git a/pkg/validation/validators/validators.go b/pkg/validation/validators/validators.go
index 0cf78c1..ad817a8 100644
--- a/pkg/validation/validators/validators.go
+++ b/pkg/validation/validators/validators.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"regexp"
 	"strings"
+	"unicode/utf8"
 
 	"github.com/editorconfig-checker/editorconfig-checker/pkg/config"
 	"github.com/editorconfig-checker/editorconfig-checker/pkg/utils"
@@ -141,8 +142,19 @@ func LineEnding(fileContent string, endOfLine string) error {
 	return nil
 }
 
-func MaxLineLength(line string, maxLineLength int) error {
-	length := len(line)
+func MaxLineLength(line string, maxLineLength int, charSet string) error {
+	var length int
+	if charSet == "utf-8" || charSet == "utf-8-bom" {
+		if charSet == "utf-8-bom" && strings.HasPrefix(line, "\xEF\xBB\xBF") {
+			line = line[3:] //strip BOM
+		}
+		length = utf8.RuneCountInString(line)
+	} else {
+		// TODO: Handle utf-16be and utf-16le properly. Unfortunately, Go doesn't provide a utf16.RuneCountinString() function
+		// Just go with byte count
+		length = len(line)
+	}
+
 	if length > maxLineLength {
 		return fmt.Errorf("Line too long (%d instead of %d)", length, maxLineLength)
 	}


Test patch:
diff --git a/pkg/validation/validators/validators_test.go b/pkg/validation/validators/validators_test.go
index 7dcdbda..54ce10d 100644
--- a/pkg/validation/validators/validators_test.go
+++ b/pkg/validation/validators/validators_test.go
@@ -244,16 +244,21 @@ func TestMaxLineLength(t *testing.T) {
 	maxLineLengthTest := []struct {
 		line          string
 		maxLineLength int
+		charSet       string
 		expected      error
 	}{
-		{"", 80, nil},
-		{"abc", 2, errors.New("Line too long (3 instead of 2)")},
-		{"   ", 2, errors.New("Line too long (3 instead of 2)")},
-		{"xx", 2, nil},
+		{"検索は次の", 5, "utf-8", nil},
+		{"検索は次の", 2, "utf-8", errors.New("Line too long (5 instead of 2)")},
+		{"\xEF\xBB\xBF検索は次の", 5, "utf-8-bom", nil},
+		{"検索は次の", 5, "latin1", errors.New("Line too long (15 instead of 5)")},
+		{"", 80, "latin1", nil},
+		{"abc", 2, "latin1", errors.New("Line too long (3 instead of 2)")},
+		{"   ", 2, "latin1", errors.New("Line too long (3 instead of 2)")},
+		{"xx", 2, "latin1", nil},
 	}
 
 	for _, tt := range maxLineLengthTest {
-		actual := MaxLineLength(tt.line, tt.maxLineLength)
+		actual := MaxLineLength(tt.line, tt.maxLineLength, tt.charSet)
 		if !reflect.DeepEqual(actual, tt.expected) {
 			t.Errorf("Max(%s, %v): expected: %v, got: %v", tt.line, tt.maxLineLength, tt.expected, actual)
 		}