Repository: gokcehan/lf (go)

Issue:
Request - sort by locale
Hey there,

Really cool program, it would be even better if we could use it manage non English files, such as those with CJK file names.

Right now `lf` does not respect the `LC_*` env variables, so for some languages it becomes very painful to browser larger directories.

Thanks :)

Pull request description:
Add option to sort base on certain locale.
This pull request adds implementation for sorting with user provided locale (Fixes #323).

An new option `locale` is introduced, which has default value `''`. And this option supports `setlocal`.

- When this option is set to empty string, localized sorting is disabled, and program should behave the same as before.
- When set to certain IETF BCP 47 language tag, file list will be sort with cultural habits of specified language.
- Special value `*` is used to indicate reading locale value from environment.

Patch:
diff --git a/doc.md b/doc.md
index cb0fbdb..9049add 100644
--- a/doc.md
+++ b/doc.md
@@ -169,6 +169,7 @@ The following options can be used to customize the behavior of lf:
 	info              []string  (default '')
 	infotimefmtnew    string    (default 'Jan _2 15:04')
 	infotimefmtold    string    (default 'Jan _2  2006')
+    locale            string    (default '')
 	mouse             bool      (default false)
 	number            bool      (default false)
 	numberfmt         string    (default "\033[33m")
@@ -823,6 +824,14 @@ Format string of the file time shown in the info column when it matches this yea
 
 Format string of the file time shown in the info column when it doesn't match this year.
 
+## locale (string) (default `''`)
+
+An IETF BCP 47 language tag (e.g. `zh-CN`) for specifying the locale used when using sort type
+`natural` and `name`.
+
+An empty string means disable locale ordering, and the special value `'*'` is used
+to indicate reading the locale setting from the system environment.
+
 ## mouse (bool) (default false)
 
 Send mouse events as input.
@@ -1184,7 +1193,7 @@ Command `set` is used to set an option which can be a boolean, integer, or strin
 	set sortby "time"  # string value with double quotes (backslash escapes)
 
 Command `setlocal` is used to set a local option for a directory which can be a boolean or string.
-Currently supported local options are `dirfirst`, `dironly`, `hidden`, `info`, `reverse`, and `sortby`.
+Currently supported local options are `dirfirst`, `dironly`, `hidden`, `info`, `reverse`, `sortby` and `locale`.
 Adding a trailing path separator (i.e. `/` for Unix and `\` for Windows) sets the option for the given directory along with its subdirectories:
 
 	setlocal /foo/bar hidden         # boolean enable
diff --git a/eval.go b/eval.go
index f68eb22..b755d01 100644
--- a/eval.go
+++ b/eval.go
@@ -269,6 +269,17 @@ func (e *setExpr) eval(app *app, args []string) {
 			}
 		}
 		gOpts.info = toks
+	case "locale":
+		localeStr := e.val
+		if localeStr != localeStrDisable {
+			if _, err = getLocaleTag(localeStr); err != nil {
+				app.ui.echoerrf("locale: %s", err.Error())
+				return
+			}
+		}
+		gOpts.locale = localeStr
+		app.nav.sort()
+		app.ui.sort()
 	case "rulerfmt":
 		gOpts.rulerfmt = e.val
 	case "preserve":
@@ -500,6 +511,17 @@ func (e *setLocalExpr) eval(app *app, args []string) {
 		gLocalOpts.sortbys[e.path] = method
 		app.nav.sort()
 		app.ui.sort()
+	case "locale":
+		localeStr := e.val
+		if localeStr != localeStrDisable {
+			if _, err = getLocaleTag(localeStr); err != nil {
+				app.ui.echoerrf("locale: %s", err.Error())
+				return
+			}
+		}
+		gLocalOpts.locales[e.path] = localeStr
+		app.nav.sort()
+		app.ui.sort()
 	default:
 		err = fmt.Errorf("unknown option: %s", e.opt)
 	}
diff --git a/go.mod b/go.mod
index c221ec4..9ef5c20 100644
--- a/go.mod
+++ b/go.mod
@@ -3,17 +3,18 @@ module github.com/gokcehan/lf
 go 1.18
 
 require (
+	github.com/Xuanwo/go-locale v1.1.2
 	github.com/djherbis/times v1.6.0
-	github.com/fsnotify/fsnotify v1.7.0
+	github.com/fsnotify/fsnotify v1.8.0
 	github.com/gdamore/tcell/v2 v2.7.4
 	github.com/mattn/go-runewidth v0.0.16
 	golang.org/x/sys v0.26.0
 	golang.org/x/term v0.25.0
+	golang.org/x/text v0.18.0
 )
 
 require (
 	github.com/gdamore/encoding v1.0.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/rivo/uniseg v0.4.3 // indirect
-	golang.org/x/text v0.14.0 // indirect
 )
diff --git a/go.sum b/go.sum
index 1123cf5..f7233f7 100644
--- a/go.sum
+++ b/go.sum
@@ -1,7 +1,9 @@
+github.com/Xuanwo/go-locale v1.1.2 h1:6H+olvrQcyVOZ+GAC2rXu4armacTT4ZrFCA0mB24XVo=
+github.com/Xuanwo/go-locale v1.1.2/go.mod h1:1JBER4QV7Ji39GJ4AvVlfvqmTUqopzxQxdg2mXYOw94=
 github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
 github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
-github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
-github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
+github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko=
 github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
 github.com/gdamore/tcell/v2 v2.7.4 h1:sg6/UnTM9jGpZU+oFYAsDahfchWAFW8Xx2yFinNSAYU=
@@ -46,8 +48,9 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
+golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
diff --git a/lf.1 b/lf.1
index a21fc06..5f59db8 100644
--- a/lf.1
+++ b/lf.1
@@ -1,6 +1,6 @@
 .\" Automatically generated by Pandoc 2.11.4
 .\"
-.TH "LF" "1" "2024-10-22" "" "DOCUMENTATION"
+.TH "LF" "1" "2024-11-01" "" "DOCUMENTATION"
 .hy
 .SH NAME
 .PP
diff --git a/misc.go b/misc.go
index d877d64..0908a75 100644
--- a/misc.go
+++ b/misc.go
@@ -11,7 +11,15 @@ import (
 	"strings"
 	"unicode"
 
+	"github.com/Xuanwo/go-locale"
 	"github.com/mattn/go-runewidth"
+	"golang.org/x/text/collate"
+	"golang.org/x/text/language"
+)
+
+const (
+	localeStrDisable = ""  // disable locale ordering for this locale value
+	localeStrSys     = "*" // replace this locale value with locale value read from environment
 )
 
 func isRoot(name string) bool { return filepath.Dir(name) == name }
@@ -338,6 +346,40 @@ func max(a, b int) int {
 	return b
 }
 
+// This function parses given locale string into language tag value. Passing empty
+// string as locale means reading locale value from environment.
+func getLocaleTag(localeStr string) (language.Tag, error) {
+	if localeStr == localeStrSys {
+		// read environment locale
+		return locale.Detect()
+	}
+
+	localeTag, err := language.Parse(localeStr)
+	if err != nil {
+		return localeTag, fmt.Errorf("invalid locale %q: %s", localeStr, err)
+	}
+
+	return localeTag, nil
+}
+
+// This function creates new collator for given locale. Passing empty string as
+// as locale means reading locale value from environment.
+//
+// *Note*: this function returns error when given `localeStr` has value `localeStrDisable`
+// or is an invalid locale tag.
+func makeCollator(localeStr string, opts ...collate.Option) (*collate.Collator, error) {
+	if localeStr == localeStrDisable {
+		return nil, fmt.Errorf("locale is disabled")
+	}
+
+	localeTag, err := getLocaleTag(localeStr)
+	if err != nil {
+		return nil, err
+	}
+
+	return collate.New(localeTag, opts...), nil
+}
+
 // We don't need no generic code
 // We don't need no type control
 // No dark templates in compiler
diff --git a/nav.go b/nav.go
index 503731c..940de91 100644
--- a/nav.go
+++ b/nav.go
@@ -16,6 +16,7 @@ import (
 	"time"
 
 	"github.com/djherbis/times"
+	"golang.org/x/text/collate"
 )
 
 type linkState byte
@@ -171,6 +172,7 @@ type dir struct {
 	filter      []string   // last filter for this directory
 	ignorecase  bool       // ignorecase value from last sort
 	ignoredia   bool       // ignoredia value from last sort
+	locale      string     // locale value from last sort
 	noPerm      bool       // whether lf has no permission to open the directory
 	lines       []string   // lines of text to display if directory previews are enabled
 }
@@ -211,6 +213,7 @@ func (dir *dir) sort() {
 	dir.dironly = getDirOnly(dir.path)
 	dir.hidden = getHidden(dir.path)
 	dir.reverse = getReverse(dir.path)
+	dir.locale = getLocale(dir.path)
 	dir.hiddenfiles = gOpts.hiddenfiles
 	dir.ignorecase = gOpts.ignorecase
 	dir.ignoredia = gOpts.ignoredia
@@ -221,23 +224,47 @@ func (dir *dir) sort() {
 	// of equivalent elements will be reversed
 	switch dir.sortby {
 	case naturalSort:
-		sort.SliceStable(dir.files, func(i, j int) bool {
-			s1, s2 := normalize(dir.files[i].Name(), dir.files[j].Name(), dir.ignorecase, dir.ignoredia)
-			if !dir.reverse {
-				return naturalLess(s1, s2)
-			} else {
-				return naturalLess(s2, s1)
-			}
-		})
+		if collator, err := makeCollator(dir.locale, collate.Numeric); err == nil {
+			sort.SliceStable(dir.files, func(i, j int) bool {
+				s1, s2 := normalize(dir.files[i].Name(), dir.files[j].Name(), dir.ignorecase, dir.ignoredia)
+				result := collator.CompareString(s1, s2)
+				if !dir.reverse {
+					return result < 0
+				} else {
+					return result > 0
+				}
+			})
+		} else {
+			sort.SliceStable(dir.files, func(i, j int) bool {
+				s1, s2 := normalize(dir.files[i].Name(), dir.files[j].Name(), dir.ignorecase, dir.ignoredia)
+				if !dir.reverse {
+					return naturalLess(s1, s2)
+				} else {
+					return naturalLess(s2, s1)
+				}
+			})
+		}
 	case nameSort:
-		sort.SliceStable(dir.files, func(i, j int) bool {
-			s1, s2 := normalize(dir.files[i].Name(), dir.files[j].Name(), dir.ignorecase, dir.ignoredia)
-			if !dir.reverse {
-				return s1 < s2
-			} else {
-				return s2 < s1
-			}
-		})
+		if collator, err := makeCollator(dir.locale); err == nil {
+			sort.SliceStable(dir.files, func(i, j int) bool {
+				s1, s2 := normalize(dir.files[i].Name(), dir.files[j].Name(), dir.ignorecase, dir.ignoredia)
+				result := collator.CompareString(s1, s2)
+				if !dir.reverse {
+					return result < 0
+				} else {
+					return result > 0
+				}
+			})
+		} else {
+			sort.SliceStable(dir.files, func(i, j int) bool {
+				s1, s2 := normalize(dir.files[i].Name(), dir.files[j].Name(), dir.ignorecase, dir.ignoredia)
+				if !dir.reverse {
+					return s1 < s2
+				} else {
+					return s2 < s1
+				}
+			})
+		}
 	case sizeSort:
 		sort.SliceStable(dir.files, func(i, j int) bool {
 			if !dir.reverse {
@@ -463,6 +490,7 @@ func (nav *nav) loadDirInternal(path string) *dir {
 		dironly:     getDirOnly(path),
 		hidden:      getHidden(path),
 		reverse:     getReverse(path),
+		locale:      getLocale(path),
 		hiddenfiles: gOpts.hiddenfiles,
 		ignorecase:  gOpts.ignorecase,
 		ignoredia:   gOpts.ignoredia,
@@ -527,6 +555,7 @@ func (nav *nav) checkDir(dir *dir) {
 		dir.dironly != getDirOnly(dir.path) ||
 		dir.hidden != getHidden(dir.path) ||
 		dir.reverse != getReverse(dir.path) ||
+		dir.locale != getLocale(dir.path) ||
 		!reflect.DeepEqual(dir.hiddenfiles, gOpts.hiddenfiles) ||
 		dir.ignorecase != gOpts.ignorecase ||
 		dir.ignoredia != gOpts.ignoredia:
diff --git a/opts.go b/opts.go
index 2a7fdb9..3431a33 100644
--- a/opts.go
+++ b/opts.go
@@ -53,6 +53,7 @@ var gOpts struct {
 	ignoredia        bool
 	incfilter        bool
 	incsearch        bool
+	locale           string
 	mouse            bool
 	number           bool
 	preview          bool
@@ -111,6 +112,7 @@ var gLocalOpts struct {
 	hiddens   map[string]bool
 	reverses  map[string]bool
 	infos     map[string][]string
+	locales   map[string]string
 }
 
 func localOptPaths(path string) []string {
@@ -176,6 +178,15 @@ func getSortBy(path string) sortMethod {
 	return gOpts.sortby
 }
 
+func getLocale(path string) string {
+	for _, key := range localOptPaths(path) {
+		if val, ok := gLocalOpts.locales[key]; ok {
+			return val
+		}
+	}
+	return gOpts.locale
+}
+
 func init() {
 	gOpts.anchorfind = true
 	gOpts.autoquit = false
@@ -200,6 +211,7 @@ func init() {
 	gOpts.ignoredia = true
 	gOpts.incfilter = false
 	gOpts.incsearch = false
+	gOpts.locale = localeStrDisable
 	gOpts.mouse = false
 	gOpts.number = false
 	gOpts.preview = true
@@ -370,6 +382,7 @@ func init() {
 	gLocalOpts.hiddens = make(map[string]bool)
 	gLocalOpts.reverses = make(map[string]bool)
 	gLocalOpts.infos = make(map[string][]string)
+	gLocalOpts.locales = make(map[string]string)
 
 	setDefaults()
 }


Test patch:
diff --git a/misc_test.go b/misc_test.go
index f409560..41d6eb9 100644
--- a/misc_test.go
+++ b/misc_test.go
@@ -6,6 +6,8 @@ import (
 	"strings"
 	"testing"
 	"time"
+
+	"golang.org/x/text/collate"
 )
 
 func TestIsRoot(t *testing.T) {
@@ -335,3 +337,52 @@ func TestGetFileExtension(t *testing.T) {
 		})
 	}
 }
+
+func TestLocaleNaturalLess(t *testing.T) {
+	tests := []struct {
+		s1  string
+		s2  string
+		exp bool
+	}{
+		// preserving behavior of `naturalLess`
+		{"foo", "bar", false},
+		{"bar", "baz", true},
+		{"foo", "123", false},
+		{"foo1", "foobar", true},
+		{"foo1", "foo10", true},
+		{"foo2", "foo10", true},
+		{"foo1", "foo10bar", true},
+		{"foo2", "foo10bar", true},
+		{"foo1bar", "foo10bar", true},
+		{"foo2bar", "foo10bar", true},
+		{"foo1bar", "foo10", true},
+		{"foo2bar", "foo10", true},
+
+		// locale sort
+		{"你好", "他好", true},     // \u4F60\u597D, \u4ED6\u597D
+		{"到这", "到那", false},    // \u5230\u8FD9, \u5230\u90A3
+		{"你说", "什么", true},     // \u4f60\u8bf4, \u4ec0\u4e48
+		{"你好", "World", false}, // \u4F60\u597D, \u57\u6f\u72\u6c\u64
+		{"甲1", "甲乙", true},
+		{"甲1", "甲10", true},
+		{"甲2", "甲10", true},
+		{"甲1", "甲10乙", true},
+		{"甲2", "甲10乙", true},
+		{"甲1乙", "甲10乙", true},
+		{"甲2乙", "甲10乙", true},
+		{"甲1乙", "甲10", true},
+		{"甲2乙", "甲10", true},
+	}
+
+	localeStr := "zh-CN"
+	collator, err := makeCollator(localeStr, collate.Numeric)
+	if err != nil {
+		t.Fatalf("failed to create collator for %q: %s", localeStr, err)
+	}
+
+	for _, test := range tests {
+		if got := collator.CompareString(test.s1, test.s2) < 0; got != test.exp {
+			t.Errorf("at input '%s' and '%s' expected '%t' but got '%t'", test.s1, test.s2, test.exp, got)
+		}
+	}
+}