Repository: bitfield/script
Pull request #10 (2019-07-01 13:43:50)

Add Freq() filter
Implements part of what's required for #7.


Patch:
diff --git a/README.md b/README.md
index 710d308..e76753d 100644
--- a/README.md
+++ b/README.md
@@ -334,7 +334,45 @@ fmt.Println(output)
 `First()` reads its input and passes on the first N lines of it (like Unix [`head`](examples/head/main.go)):
 
 ```go
-Stdin().First(10).Stdout()
+script.Stdin().First(10).Stdout()
+```
+
+### Freq
+
+`Freq()` counts the frequencies of input lines, and outputs only the unique lines in the input, each prefixed with a count of its frequency, in descending order of frequency (that is, most frequent lines first). Lines with the same frequency will be sorted alphabetically. For example, given this input:
+
+```
+banana
+apple
+orange
+apple
+banana
+```
+
+and a program like:
+
+```go
+script.Stdin().Freq().Stdout()
+```
+
+the output will be:
+
+```
+2 apple
+2 banana
+1 orange
+```
+
+This is a common pattern in shell scripts to find the most frequently-occurring lines in a file:
+
+```sh
+sort testdata/freq.input.txt |uniq -c |sort -rn
+```
+
+`Freq()`'s behaviour is like the combination of Unix `sort`, `uniq -c`, and `sort -rn` used here. You can use `Freq()` in combination with `First()` to get, for example, the ten most common lines in a file:
+
+```go
+script.Stdin().Freq().First(10).Stdout()
 ```
 
 ### Join
diff --git a/filters.go b/filters.go
index 87b3ba6..8095180 100644
--- a/filters.go
+++ b/filters.go
@@ -3,10 +3,12 @@ package script
 import (
 	"bufio"
 	"bytes"
+	"fmt"
 	"io"
 	"os"
 	"os/exec"
 	"regexp"
+	"sort"
 	"strings"
 )
 
@@ -157,3 +159,38 @@ func (p *Pipe) First(lines int) *Pipe {
 	}
 	return Echo(output.String())
 }
+
+// Freq reads from the pipe, and returns a new pipe containing only unique lines
+// from the input, prefixed with a frequency count, in descending numerical
+// order (most frequent lines first). Lines with equal frequency will be sorted
+// alphabetically. If there is an error reading the pipe, the pipe's error
+// status is also set.
+func (p *Pipe) Freq() *Pipe {
+	if p == nil || p.Error() != nil {
+		return p
+	}
+	var freq = map[string]int{}
+	p.EachLine(func(line string, out *strings.Builder) {
+		freq[line]++
+	})
+	type frequency struct {
+		line  string
+		count int
+	}
+	var freqs = make([]frequency, 0, len(freq))
+	for line, count := range freq {
+		freqs = append(freqs, frequency{line, count})
+	}
+	sort.Slice(freqs, func(i, j int) bool {
+		if freqs[i].count == freqs[j].count {
+			return freqs[i].line < freqs[j].line
+		}
+		return freqs[i].count > freqs[j].count
+	})
+	var output strings.Builder
+	for _, item := range freqs {
+		output.WriteString(fmt.Sprintf("%d %s", item.count, item.line))
+		output.WriteRune('\n')
+	}
+	return Echo(output.String())
+}


Test patch:
diff --git a/filters_test.go b/filters_test.go
index 15380c4..7d51b01 100644
--- a/filters_test.go
+++ b/filters_test.go
@@ -255,3 +255,18 @@ func TestFirst(t *testing.T) {
 		t.Errorf("want %q, got %q", want, got)
 	}
 }
+
+func TestFreq(t *testing.T) {
+	t.Parallel()
+	want, err := ioutil.ReadFile("testdata/freq.golden.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := File("testdata/freq.input.txt").Freq().Bytes()
+	if err != nil {
+		t.Error(err)
+	}
+	if !bytes.Equal(got, want) {
+		t.Errorf("want %q, got %q", want, got)
+	}
+}
diff --git a/pipes_test.go b/pipes_test.go
index 5e93668..01febbd 100644
--- a/pipes_test.go
+++ b/pipes_test.go
@@ -102,6 +102,8 @@ func doMethodsOnPipe(t *testing.T, p *Pipe, kind string) {
 	p.Read([]byte{})
 	action = "First()"
 	p.First(1)
+	action = "Freq()"
+	p.Freq()
 }
 
 func TestNilPipes(t *testing.T) {
diff --git a/testdata/freq.golden.txt b/testdata/freq.golden.txt
new file mode 100644
index 0000000..ca7d8b2
--- /dev/null
+++ b/testdata/freq.golden.txt
@@ -0,0 +1,4 @@
+4 apple
+4 banana
+2 orange
+1 kumquat
diff --git a/testdata/freq.input.txt b/testdata/freq.input.txt
new file mode 100644
index 0000000..e15ba3e
--- /dev/null
+++ b/testdata/freq.input.txt
@@ -0,0 +1,11 @@
+apple
+orange
+banana
+banana
+apple
+orange
+kumquat
+apple
+apple
+banana
+banana
\ No newline at end of file