Repository: smallstep/cli
Pull request #33 (2018-10-29 20:17:24)

Add simple quality gates to integration tests 
### Description
This PR makes it easier to continuously and quickly gauge simple quality gates such as 
* minimum word/line count per help sections, 
* headline consistency in sections across the documentation, and 
* surfacing any unresolved TODOs.

Minor improvements:
* Correction of a headline to enforce consistency.
* Removes dependency to external library to strip tags, use regex instead


Patch:
diff --git a/Gopkg.lock b/Gopkg.lock
index e633ae4b..b937053f 100644
--- a/Gopkg.lock
+++ b/Gopkg.lock
@@ -117,14 +117,6 @@
   pruneopts = "UT"
   revision = "3fd9b69f2fb179405773f03d33c68a00f3a1ca4a"
 
-[[projects]]
-  branch = "master"
-  digest = "1:8dbe76014be3c83806abc61befcb5e1789d2d872bc8f98a8fb955405550c63be"
-  name = "github.com/grokify/html-strip-tags-go"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "e9e44961e26f513866063f54bf85070db95600f7"
-
 [[projects]]
   branch = "master"
   digest = "1:22725c01ecd8ed0c0f0078944305a57053340d92878b02db925c660cc4accf64"
@@ -426,7 +418,6 @@
     "github.com/google/go-cmp/cmp",
     "github.com/google/go-cmp/cmp/cmpopts",
     "github.com/gordonklaus/ineffassign",
-    "github.com/grokify/html-strip-tags-go",
     "github.com/icrowley/fake",
     "github.com/pkg/errors",
     "github.com/pquerna/otp",
diff --git a/command/crypto/otp/otp.go b/command/crypto/otp/otp.go
index fdc9ad20..4f2805da 100644
--- a/command/crypto/otp/otp.go
+++ b/command/crypto/otp/otp.go
@@ -13,7 +13,7 @@ func Command() cli.Command {
 		Description: `**step crypto otp** command group implements TOTP and HOTP one-time passwords
 (mention RFCs)
 
-## EXAMPLE
+## EXAMPLES
 
 Generate a new TOTP token and it's QR Code to scan:
 '''
diff --git a/usage/report.go b/usage/report.go
index 2a0b2f32..cd87831b 100644
--- a/usage/report.go
+++ b/usage/report.go
@@ -6,9 +6,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"regexp"
 	"strings"
 
-	"github.com/grokify/html-strip-tags-go"
 	"golang.org/x/net/html"
 )
 
@@ -102,12 +102,14 @@ func (report *Report) processNode(node *html.Node) (string, *html.Node) {
 	text := ""
 	current := node.NextSibling
 
+	r, _ := regexp.Compile("<[^>]*>")
+
 	for current != nil {
 		var buf bytes.Buffer
 		w := io.Writer(&buf)
 		html.Render(w, current)
 
-		notags := strip.StripTags(buf.String())
+		notags := r.ReplaceAllString(buf.String(), "")
 		clean := strings.TrimSpace(notags)
 
 		if len(text) > 0 && len(clean) > 0 {
@@ -128,3 +130,19 @@ func (report *Report) processNode(node *html.Node) (string, *html.Node) {
 
 	return text, node
 }
+
+// PerHeadline returns all sections across commands/pages with the same headline
+func (report *Report) PerHeadline(headline string) []Section {
+	var results []Section
+	for _, top := range report.Report {
+		for _, section := range top.Sections {
+			if section.Name != headline {
+				continue
+			}
+
+			results = append(results, *section)
+		}
+	}
+
+	return results
+}


Test patch:
diff --git a/integration/help_quality_test.go b/integration/help_quality_test.go
new file mode 100644
index 00000000..fbf921ff
--- /dev/null
+++ b/integration/help_quality_test.go
@@ -0,0 +1,88 @@
+// +build integration
+
+package integration
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/smallstep/assert"
+	"github.com/smallstep/cli/usage"
+)
+
+func TestHelpQuality(t *testing.T) {
+	cmd := NewCLICommand().setCommand("../bin/step help").setFlag("html", "./html").setFlag("report", "")
+	cmd.run()
+
+	raw, _ := ioutil.ReadFile("./html/report.json")
+	var report *usage.Report
+	json.Unmarshal([]byte(raw), &report)
+
+	expectations := make(map[string]usage.Section)
+	expectations["COMMANDS"] = usage.Section{Name: "COMMANDS", Words: 0, Lines: 0}
+	expectations["COPYRIGHT"] = usage.Section{Name: "COPYRIGHT", Words: 5, Lines: 1}
+	expectations["DESCRIPTION"] = usage.Section{Name: "DESCRIPTION", Words: 8, Lines: 1}
+	expectations["EXAMPLES"] = usage.Section{Name: "EXAMPLES", Words: 10, Lines: 1}
+	expectations["EXIT CODES"] = usage.Section{Name: "EXIT CODES", Words: 12, Lines: 1}
+	expectations["ONLINE"] = usage.Section{Name: "ONLINE", Words: 7, Lines: 1}
+	expectations["OPTIONS"] = usage.Section{Name: "OPTIONS", Words: 6, Lines: 2}
+	expectations["POSITIONAL ARGUMENTS"] = usage.Section{Name: "POSITIONAL ARGUMENTS", Words: 6, Lines: 2}
+	expectations["PRINTING"] = usage.Section{Name: "PRINTING", Words: 23, Lines: 1}
+	expectations["SECURITY CONSIDERATIONS"] = usage.Section{Name: "SECURITY CONSIDERATIONS", Words: 220, Lines: 25}
+	expectations["STANDARDS"] = usage.Section{Name: "STANDARDS", Words: 45, Lines: 10}
+	expectations["USAGE"] = usage.Section{Name: "USAGE", Words: 3, Lines: 1}
+	expectations["VERSION"] = usage.Section{Name: "VERSION", Words: 3, Lines: 1}
+
+	t.Run("Headlines consistency", func(t *testing.T) {
+		var headlines []string
+		for _, top := range report.Report {
+			for _, section := range top.Sections {
+				headlines = append(headlines, section.Name)
+			}
+		}
+		sort.Strings(headlines)
+
+		grouped := make(map[string]int)
+		for _, line := range headlines {
+			grouped[line]++
+		}
+
+		for item, count := range grouped {
+			fmt.Printf("%s: %d\n", item, count)
+
+			// Let's say only upper case considered headlines
+			if strings.ToUpper(item) == item {
+
+				_, ok := expectations[item]
+				msg := fmt.Sprintf("Unexpected headline %s might lead to inconsistent docs", item)
+				assert.Equals(t, ok, true, msg)
+			}
+		}
+	})
+
+	t.Run("Thresholds", func(t *testing.T) {
+		for _, expected := range expectations {
+			entries := report.PerHeadline(expected.Name)
+
+			for _, entry := range entries {
+				msgw := fmt.Sprintf("Short on words (%d < %d) in %s (%s)", entry.Words, expected.Words, entry.Command, expected.Name)
+				msgl := fmt.Sprintf("Short on lines (%d < %d) in %s (%s)", entry.Lines, expected.Lines, entry.Command, expected.Name)
+				assert.True(t, entry.Words >= expected.Words, msgw)
+				assert.True(t, entry.Lines >= expected.Lines, msgl)
+			}
+		}
+	})
+
+	t.Run("No TODOs", func(t *testing.T) {
+		for _, top := range report.Report {
+			for _, section := range top.Sections {
+				msg := fmt.Sprintf("TODO found in %s (%s)", section.Command, section.Name)
+				assert.False(t, strings.Contains(strings.ToUpper(section.Text), "TODO"), msg)
+			}
+		}
+	})
+}