Repository: projectdiscovery/goflags
Pull request #19 (2021-07-28 20:10:50)

Removing normalization of case-sensitive parameters


Patch:
diff --git a/goflags.go b/goflags.go
index 8ff9ffd..6b05ad1 100644
--- a/goflags.go
+++ b/goflags.go
@@ -321,7 +321,48 @@ func (flagSet *FlagSet) IntVar(field *int, long string, defaultValue int, usage
 	return flagData
 }
 
+// NormalizedStringSliceVarP adds a path slice flag with a shortname and longname.
+// It supports comma separated values, that are normalized (lower-cased, stripped of any leading and trailing whitespaces and quotes)
+func (flagSet *FlagSet) NormalizedStringSliceVarP(field *NormalizedStringSlice, long, short string, defaultValue []string, usage string) *FlagData {
+	for _, item := range defaultValue {
+		_ = field.Set(item)
+	}
+
+	flag.Var(field, short, usage)
+	flag.Var(field, long, usage)
+
+	flagData := &FlagData{
+		usage:        usage,
+		short:        short,
+		long:         long,
+		defaultValue: field.createStringArrayDefaultValue(),
+	}
+	flagSet.flagKeys.Set(short, flagData)
+	flagSet.flagKeys.Set(long, flagData)
+	return flagData
+}
+
+// NormalizedStringSliceVar adds a path slice flag with a long name
+// It supports comma separated values, that are normalized (lower-cased, stripped of any leading and trailing whitespaces and quotes)
+func (flagSet *FlagSet) NormalizedStringSliceVar(field *NormalizedStringSlice, long string, defaultValue []string, usage string) *FlagData {
+	for _, item := range defaultValue {
+		_ = field.Set(item)
+	}
+
+	flag.Var(field, long, usage)
+
+	flagData := &FlagData{
+		usage:        usage,
+		long:         long,
+		defaultValue: field.createStringArrayDefaultValue(),
+	}
+	flagSet.flagKeys.Set(long, flagData)
+	return flagData
+}
+
 // StringSliceVarP adds a string slice flag with a shortname and longname
+// Supports ONE value at a time. Adding multiple values require repeating the argument (-flag value1 -flag value2)
+// No value normalization is happening.
 func (flagSet *FlagSet) StringSliceVarP(field *StringSlice, long, short string, defaultValue []string, usage string) *FlagData {
 	for _, item := range defaultValue {
 		_ = field.Set(item)
@@ -342,6 +383,8 @@ func (flagSet *FlagSet) StringSliceVarP(field *StringSlice, long, short string,
 }
 
 // StringSliceVar adds a string slice flag with a longname
+// Supports ONE value at a time. Adding multiple values require repeating the argument (-flag value1 -flag value2)
+// No value normalization is happening.
 func (flagSet *FlagSet) StringSliceVar(field *StringSlice, long string, defaultValue []string, usage string) *FlagData {
 	for _, item := range defaultValue {
 		_ = field.Set(item)
@@ -358,21 +401,6 @@ func (flagSet *FlagSet) StringSliceVar(field *StringSlice, long string, defaultV
 	return flagData
 }
 
-func (stringSlice *StringSlice) createStringArrayDefaultValue() string {
-	defaultBuilder := &strings.Builder{}
-	defaultBuilder.WriteString("[")
-	for i, k := range *stringSlice {
-		defaultBuilder.WriteString("\"")
-		defaultBuilder.WriteString(k)
-		defaultBuilder.WriteString("\"")
-		if i != len(*stringSlice)-1 {
-			defaultBuilder.WriteString(", ")
-		}
-	}
-	defaultBuilder.WriteString("]")
-	return defaultBuilder.String()
-}
-
 func (flagSet *FlagSet) usageFunc() {
 	cliOutput := flag.CommandLine.Output()
 	fmt.Fprintf(cliOutput, "%s\n\n", flagSet.description)
@@ -415,9 +443,8 @@ func (flagSet *FlagSet) usageFuncForGroups(cliOutput io.Writer, writer *tabwrite
 		flagSet.flagKeys.forEach(func(key string, data *FlagData) {
 			currentFlag := flag.CommandLine.Lookup(key)
 
-			isUnique := uniqueDeduper.isUnique(data)
 			if data.group == "" {
-				if !isUnique {
+				if !uniqueDeduper.isUnique(data) {
 					return
 				}
 				otherOptions = append(otherOptions, createUsageString(data, currentFlag))
@@ -427,7 +454,7 @@ func (flagSet *FlagSet) usageFuncForGroups(cliOutput io.Writer, writer *tabwrite
 			if !strings.EqualFold(data.group, group.name) {
 				return
 			}
-			if !isUnique {
+			if !uniqueDeduper.isUnique(data) {
 				return
 			}
 			result := createUsageString(data, currentFlag)
diff --git a/misc.go b/misc.go
deleted file mode 100644
index 33b60e0..0000000
--- a/misc.go
+++ /dev/null
@@ -1,46 +0,0 @@
-package goflags
-
-import (
-	"github.com/pkg/errors"
-	"regexp"
-	"strings"
-)
-
-// StringSlice is a slice of strings
-type StringSlice []string
-
-func (stringSlice *StringSlice) String() string {
-	return strings.Join(*stringSlice, " ")
-}
-
-// Set appends a value to the string slice.
-func (stringSlice *StringSlice) Set(value string) error {
-	if slice, err := ToStringSlice(value); err != nil {
-		return err
-	} else {
-		*stringSlice = append(*stringSlice, slice...)
-		return nil
-	}
-}
-
-var multiValueValidator = regexp.MustCompile("('[^',]+?,.*?')|(\"[^\",]+?,.*?\")|(`[^,]+?,.*?`)")
-
-func ToStringSlice(value string) ([]string, error) {
-	if multiValueValidator.FindString(value) != "" {
-		return nil, errors.New("Supported values are: value1,value2 etc")
-	}
-
-	value = strings.ToLower(value)
-
-	var result []string
-	if strings.Contains(value, ",") {
-		slices := strings.Split(value, ",")
-		result = make([]string, 0, len(slices))
-		for _, slice := range slices {
-			result = append(result, strings.TrimSpace(strings.Trim(strings.TrimSpace(slice), "\"'`")))
-		}
-	} else {
-		result = []string{value}
-	}
-	return result, nil
-}
diff --git a/normalized_slice.go b/normalized_slice.go
new file mode 100644
index 0000000..afe1cc1
--- /dev/null
+++ b/normalized_slice.go
@@ -0,0 +1,99 @@
+package goflags
+
+import (
+	"github.com/pkg/errors"
+	"strings"
+)
+
+var quotes = []rune{'"', '\'', '`'}
+
+// NormalizedStringSlice is a slice of strings
+type NormalizedStringSlice []string
+
+func (normalizedStringSlice *NormalizedStringSlice) String() string {
+	return strings.Join(*normalizedStringSlice, " ")
+}
+
+//Set appends a value to the string slice.
+func (normalizedStringSlice *NormalizedStringSlice) Set(value string) error {
+	if slice, err := ToNormalizedStringSlice(value); err != nil {
+		return err
+	} else {
+		*normalizedStringSlice = append(*normalizedStringSlice, slice...)
+		return nil
+	}
+}
+
+func (normalizedStringSlice *NormalizedStringSlice) createStringArrayDefaultValue() string {
+	defaultBuilder := &strings.Builder{}
+	defaultBuilder.WriteString("[")
+	for i, k := range *normalizedStringSlice {
+		defaultBuilder.WriteString("\"")
+		defaultBuilder.WriteString(k)
+		defaultBuilder.WriteString("\"")
+		if i != len(*normalizedStringSlice)-1 {
+			defaultBuilder.WriteString(", ")
+		}
+	}
+	defaultBuilder.WriteString("]")
+	return defaultBuilder.String()
+}
+
+func ToNormalizedStringSlice(value string) ([]string, error) {
+	var result []string
+
+	addPartToResult := func(part string) {
+		if strings.TrimSpace(part) != "" {
+			result = append(result, strings.TrimSpace(strings.Trim(strings.TrimSpace(strings.ToLower(part)), string(quotes))))
+		}
+	}
+
+	index := 0
+	for index < len(value) {
+		char := rune(value[index])
+		if isQuote, quote := isQuote(char); isQuote {
+			quoteFound, part := searchPart(value[index+1:], quote)
+
+			if !quoteFound {
+				return nil, errors.New("Unclosed quote in path")
+			}
+
+			index += len(part) + 2
+
+			addPartToResult(part)
+		} else {
+			commaFound, part := searchPart(value[index:], ',')
+
+			if commaFound {
+				index += len(part) + 1
+			} else {
+				index += len(part)
+			}
+
+			addPartToResult(part)
+		}
+	}
+
+	return result, nil
+}
+
+func isQuote(char rune) (bool, rune) {
+	for _, quote := range quotes {
+		if quote == char {
+			return true, quote
+		}
+	}
+	return false, 0
+}
+
+func searchPart(value string, stop rune) (bool, string) {
+	var result string
+	for _, char := range value {
+		if char != stop {
+			result += string(char)
+		} else {
+			return true, result
+		}
+	}
+	return false, result
+}
diff --git a/string_slice.go b/string_slice.go
new file mode 100644
index 0000000..58a49b2
--- /dev/null
+++ b/string_slice.go
@@ -0,0 +1,33 @@
+package goflags
+
+import (
+	"strings"
+)
+
+// StringSlice is a slice of strings
+type StringSlice []string
+
+func (stringSlice *StringSlice) String() string {
+	return strings.Join(*stringSlice, " ")
+}
+
+// Set appends a value to the string slice.
+func (stringSlice *StringSlice) Set(value string) error {
+	*stringSlice = append(*stringSlice, value)
+	return nil
+}
+
+func (stringSlice *StringSlice) createStringArrayDefaultValue() string {
+	defaultBuilder := &strings.Builder{}
+	defaultBuilder.WriteString("[")
+	for i, k := range *stringSlice {
+		defaultBuilder.WriteString("\"")
+		defaultBuilder.WriteString(k)
+		defaultBuilder.WriteString("\"")
+		if i != len(*stringSlice)-1 {
+			defaultBuilder.WriteString(", ")
+		}
+	}
+	defaultBuilder.WriteString("]")
+	return defaultBuilder.String()
+}


Test patch:
diff --git a/goflags_test.go b/goflags_test.go
index 3612494..595ad88 100644
--- a/goflags_test.go
+++ b/goflags_test.go
@@ -1,6 +1,7 @@
 package goflags
 
 import (
+	"bytes"
 	"flag"
 	"io/ioutil"
 	"os"
@@ -82,25 +83,62 @@ func TestUsageOrder(t *testing.T) {
 	var intData int
 	var boolData bool
 
-	flagSet.StringVar(&stringData, "string-value", "", "String example value example")
-	flagSet.StringVarP(&stringData, "", "ts2", "test-string", "String with default value example #2")
-	flagSet.StringVar(&stringData, "string-with-default-value", "test-string", "String with default value example")
-	flagSet.StringVarP(&stringData, "string-with-default-value2", "ts", "test-string", "String with default value example #2")
-	flagSet.StringSliceVar(&stringSliceData, "slice-value", []string{}, "String slice flag example value")
-	flagSet.StringSliceVarP(&stringSliceData, "slice-value2", "sv", []string{}, "String slice flag example value #2")
-	flagSet.StringSliceVar(&stringSliceData, "slice-with-default-value", []string{"a", "b", "c"}, "String slice flag with default example values")
-	flagSet.StringSliceVarP(&stringSliceData2, "slice-with-default-value2", "swdf", []string{"a", "b", "c"}, "String slice flag with default example values #2")
-	flagSet.IntVar(&intData, "int-value", 0, "Int value example")
-	flagSet.IntVarP(&intData, "int-value2", "iv", 0, "Int value example #2")
-	flagSet.IntVar(&intData, "int-with-default-value", 12, "Int with default value example")
-	flagSet.IntVarP(&intData, "int-with-default-value2", "iwdv", 12, "Int with default value example #2")
-	flagSet.BoolVar(&boolData, "bool-value", false, "Bool value example")
-	flagSet.BoolVarP(&boolData, "bool-value2", "bv", false, "Bool value example #2")
-	flagSet.BoolVar(&boolData, "bool-with-default-value", true, "Bool with default value example")
-	flagSet.BoolVarP(&boolData, "bool-with-default-value2", "bwdv", true, "Bool with default value example #2")
+	flagSet.SetGroup("String", "String")
+	flagSet.StringVar(&stringData, "string-value", "", "String example value example").Group("String")
+	flagSet.StringVarP(&stringData, "", "ts2", "test-string", "String with default value example #2").Group("String")
+	flagSet.StringVar(&stringData, "string-with-default-value", "test-string", "String with default value example").Group("String")
+	flagSet.StringVarP(&stringData, "string-with-default-value2", "ts", "test-string", "String with default value example #2").Group("String")
+
+	flagSet.SetGroup("StringSlice", "StringSlice")
+	flagSet.StringSliceVar(&stringSliceData, "slice-value", []string{}, "String slice flag example value").Group("StringSlice")
+	flagSet.StringSliceVarP(&stringSliceData, "slice-value2", "sv", []string{}, "String slice flag example value #2").Group("StringSlice")
+	flagSet.StringSliceVar(&stringSliceData, "slice-with-default-value", []string{"a", "b", "c"}, "String slice flag with default example values").Group("StringSlice")
+	flagSet.StringSliceVarP(&stringSliceData2, "slice-with-default-value2", "swdf", []string{"a", "b", "c"}, "String slice flag with default example values #2").Group("StringSlice")
+
+	flagSet.SetGroup("Integer", "Integer")
+	flagSet.IntVar(&intData, "int-value", 0, "Int value example").Group("Integer")
+	flagSet.IntVarP(&intData, "int-value2", "iv", 0, "Int value example #2").Group("Integer")
+	flagSet.IntVar(&intData, "int-with-default-value", 12, "Int with default value example").Group("Integer")
+	flagSet.IntVarP(&intData, "int-with-default-value2", "iwdv", 12, "Int with default value example #2").Group("Integer")
+
+	flagSet.SetGroup("Bool", "Boolean")
+	flagSet.BoolVar(&boolData, "bool-value", false, "Bool value example").Group("Bool")
+	flagSet.BoolVarP(&boolData, "bool-value2", "bv", false, "Bool value example #2").Group("Bool")
+	flagSet.BoolVar(&boolData, "bool-with-default-value", true, "Bool with default value example").Group("Bool")
+	flagSet.BoolVarP(&boolData, "bool-with-default-value2", "bwdv", true, "Bool with default value example #2").Group("Bool")
+
+	output := &bytes.Buffer{}
+	flag.CommandLine.SetOutput(output)
 
 	flagSet.usageFunc()
-	// TODO try to retrieve the data written to the stdout/err and do some assertions on it
+
+	resultOutput := output.String()
+	actual := resultOutput[strings.Index(resultOutput, "Flags:\n"):]
+
+	expected :=
+		`Flags:
+STRING:
+   -string-value string                     String example value example
+   -ts2 string                              String with default value example #2 (default "test-string")
+   -string-with-default-value string        String with default value example (default "test-string")
+   -ts, -string-with-default-value2 string  String with default value example #2 (default "test-string")
+STRINGSLICE:
+   -slice-value string[]                       String slice flag example value
+   -sv, -slice-value2 string[]                 String slice flag example value #2
+   -slice-with-default-value string[]          String slice flag with default example values (default ["a", "b", "c"])
+   -swdf, -slice-with-default-value2 string[]  String slice flag with default example values #2 (default ["a", "b", "c"])
+INTEGER:
+   -int-value int                       Int value example
+   -iv, -int-value2 int                 Int value example #2
+   -int-with-default-value int          Int with default value example (default 12)
+   -iwdv, -int-with-default-value2 int  Int with default value example #2 (default 12)
+BOOLEAN:
+   -bool-value                       Bool value example
+   -bv, -bool-value2                 Bool value example #2
+   -bool-with-default-value          Bool with default value example (default true)
+   -bwdv, -bool-with-default-value2  Bool with default value example #2 (default true)
+`
+	assert.Equal(t, actual, expected)
 
 	tearDown(t.Name())
 }
@@ -112,6 +150,8 @@ func TestIncorrectStringFlagsCausePanic(t *testing.T) {
 	flagSet.StringVar(&stringData, "", "test-string", "String with default value example")
 	assert.Panics(t, flagSet.usageFunc)
 
+	// env GOOS=linux GOARCH=amd64 go build main.go -o nuclei
+
 	tearDown(t.Name())
 }
 
@@ -154,7 +194,7 @@ type testSliceValue []interface{}
 func (value testSliceValue) String() string   { return "" }
 func (value testSliceValue) Set(string) error { return nil }
 
-func TestMe(t *testing.T) {
+func TestCustomSliceUsageType(t *testing.T) {
 	testCases := map[string]flag.Flag{
 		"string[]": {Value: &StringSlice{}},
 		"value[]":  {Value: &testSliceValue{}},
@@ -166,6 +206,30 @@ func TestMe(t *testing.T) {
 	}
 }
 
+func TestParseStringSlice(t *testing.T) {
+	flagSet := NewFlagSet()
+
+	var stringSlice StringSlice
+	flagSet.StringSliceVarP(&stringSlice, "header", "H", []string{}, "Header values. Expected usage: -H \"header1\":\"value1\" -H \"header2\":\"value2\"")
+
+	header1 := "\"header1:value1\""
+	header2 := "\" HEADER 2: VALUE2  \""
+	header3 := "\"header3\":\"value3, value4\""
+
+	os.Args = []string{
+		"./appName",
+		"-H", header1,
+		"-header", header2,
+		"-H", header3,
+	}
+
+	err := flagSet.Parse()
+	assert.Nil(t, err)
+
+	assert.Equal(t, stringSlice, StringSlice{header1, header2, header3})
+}
+
 func tearDown(uniqueValue string) { // sadly there is no official support for setup/teardown/test
-	flag.CommandLine = flag.NewFlagSet(uniqueValue, flag.PanicOnError)
+	flag.CommandLine = flag.NewFlagSet(uniqueValue, flag.ContinueOnError)
+	flag.CommandLine.Usage = flag.Usage
 }
diff --git a/misc_test.go b/misc_test.go
deleted file mode 100644
index ab90e47..0000000
--- a/misc_test.go
+++ /dev/null
@@ -1,36 +0,0 @@
-package goflags
-
-import (
-	"github.com/stretchr/testify/assert"
-	"testing"
-)
-
-func Test_toStringSlice(t *testing.T) {
-	expected := []string{"aa", "bb", "cc"}
-	values := map[string]interface{}{
-		"aa,bb,cc":                       expected,
-		"  aa, bb,  cc   ":               expected,
-		"  `aa`, 'bb',  \"cc\"   ":       expected,
-		"  `aa`, bb,  \"cc\"   ":         expected,
-		"  `aa, bb,  cc\"   ":            expected,
-		"  \"aa\", bb,  cc\"   ":         expected,
-		"\n  aa, \tbb,  cc\r   ":         expected,
-		"\n  \"aa', \t`bb',  \"cc\r`   ": nil,
-		"\"\n  aa', `\tbb',  \"cc\r`   ": nil,
-		"'aa,bb,cc'":                     nil,
-		"`aa,bb,cc`":                     nil,
-		"\"aa,bb,cc\"":                   nil,
-	}
-	for input, expectedValue := range values {
-		t.Run(input, func(t *testing.T) {
-			slice, err := ToStringSlice(input)
-			if expectedValue == nil {
-				if err == nil {
-					t.Fail()
-				}
-			} else {
-				assert.Equal(t, expectedValue, slice)
-			}
-		})
-	}
-}
diff --git a/normalized_slice_test.go b/normalized_slice_test.go
new file mode 100644
index 0000000..3808323
--- /dev/null
+++ b/normalized_slice_test.go
@@ -0,0 +1,67 @@
+package goflags
+
+import (
+	"fmt"
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestNormalizedStringSlicePositive(t *testing.T) {
+	expectedABC := []string{"aa", "bb", "cc"}
+	expectedFilePath := []string{"/root/home/file0"}
+
+	slices := map[string][]string{
+		"aa,bb,cc":                 expectedABC,
+		"  aa, bb,  cc   ":         expectedABC,
+		"  `aa`, 'bb',  \"cc\"   ": expectedABC,
+		"  `aa`, bb,  \"cc\"   ":   expectedABC,
+		"  `aa, bb,  cc\"   ":      expectedABC,
+		"  \"aa\", bb,  cc\"   ":   expectedABC,
+		"\n  aa, \tbb,  cc\r   ":   expectedABC,
+
+		"\"value1\",value,'value3'": {"value1", "value", "value3"},
+		"\"value1\",VALUE,'value3'": {"value1", "value", "value3"},
+
+		"\"/root/home/file0\"":       expectedFilePath,
+		"'/root/home/file0'":         expectedFilePath,
+		"`/root/home/file0`":         expectedFilePath,
+		"\"/root/home/file0\",":      expectedFilePath,
+		",\"/root/home/file0\",":     expectedFilePath,
+		",\"/root/home/file0\"":      expectedFilePath,
+		",,\"/root/home/file0\"":     expectedFilePath,
+		"\"\",,\"/root/home/file0\"": expectedFilePath,
+		"\" \",\"/root/home/file0\"": expectedFilePath,
+		"\"/root/home/file0\",\"\"":  expectedFilePath,
+		"/root/home/file0":           expectedFilePath,
+
+		"\"/root/home/file2\",\"/root/home/file3\"":             {"/root/home/file2", "/root/home/file3"},
+		"/root/home/file4,/root/home/file5":                     {"/root/home/file4", "/root/home/file5"},
+		"\"/root/home/file4,/root/home/file5\"":                 {"/root/home/file4,/root/home/file5"},
+		"\"/root/home/file6\",/root/home/file7":                 {"/root/home/file6", "/root/home/file7"},
+		"\"c:\\my files\\bug,bounty\"":                          {"c:\\my files\\bug,bounty"},
+		"\"c:\\my files\\bug,bounty\",c:\\my_files\\bug bounty": {"c:\\my files\\bug,bounty", "c:\\my_files\\bug bounty"},
+	}
+
+	for value, expected := range slices {
+		result, err := ToNormalizedStringSlice(value)
+		fmt.Println(result)
+		assert.Nil(t, err)
+		assert.Equal(t, result, expected)
+	}
+}
+
+func TestNormalizedStringSliceNegative(t *testing.T) {
+	slices := []string{
+		"\"/root/home/file0",
+		"'/root/home/file0",
+		"`/root/home/file0",
+		"\"/root/home/file0'",
+		"\"/root/home/file0`",
+	}
+
+	for _, value := range slices {
+		result, err := ToNormalizedStringSlice(value)
+		assert.Nil(t, result)
+		assert.NotNil(t, err)
+	}
+}