Repository: projectdiscovery/httpx (language: go)

Issue:
In httpx is it intentional that the timestamp format is different between csv and json?
When outputting jusing -csv the timestamp looks like this 2025-02-25 16:24:45.722337 -0500 EST m=+1.690385793
Json looks like this "timestamp":"2025-02-25T16:25:37.330406-05:00"

CSV breaks tooling due to non standard format.

> _Originally posted by failOpen in https://discord.com/channels/695645237418131507/1344058703057260617_.


Discussion / hints:

Fix patch:
diff --git a/go.mod b/go.mod
index eb1e4bec..0e9e746f 100644
--- a/go.mod
+++ b/go.mod
@@ -54,6 +54,7 @@ require (
 
 require (
 	github.com/go-viper/mapstructure/v2 v2.2.1
+	github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1
 	github.com/weppos/publicsuffix-go v0.40.2
 )
 
diff --git a/go.sum b/go.sum
index 507922d0..22ee0058 100644
--- a/go.sum
+++ b/go.sum
@@ -100,6 +100,8 @@ github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlnd
 github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
 github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
 github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
+github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 h1:FWNFq4fM1wPfcK40yHE5UO3RUdSNPaBC+j3PokzA6OQ=
+github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI=
 github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
diff --git a/runner/runner.go b/runner/runner.go
index 1ec9bb24..00579f43 100644
--- a/runner/runner.go
+++ b/runner/runner.go
@@ -16,7 +16,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"reflect"
 	"regexp"
 	"slices"
 	"sort"
@@ -29,6 +28,7 @@ import (
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/corona10/goimagehash"
+	"github.com/gocarina/gocsv"
 	"github.com/mfonda/simhash"
 	asnmap "github.com/projectdiscovery/asnmap/libs"
 	"github.com/projectdiscovery/fastdialer/fastdialer"
@@ -2466,57 +2466,51 @@ func (r Result) JSON(scanopts *ScanOptions) string { //nolint
 
 // CSVHeader the CSV headers
 func (r Result) CSVHeader() string { //nolint
-	buffer := bytes.Buffer{}
-	writer := csv.NewWriter(&buffer)
-
-	var headers []string
-	ty := reflect.TypeOf(r)
-	for i := 0; i < ty.NumField(); i++ {
-		tag := ty.Field(i).Tag.Get("csv")
-		if ignored := (tag == "" || tag == "-"); ignored {
-			continue
-		}
+	var header string
 
-		headers = append(headers, tag)
+	if h, err := gocsv.MarshalString([]Result{}); err == nil {
+		header = h
 	}
-	_ = writer.Write(headers)
-	writer.Flush()
+	header = strings.TrimSpace(header)
 
-	return strings.TrimSpace(buffer.String())
+	return header
 }
 
 // CSVRow the CSV Row
 func (r Result) CSVRow(scanopts *ScanOptions) string { //nolint
+	var res string
+
 	if scanopts != nil && len(r.ResponseBody) > scanopts.MaxResponseBodySizeToSave {
 		r.ResponseBody = r.ResponseBody[:scanopts.MaxResponseBodySizeToSave]
 	}
 
-	buffer := bytes.Buffer{}
-	writer := csv.NewWriter(&buffer)
-
-	var cells []string
-	elem := reflect.ValueOf(r)
-	for i := 0; i < elem.NumField(); i++ {
-		value := elem.Field(i)
-		tag := elem.Type().Field(i).Tag.Get(`csv`)
-		if ignored := (tag == "" || tag == "-"); ignored {
-			continue
-		}
-
-		str := fmt.Sprintf("%v", value.Interface())
-
-		// defense against csv injection
-		startWithRiskyChar, _ := regexp.Compile(`^([=+\-@])`)
-		if startWithRiskyChar.Match([]byte(str)) {
-			str = "'" + str
+	if row, err := gocsv.MarshalStringWithoutHeaders([]Result{r}); err == nil {
+		reader := csv.NewReader(strings.NewReader(row))
+		records, err := reader.ReadAll()
+		if err == nil && len(records) > 0 {
+			buf := &bytes.Buffer{}
+			writer := csv.NewWriter(buf)
+			for _, record := range records {
+				for i, field := range record {
+					if len(field) > 0 {
+						firstChar := field[0]
+						// NOTE(dwisiswant0): Sanitize (prevent CSV injection).
+						if firstChar == '=' || firstChar == '+' || firstChar == '-' || firstChar == '@' {
+							record[i] = "'" + field
+						}
+					}
+				}
+				_ = writer.Write(record) //nolint
+			}
+			writer.Flush()
+			res = buf.String()
+		} else {
+			res = row
 		}
-
-		cells = append(cells, str)
+		res = strings.TrimSpace(res)
 	}
-	_ = writer.Write(cells)
-	writer.Flush()
 
-	return strings.TrimSpace(buffer.String()) // remove "\n" in the end
+	return res
 }
 
 func (r *Runner) skipCDNPort(host string, port string) bool {
diff --git a/runner/types.go b/runner/types.go
index c3eeae3b..724e8697 100644
--- a/runner/types.go
+++ b/runner/types.go
@@ -34,11 +34,11 @@ func (o AsnResponse) String() string {
 // Result of a scan
 type Result struct {
 	Timestamp          time.Time                     `json:"timestamp,omitempty" csv:"timestamp" mapstructure:"timestamp"`
-	ASN                *AsnResponse                  `json:"asn,omitempty" csv:"asn" mapstructure:"asn"`
+	ASN                *AsnResponse                  `json:"asn,omitempty" csv:"-" mapstructure:"asn"`
 	Err                error                         `json:"-" csv:"-" mapstructure:"-"`
-	CSPData            *httpx.CSPData                `json:"csp,omitempty" csv:"csp" mapstructure:"csp"`
-	TLSData            *clients.Response             `json:"tls,omitempty" csv:"tls" mapstructure:"tls"`
-	Hashes             map[string]interface{}        `json:"hash,omitempty" csv:"hash" mapstructure:"hash"`
+	CSPData            *httpx.CSPData                `json:"csp,omitempty" csv:"-" mapstructure:"csp"`
+	TLSData            *clients.Response             `json:"tls,omitempty" csv:"-" mapstructure:"tls"`
+	Hashes             map[string]interface{}        `json:"hash,omitempty" csv:"-" mapstructure:"hash"`
 	ExtractRegex       []string                      `json:"extract_regex,omitempty" csv:"extract_regex" mapstructure:"extract_regex"`
 	CDNName            string                        `json:"cdn_name,omitempty" csv:"cdn_name" mapstructure:"cdn_name"`
 	CDNType            string                        `json:"cdn_type,omitempty" csv:"cdn_type" mapstructure:"cdn_type"`
@@ -49,7 +49,7 @@ type Result struct {
 	Input              string                        `json:"input,omitempty" csv:"input" mapstructure:"input"`
 	Location           string                        `json:"location,omitempty" csv:"location" mapstructure:"location"`
 	Title              string                        `json:"title,omitempty" csv:"title" mapstructure:"title"`
-	str                string                        `mapstructure:"-"`
+	str                string                        `json:"-" csv:"-" mapstructure:"-"`
 	Scheme             string                        `json:"scheme,omitempty" csv:"scheme" mapstructure:"scheme"`
 	Error              string                        `json:"error,omitempty" csv:"error" mapstructure:"error"`
 	WebServer          string                        `json:"webserver,omitempty" csv:"webserver" mapstructure:"webserver"`
@@ -74,8 +74,8 @@ type Result struct {
 	AAAA               []string                      `json:"aaaa,omitempty" csv:"aaaa" mapstructure:"aaaa"`
 	CNAMEs             []string                      `json:"cname,omitempty" csv:"cname" mapstructure:"cname"`
 	Technologies       []string                      `json:"tech,omitempty" csv:"tech" mapstructure:"tech"`
-	Extracts           map[string][]string           `json:"extracts,omitempty" csv:"extracts" mapstructure:"extracts"`
-	Chain              []httpx.ChainItem             `json:"chain,omitempty" csv:"chain" mapstructure:"chain"`
+	Extracts           map[string][]string           `json:"extracts,omitempty" csv:"-" mapstructure:"extracts"`
+	Chain              []httpx.ChainItem             `json:"chain,omitempty" csv:"-" mapstructure:"chain"`
 	Words              int                           `json:"words" csv:"words" mapstructure:"words"`
 	Lines              int                           `json:"lines" csv:"lines" mapstructure:"lines"`
 	StatusCode         int                           `json:"status_code" csv:"status_code" mapstructure:"status_code"`
@@ -91,15 +91,15 @@ type Result struct {
 	StoredResponsePath string                        `json:"stored_response_path,omitempty" csv:"stored_response_path" mapstructure:"stored_response_path"`
 	ScreenshotPath     string                        `json:"screenshot_path,omitempty" csv:"screenshot_path" mapstructure:"screenshot_path"`
 	ScreenshotPathRel  string                        `json:"screenshot_path_rel,omitempty" csv:"screenshot_path_rel" mapstructure:"screenshot_path_rel"`
-	KnowledgeBase      map[string]interface{}        `json:"knowledgebase,omitempty" csv:"knowledgebase" mapstructure:"knowledgebase"`
+	KnowledgeBase      map[string]interface{}        `json:"knowledgebase,omitempty" csv:"-" mapstructure:"knowledgebase"`
 	Resolvers          []string                      `json:"resolvers,omitempty" csv:"resolvers" mapstructure:"resolvers"`
-	Fqdns              []string                      `json:"body_fqdn,omitempty" mapstructure:"body_fqdn"`
-	Domains            []string                      `json:"body_domains,omitempty" mapstructure:"body_domains"`
+	Fqdns              []string                      `json:"body_fqdn,omitempty" csv:"body_fqdn" mapstructure:"body_fqdn"`
+	Domains            []string                      `json:"body_domains,omitempty" csv:"body_domains" mapstructure:"body_domains"`
 	TechnologyDetails  map[string]wappalyzer.AppInfo `json:"-" csv:"-" mapstructure:"-"`
 	RequestRaw         []byte                        `json:"-" csv:"-" mapstructure:"-"`
 	Response           *httpx.Response               `json:"-" csv:"-" mapstructure:"-"`
 	FaviconData        []byte                        `json:"-" csv:"-" mapstructure:"-"`
-	Trace              *retryablehttp.TraceInfo      `json:"trace,omitempty" csv:"trace"  mapstructure:"trace"`
+	Trace              *retryablehttp.TraceInfo      `json:"trace,omitempty" csv:"-"  mapstructure:"trace"`
 }
 
 type Trace struct {


Test patch:
diff --git a/runner/runner_test.go b/runner/runner_test.go
index 82e46eb1..f88e012f 100644
--- a/runner/runner_test.go
+++ b/runner/runner_test.go
@@ -1,9 +1,11 @@
 package runner
 
 import (
+	"fmt"
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	_ "github.com/projectdiscovery/fdmax/autofdmax"
 	"github.com/projectdiscovery/httpx/common/httpx"
@@ -181,3 +183,42 @@ func TestRunner_urlWithComma_targets(t *testing.T) {
 	}
 	require.ElementsMatch(t, expected, got, "could not expected output")
 }
+
+func TestRunner_CSVRow(t *testing.T) {
+	// Create a result with fields that would be vulnerable to CSV injection
+	result := Result{
+		URL:         `=HYPERLINK('https://evil.com','click me')`,
+		Title:       `+CMD('calc')`,
+		ContentType: `-SUM(1+1)`,
+		WebServer:   `@MACRO=Virus()`,
+		StatusCode:  200,
+		Timestamp:   time.Now(),
+	}
+
+	// Call CSVRow to get the sanitized output
+	csvOutput := result.CSVRow(nil)
+
+	// Check that vulnerable fields are properly sanitized with a prefix quote
+	tests := []struct {
+		fieldName string
+		original  string
+		expected  string
+	}{
+		{"URL", result.URL, fmt.Sprintf("'%s", result.URL)},
+		{"Title", result.Title, fmt.Sprintf("'%s", result.Title)},
+		{"ContentType", result.ContentType, fmt.Sprintf("'%s", result.ContentType)},
+		{"WebServer", result.WebServer, fmt.Sprintf("'%s", result.WebServer)},
+	}
+
+	for _, tc := range tests {
+		if !strings.Contains(csvOutput, tc.expected) {
+			t.Errorf("CSV sanitization failed for %s field: expected %q but sanitized value not found in output: %s",
+				tc.fieldName, tc.expected, csvOutput)
+		}
+	}
+
+	// Also check that normal fields remain unsanitized
+	if strings.Contains(csvOutput, "'200") {
+		t.Error("CSV sanitization incorrectly modified non-vulnerable field")
+	}
+}