Repository: ossf/package-analysis (language: python)

Issue:
Change static analysis JSON schema to support BigQuery import
BigQuery doesn't support importing JSON that contains dictionaries, i.e. when the keys are not static. The static analysis JSON output currently contains several dicts; these need to be converted into lists of objects by merging the key and the value(s) together in a struct/object


Discussion / hints:
Required by #703 
This will also require converting historical static analysis JSON data to the new format

Fix patch:
diff --git a/internal/staticanalysis/analyze.go b/internal/staticanalysis/analyze.go
index 7c216823..ed9a5dcb 100644
--- a/internal/staticanalysis/analyze.go
+++ b/internal/staticanalysis/analyze.go
@@ -96,16 +96,15 @@ func AnalyzePackageFiles(extractDir string, jsParserConfig parsing.ParserConfig,
 
 		input := externalcmd.MultipleFileInput(fileList)
 		parsingResults, err := parsing.Analyze(jsParserConfig, input, false)
-		result.ParsingData = parsing.PackageResult{}
 
 		if err != nil {
 			log.Error("static analysis error", log.Label("task", string(Parsing)), "error", err)
 		} else {
 			// change absolute path in parsingResults to package-relative path
-			for path, parseResult := range parsingResults {
-				pathInArchive := archivePath[path]
-				result.ParsingData[pathInArchive] = parseResult
+			for _, parseResult := range parsingResults {
+				parseResult.Filename = archivePath[parseResult.Filename]
 			}
+			result.ParsingData = parsingResults
 		}
 	}
 
diff --git a/internal/staticanalysis/basic_data.go b/internal/staticanalysis/basic_data.go
index 16d4dd00..f6b92b95 100644
--- a/internal/staticanalysis/basic_data.go
+++ b/internal/staticanalysis/basic_data.go
@@ -13,29 +13,33 @@ import (
 	"github.com/ossf/package-analysis/internal/staticanalysis/externalcmd"
 	"github.com/ossf/package-analysis/internal/staticanalysis/linelengths"
 	"github.com/ossf/package-analysis/internal/utils"
+	"github.com/ossf/package-analysis/internal/utils/valuecounts"
 )
 
 // BasicPackageData records basic information about files in a package,
 // mapping file path within the archive to BasicFileData about that file.
 type BasicPackageData struct {
-	Files map[string]BasicFileData `json:"files"`
+	Files []BasicFileData `json:"files"`
 }
 
 // BasicFileData records various information about a file that can be determined
 // without parsing it using a programming language parser.
 type BasicFileData struct {
+	// Filename records the path to the file within the package archive
+	Filename string `json:"filename"`
+
 	// FileType records the output of the `file` command run on that file.
-	FileType string
+	FileType string `json:"filetype"`
 
 	// Size records the size of the file (as reported by the filesystem).
-	Size int64
+	Size int64 `json:"size"`
 
 	// Hash records the SHA256sum hash of the file.
-	Hash string
+	Hash string `json:"hash"`
 
 	// LineLengthCounts records the count of lines of each length in the file,
 	// where a line is defined as all characters up to a newline.
-	LineLengthCounts map[int]int
+	LineLengthCounts valuecounts.ValueCounts `json:"line_length_counts"`
 }
 
 func (bd BasicFileData) String() string {
@@ -128,7 +132,7 @@ func GetBasicData(fileList []string, pathInArchive map[string]string) (*BasicPac
 	}
 
 	result := BasicPackageData{
-		Files: map[string]BasicFileData{},
+		Files: []BasicFileData{},
 	}
 
 	for index, filePath := range fileList {
@@ -152,19 +156,20 @@ func GetBasicData(fileList []string, pathInArchive map[string]string) (*BasicPac
 			fileHash = "sha256:" + hash
 		}
 
-		var lineLengthCounts map[int]int
+		var lineLengthCounts valuecounts.ValueCounts
 		if lineLengths, err := linelengths.GetLineLengths(filePath, ""); err != nil {
 			log.Error("Error collecting line lengths", "path", archivePath, "error", err)
 		} else {
 			lineLengthCounts = lineLengths
 		}
 
-		result.Files[archivePath] = BasicFileData{
+		result.Files = append(result.Files, BasicFileData{
+			Filename:         archivePath,
 			FileType:         fileType,
 			Size:             fileSize,
 			Hash:             fileHash,
 			LineLengthCounts: lineLengthCounts,
-		}
+		})
 	}
 
 	return &result, nil
diff --git a/internal/staticanalysis/linelengths/line_lengths.go b/internal/staticanalysis/linelengths/line_lengths.go
index 948ee04d..821c30e9 100644
--- a/internal/staticanalysis/linelengths/line_lengths.go
+++ b/internal/staticanalysis/linelengths/line_lengths.go
@@ -5,6 +5,8 @@ import (
 	"io"
 	"os"
 	"strings"
+
+	"github.com/ossf/package-analysis/internal/utils/valuecounts"
 )
 
 /*
@@ -22,7 +24,7 @@ Note: there may not be much useful information to be gathered by distinguishing
 between line lengths when they get very long. It may be pragmatic to just report
 all lines above e.g. 64K as 64K long.
 */
-func GetLineLengths(filePath string, sourceString string) (map[int]int, error) {
+func GetLineLengths(filePath string, sourceString string) (valuecounts.ValueCounts, error) {
 	var reader *bufio.Reader
 	if len(filePath) > 0 {
 		file, err := os.Open(filePath)
@@ -36,7 +38,7 @@ func GetLineLengths(filePath string, sourceString string) (map[int]int, error) {
 		reader = bufio.NewReader(strings.NewReader(sourceString))
 	}
 
-	lengths := map[int]int{}
+	lengths := valuecounts.ValueCounts{}
 	for {
 		/* Normally bufio.Scanner would be more convenient to use here, however by default
 		it uses a fixed maximum buffer size (MaxScanTokenSize = 64 * 1024). Since some
diff --git a/internal/staticanalysis/obfuscation/analyze.go b/internal/staticanalysis/obfuscation/analyze.go
index 2297fba7..32e3c971 100644
--- a/internal/staticanalysis/obfuscation/analyze.go
+++ b/internal/staticanalysis/obfuscation/analyze.go
@@ -2,23 +2,29 @@ package obfuscation
 
 import "github.com/ossf/package-analysis/internal/staticanalysis/parsing"
 
-// Analyze performs obfuscation analysis for a package, operating on the data
-// obtained from parsing each file in the package.
-func Analyze(fileParseData parsing.PackageResult) Result {
+/*
+Analyze performs obfuscation analysis for a package, operating on the data
+obtained from parsing each file in the package.
+If Language is empty (== NoLanguage), it means that the file could not be
+parsed with any parser.
+*/
+func Analyze(parseData []*parsing.SingleResult) Result {
 	result := Result{
-		ExcludedFiles:   []string{},
-		Signals:         map[string]FileSignals{},
+		ExcludedFiles: []string{},
+		Signals:       []FileSignals{},
 	}
 
-	for pathInArchive, parseData := range fileParseData {
-		if parseData == nil || parseData[parsing.JavaScript] == nil {
+	for _, fileData := range parseData {
+		switch fileData.Language {
+		case parsing.NoLanguage:
 			// couldn't be parsed
-			result.ExcludedFiles = append(result.ExcludedFiles, pathInArchive)
-		} else {
-			signals := ComputeFileSignals(*parseData[parsing.JavaScript])
+			result.ExcludedFiles = append(result.ExcludedFiles, fileData.Filename)
+		case parsing.JavaScript:
+			signals := ComputeFileSignals(*fileData)
+			signals.Filename = fileData.Filename
 			// remove NaNs for JSON
 			RemoveNaNs(&signals)
-			result.Signals[pathInArchive] = signals
+			result.Signals = append(result.Signals, signals)
 		}
 	}
 
diff --git a/internal/staticanalysis/obfuscation/file_signals.go b/internal/staticanalysis/obfuscation/file_signals.go
index 70096a35..e7d225ce 100644
--- a/internal/staticanalysis/obfuscation/file_signals.go
+++ b/internal/staticanalysis/obfuscation/file_signals.go
@@ -11,14 +11,18 @@ import (
 	"github.com/ossf/package-analysis/internal/staticanalysis/parsing"
 	"github.com/ossf/package-analysis/internal/staticanalysis/token"
 	"github.com/ossf/package-analysis/internal/utils"
+	"github.com/ossf/package-analysis/internal/utils/valuecounts"
 )
 
 // FileSignals holds information related to the presence of obfuscated code in a single file.
 type FileSignals struct {
-	// StringLengths is a map from length (in characters) to number of
-	// string literals in the file having that length. If a length key is
-	// missing, it is assumed to be zero.
-	StringLengths map[int]int
+	// Filename is the path in the package
+	Filename string
+
+	// StringLengths counts the lengths of string literals in the file.
+	// If there is no count for a given string length, it means that there
+	// were no strings of that length in the file
+	StringLengths valuecounts.ValueCounts
 
 	// StringEntropySummary provides sample statistics for the set of entropy
 	// values calculated on each string literal. Character probabilities for the
@@ -31,10 +35,10 @@ type FileSignals struct {
 	// to normalise the values in StringEntropySummary.
 	CombinedStringEntropy float64
 
-	// IdentifierLengths is a map from length (in characters) to number of
-	// identifiers in the file having that length. If a length key is missing,
-	// it is assumed to be zero.
-	IdentifierLengths map[int]int
+	// IdentifierLengths counts the lengths of identifiers in the file.
+	// If there is no entry for a given length, it means that there
+	// were no identifiers of that length in the file
+	IdentifierLengths valuecounts.ValueCounts
 
 	// IdentifierEntropySummary provides sample statistics for the set of entropy
 	// values calculated on each identifier. Character probabilities for the
@@ -77,6 +81,7 @@ type FileSignals struct {
 
 func (s FileSignals) String() string {
 	parts := []string{
+		fmt.Sprintf("filename: %s", s.Filename),
 		fmt.Sprintf("string lengths: %v", s.StringLengths),
 		fmt.Sprintf("string entropy: %s", s.StringEntropySummary),
 		fmt.Sprintf("combined string entropy: %f", s.CombinedStringEntropy),
diff --git a/internal/staticanalysis/obfuscation/result.go b/internal/staticanalysis/obfuscation/result.go
index d8a0d1d2..706d6403 100644
--- a/internal/staticanalysis/obfuscation/result.go
+++ b/internal/staticanalysis/obfuscation/result.go
@@ -3,6 +3,8 @@ package obfuscation
 import (
 	"fmt"
 	"strings"
+
+	"github.com/ossf/package-analysis/internal/utils"
 )
 
 // Result holds all data produced by obfuscation analysis (see Analyze() in analyze.go).
@@ -11,21 +13,16 @@ type Result struct {
 	// e.g. because they could not be parsed by any supported parser.
 	ExcludedFiles []string `json:"excluded_files"`
 
-	// Signals maps package file names to corresponding obfuscation.FileSignals
-	// that are used to detect suspicious files.
-	Signals map[string]FileSignals `json:"signals"`
+	// Signals contains an obfuscation.FileSignals object that is useful for detecting suspicious files.
+	Signals []FileSignals `json:"signals"`
 }
 
 func (r Result) String() string {
-	fileSignalsStrings := make([]string, 0)
-
-	for filename, signals := range r.Signals {
-		fileSignalsStrings = append(fileSignalsStrings, fmt.Sprintf("== %s ==\n%s\n", filename, signals))
-	}
+	signalsStrings := utils.Transform(r.Signals, func(s FileSignals) string { return s.String() })
 
 	parts := []string{
-		fmt.Sprintf("excluded files:\n%v", r.ExcludedFiles),
-		fmt.Sprintf("file signals\n%s", strings.Join(fileSignalsStrings, "\n\n")),
+		fmt.Sprintf("excluded files:\n%s", strings.Join(r.ExcludedFiles, ", ")),
+		fmt.Sprintf("file signals:\n%s", strings.Join(signalsStrings, "\n==================\n")),
 	}
 
 	return strings.Join(parts, "\n\n-----------------------------\n\n")
diff --git a/internal/staticanalysis/parsing/analyze.go b/internal/staticanalysis/parsing/analyze.go
index deed03ac..4dbc956b 100644
--- a/internal/staticanalysis/parsing/analyze.go
+++ b/internal/staticanalysis/parsing/analyze.go
@@ -9,58 +9,55 @@ import (
 	"github.com/ossf/package-analysis/internal/staticanalysis/token"
 )
 
-func (p PackageResult) processAndAdd(language Language, result languageResult) {
-	for fileName, fileResult := range result {
-		if p[fileName] == nil {
-			p[fileName] = FileResult{}
-		}
-
-		if !fileResult.ValidInput {
-			p[fileName][language] = nil
-			continue
+func processJsData(filename string, fileData singleParseData) *SingleResult {
+	if !fileData.ValidInput {
+		return &SingleResult{
+			Filename: filename,
+			Language: NoLanguage,
 		}
+	}
 
+	result := &SingleResult{
+		Filename: filename,
+		Language: JavaScript,
 		// Initialise with empty slices to avoid null values in JSON
-		data := &SingleResult{
-			Identifiers:    []token.Identifier{},
-			StringLiterals: []token.String{},
-			IntLiterals:    []token.Int{},
-			FloatLiterals:  []token.Float{},
-			Comments:       []token.Comment{},
-		}
-
-		for _, d := range fileResult.Literals {
-			if d.GoType == "string" {
-				data.StringLiterals = append(data.StringLiterals, token.String{Value: d.Value.(string), Raw: d.RawValue})
-			} else if d.GoType == "float64" {
-				if intValue, err := strconv.ParseInt(d.RawValue, 0, 64); err == nil {
-					data.IntLiterals = append(data.IntLiterals, token.Int{Value: intValue, Raw: d.RawValue})
-				} else {
-					data.FloatLiterals = append(data.FloatLiterals, token.Float{Value: d.Value.(float64), Raw: d.RawValue})
-				}
-			}
-		}
+		Identifiers:    []token.Identifier{},
+		StringLiterals: []token.String{},
+		IntLiterals:    []token.Int{},
+		FloatLiterals:  []token.Float{},
+		Comments:       []token.Comment{},
+	}
 
-		for _, ident := range fileResult.Identifiers {
-			switch ident.Type {
-			case token.Function, token.Class, token.Parameter, token.Property, token.Variable:
-				data.Identifiers = append(data.Identifiers, token.Identifier{Name: ident.Name, Type: ident.Type})
+	for _, d := range fileData.Literals {
+		if d.GoType == "string" {
+			result.StringLiterals = append(result.StringLiterals, token.String{Value: d.Value.(string), Raw: d.RawValue})
+		} else if d.GoType == "float64" {
+			if intValue, err := strconv.ParseInt(d.RawValue, 0, 64); err == nil {
+				result.IntLiterals = append(result.IntLiterals, token.Int{Value: intValue, Raw: d.RawValue})
+			} else {
+				result.FloatLiterals = append(result.FloatLiterals, token.Float{Value: d.Value.(float64), Raw: d.RawValue})
 			}
 		}
+	}
 
-		for _, c := range fileResult.Comments {
-			data.Comments = append(data.Comments, token.Comment{Value: c.Data})
+	for _, ident := range fileData.Identifiers {
+		switch ident.Type {
+		case token.Function, token.Class, token.Parameter, token.Property, token.Variable:
+			result.Identifiers = append(result.Identifiers, token.Identifier{Name: ident.Name, Type: ident.Type})
 		}
+	}
 
-		p[fileName][language] = data
+	for _, c := range fileData.Comments {
+		result.Comments = append(result.Comments, token.Comment{Value: c.Data})
 	}
+
+	return result
 }
 
 /*
 Analyze (parsing.Analyze) parses the specified list of files using all supported parsers
-and returns a map of file path to parsing.FileResult, which in turn is a map of language
-to parsing.SingleResult. Each parsing.SingleResult holds information about source code
-tokens found for that language.
+and returns a list of parsing.SingleResult, each of which holds information about source code
+tokens found for a given file and langauge parser combination.
 
 Currently, the only supported language is JavaScript, however more language parsers will
 be added in the future.
@@ -79,9 +76,7 @@ they are normally both parsed as floating point. This function records a numeric
 as an integer if it can be converted using strconv.Atoi(), otherwise it is recorded as
 floating point.
 */
-func Analyze(parserConfig ParserConfig, input externalcmd.Input, printDebug bool) (PackageResult, error) {
-	resultsByLanguage := map[Language]languageResult{}
-
+func Analyze(parserConfig ParserConfig, input externalcmd.Input, printDebug bool) ([]*SingleResult, error) {
 	// JavaScript parsing
 	jsResults, rawOutput, err := parseJS(parserConfig, input)
 	if printDebug {
@@ -91,10 +86,9 @@ func Analyze(parserConfig ParserConfig, input externalcmd.Input, printDebug bool
 		return nil, err
 	}
 
-	packageResult := PackageResult{}
-	resultsByLanguage[JavaScript] = jsResults
-	for language, result := range resultsByLanguage {
-		packageResult.processAndAdd(language, result)
+	packageResult := make([]*SingleResult, 0, len(jsResults))
+	for filename, jsData := range jsResults {
+		packageResult = append(packageResult, processJsData(filename, jsData))
 	}
 
 	return packageResult, nil
diff --git a/internal/staticanalysis/parsing/js_parsing.go b/internal/staticanalysis/parsing/js_parsing.go
index 2ad9d859..3274134d 100644
--- a/internal/staticanalysis/parsing/js_parsing.go
+++ b/internal/staticanalysis/parsing/js_parsing.go
@@ -98,8 +98,8 @@ func runParser(parserPath string, input externalcmd.Input, extraArgs ...string)
 	}
 }
 
-func (pd parseDataJSON) process() languageData {
-	processed := languageData{
+func (pd parseDataJSON) process() singleParseData {
+	processed := singleParseData{
 		ValidInput: true,
 	}
 
@@ -173,12 +173,12 @@ parseJS extracts source code identifiers and string literals from JavaScript cod
 
 parserConfig specifies options relevant to the parser itself, and is produced by InitParser
 
-If internal errors occurred during parsing, then a nil languageResult pointer is returned.
+If internal errors occurred during parsing, then a nil map is returned.
 The other two return values are the raw parser output and the error respectively.
 Otherwise, the first return value points to the parsing result object while the second
 contains the raw JSON output from the parser.
 */
-func parseJS(parserConfig ParserConfig, input externalcmd.Input) (languageResult, string, error) {
+func parseJS(parserConfig ParserConfig, input externalcmd.Input) (map[string]singleParseData, string, error) {
 	rawOutput, err := runParser(parserConfig.ParserPath, input)
 	if err != nil {
 		if exitErr, ok := err.(*exec.ExitError); ok {
@@ -195,7 +195,7 @@ func parseJS(parserConfig ParserConfig, input externalcmd.Input) (languageResult
 	}
 
 	// convert the elements into more natural data structure
-	result := languageResult{}
+	result := map[string]singleParseData{}
 	for filename, data := range parseOutput {
 		result[filename] = data.process()
 	}
diff --git a/internal/staticanalysis/parsing/parsing_types.go b/internal/staticanalysis/parsing/parsing_types.go
index d5f5e17b..379e6dae 100644
--- a/internal/staticanalysis/parsing/parsing_types.go
+++ b/internal/staticanalysis/parsing/parsing_types.go
@@ -12,6 +12,7 @@ import (
 type Language string
 
 const (
+	NoLanguage Language = ""
 	JavaScript Language = "JavaScript"
 )
 
@@ -83,11 +84,8 @@ func (s parserStatus) String() string {
 	return fmt.Sprintf("[%s] %s: %s pos %d:%d", s.Type, s.Name, s.Message, s.Pos.Row(), s.Pos.Col())
 }
 
-// languageResult maps filenames to languageData for that file (i.e. parsing results for a single language).
-type languageResult map[string]languageData
-
-// languageData holds data for a single file processed by a single language parser.
-type languageData struct {
+// singleParseData holds package-internal data for a single file processed by a single language parser.
+type singleParseData struct {
 	ValidInput  bool
 	Identifiers []parsedIdentifier
 	Literals    []parsedLiteral[any]
@@ -96,12 +94,12 @@ type languageData struct {
 	Errors      []parserStatus
 }
 
-func (p languageData) String() string {
-	identifiers := utils.Transform(p.Identifiers, func(pi parsedIdentifier) string { return pi.String() })
-	literals := utils.Transform(p.Literals, func(pl parsedLiteral[any]) string { return pl.String() })
-	comments := utils.Transform(p.Comments, func(c parsedComment) string { return c.String() })
-	info := utils.Transform(p.Info, func(i parserStatus) string { return i.String() })
-	errors := utils.Transform(p.Errors, func(e parserStatus) string { return e.String() })
+func (d singleParseData) String() string {
+	identifiers := utils.Transform(d.Identifiers, func(pi parsedIdentifier) string { return pi.String() })
+	literals := utils.Transform(d.Literals, func(pl parsedLiteral[any]) string { return pl.String() })
+	comments := utils.Transform(d.Comments, func(c parsedComment) string { return c.String() })
+	info := utils.Transform(d.Info, func(i parserStatus) string { return i.String() })
+	errors := utils.Transform(d.Errors, func(e parserStatus) string { return e.String() })
 
 	parts := []string{
 		"== Identifiers ==",
diff --git a/internal/staticanalysis/parsing/result.go b/internal/staticanalysis/parsing/result.go
index adf1fd12..6781041b 100644
--- a/internal/staticanalysis/parsing/result.go
+++ b/internal/staticanalysis/parsing/result.go
@@ -7,16 +7,11 @@ import (
 	"github.com/ossf/package-analysis/internal/staticanalysis/token"
 )
 
-// PackageResult maps file paths (e.g. in a package) to the FileResult for that file.
-type PackageResult map[string]FileResult
-
-// FileResult holds the full set of parsing data for a single file, which may
-// contain valid parsing results for multiple languages.
-type FileResult map[Language]*SingleResult
-
-// SingleResult holds information about source code tokens found in a file,
-// when parsed as particular language.
+// SingleResult holds processed information about source code tokens
+// found in a single file by a single language parser
 type SingleResult struct {
+	Filename       string             `json:"filename"`
+	Language       Language           `json:"language"`
 	Identifiers    []token.Identifier `json:"identifiers"`
 	StringLiterals []token.String     `json:"string_literals"`
 	IntLiterals    []token.Int        `json:"int_literals"`
@@ -24,13 +19,15 @@ type SingleResult struct {
 	Comments       []token.Comment    `json:"comments"`
 }
 
-func (rd SingleResult) String() string {
+func (r SingleResult) String() string {
 	parts := []string{
-		fmt.Sprintf("identifiers\n%v\n", rd.Identifiers),
-		fmt.Sprintf("string literals\n%v\n", rd.StringLiterals),
-		fmt.Sprintf("integer literals\n%v\n", rd.IntLiterals),
-		fmt.Sprintf("float literals\n%v\n", rd.FloatLiterals),
-		fmt.Sprintf("comments\n%v\n", rd.Comments),
+		fmt.Sprintf("filename: %s\n", r.Filename),
+		fmt.Sprintf("language: %s\n", r.Language),
+		fmt.Sprintf("identifiers\n%v\n", r.Identifiers),
+		fmt.Sprintf("string literals\n%v\n", r.StringLiterals),
+		fmt.Sprintf("integer literals\n%v\n", r.IntLiterals),
+		fmt.Sprintf("float literals\n%v\n", r.FloatLiterals),
+		fmt.Sprintf("comments\n%v\n", r.Comments),
 	}
 	return strings.Join(parts, "\n-------------------\n")
 }
diff --git a/internal/staticanalysis/result.go b/internal/staticanalysis/result.go
index ce743d60..e769dd80 100644
--- a/internal/staticanalysis/result.go
+++ b/internal/staticanalysis/result.go
@@ -6,6 +6,7 @@ import (
 
 	"github.com/ossf/package-analysis/internal/staticanalysis/obfuscation"
 	"github.com/ossf/package-analysis/internal/staticanalysis/parsing"
+	"github.com/ossf/package-analysis/internal/utils"
 )
 
 /*
@@ -18,20 +19,17 @@ type Result struct {
 	// NOTE: the JSON names below should match the values in task.go
 	BasicData *BasicPackageData `json:"basic,omitempty"`
 
-	ParsingData parsing.PackageResult `json:"parsing,omitempty"`
+	ParsingData []*parsing.SingleResult `json:"parsing,omitempty"`
 
 	ObfuscationData *obfuscation.Result `json:"obfuscation,omitempty"`
 }
 
 func (ar Result) String() string {
-	parseDataStrings := make([]string, 0)
-	for filename, parseData := range ar.ParsingData {
-		parseDataStrings = append(parseDataStrings, fmt.Sprintf("== %s ==\n%s", filename, parseData))
-	}
+	parsingDataStrings := utils.Transform(ar.ParsingData, func(d *parsing.SingleResult) string { return d.String() })
 
 	parts := []string{
 		fmt.Sprintf("File basic data\n%v", ar.BasicData),
-		fmt.Sprintf("File parse data\n%s", strings.Join(parseDataStrings, "\n\n")),
+		fmt.Sprintf("File parse data\n%s", strings.Join(parsingDataStrings, "\n\n")),
 		fmt.Sprintf("File obfuscation data\n%s", ar.ObfuscationData),
 	}
 
diff --git a/internal/utils/equals.go b/internal/utils/equals.go
new file mode 100644
index 00000000..405abb4d
--- /dev/null
+++ b/internal/utils/equals.go
@@ -0,0 +1,32 @@
+package utils
+
+import (
+	"encoding/json"
+	"math"
+	"reflect"
+)
+
+// FloatEquals compares two floats and returns true if they are both
+// within absTol of each other, or are both NaN.
+// Note that normally NaN != NaN, but we define it as true because it's
+// convenient for comparing arrays and structs that contain floats.
+func FloatEquals(x1, x2, absTol float64) bool {
+	return x1 == x2 || math.Abs(x1-x2) < absTol || (math.IsNaN(x1) && math.IsNaN(x2))
+}
+
+// JSONEquals compares two byte sequences containing JSON data and returns true if
+// 1) both j1 and j2 contain valid JSON data, and
+// 2) the JSON objects that they represent are equal.
+// If j1 or j2 contain invalid JSON data, an error is returned.
+func JSONEquals(j1, j2 []byte) (bool, error) {
+	// Adapted from https://stackoverflow.com/a/32409106
+	var o1, o2 interface{}
+	if err := json.Unmarshal(j1, &o1); err != nil {
+		return false, err
+	}
+	if err := json.Unmarshal(j2, &o2); err != nil {
+		return false, err
+	}
+	return reflect.DeepEqual(o1, o2), nil
+
+}
diff --git a/internal/utils/float_equals.go b/internal/utils/float_equals.go
deleted file mode 100644
index 7a24914d..00000000
--- a/internal/utils/float_equals.go
+++ /dev/null
@@ -1,7 +0,0 @@
-package utils
-
-import "math"
-
-func FloatEquals(x1, x2, absTol float64) bool {
-	return x1 == x2 || math.Abs(x1-x2) < absTol || (math.IsNaN(x1) && math.IsNaN(x2))
-}
diff --git a/internal/utils/valuecounts/value_counts.go b/internal/utils/valuecounts/value_counts.go
new file mode 100644
index 00000000..cc932620
--- /dev/null
+++ b/internal/utils/valuecounts/value_counts.go
@@ -0,0 +1,92 @@
+package valuecounts
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"golang.org/x/exp/maps"
+	"golang.org/x/exp/slices"
+)
+
+// ValueCounts stores unordered counts of integer values as a map
+// from value (int) to count (int). It can be serialized to JSON
+// as an array of (value, count) pairs.
+type ValueCounts map[int]int
+
+// Aside: I know using 'value' to refer to map keys is not great, but the
+// other names I came up with like 'size' and 'length' were all usage-specific.
+
+// Pair stores a single value and associated count pair
+type Pair struct {
+	Value int `json:"value"`
+	Count int `json:"count"`
+}
+
+func New() ValueCounts {
+	return ValueCounts{}
+}
+
+// ToPairs converts this ValueCounts into a list of (value, count) pairs.
+// The values are sorted in increasing order so that the output is deterministic.
+// If this ValueCounts is empty, returns an empty slice.
+func (vc ValueCounts) ToPairs() []Pair {
+	pairs := make([]Pair, 0, len(vc))
+
+	// sort the values so that the output is in a deterministic order
+	values := maps.Keys(vc)
+	slices.Sort(values)
+
+	for _, value := range values {
+		count := vc[value]
+		pairs = append(pairs, Pair{Value: value, Count: count})
+	}
+
+	return pairs
+}
+
+// FromPairs converts a list of (value, count) pairs back into ValueCounts.
+// If the same value occurs multiple times in the list, an error is raised.
+// If pairs is nil or empty but non-nil, an empty ValueCounts is returned.
+func FromPairs(pairs []Pair) (ValueCounts, error) {
+	valueCounts := New()
+
+	for _, item := range pairs {
+		if _, seen := valueCounts[item.Value]; seen {
+			return nil, fmt.Errorf("value occurs multiple times: %d", item.Value)
+		}
+		valueCounts[item.Value] = item.Count
+	}
+
+	return valueCounts, nil
+}
+
+// MarshalJSON serialises this ValueCounts into a JSON array of {value, count} pairs.
+func (vc ValueCounts) MarshalJSON() ([]byte, error) {
+	return json.Marshal(vc.ToPairs())
+}
+
+/*
+UnmarshalJSON converts a JSON-serialised ValueCounts object, serialised
+using MarshalJSON, back into a Go object. Existing counts are discarded.
+
+Note that the serialised data is formatted as an array of (value, count) pairs,
+and it is possible to create an array with multiple counts for the same value.
+If this occurs, an error will be raised.
+
+If any error is encountered, the original countData object is not modified.
+*/
+func (vc *ValueCounts) UnmarshalJSON(data []byte) error {
+	var pair []Pair
+
+	if err := json.Unmarshal(data, &pair); err != nil {
+		return err
+	}
+
+	valueCounts, err := FromPairs(pair)
+	if err != nil {
+		return err
+	}
+
+	*vc = valueCounts
+	return nil
+}
diff --git a/sandboxes/staticanalysis/staticanalyze.go b/sandboxes/staticanalysis/staticanalyze.go
index acb3ea0e..8a7bbe6e 100644
--- a/sandboxes/staticanalysis/staticanalyze.go
+++ b/sandboxes/staticanalysis/staticanalyze.go
@@ -180,7 +180,8 @@ func run() (err error) {
 
 	jsonResult, err := json.Marshal(results)
 	if err != nil {
-		return fmt.Errorf("JSON marshall error: %v", err)
+		log.Debug(fmt.Sprintf("unserialisable JSON: %v", results))
+		return fmt.Errorf("JSON marshal error: %v", err)
 	}
 
 	outputFile := os.Stdout


Test patch:
diff --git a/internal/staticanalysis/linelengths/line_lengths_test.go b/internal/staticanalysis/linelengths/line_lengths_test.go
index c06da946..b976cbe6 100644
--- a/internal/staticanalysis/linelengths/line_lengths_test.go
+++ b/internal/staticanalysis/linelengths/line_lengths_test.go
@@ -3,13 +3,15 @@ package linelengths
 import (
 	"reflect"
 	"testing"
+
+	"github.com/ossf/package-analysis/internal/utils/valuecounts"
 )
 
 func TestSourceStringLineLengths(t *testing.T) {
 	tests := []struct {
 		name    string
 		source  string
-		want    map[int]int
+		want    valuecounts.ValueCounts
 		wantErr bool
 	}{
 		{
diff --git a/internal/staticanalysis/parsing/analyze_test.go b/internal/staticanalysis/parsing/analyze_test.go
index 21d7ff36..edadee36 100644
--- a/internal/staticanalysis/parsing/analyze_test.go
+++ b/internal/staticanalysis/parsing/analyze_test.go
@@ -83,7 +83,7 @@ func TestCollectData(t *testing.T) {
 				t.Errorf("%v", err)
 				return
 			}
-			got := result["stdin"][JavaScript]
+			got := result[0]
 
 			if !reflect.DeepEqual(got.Identifiers, tt.expectedData.Identifiers) {
 				t.Errorf("Identifiers mismatch: got %v, want %v", got.Identifiers, tt.expectedData.Identifiers)
diff --git a/internal/staticanalysis/parsing/js_parsing_test.go b/internal/staticanalysis/parsing/js_parsing_test.go
index c8b2b47b..7445f467 100644
--- a/internal/staticanalysis/parsing/js_parsing_test.go
+++ b/internal/staticanalysis/parsing/js_parsing_test.go
@@ -6,7 +6,6 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/ossf/package-analysis/internal/log"
 	"github.com/ossf/package-analysis/internal/staticanalysis/externalcmd"
 	"github.com/ossf/package-analysis/internal/staticanalysis/token"
 )
@@ -14,7 +13,7 @@ import (
 type jsTestCase struct {
 	name      string
 	inputJS   string
-	want      languageData
+	want      singleParseData
 	printJSON bool // set to true to see raw parser output
 }
 
@@ -37,7 +36,7 @@ function test() {
 //"'11"'` + "`" + `;
 	var mystring12 = ` + "`hello\"'${5.6 + 6.4}\"'`" + `;
 }`,
-		want: languageData{
+		want: singleParseData{
 			Identifiers: []parsedIdentifier{
 				{token.Function, "test", token.Position{2, 9}},
 				{token.Variable, "mystring1", token.Position{3, 8}},
@@ -81,7 +80,7 @@ function test() {
 function test2(param1, param2, param3 = "ahd") {
 	return param1 + param2 + param3;
 }`,
-		want: languageData{
+		want: singleParseData{
 			Identifiers: []parsedIdentifier{
 				{token.Function, "test2", token.Position{2, 9}},
 				{token.Parameter, "param1", token.Position{2, 15}},
@@ -114,7 +113,7 @@ outer:
     }
     console.log("End");
 }`,
-		want: languageData{
+		want: singleParseData{
 			Identifiers: []parsedIdentifier{
 				{token.Function, "test3", token.Position{2, 9}},
 				{token.Parameter, "a", token.Position{2, 15}},
@@ -166,7 +165,7 @@ function test4() {
             break;
     }
 }`,
-		want: languageData{
+		want: singleParseData{
 			Identifiers: []parsedIdentifier{
 				{token.Function, "test4", token.Position{2, 9}},
 				{token.Variable, "a", token.Position{3, 10}},
@@ -222,7 +221,7 @@ Rectangle = class Rectangle2 {
 console.log(Rectangle.name);
 // output: "Rectangle2"
 `,
-		want: languageData{
+		want: singleParseData{
 			Identifiers: []parsedIdentifier{
 				{token.Variable, "Rectangle", token.Position{3, 4}},
 				{token.Parameter, "height", token.Position{4, 16}},
@@ -250,7 +249,7 @@ console.log(Rectangle.name);
 'use strict';
 console.log("Hello");
 `,
-		want: languageData{
+		want: singleParseData{
 			Identifiers: []parsedIdentifier{
 				{token.Member, "log", token.Position{3, 8}},
 			},
@@ -270,7 +269,7 @@ var index = 0,
     {length, width} = 10,
     cancelled = false;
 `,
-		want: languageData{
+		want: singleParseData{
 			Identifiers: []parsedIdentifier{
 				{token.Variable, "a", token.Position{2, 5}},
 				{token.Variable, "b", token.Position{2, 8}},
@@ -305,7 +304,7 @@ function validateIPAddress(ipaddress) {
     return (false)
 }
 `,
-		want: languageData{
+		want: singleParseData{
 			ValidInput: true,
 			Identifiers: []parsedIdentifier{
 				{token.Function, "validateIPAddress", token.Position{2, 9}},
@@ -337,7 +336,7 @@ let b = 0o777777777777n;         // 68719476735
 let c = 0x123456789ABCDEFn;      // 81985529216486895
 let d = 0b11101001010101010101n; // 955733
 `,
-		want: languageData{
+		want: singleParseData{
 			ValidInput: true,
 			Identifiers: []parsedIdentifier{
 				{token.Variable, "a", token.Position{2, 4}},
@@ -359,7 +358,7 @@ let d = 0b11101001010101010101n; // 955733
 		inputJS: `
 a = w w;
 `,
-		want: languageData{
+		want: singleParseData{
 			ValidInput:  false,
 			Identifiers: []parsedIdentifier{},
 			Errors: []parserStatus{
@@ -375,10 +374,6 @@ a = w w;
 	},
 }
 
-func init() {
-	log.Initialize("")
-}
-
 func TestParseJS(t *testing.T) {
 	const printAllJSON = false
 
diff --git a/internal/utils/valuecounts/value_counts_test.go b/internal/utils/valuecounts/value_counts_test.go
new file mode 100644
index 00000000..e1e1f66b
--- /dev/null
+++ b/internal/utils/valuecounts/value_counts_test.go
@@ -0,0 +1,194 @@
+package valuecounts
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/ossf/package-analysis/internal/utils"
+)
+
+func TestCountData_ToValueCountPairs(t *testing.T) {
+	tests := []struct {
+		name string
+		vc   ValueCounts
+		want []Pair
+	}{
+		{
+			"nil",
+			nil,
+			[]Pair{},
+		},
+		{
+			"empty",
+			ValueCounts{},
+			[]Pair{},
+		},
+		{
+			"single item",
+			ValueCounts{0: 1},
+			[]Pair{{0, 1}},
+		},
+		{
+			"multiple items",
+			ValueCounts{0: 1, 1: 2, 2: 3},
+			[]Pair{{0, 1}, {1, 2}, {2, 3}},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.vc.ToPairs(); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("ToPairs() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestFromValueCountPairs(t *testing.T) {
+	tests := []struct {
+		name    string
+		pairs   []Pair
+		want    ValueCounts
+		wantErr bool
+	}{
+		{
+			"nil",
+			nil,
+			ValueCounts{},
+			false,
+		},
+		{
+			"empty non-nil",
+			[]Pair{},
+			ValueCounts{},
+			false,
+		},
+		{
+			"single item",
+			[]Pair{{0, 1}},
+			ValueCounts{0: 1},
+			false,
+		},
+		{
+			"multiple items",
+			[]Pair{{0, 1}, {1, 2}},
+			ValueCounts{0: 1, 1: 2},
+			false,
+		},
+		{
+			"repeated items",
+			[]Pair{{0, 1}, {0, 1}},
+			nil,
+			true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := FromPairs(tt.pairs)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("FromPairs() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("FromPairs() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestCountData_MarshalJSON(t *testing.T) {
+	tests := []struct {
+		name    string
+		vc      ValueCounts
+		want    string
+		wantErr bool
+	}{
+		{
+			"nil",
+			nil,
+			"[]",
+			false,
+		},
+		{
+			"empty",
+			ValueCounts{},
+			"[]",
+			false,
+		},
+		{
+			"single item",
+			ValueCounts{0: 1},
+			`[ {"value": 0, "count": 1} ]`,
+			false,
+		},
+		{
+			"multiple items",
+			ValueCounts{0: 1, 1: 2, 2: 3},
+			`[ {"value":0, "count": 1}, {"value": 1, "count": 2}, {"value": 2, "count": 3} ]`,
+			false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotBytes, err := tt.vc.MarshalJSON()
+			got := string(gotBytes)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("MarshalJSON() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+
+			if equal, err := utils.JSONEquals(gotBytes, []byte(tt.want)); err != nil {
+				t.Errorf("MarshalJSON() error decoding JSON: %v", err)
+			} else if !equal {
+				t.Errorf("MarshalJSON() got = %s, want %s", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestCountData_UnmarshalJSON(t *testing.T) {
+	tests := []struct {
+		name    string
+		json    string
+		want    ValueCounts
+		wantErr bool
+	}{
+		{
+			"null",
+			"null",
+			ValueCounts{},
+			false,
+		},
+		{
+			"empty",
+			"[]",
+			ValueCounts{},
+			false,
+		},
+		{
+			"single item",
+			`[{"value": 0, "count": 1}]`,
+			ValueCounts{0: 1},
+			false,
+		},
+		{
+			"multiple items",
+			`[{"value":0,"count":1},{"value":1,"count":2},{"value":2,"count":3}]`,
+			ValueCounts{0: 1, 1: 2, 2: 3},
+			false,
+		},
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			valueCounts := New()
+
+			if err := valueCounts.UnmarshalJSON([]byte(tt.json)); (err != nil) != tt.wantErr {
+				t.Errorf("UnmarshalJSON() error = %v, wantErr %v", err, tt.wantErr)
+			}
+
+			if !reflect.DeepEqual(valueCounts, tt.want) {
+				t.Errorf("UnmarshalJSON(): got %v, want %v", valueCounts, tt.want)
+			}
+		})
+	}
+}