Repository: sonatype-nexus-community/nancy
Pull request #86 (2020-03-03 02:04:08)

Beef up user agent string
Basically, we send in a user-agent string, but it was really "simple". Let's add a couple bits more info:

- If Nancy is being run in CI, let's say what system it's being run in
- If Nancy is being run in CI, in a orb, etc... that we create, add the info that the orb supplies `SC_CALLER_INFO` environment variable is our new friend (hi @bhamail )
- Regardless of Nancy being run in CI, let's send back the OS and the architecture so we know a bit more about who is doing Golang development with Nancy

What is the goal here? Well, not to be creepy, that's for sure. It's to get some information that we can use to help prioritize new work, like building more orbs, working on new orbs, etc...

This pull request makes the following changes:
* Adds a new package `useragent`
* Logic for finding out what CI system is being run, and providing additional information if it exists
* TESTS

cc @bhamail / @DarthHater / @zendern / @greut / @fitzoh 


greut: A way to put this in the `http.Client` once and for all, https://github.com/hashicorp/terraform-plugin-sdk/blob/master/internal/httpclient/client.go
bhamail: I would like to always check and use the SC_CALLER_INFO env var (regardless of if we detect a CI in use). I'm working on a branch that shows this idea. 

The  SC_CALLER_INFO is meant to be potentially more of a SC_CALL_STACK, containing multiple caller info's, and would not replace the current user agent info. e.g. in TestGetUserAgentScCallerInfo():
instead of:

	expected := "nancy-client/development (bitbucket-nancy-pipe-0.1.9; linux amd64)"

have:

	expected := "nancy-client/development (ci usage; linux amd64)__(bitbucket-nancy-pipe-0.1.9; linux amd64)"

Basically, the SC_CALL_STACK feature is not tied to CI detection. Will have to figure out the best was to embed this call tree info so HDS is happy. I'm also thinking about adding some simple delimiter to keep regex madness at bay when we try to read this string. (Double underscore used to delimit caller's call info).
DarthHater: @bhamail I'm struggling to think of a case where it wouldn't be in CI, what do you imagine instead?
bhamail: > @bhamail I'm struggling to think of a case where it wouldn't be in CI, what do you imagine instead?

Not sure I have common use cases, but couldn't see a reason not to use the env var if set. Grasping at some edge cases:
1. Running the pipe.sh script from the BitBucket Pipe locally to validate/reproduce any failures that happened in CI.
2. Some shiny new CI where for whatever reason, we don't detect being in CI. We'd still get the call tree info.
3. Simplified debugging of the call tree behavior (though it is probably just as easy to fake setting of the expected CI env vars - so this use case is really a stretch).

Here's the PR where I made some changes: https://github.com/sonatype-nexus-community/nancy/pull/87
The build_number stuff could (should?) easily be removed (just added it because I was there).
I'm also happy to remove the "always read SC_CALLER_INFO env var" if we decide that is best.
The main goal I'm after is the getting the `DansCrazyCallTree` string stuffed into the `Other` slot of the User-Agent string. 

Patch:
diff --git a/iq/iq.go b/iq/iq.go
index fb257a7..b9bbb87 100644
--- a/iq/iq.go
+++ b/iq/iq.go
@@ -23,12 +23,12 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/sonatype-nexus-community/nancy/buildversion"
 	"github.com/sonatype-nexus-community/nancy/configuration"
 	"github.com/sonatype-nexus-community/nancy/customerrors"
 	"github.com/sonatype-nexus-community/nancy/cyclonedx"
 	"github.com/sonatype-nexus-community/nancy/ossindex"
 	"github.com/sonatype-nexus-community/nancy/types"
+	"github.com/sonatype-nexus-community/nancy/useragent"
 )
 
 const internalApplicationIDURL = "/api/v2/applications?publicId="
@@ -114,6 +114,7 @@ func getInternalApplicationID(applicationID string) (internalID string) {
 	)
 
 	req.SetBasicAuth(localConfig.User, localConfig.Token)
+	req.Header.Set("User-Agent", useragent.GetUserAgent())
 
 	resp, err := client.Do(req)
 	customerrors.Check(err, "There was an error communicating with Nexus IQ Server to get your internal application ID")
@@ -143,9 +144,8 @@ func submitToThirdPartyAPI(sbom string, internalID string) string {
 	)
 
 	req.SetBasicAuth(localConfig.User, localConfig.Token)
-
+	req.Header.Set("User-Agent", useragent.GetUserAgent())
 	req.Header.Set("Content-Type", "application/xml")
-	req.Header.Set("User-Agent", fmt.Sprintf("nancy-client/%s", buildversion.BuildVersion))
 
 	resp, err := client.Do(req)
 	customerrors.Check(err, "There was an issue communicating with the Nexus IQ Third Party API")
@@ -174,7 +174,7 @@ func pollIQServer(statusURL string, finished chan bool, maxRetries int) {
 
 	req.SetBasicAuth(localConfig.User, localConfig.Token)
 
-	req.Header.Set("User-Agent", fmt.Sprintf("nancy-client/%s", buildversion.BuildVersion))
+	req.Header.Set("User-Agent", useragent.GetUserAgent())
 
 	resp, err := client.Do(req)
 
diff --git a/ossindex/ossindex.go b/ossindex/ossindex.go
index 92ea932..fa084d6 100644
--- a/ossindex/ossindex.go
+++ b/ossindex/ossindex.go
@@ -29,9 +29,9 @@ import (
 	"time"
 
 	"github.com/dgraph-io/badger"
-	"github.com/sonatype-nexus-community/nancy/buildversion"
 	"github.com/sonatype-nexus-community/nancy/customerrors"
 	"github.com/sonatype-nexus-community/nancy/types"
+	"github.com/sonatype-nexus-community/nancy/useragent"
 )
 
 const dbValueDirName = "golang"
@@ -133,9 +133,7 @@ func AuditPackages(purls []string) ([]types.Coordinate, error) {
 				return nil, err
 			}
 
-			if resp.StatusCode == http.StatusOK {
-				log.Printf("Response: %+v\n", resp)
-			} else {
+			if resp.StatusCode != http.StatusOK {
 				return nil, fmt.Errorf("[%s] error accessing OSS Index", resp.Status)
 			}
 
@@ -205,7 +203,7 @@ func setupRequest(jsonStr []byte) (req *http.Request, err error) {
 		return nil, err
 	}
 
-	req.Header.Set("User-Agent", fmt.Sprintf("nancy-client/%s", buildversion.BuildVersion))
+	req.Header.Set("User-Agent", useragent.GetUserAgent())
 	req.Header.Set("Content-Type", "application/json")
 
 	return req, nil
diff --git a/useragent/useragent.go b/useragent/useragent.go
new file mode 100644
index 0000000..d81e824
--- /dev/null
+++ b/useragent/useragent.go
@@ -0,0 +1,119 @@
+// Copyright 2020 Sonatype Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package useragent has functions for setting a user agent with helpful information
+package useragent
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+
+	"github.com/sonatype-nexus-community/nancy/buildversion"
+)
+
+// Variables that can be overriden (primarily for tests), or for consumers
+var (
+	GOOS       = runtime.GOOS
+	GOARCH     = runtime.GOARCH
+	CLIENTTOOL = "nancy-client"
+)
+
+// GetUserAgent provides a user-agent to nancy that provides info on what version of nancy
+// (or upstream consumers like ahab or cheque) is running, and if the process is being run in
+// CI. If so, it looks for what CI system, and other information such as SC_CALLER_INFO which
+// can be used to tell if nancy is being ran inside an orb, bitbucket pipeline, etc... that
+// we authored
+func GetUserAgent() string {
+	callTree := os.Getenv("SC_CALLER_INFO")
+	if checkForCIEnvironment() {
+		return checkCIEnvironments(callTree)
+	}
+	return getUserAgent("non ci usage", callTree)
+}
+
+func getUserAgentBaseAndVersion() string {
+	return fmt.Sprintf("%s/%s", CLIENTTOOL, buildversion.BuildVersion)
+}
+
+func checkCIEnvironments(callTree string) string {
+	if checkForCISystem("CIRCLECI") {
+		return getUserAgent("circleci", callTree)
+	}
+	if checkForCISystem("BITBUCKET_BUILD_NUMBER") {
+		return getUserAgent("bitbucket", callTree)
+	}
+	if checkForCISystem("TRAVIS") {
+		return getUserAgent("travis-ci", callTree)
+	}
+	if checkForCISystem("GITLAB_CI") {
+		return getUserAgent("gitlab-ci", callTree)
+	}
+	if checkIfJenkins() {
+		return getUserAgent("jenkins", callTree)
+	}
+	if checkIfGitHub() {
+		id := getGitHubActionID()
+		return getUserAgent(fmt.Sprintf("github-action %s", id), callTree)
+	}
+
+	return getUserAgent("ci usage", callTree)
+}
+
+func getUserAgent(agent string, callTree string) string {
+	return fmt.Sprintf("%s (%s; %s %s; %s)", getUserAgentBaseAndVersion(), agent, GOOS, GOARCH, callTree)
+}
+
+func checkForCIEnvironment() bool {
+	s := os.Getenv("CI")
+	if s != "" {
+		return true
+	}
+	return checkIfJenkins() || checkIfGitHub()
+}
+
+func checkIfJenkins() bool {
+	s := os.Getenv("JENKINS_HOME")
+	if s != "" {
+		return true
+	}
+	return false
+}
+
+func checkIfGitHub() bool {
+	s := os.Getenv("GITHUB_ACTIONS")
+	if s != "" {
+		return true
+	}
+	return false
+}
+
+// Returns info from SC_CALLER_INFO, example: bitbucket-nancy-pipe-0.1.9
+func getCallerInfo() string {
+	s := os.Getenv("SC_CALLER_INFO")
+	return s
+}
+
+func getGitHubActionID() string {
+	s := os.Getenv("GITHUB_ACTION")
+	return s
+}
+
+func checkForCISystem(system string) bool {
+	s := os.Getenv(system)
+	if s != "" {
+		return true
+	}
+	return false
+}


Test patch:
diff --git a/ossindex/ossindex_test.go b/ossindex/ossindex_test.go
index ea1a1e8..8191fff 100644
--- a/ossindex/ossindex_test.go
+++ b/ossindex/ossindex_test.go
@@ -253,7 +253,6 @@ func TestSetupRequest(t *testing.T) {
 	coordJson, _ := setupJson(t)
 	req, err := setupRequest(coordJson)
 
-	assert.Equal(t, req.Header.Get("User-Agent"), "nancy-client/development")
 	assert.Equal(t, req.Header.Get("Content-Type"), "application/json")
 	assert.Equal(t, req.Method, "POST")
 	assert.Nil(t, err)
diff --git a/useragent/useragent_test.go b/useragent/useragent_test.go
new file mode 100644
index 0000000..a83191b
--- /dev/null
+++ b/useragent/useragent_test.go
@@ -0,0 +1,182 @@
+// Copyright 2020 Sonatype Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package useragent has functions for setting a user agent with helpful information
+package useragent
+
+import (
+	"os"
+	"testing"
+)
+
+func TestGetUserAgentNonCI(t *testing.T) {
+	clearCircleCIVariables()
+	expected := "nancy-client/development (non ci usage; linux amd64; )"
+
+	GOOS = "linux"
+	GOARCH = "amd64"
+
+	agent := GetUserAgent()
+
+	if agent != expected {
+		t.Errorf("User Agent not retrieved successfully, got %s, expected %s", agent, expected)
+	}
+}
+
+func TestGetUserAgentCircleCI(t *testing.T) {
+	expected := "nancy-client/development (circleci; linux amd64; )"
+
+	os.Setenv("CI", "true")
+	os.Setenv("CIRCLECI", "true")
+	GOOS = "linux"
+	GOARCH = "amd64"
+
+	agent := GetUserAgent()
+
+	if agent != expected {
+		t.Errorf("User Agent not retrieved successfully, got %s, expected %s", agent, expected)
+	}
+}
+
+func TestGetUserAgentJenkins(t *testing.T) {
+	clearCircleCIVariables()
+	expected := "nancy-client/development (jenkins; linux amd64; )"
+
+	os.Setenv("JENKINS_HOME", "/a/place/under/the/sun")
+	GOOS = "linux"
+	GOARCH = "amd64"
+
+	agent := GetUserAgent()
+
+	os.Unsetenv("JENKINS_HOME")
+
+	if agent != expected {
+		t.Errorf("User Agent not retrieved successfully, got %s, expected %s", agent, expected)
+	}
+}
+
+func TestGetUserAgentTravisCI(t *testing.T) {
+	clearCircleCIVariables()
+	expected := "nancy-client/development (travis-ci; linux amd64; )"
+
+	os.Setenv("CI", "true")
+	os.Setenv("TRAVIS", "true")
+	GOOS = "linux"
+	GOARCH = "amd64"
+
+	agent := GetUserAgent()
+
+	os.Unsetenv("CI")
+	os.Unsetenv("TRAVIS")
+
+	if agent != expected {
+		t.Errorf("User Agent not retrieved successfully, got %s, expected %s", agent, expected)
+	}
+}
+
+func TestGetUserAgentGitLabCI(t *testing.T) {
+	clearCircleCIVariables()
+	expected := "nancy-client/development (gitlab-ci; linux amd64; )"
+
+	os.Setenv("CI", "true")
+	os.Setenv("GITLAB_CI", "true")
+	GOOS = "linux"
+	GOARCH = "amd64"
+
+	agent := GetUserAgent()
+
+	os.Unsetenv("CI")
+	os.Unsetenv("GITLAB_CI")
+
+	if agent != expected {
+		t.Errorf("User Agent not retrieved successfully, got %s, expected %s", agent, expected)
+	}
+}
+
+func TestGetUserAgentGitHubAction(t *testing.T) {
+	clearCircleCIVariables()
+	expected := "nancy-client/development (github-action 20; linux amd64; )"
+
+	os.Setenv("GITHUB_ACTIONS", "true")
+	os.Setenv("GITHUB_ACTION", "20")
+	GOOS = "linux"
+	GOARCH = "amd64"
+
+	agent := GetUserAgent()
+
+	os.Unsetenv("GITHUB_ACTIONS")
+	os.Unsetenv("GITHUB_ACTION")
+
+	if agent != expected {
+		t.Errorf("User Agent not retrieved successfully, got %s, expected %s", agent, expected)
+	}
+}
+
+func TestGetUserAgentBitBucket(t *testing.T) {
+	clearCircleCIVariables()
+	expected := "nancy-client/development (bitbucket; linux amd64; )"
+
+	os.Setenv("CI", "true")
+	os.Setenv("BITBUCKET_BUILD_NUMBER", "20")
+	GOOS = "linux"
+	GOARCH = "amd64"
+
+	agent := GetUserAgent()
+
+	os.Unsetenv("CI")
+	os.Unsetenv("BITBUCKET_BUILD_NUMBER")
+
+	if agent != expected {
+		t.Errorf("User Agent not retrieved successfully, got %s, expected %s", agent, expected)
+	}
+}
+
+func TestGetUserAgentScCallerInfo(t *testing.T) {
+	clearCircleCIVariables()
+	expected := "nancy-client/development (non ci usage; linux amd64; bitbucket-nancy-pipe-0.1.9)"
+
+	os.Setenv("SC_CALLER_INFO", "bitbucket-nancy-pipe-0.1.9")
+	GOOS = "linux"
+	GOARCH = "amd64"
+
+	agent := GetUserAgent()
+
+	os.Unsetenv("SC_CALLER_INFO")
+
+	if agent != expected {
+		t.Errorf("User Agent not retrieved successfully, got %s, expected %s", agent, expected)
+	}
+}
+
+func TestGetUserAgentCINoClueWhatSystem(t *testing.T) {
+	clearCircleCIVariables()
+	expected := "nancy-client/development (ci usage; linux amd64; )"
+
+	os.Setenv("CI", "true")
+	GOOS = "linux"
+	GOARCH = "amd64"
+
+	agent := GetUserAgent()
+
+	os.Unsetenv("CI")
+
+	if agent != expected {
+		t.Errorf("User Agent not retrieved successfully, got %s, expected %s", agent, expected)
+	}
+}
+
+func clearCircleCIVariables() {
+	os.Unsetenv("CI")
+	os.Unsetenv("CIRCLECI")
+}