Repository: kubernetes-sigs/security-profiles-operator
Pull request #1069 (2022-08-04 07:57:51)

Run metrics e2e tests in single-node environments only (cherry-pick)
<!--  Thanks for sending a pull request!  Here are some tips for you:

- If this is your first time, please read our contributor guidelines: https://git.k8s.io/community/contributors/guide#your-first-contribution and developer guide https://git.k8s.io/community/contributors/devel/development.md#development-guide
- Please label this pull request according to what type of issue you are addressing, especially if this is a release targeted pull request. For reference on required PR/issue labels, read here:
https://git.k8s.io/community/contributors/devel/sig-release/release.md#issuepr-kind-label
- If you want *faster* PR reviews, read how: https://git.k8s.io/community/contributors/guide/pull-requests.md#best-practices-for-faster-reviews
- If the PR is unfinished, see how to mark it: https://git.k8s.io/community/contributors/guide/pull-requests.md#marking-unfinished-pull-requests
-->

#### What type of PR is this?
/kind flake

<!--
Add one of the following kinds:
/kind cleanup
/kind documentation
/kind feature
/kind design

Optionally add one or more of the following kinds if applicable:
/kind api-change
/kind deprecation
/kind failing-test
/kind regression
-->

#### What this PR does / why we need it:
This PR cherry-picks one patch from PR #1059 and adds one more patch in
a similar manner, also running a metrics check in a single-node environment
only.

The second patch prevents frequent flakes in an OCP six-node environment.


#### Which issue(s) this PR fixes:
None

<!--
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.

Fixes #

or

None
-->

#### Does this PR have test?
N/A

<!--
If tests aren't applicable just write N/A.
-->

#### Special notes for your reviewer:
N/A

#### Does this PR introduce a user-facing change?

<!--
If no, just write "NONE" in the release-note block below.
If yes, a release note is required:
Enter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string "action required".

For more information on release notes see: https://git.k8s.io/community/contributors/guide/release-notes.md
-->

```release-note
NONE
```


k8s-ci-robot: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/kubernetes-sigs/security-profiles-operator/pull/1069#" title="Author self-approved">jhrozek</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=kubernetes-sigs%2Fsecurity-profiles-operator).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/kubernetes-sigs/security-profiles-operator/blob/main/OWNERS)~~ [jhrozek]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
codecov-commenter: # [Codecov](https://codecov.io/gh/kubernetes-sigs/security-profiles-operator/pull/1069?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None) Report
> Merging [#1069](https://codecov.io/gh/kubernetes-sigs/security-profiles-operator/pull/1069?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None) (473872f) into [main](https://codecov.io/gh/kubernetes-sigs/security-profiles-operator/commit/3d7d017d9724fd67ad844df904d51a5ba1ebfccd?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None) (3d7d017) will **not change** coverage.
> The diff coverage is `n/a`.

```diff
@@           Coverage Diff           @@
##             main    #1069   +/-   ##
=======================================
  Coverage   50.43%   50.43%           
=======================================
  Files          42       42           
  Lines        4761     4761           
=======================================
  Hits         2401     2401           
  Misses       2281     2281           
  Partials       79       79           
```

k8s-ci-robot: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/kubernetes-sigs/security-profiles-operator/pull/1069#" title="Author self-approved">jhrozek</a>*, *<a href="https://github.com/kubernetes-sigs/security-profiles-operator/pull/1069#pullrequestreview-1061458661" title="Approved">saschagrunert</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=kubernetes-sigs%2Fsecurity-profiles-operator).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/kubernetes-sigs/security-profiles-operator/blob/main/OWNERS)~~ [jhrozek,saschagrunert]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->

Patch:
diff --git a/.github/workflows/helm-chart-package.yaml b/.github/workflows/helm-chart-package.yaml
new file mode 100644
index 000000000..ca4b95a0d
--- /dev/null
+++ b/.github/workflows/helm-chart-package.yaml
@@ -0,0 +1,53 @@
+name: helm-chart-package
+on:
+  pull_request:
+  release:
+    types: [published]
+permissions:
+  actions: none
+  checks: none
+  contents: write
+  deployments: none
+  id-token: none
+  issues: none
+  discussions: none
+  packages: none
+  pull-requests: none
+  repository-projects: none
+  security-events: none
+  statuses: none
+jobs:
+  helm-package:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Set up Helm
+        uses: azure/setup-helm@v1
+        with:
+          version: v3.8.1
+      - run: mkdir -p build
+      - id: package
+        run: |
+          helm_output="$(helm package -d build deploy/helm)"
+          tgz_path="${helm_output##*saved it to: }"
+          echo "helm chart tgz path: '${tgz_path}'"
+          [ ! -f "${tgz_path}" ] && echo "failed to find helm chart from 'helm package' stdout" && exit 1
+          echo "::set-output name=helm_tgz_path::${tgz_path}"
+          echo "::set-output name=helm_tgz_name::${tgz_path##*/}"
+      - uses: actions/upload-artifact@v3
+        with:
+          name: ${{ steps.package.outputs.helm_tgz_name}}
+          path: ${{ steps.package.outputs.helm_tgz_path}}
+          if-no-files-found: error
+      - uses: actions/upload-release-asset@v1
+        if: github.event_name == 'release'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ github.event.release.upload_url }}
+          asset_path: ${{ steps.package.outputs.helm_tgz_path}}
+          asset_name: ${{ steps.package.outputs.helm_tgz_name}}
+          asset_content_type: 'application/x-tar'


Test patch:
diff --git a/test/suite_test.go b/test/suite_test.go
index 7db0aa411..695d982e2 100644
--- a/test/suite_test.go
+++ b/test/suite_test.go
@@ -95,6 +95,7 @@ type e2e struct {
 	bpfRecorderEnabled     bool
 	skipNamespacedTests    bool
 	testWebhookConfig      bool
+	singleNodeEnvironment  bool
 	logger                 logr.Logger
 	execNode               func(node string, args ...string) string
 	waitForReadyPods       func()
@@ -254,6 +255,9 @@ func TestSuite(t *testing.T) {
 				skipNamespacedTests:    skipNamespacedTests,
 				operatorManifest:       operatorManifest,
 				testWebhookConfig:      testWebhookConfig,
+				// NOTE(jaosorior): Our current vanilla jobs are
+				// single-node only.
+				singleNodeEnvironment: true,
 			},
 		})
 	default:
@@ -691,6 +695,12 @@ func (e *e2e) bpfRecorderOnlyTestCase() {
 	e.enableBpfRecorderInSpod()
 }
 
+func (e *e2e) singleNodeTestCase() {
+	if !e.singleNodeEnvironment {
+		e.T().Skip("Skipping test because we're in a multi-node environment.")
+	}
+}
+
 func (e *e2e) enableBpfRecorderInSpod() {
 	e.logf("Enable bpf recorder in SPOD")
 	e.kubectlOperatorNS("patch", "spod", "spod", "-p", `{"spec":{"enableBpfRecorder": true}}`, "--type=merge")
diff --git a/test/tc_log_enricher_test.go b/test/tc_log_enricher_test.go
index df193d997..ed4e6a5f6 100644
--- a/test/tc_log_enricher_test.go
+++ b/test/tc_log_enricher_test.go
@@ -106,14 +106,19 @@ spec:
 	e.Contains(output, `"syscallName"="listen"`)
 	e.Contains(output, `"syscallID"=50`)
 
-	metrics := e.getSpodMetrics()
-	e.Regexp(fmt.Sprintf(`(?m)security_profiles_operator_seccomp_profile_audit_total{`+
-		`container="%s",`+
-		`executable="/usr/sbin/nginx",`+
-		`namespace="%s",`+
-		`node=".*",`+
-		`pod="%s",`+
-		`syscall="listen"} \d+`,
-		containerName, namespace, podName,
-	), metrics)
+	if e.singleNodeEnvironment {
+		// we only run the metrics checks in a single node environment because otherwise it's a lottery
+		// which spod instance do we hit and the test is not stable
+
+		metrics := e.getSpodMetrics()
+		e.Regexp(fmt.Sprintf(`(?m)security_profiles_operator_seccomp_profile_audit_total{`+
+			`container="%s",`+
+			`executable="/usr/sbin/nginx",`+
+			`namespace="%s",`+
+			`node=".*",`+
+			`pod="%s",`+
+			`syscall="listen"} \d+`,
+			containerName, namespace, podName,
+		), metrics)
+	}
 }
diff --git a/test/tc_metrics_test.go b/test/tc_metrics_test.go
index bc9529fe6..aca380b18 100644
--- a/test/tc_metrics_test.go
+++ b/test/tc_metrics_test.go
@@ -27,6 +27,7 @@ const profileName = "metrics-profile"
 
 func (e *e2e) testCaseSeccompMetrics(nodes []string) {
 	e.seccompOnlyTestCase()
+	e.singleNodeTestCase()
 
 	const (
 		operationDelete = `security_profiles_operator_seccomp_profile_total{operation="delete"}`
@@ -72,6 +73,7 @@ spec:
 
 func (e *e2e) testCaseSelinuxMetrics(nodes []string) {
 	e.selinuxOnlyTestCase()
+	e.singleNodeTestCase()
 
 	const (
 		operationDelete = `security_profiles_operator_selinux_profile_total{operation="delete"}`