<ins> #!/usr/bin/env bash # Copyright 2014 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # This script verifies the following e2e test ownership policies # - tests MUST start with [sig-foo] # - tests MUST use top-level SIGDescribe # - tests SHOULD NOT have multiple [sig-foo] tags # - tests MUST NOT use nested SIGDescribe # TODO: these two can be dropped if KubeDescribe is gone from codebase # - tests MUST NOT have [k8s.io] in test names # - tests MUST NOT use KubeDescribe set -o errexit set -o nounset set -o pipefail # This will canonicalize the path KUBE_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd -P) source "${KUBE_ROOT}/hack/lib/init.sh" # Set REUSE_BUILD_OUTPUT=y to skip rebuilding dependencies if present REUSE_BUILD_OUTPUT=${REUSE_BUILD_OUTPUT:-n} # set VERBOSE_OUTPUT=y to output .jq files and shell commands VERBOSE_OUTPUT=${VERBOSE_OUTPUT:-n} if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then set -x fi pushd "${KUBE_ROOT}" > /dev/null # Setup a tmpdir to hold generated scripts and results readonly tmpdir=$(mktemp -d -t verify-e2e-test-ownership.XXXX) trap 'rm -rf ${tmpdir}' EXIT # input spec_summaries="${KUBE_ROOT}/_output/specsummaries.json" # output results_json="${tmpdir}/results.json" summary_json="${tmpdir}/summary.json" failures_json="${tmpdir}/failures.json" # rebuild dependencies if necessary function ensure_dependencies() { local -r ginkgo="${KUBE_ROOT}/_output/bin/ginkgo" local -r e2e_test="${KUBE_ROOT}/_output/bin/e2e.test" if ! { [ -f "${ginkgo}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then make ginkgo fi if ! { [ -f "${e2e_test}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then hack/make-rules/build.sh test/e2e/e2e.test fi if ! { [ -f "${spec_summaries}" ] && [[ "${REUSE_BUILD_OUTPUT}" =~ ^[yY]$ ]]; }; then "${ginkgo}" --dryRun=true "${e2e_test}" -- --spec-dump "${spec_summaries}" > /dev/null fi } # evaluate ginkgo spec summaries against e2e test ownership polices # output to ${results_json} function generate_results_json() { readonly results_jq=${tmpdir}/results.jq cat >"${results_jq}" <<EOS [.[] | . as { ComponentTexts: $text, ComponentCodeLocations: $code } | { calls: [ $text | range(1;length) as $i | { sig: (($text[$i] | match("[(sig-[^]]+)]") | .captures[0].string) // "unknown"), text: $text[$i], # unused, but if we ever wanted to have policies based on other tags... # tags: $text[$i] | [match("([[^]]+])"; "g").string], line: $code[$i] | "(.FileName):(.LineNumber)", } + ($code[$i] | .FullStackTrace | match("^(.*).(.+)(.*n") | .captures | { package: .[0].string, func: .[1].string })], } | { owner: .calls[0].sig, testname: .calls | map(.text) | join(" "), calls, policies: [( .calls[0] | { fail: (.sig == "unknown"), level: "FAIL", category: "unowned_test", reason: "must start with [sig-foo]", found: ., }, { fail: (.func != "SIGDescribe"), level: "FAIL", category: "no_sig_describe", reason: "must use top-level SIGDescribe", found: ., } ), ( .calls[1:] | (map(select(.sig != "unknown")) // [] | { fail: . | any, level: "WARN", category: "too_many_sigs", reason: "should not have multiple [sig-foo] tags", found: ., }), (map(select(.func == "SIGDescribe")) // [] | { fail: . | any, level: "FAIL", category: "nested_sig_describe", reason: "must not use nested SIGDescribe", found: ., }) ) ] }] EOS if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then echo "about to  ${results_jq}..." cat -n "${results_jq}" echo fi <"${spec_summaries}" jq --slurp --from-file "${results_jq}" > "${results_json}" } # summarize e2e test policy results # output to ${summary_json} function generate_summary_json() { summary_jq=${tmpdir}/summary.jq cat >"${summary_jq}" <<EOS . as $results | # for each policy category reduce $results[0].policies[] as $p ({}; . + { # add a convenience .policy field containing that policy's result ($p.category): $results | map(. + {policy: .policies[] | select(.category == $p.category)}) | { level: $p.level, reason: $p.reason, passing: map(select(.policy.fail | not)) | length, failing: map(select(.policy.fail)) | length, testnames: map(select(.policy.fail) | .testname), } }) # add a meta policy based on whether any policy failed + { all_policies: $results | { level: "WARN", reason: "should pass all policies", passing: map(select(.policies | map(.fail) | any | not)) | length, failing: map(select(.policies | map(.fail) | any)) | length, testnames: map(select(.policies | map(.fail) | any) | .testname), } } # if a policy has no failing tests, change its log output to PASS | with_entries(.value += { log: (if (.value.failing == 0) then "PASS" else .value.level end) }) # sort by policies with the most failing tests first | to_entries | sort_by(.value.failing) | reverse | from_entries EOS if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then echo "about to run ${results_jq}..." cat -n "${summary_jq}" echo fi <"${results_json}" jq --from-file "${summary_jq}" > "${summary_json}" } # filter e2e policy tests results to tests that failed, with the policies they failed # output to ${failures_json} function generate_failures_json() { local -r failures_jq="${tmpdir}/failures.jq" cat >"${failures_jq}" <<EOS . # for each test | map( # filter down to failing policies; trim category, .reason is more verbose .policies |= map(select(.fail) | del(.category)) # trim the full callstack, .found will contain the relevant call | del(.calls) ) # filter down to tests that have failed policies | map(select(.policies | map (.fail) | any)) EOS if [[ ${VERBOSE_OUTPUT} =~ ^[yY]$ ]]; then echo "about to run ${failures_jq}..." cat -n "${failures_jq}" echo fi <"${results_json}" jq --from-file "${failures_jq}" > "${failures_json}" } function output_results_and_exit_if_failed() { local -r total_tests=$(<"${spec_summaries}" wc -l | awk '{print $1}') # output results to console ( echo "run at datetime: $(date -u +%Y-%m-%dT%H:%M:%SZ)" echo "based on commit: $(git log -n1 --date=iso-strict --pretty='%h - %cd - %s')" echo <"${failures_json}" cat printf "%4s: e2e tests %-40s: %-4dn" "INFO" "in total" "${total_tests}" <"${summary_json}" jq -r 'to_entries[].value | "printf "%4s: ..failing %-40s: %-4dn" "(.log)" "(.reason)" "(.failing)""' | sh ) | tee "${tmpdir}/output.txt" # if we said "FAIL" in that output, we should fail if <"${tmpdir}/output.txt" grep -q "^FAIL"; then echo "FAIL" exit 1 fi } ensure_dependencies generate_results_json generate_failures_json generate_summary_json output_results_and_exit_if_failed echo "PASS" </ins>