Repository: scionproto/scion
Pull request #3751 (2020-05-07 11:19:10)

ci: general improvements
This commit also adds integration test for file logging
to verify that file logging is working.

This will pave the way for removing file logging from our testing
infrastructure.

Co-authored-by: Lukas Vogel <lukedirtwalker@gmail.com>

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/scionproto/scion/3751)
<!-- Reviewable:end -->



Patch:
diff --git a/.buildkite/hooks/pre-artifact b/.buildkite/hooks/pre-artifact
index 914e139e5..e7ae12b1f 100755
--- a/.buildkite/hooks/pre-artifact
+++ b/.buildkite/hooks/pre-artifact
@@ -2,6 +2,33 @@
 
 set -eo pipefail
 
+# Attempt to do clean docker topology shutdown.
+# This lets applications flush the logs to avoid cutting them off.
+if [ -f "gen/scion-dc.yml" ]; then
+    # This hook is global, therefore we need to disable fail on error because
+    # not all docker-compose tests are following the pattern
+    # COMPOSE_PROJECT_NAME=scion, COMPOSE_FILE=gen/scion-dc.yml
+    # and logs in stdout.
+    set +e
+    docker-compose -f gen/scion-dc.yml -p scion stop
+
+    docker-compose -f gen/scion-dc.yml -p scion logs --no-color > logs/scion-dc.log
+
+    for s in $(docker-compose -f gen/scion-dc.yml -p scion ps --services); do
+        cat logs/scion-dc.log | grep $s| cut -f2 -d"|" > logs/${s#"scion_"}.log
+    done
+
+    docker-compose -f gen/scion-dc.yml -p scion down -v
+
+    # a subset of tests are using testgen and they do a collect log and dc stop
+    # on their own. Therefore the above code produces empty files that are confusing.
+    # Given the limitation of buildkite that the hook is global we just cleanup
+    # the empty files.
+    find . -type f -empty -delete
+
+    set -e
+fi
+
 # Now we build the artifact name next, for this we first need TARGET and BUILD,
 # see below.
 #
@@ -21,6 +48,7 @@ echo "\$BUILD=$BUILD"
 ARTIFACTS="buildkite.${BUILDKITE_ORGANIZATION_SLUG}.${TARGET}.${BUILD}.${BUILDKITE_STEP_KEY:-unset}.${BUILDKITE_JOB_ID}"
 mkdir -p "artifacts/$ARTIFACTS" artifacts.out
 
+set -x
 function save {
     if [ -d "$1" ]; then
         echo Found artifacts: "$1"
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 91d262be6..92b763fad 100755
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -2,7 +2,6 @@
 
 set -euo pipefail
 
-
 # ACCEPTANCE_ARTIFACTS is used for acceptance tests built with the "old"
 # acceptance framework
 export ACCEPTANCE_ARTIFACTS=/tmp/test-artifacts
@@ -16,7 +15,7 @@ if [ -z ${BAZEL_REMOTE_S3_ACCESS_KEY_ID+x} ]; then
 fi
 
 # Add bazelrc for CI to home directory, so that remote_cache is picked up.
-mv -f .bazelrc_ci $HOME/.bazelrc
+cp -f .bazelrc_ci $HOME/.bazelrc
 
 echo "Starting bazel remote cache proxy"
 
diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
index 18bb6b211..97aa9b830 100644
--- a/.buildkite/hooks/pre-exit
+++ b/.buildkite/hooks/pre-exit
@@ -10,4 +10,5 @@ docker network prune -f
 echo "Remove leftover volumes"
 docker volume prune -f
 
+rm -f scion-utils-{rpm,deb}.{version,release}
 rm -rf bazel-testlogs logs/* traces gen gen-cache /tmp/test-artifacts
diff --git a/.buildkite/pipeline.sh b/.buildkite/pipeline.sh
index 2c32ad9dd..058729ec4 100755
--- a/.buildkite/pipeline.sh
+++ b/.buildkite/pipeline.sh
@@ -1,51 +1,9 @@
 #!/bin/bash
 
-set -e
+set -eou pipefail
 
-# gen_acceptance generates all the acceptance steps.
-gen_acceptance() {
-    for test in ./acceptance/*_acceptance; do
-        name="$(basename ${test%_acceptance})"
-        echo "  - label: \"Acceptance: $name\""
-        echo "    command:"
-        echo "      - ./acceptance/ctl gsetup"
-        echo "      - ./acceptance/ctl grun $name"
-        echo "    key: ${name}_acceptance"
-        echo "    env:"
-        echo "      PYTHONPATH: \"python/:.\""
-        echo "    artifact_paths:"
-        echo "      - \"artifacts.out/**/*\""
-        echo "    retry:"
-        echo "      automatic:"
-        echo "        - exit_status: -1 # Agent was lost"
-        echo "        - exit_status: 255 # Forced agent shutdown"
-    done
-}
-
-# gen_bazel_acceptance generates steps for bazel tests in acceptance folder.
-gen_bazel_acceptance() {
-    for test in $(bazel query 'kind(test, //acceptance/...)' 2>/dev/null); do
-        # test has the format //acceptance/<name>:<name>_test
-        name=$(echo $test | cut -d ':' -f 1)
-        name=${name#'//acceptance/'}
-        echo "  - label: \":bazel: Acceptance: $name\""
-        echo "    command:"
-        if [[ "$test" =~ "go" ]]; then
-            # for go tests add verbose flag.
-            echo "      - bazel test $test --test_arg=-test.v"
-        else
-            echo "      - bazel test $test"
-        fi
-        echo "    key: ${name}_acceptance"
-        echo "    artifact_paths:"
-        echo "      - \"artifacts.out/**/*\""
-        echo "    retry:"
-        echo "      automatic:"
-        echo "        - exit_status: -1 # Agent was lost"
-        echo "        - exit_status: 255 # Forced agent shutdown"
-    done
-}
+. ./.buildkite/pipeline_lib.sh
 
 cat .buildkite/pipeline.yml
-gen_bazel_acceptance
-gen_acceptance
+gen_bazel_test_steps //acceptance
+gen_acceptance ./acceptance
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index 59ce111f5..a8e9ec442 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -1,6 +1,9 @@
 steps:
   - label: ":bazel: Build scion code"
+    if: build.message !~ /\[doc\]/
     command:
+      # create version files so that deb and rpm release targets work.
+      - ./tools/package-version 0.1.0-citest
       - bazel build --announce_rc //:all
     key: build
     retry:
@@ -9,6 +12,7 @@ steps:
         - exit_status: 255 # Forced agent shutdown
     timeout_in_minutes: 10
   - label: ":bazel: Go tests"
+    if: build.message !~ /\[doc\]/
     command:
       - bazel test //go/... --print_relative_test_log_paths
     key: go_tests
@@ -20,6 +24,7 @@ steps:
         - exit_status: 255 # Forced agent shutdown
     timeout_in_minutes: 10
   - label: ":python: Python tests"
+    if: build.message !~ /\[doc\]/
     command: ./scion.sh test py
     key: py_tests
     retry:
@@ -37,6 +42,7 @@ steps:
         - exit_status: -1  # Agent was lost
         - exit_status: 255 # Forced agent shutdown
   - label: "Check generated go_deps.bzl file is up to date with go.mod"
+    if: build.message !~ /\[doc\]/
     command:
       - mkdir -p /tmp/test-artifacts
       - cp go.mod go.sum go_deps.bzl /tmp/test-artifacts/
@@ -51,6 +57,7 @@ steps:
         - exit_status: -1  # Agent was lost
         - exit_status: 255 # Forced agent shutdown
   - label: "Check generated go/proto files in git"
+    if: build.message !~ /\[doc\]/
     command:
       - mkdir -p /tmp/test-artifacts
       - cp -R go/proto/ /tmp/test-artifacts/
@@ -68,7 +75,19 @@ steps:
       automatic:
         - exit_status: -1  # Agent was lost
         - exit_status: 255 # Forced agent shutdown
+  - label: ":bazel: Integration tests"
+    if: build.message !~ /\[doc\]/
+    command:
+    - bazel test //integration/...
+    key: integration_tests
+    artifact_paths:
+      - "artifacts.out/**/*"
+    retry:
+      automatic:
+        - exit_status: -1  # Agent was lost
+        - exit_status: 255 # Forced agent shutdown
   - label: "Revocation tests"
+    if: build.message !~ /\[doc\]/
     command:
       - bazel build //:scion //:scion-ci >/dev/null 2>&1
       - tar -kxf bazel-bin/scion.tar -C bin --overwrite
@@ -90,14 +109,16 @@ steps:
       automatic:
         - exit_status: -1  # Agent was lost
         - exit_status: 255 # Forced agent shutdown
-  - label: "Integration: {cert_req,pp,end2end,scmp}_integration"
+  - label: "E2E-Integration: {cert_req,pp,end2end,scmp}_integration"
+    if: build.message !~ /\[doc\]/
     command:
     - bazel build //:scion //:scion-ci >/dev/null 2>&1
     - tar -kxf bazel-bin/scion.tar -C bin --overwrite
     - tar -kxf bazel-bin/scion-ci.tar -C bin --overwrite
     - ./scion.sh topology -t
     - ./scion.sh run && sleep 10
-    - ./bin/cert_req_integration -log.console warn
+    # FIXME(roosd): enable again
+    #- ./bin/cert_req_integration -log.console warn
     - ./bin/pp_integration -log.console warn
     - ./bin/scmp_integration -log.console warn
     - ./bin/end2end_integration -log.console warn
@@ -110,7 +131,7 @@ steps:
     artifact_paths:
       - "artifacts.out/**/*"
     timeout_in_minutes: 10
-    key: integration_tests
+    key: e2e_integration_tests
     retry:
       automatic:
         - exit_status: -1  # Agent was lost
@@ -121,7 +142,7 @@ steps:
     - ./scion.sh run
     - docker-compose -f gen/scion-dc.yml -p scion up -d $(docker-compose -f gen/scion-dc.yml config --services | grep tester)
     - sleep 10
-    - ./bin/cert_req_integration -d -log.console warn
+    # - ./bin/cert_req_integration -d -log.console warn
     - ./bin/pp_integration -d -log.console warn
     - ./bin/scmp_integration -d -log.console warn
     - ./bin/end2end_integration -d -log.console warn
diff --git a/.buildkite/pipeline_lib.sh b/.buildkite/pipeline_lib.sh
new file mode 100644
index 000000000..548bbe0e7
--- /dev/null
+++ b/.buildkite/pipeline_lib.sh
@@ -0,0 +1,57 @@
+# gen_acceptance generates all the acceptance steps in a given directory.
+# args:
+#   -1: the directory in which the acceptance tests are.
+#   -2: the tests which don't need any setup. (default: none)
+gen_acceptance() {
+    local accept_dir=${1}
+    local no_setup_tests=${2:-""}
+    for test in "$accept_dir"/*_acceptance; do
+        name="$(basename ${test%_acceptance})"
+        echo "  - label: \"Acceptance: $name\""
+        echo "    if: build.message !~ /\[doc\]/"
+        echo "    command:"
+        if [[ ! "${no_setup_tests}" == *"${name}"* ]]; then
+            # some tests don't need the global setup, they are just starting a
+            # (few) docker container(s) and run a bazel test against it. So no
+            # prebuilding of all docker containers is needed.
+            echo "      - $accept_dir/ctl gsetup"
+        fi
+        echo "      - $accept_dir/ctl grun $name"
+        echo "    key: ${name}_acceptance"
+        echo "    env:"
+        echo "      PYTHONPATH: \"python/:.\""
+        echo "    artifact_paths:"
+        echo "      - \"artifacts.out/**/*\""
+        echo "    retry:"
+        echo "      automatic:"
+        echo "        - exit_status: -1 # Agent was lost"
+        echo "        - exit_status: 255 # Forced agent shutdown"
+    done
+}
+
+# gen_bazel_test_steps generates steps for bazel tests in the given directory.
+# args:
+#   -1: the bazel directory in which the tests are.
+gen_bazel_test_steps() {
+    for test in $(bazel query "kind(test, ${1}/...)" 2>/dev/null); do
+        # test has the format //acceptance/<name>:<name>_test
+        name=$(echo $test | cut -d ':' -f 1)
+        name=${name#"$1/"}
+        echo "  - label: \":bazel: Acceptance: $name\""
+        echo "    if: build.message !~ /\[doc\]/"
+        echo "    command:"
+        if [[ "$test" =~ "go" ]]; then
+            # for go tests add verbose flag.
+            echo "      - bazel test $test --test_arg=-test.v"
+        else
+            echo "      - bazel test $test"
+        fi
+        echo "    key: ${name}_acceptance"
+        echo "    artifact_paths:"
+        echo "      - \"artifacts.out/**/*\""
+        echo "    retry:"
+        echo "      automatic:"
+        echo "        - exit_status: -1 # Agent was lost"
+        echo "        - exit_status: 255 # Forced agent shutdown"
+    done
+}
diff --git a/integration/service_filelog/BUILD.bazel b/integration/service_filelog/BUILD.bazel
new file mode 100644
index 000000000..0d7003bfa
--- /dev/null
+++ b/integration/service_filelog/BUILD.bazel
@@ -0,0 +1,24 @@
+load("@io_bazel_rules_docker//container:container.bzl", "container_image")
+load("@rules_python//python:defs.bzl", "py_test")
+load("@pip3_deps//:requirements.bzl", "requirement")
+
+py_test(
+    name = "test",
+    srcs = ["test.py"],
+    data = [":cs.tar"],
+    deps = [
+        requirement("plumbum"),
+    ],
+)
+
+container_image(
+    name = "cs",
+    base = "//docker:cs_debug",
+    entrypoint = [
+        "/app/cs",
+        "-config",
+        "/cs.toml",
+    ],
+    files = ["testdata/cs.toml"],
+    stamp = False,
+)
diff --git a/tools/package-version b/tools/package-version
new file mode 100755
index 000000000..9a86df98a
--- /dev/null
+++ b/tools/package-version
@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+# This script converts a version from ./tools/git-version to debian and rpm 
+# version files that can be consumed by bazel.
+# Examples:
+# - prefix-v0.8.0-rc1-1-g0aec09a 
+#   -> scion-utils-deb.version: 0.8.0~rc1~1~g0aec09a
+#   -> scion-utils-rpm.version: 0.8.0
+#   -> scion-utils-rpm.release: rc1-1-g0aec09a
+# - prefix-v0.8.0 
+#   -> scion-utils-deb.version: 0.8.0
+#   -> scion-utils-rpm.version: 0.8.0
+#   -> scion-utils-rpm.release: 1
+
+from pathlib import Path
+import re
+import sys
+
+DEB_VERSION = 'scion-utils-deb.version'
+RPM_VERSION = 'scion-utils-rpm.version'
+RPM_RELEASE = 'scion-utils-rpm.release'
+
+if len(sys.argv) < 2:
+    print('Requires version as first argument')
+    sys.exit(1)
+
+for f in [DEB_VERSION, RPM_VERSION, RPM_RELEASE]:
+    if Path(f).is_file():
+        print('%s exists' % f)
+        sys.exit(1)
+
+version = sys.argv[1]
+m = re.search(r'\d', version)
+if m is None:
+    print('Invalid version number: %s' % version)
+    sys.exit(1)
+
+version = version[m.start():]
+deb_version = version.replace('-', '~')
+parts = version.split('-', 1)
+rpm_version, rpm_release = parts[0], 1
+if len(parts) > 1:
+    rpm_release = parts[1]
+
+with open(DEB_VERSION, 'w') as f:
+    f.write('%s' % deb_version)
+
+with open(RPM_VERSION, 'w') as f:
+    f.write('%s' % rpm_version)
+
+with open(RPM_RELEASE, 'w') as f:
+    f.write('%s' % rpm_release)


Test patch:
diff --git a/integration/service_filelog/test.py b/integration/service_filelog/test.py
new file mode 100644
index 000000000..4c08074cc
--- /dev/null
+++ b/integration/service_filelog/test.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+
+# Copyright 2020 Anapaya Systems
+
+from pathlib import Path
+import shutil
+import sys
+import tempfile
+
+from plumbum import cli, local
+from plumbum.cmd import docker
+
+
+class Test(cli.Application):
+    """
+    Tests that the CS application logs to a file.
+    """
+    def main(self):
+        tmpdir = tempfile.mkdtemp(prefix='service_filelog.')
+        print(docker('image', 'load', '-i', './integration/service_filelog/cs.tar'))
+        name = 'service_filelog_test'
+        try:
+            args = ['run', '--name', name, '-v', '{}:/share/logs'.format(tmpdir),
+                    'bazel/integration/service_filelog:cs']
+            print('docker', *args)
+            print(docker(*args, retcode=1))
+            logfile = Path('{}/cs.log'.format(tmpdir))
+            if not logfile.is_file():
+                print('{} is not a file'.format(logfile))
+                sys.exit(1)
+            if logfile.stat().st_size == 0:
+                print('{} is empty'.format(logfile))
+                sys.exit(1)
+        finally:
+            docker('logs', name)
+            docker('rm', '-f', name)
+            docker('rmi', 'bazel/integration/service_filelog:cs')
+            shutil.rmtree(tmpdir, ignore_errors=True)
+
+if __name__ == "__main__":
+    Test.run()
diff --git a/integration/service_filelog/testdata/cs.toml b/integration/service_filelog/testdata/cs.toml
new file mode 100644
index 000000000..a3cd313a2
--- /dev/null
+++ b/integration/service_filelog/testdata/cs.toml
@@ -0,0 +1,4 @@
+[general]
+    id = "cs-1"
+[log.file]
+    path = "/share/logs/cs.log"