Repository: scionproto/scion
Pull request #3566 (2020-01-03 14:14:35)

SH: Only build necessary binaries
The topology generator only uses two binaries when running.
With this change, scion.sh only builds the necessary binaries instead
of all binaries.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/scionproto/scion/3566)
<!-- Reviewable:end -->



Patch:
diff --git a/.buildkite/pipeline_buildlint.yml b/.buildkite/pipeline_buildlint.yml
index f0416b7ac..5b05c495c 100644
--- a/.buildkite/pipeline_buildlint.yml
+++ b/.buildkite/pipeline_buildlint.yml
@@ -61,7 +61,7 @@ steps:
       - bazel --bazelrc=.bazelrc_ci build //:scion //:scion-ci >/dev/null 2>&1
       - tar -kxf bazel-bin/scion.tar -C bin --overwrite
       - tar -kxf bazel-bin/scion-ci.tar -C bin --overwrite
-      - ./scion.sh topology nobuild
+      - ./scion.sh topology
       - ./scion.sh run nobuild && sleep 10
       - ./bin/end2end_integration -log.console warn
       - ./integration/revocation_test.sh
@@ -77,7 +77,7 @@ steps:
     - bazel --bazelrc=.bazelrc_ci build //:scion //:scion-ci >/dev/null 2>&1
     - tar -kxf bazel-bin/scion.tar -C bin --overwrite
     - tar -kxf bazel-bin/scion-ci.tar -C bin --overwrite
-    - ./scion.sh topology nobuild
+    - ./scion.sh topology
     - ./scion.sh run nobuild && sleep 10
     - ./bin/cert_req_integration -log.console warn
     - ./bin/pp_integration  -log.console warn
diff --git a/BUILD.bazel b/BUILD.bazel
index 46f7d14a2..9457caaf0 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -50,6 +50,16 @@ pkg_tar(
     mode = "0755",
 )
 
+# This contains all of the binaries needed to run the topology generator.
+pkg_tar(
+    name = "scion-topo",
+    package_dir = "",
+    srcs = [
+        "//go/tools/scion-custpk-load:scion-custpk-load",
+        "//go/tools/scion-pki:scion-pki",
+    ],
+    mode = "0755",
+)
 
 # This is a package of tools used for linting the source code.
 pkg_tar(
diff --git a/acceptance/common/scion.py b/acceptance/common/scion.py
index 55c9a14f6..5bd9c2a4f 100644
--- a/acceptance/common/scion.py
+++ b/acceptance/common/scion.py
@@ -111,8 +111,7 @@ class ScionDocker(Scion):
     @LogExec(logger, "creating dockerized topology")
     def topology(self, topo_file: str, *args: str):
         """ Create the dockerized topology files. """
-        self.scion_sh('topology', 'nobuild', '-c', topo_file, '-t',
-                      '-d', *args)
+        self.scion_sh('topology', '-c', topo_file, '-t', '-d', *args)
 
     def _send_signals(self, svc_names: List[str], sig: str):
         for svc_name in svc_names:
@@ -131,7 +130,7 @@ class ScionSupervisor(Scion):
     @LogExec(logger, "creating supervisor topology")
     def topology(self, topo_file: str, *args: str):
         """ Create the topology files. """
-        self.scion_sh('topology', 'nobuild', '-c', topo_file, *args)
+        self.scion_sh('topology', '-c', topo_file, *args)
 
     def _send_signals(self, svc_names: List[str], sig: str):
         for svc_name in svc_names:
diff --git a/acceptance/discovery_util/util.sh b/acceptance/discovery_util/util.sh
index 74754528a..2a8f9ed47 100644
--- a/acceptance/discovery_util/util.sh
+++ b/acceptance/discovery_util/util.sh
@@ -22,7 +22,7 @@ DYNAMIC_DEFAULT="$DYNAMIC_DIR/default.json"
 base_setup() {
     set -e
     # Create topology setup all necessary config files.
-    ./scion.sh topology nobuild -c "$TEST_TOPOLOGY" -d -t -ds
+    ./scion.sh topology -c "$TEST_TOPOLOGY" -d -t -ds
     # Create the topology directories for serving.
     mkdir -p "$STATIC_DIR"
     mkdir -p "$DYNAMIC_DIR"
diff --git a/acceptance/sigutil/common.sh b/acceptance/sigutil/common.sh
index 31d64fc87..1319d2f11 100755
--- a/acceptance/sigutil/common.sh
+++ b/acceptance/sigutil/common.sh
@@ -8,7 +8,7 @@ DST_IA=${DST_IA:-1-ff00:0:112}
 
 test_setup() {
     set -e
-    ./scion.sh topology nobuild -c $TEST_TOPOLOGY -d -t --sig -n 242.254.0.0/16
+    ./scion.sh topology -c $TEST_TOPOLOGY -d -t --sig -n 242.254.0.0/16
     for sig in gen/ISD1/*/sig*/sig.toml; do
         sed -i '/\[logging\.file\]/a FlushInterval = 1' "$sig"
     done
diff --git a/acceptance/topo_br_reload_util/util.sh b/acceptance/topo_br_reload_util/util.sh
index 280c2022b..78e927838 100644
--- a/acceptance/topo_br_reload_util/util.sh
+++ b/acceptance/topo_br_reload_util/util.sh
@@ -31,7 +31,7 @@ base_setup() {
 }
 
 base_gen_topo() {
-    ./scion.sh topology nobuild -c $TEST_TOPOLOGY -d -t
+    ./scion.sh topology -c $TEST_TOPOLOGY -d -t
     sed -i '/\[logging\.file\]/a FlushInterval = 1' gen/ISD1/*/br*/br.toml
 }
 
diff --git a/scion.sh b/scion.sh
index e8ee6dea1..8252085f9 100755
--- a/scion.sh
+++ b/scion.sh
@@ -22,16 +22,12 @@ cmd_topo_clean() {
 
 cmd_topology() {
     set -e
-    local nobuild
-    if [ "$1" = "nobuild" ]; then
-        shift
-        nobuild="y"
-    fi
     cmd_topo_clean
-    if [ -z "$nobuild" ]; then
-        echo "Compiling..."
-        cmd_build || exit 1
-    fi
+
+    # Build the necessary binaries.
+    bazel build //:scion-topo
+    tar --overwrite -xf bazel-bin/scion-topo.tar -C bin
+
     echo "Create topology, configuration, and execution files."
     is_running_in_docker && set -- "$@" --in-docker
     python/topology/generator.py "$@"
@@ -54,7 +50,7 @@ cmd_topology() {
 cmd_run() {
     if [ "$1" != "nobuild" ]; then
         echo "Compiling..."
-        cmd_build || exit 1
+        make -s || exit 1
         if is_docker_be; then
             echo "Build scion_base image"
             ./tools/quiet ./docker.sh base
@@ -353,11 +349,7 @@ cmd_version() {
 }
 
 cmd_build() {
-    if [ "$1" == "bypass" ]; then
-        USER_OPTS=-DBYPASS_ROUTERS make -s
-    else
-        make -s
-    fi
+    make -s
 }
 
 cmd_clean() {
@@ -417,11 +409,10 @@ cmd_help() {
 	echo
 	cat <<-_EOF
 	Usage:
-	    $PROGRAM topology [nobuild]
-	        Create topology, configuration, and execution files. With the 'nobuild'
-            option, don't build the code. All other arguments or options are passed
-            to topology/generator.py
-	    $PROGRAM run
+	    $PROGRAM topology
+	        Create topology, configuration, and execution files.
+	        All arguments or options are passed to topology/generator.py
+	    $PROGRAM run [nobuild]
 	        Run network.
 	    $PROGRAM sciond ISD-AS [ADDR]
 	        Start sciond with provided ISD and AS parameters, and bind to ADDR.
@@ -445,10 +436,10 @@ cmd_help() {
 	        Show this text.
 	    $PROGRAM version
 	        Show version information.
-        $PROGRAM traces [folder]
-            Serve jaeger traces from the specified folder (default: traces/)
-        $PROGRAM stop_traces
-            Stop the jaeger container started during the traces command
+	    $PROGRAM traces [folder]
+	        Serve jaeger traces from the specified folder (default: traces/)
+	    $PROGRAM stop_traces
+	        Stop the jaeger container started during the traces command
 	_EOF
 }
 # END subcommand functions
diff --git a/tools/ci/integration b/tools/ci/integration
index 34ec63210..93dd3458d 100755
--- a/tools/ci/integration
+++ b/tools/ci/integration
@@ -44,7 +44,7 @@ stop_containers() {
     fi
 }
 
-./docker.sh exec "set -x; ./scion.sh topology nobuild -t ${DOCKER_BE:+ -d -n 172.21.0.0/16 -c topology/Tiny.topo}" ||\
+./docker.sh exec "set -x; ./scion.sh topology -t ${DOCKER_BE:+ -d -n 172.21.0.0/16 -c topology/Tiny.topo}" ||\
     { echo "Failed to create topology"; stop_containers; exit 1; }
 ./docker.sh exec "set -eo pipefail; ./scion.sh run nobuild | grep -v "started" |& tee logs/integration.run"
 ./docker.sh exec "set -eo pipefail; ./scion.sh status |& tee logs/integration.run"


Test patch:
diff --git a/acceptance/cert_renewal_acceptance_disabled/test b/acceptance/cert_renewal_acceptance_disabled/test
index 79856cd09..48324c3ca 100755
--- a/acceptance/cert_renewal_acceptance_disabled/test
+++ b/acceptance/cert_renewal_acceptance_disabled/test
@@ -23,7 +23,7 @@ CORE_IA_FILE="$(ia_file $CORE_IA)"
 
 test_setup() {
     set -e
-    ./scion.sh topology nobuild -c $TEST_TOPOLOGY -d -t
+    ./scion.sh topology -c $TEST_TOPOLOGY -d -t
     sed -i -e 's/Level = .*$/Level = "trace"/g' \
         -e '/\[logging\.file\]/a FlushInterval = 1' \
         -e '/\[cs\]/a AutomaticRenewal = true' \
diff --git a/acceptance/reconnecting_acceptance/test b/acceptance/reconnecting_acceptance/test
index badfee952..a3622408f 100755
--- a/acceptance/reconnecting_acceptance/test
+++ b/acceptance/reconnecting_acceptance/test
@@ -11,7 +11,7 @@ TEST_TOPOLOGY="topology/Tiny.topo"
 
 test_setup() {
     set -e
-    ./scion.sh topology nobuild -c $TEST_TOPOLOGY -d -t
+    ./scion.sh topology -c $TEST_TOPOLOGY -d -t
     ./scion.sh run nobuild
     ./tools/dc start tester_1-ff00_0_112 tester_1-ff00_0_110
     docker_status
diff --git a/acceptance/topo_invalid_reloads_acceptance/test b/acceptance/topo_invalid_reloads_acceptance/test
index 43a6d9d6e..b08d91ebc 100755
--- a/acceptance/topo_invalid_reloads_acceptance/test
+++ b/acceptance/topo_invalid_reloads_acceptance/test
@@ -16,7 +16,7 @@ TOPO="gen/ISD1/AS$AS_FILE/topology.json"
 
 test_setup() {
     set -e
-    ./scion.sh topology nobuild -c $TEST_TOPOLOGY -d -t
+    ./scion.sh topology -c $TEST_TOPOLOGY -d -t
     sed -i '/\[logging\.file\]/a FlushInterval = 1' gen/ISD1/*/*/*.toml
     ./tools/dc start scion_ps$IA_FILE-1 scion_sd$IA_FILE
     docker_status
diff --git a/acceptance/topo_ps_reloads_br_acceptance/test b/acceptance/topo_ps_reloads_br_acceptance/test
index 7ae120d61..0d581eba6 100755
--- a/acceptance/topo_ps_reloads_br_acceptance/test
+++ b/acceptance/topo_ps_reloads_br_acceptance/test
@@ -17,7 +17,7 @@ DST_IA=${DST_IA:-1-ff00:0:111}
 
 test_setup() {
     set -e
-    ./scion.sh topology nobuild -c $TEST_TOPOLOGY -d -t
+    ./scion.sh topology -c $TEST_TOPOLOGY -d -t
     sed -i '/\[logging\.file\]/a FlushInterval = 1' gen/ISD1/*/ps*/ps.toml
     ./scion.sh run nobuild
     ./tools/dc start tester_$SRC_IA_FILE
diff --git a/acceptance/topo_ps_reloads_cs_acceptance_disabled/test b/acceptance/topo_ps_reloads_cs_acceptance_disabled/test
index 2e1ca6a1a..00bd83c78 100755
--- a/acceptance/topo_ps_reloads_cs_acceptance_disabled/test
+++ b/acceptance/topo_ps_reloads_cs_acceptance_disabled/test
@@ -15,7 +15,7 @@ AS_FILE="$(as_file $IA)"
 
 test_setup() {
     set -e
-    ./scion.sh topology nobuild -c $TEST_TOPOLOGY -d -t
+    ./scion.sh topology -c $TEST_TOPOLOGY -d -t
     sed -i -e '/\[logging\.file\]/a FlushInterval = 1'\
             -e '/\[ps\]/a\CryptoSyncInterval="1s"' gen/ISD1/*/ps*/ps.toml
     ./scion.sh run nobuild
diff --git a/acceptance/topo_sd_reloads_br_acceptance/test b/acceptance/topo_sd_reloads_br_acceptance/test
index 2d1749465..a2fa9e394 100755
--- a/acceptance/topo_sd_reloads_br_acceptance/test
+++ b/acceptance/topo_sd_reloads_br_acceptance/test
@@ -17,7 +17,7 @@ DST_IA=${DST_IA:-1-ff00:0:110}
 
 test_setup() {
     set -e
-    ./scion.sh topology nobuild -c $TEST_TOPOLOGY -d -t
+    ./scion.sh topology -c $TEST_TOPOLOGY -d -t
     sed -i '/\[logging\.file\]/a FlushInterval = 1' gen/ISD1/*/endhost/sd.toml
     ./scion.sh run nobuild
     ./tools/dc start tester_$SRC_IA_FILE
diff --git a/acceptance/topo_sd_reloads_cs_acceptance_disabled/test b/acceptance/topo_sd_reloads_cs_acceptance_disabled/test
index 89284cd92..c4aba732d 100755
--- a/acceptance/topo_sd_reloads_cs_acceptance_disabled/test
+++ b/acceptance/topo_sd_reloads_cs_acceptance_disabled/test
@@ -16,7 +16,7 @@ DST_IA=${DST_IA:-1-ff00:0:110}
 
 test_setup() {
     set -e
-    ./scion.sh topology nobuild -c $TEST_TOPOLOGY -d -t
+    ./scion.sh topology -c $TEST_TOPOLOGY -d -t
     sed -i '/\[logging\.file\]/a FlushInterval = 1' gen/ISD1/*/endhost/sd.toml
     ./scion.sh run nobuild
     ./tools/dc start tester_$SRC_IA_FILE
diff --git a/acceptance/topo_sd_reloads_ps_acceptance/test b/acceptance/topo_sd_reloads_ps_acceptance/test
index da0ac6ed3..6ee91afec 100755
--- a/acceptance/topo_sd_reloads_ps_acceptance/test
+++ b/acceptance/topo_sd_reloads_ps_acceptance/test
@@ -16,7 +16,7 @@ DST_IA=${DST_IA:-1-ff00:0:110}
 
 test_setup() {
     set -e
-    ./scion.sh topology nobuild -c $TEST_TOPOLOGY -d -t
+    ./scion.sh topology -c $TEST_TOPOLOGY -d -t
     sed -i '/\[logging\.file\]/a FlushInterval = 1' gen/ISD1/*/endhost/sd.toml
     ./scion.sh run nobuild
     ./tools/dc start tester_$SRC_IA_FILE