Repository: nasa/nos3
Pull request #609: [nasa/nos3#553] Headlesss System Tests
Author: login: rjbrown2
Created: 2025-04-03T14:01:57Z | Merged: 2025-04-08T18:47:16Z

Closes #553 

What does it do:
- Updated script for running headlessly within CI
- Can also run using GUI - but COSMOS must be started quickly.

How to test:
update config to use cosmos
make uninstall / make clean
make prep
make
make ci-launch

Verify the CI has run properly
Run the script locally /scripts/ci_launch.sh

Submodules:
COSMOS

Linked issue #553: Running System Tests
### Summary

Looking to have CI run system tests in various configurations. 

### Use Case

Ensures things are nominally operational as things get updated over time.
Linked issue comments: user: login: jlucas9
created_at: 2025-04-10T10:29:34Z
body: Merged into `dev` ^ 
author_association: CONTRIBUTOR

Diff:
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 8f547f0d1..4436320b2 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -72,5 +72,4 @@ jobs:
           uses: actions/upload-artifact@v4
           with:
             name: coverage-artifacts
-            path: docs/coverage
-    
\ No newline at end of file
+            path: docs/coverage
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 7bba4c107..9b7d9ca40 100755
--- a/Makefile
+++ b/Makefile
@@ -8,6 +8,8 @@ GSWBUILDDIR ?= $(CURDIR)/gsw/build
 SIMBUILDDIR ?= $(CURDIR)/sims/build
 COVERAGEDIR ?= $(CURDIR)/fsw/build/amd64-posix/default_cpu1
 
+export SYSTEM_TEST_FILE_PATH = /scripts/gsw/system_test.rb
+
 export CFS_APP_PATH = ../components
 export MISSION_DEFS = ../cfg/build/
 export MISSIONCONFIG = ../cfg/build/nos3
@@ -80,6 +82,16 @@ endif
 checkout:
 	./scripts/checkout.sh
 
+ci-launch:
+	@export SYSTEM_TEST_FILE_PATH=$(SYSTEM_TEST_FILE_PATH) && \
+	./scripts/ci_launch.sh && \
+	./scripts/stop.sh
+
+system-tests:
+	@export SYSTEM_TEST_FILE_PATH=../..$(SYSTEM_TEST_FILE_PATH) && \
+	./cfg/build/launch.sh && \
+	./scripts/system_tests.sh
+
 clean:
 	$(MAKE) clean-fsw
 	$(MAKE) clean-sim
diff --git a/scripts/ci_launch.sh b/scripts/ci_launch.sh
new file mode 100644
index 000000000..b4bf5fdf4
--- /dev/null
+++ b/scripts/ci_launch.sh
@@ -0,0 +1,128 @@
+#!/bin/bash -i
+set -e
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source "$SCRIPT_DIR/env.sh"
+
+if [ ! -d $USER_NOS3_DIR ]; then
+    echo ""
+    echo "    Need to run make prep first!"
+    echo ""
+    exit 1
+fi
+
+if [ ! -d $BASE_DIR/cfg/build ]; then
+    echo ""
+    echo "    Need to run make config first!"
+    echo ""
+    exit 1
+fi
+
+mkdir -p $FSW_DIR/data/{cam,evs,hk,inst}
+mkdir -p /tmp/nos3/data/{cam,evs,hk,inst} /tmp/nos3/uplink
+cp $BASE_DIR/fsw/build/exe/cpu1/cf/cfe_es_startup.scr /tmp/nos3/uplink/tmp0.so 2>/dev/null || true
+cp $BASE_DIR/fsw/build/exe/cpu1/cf/sample.so /tmp/nos3/uplink/tmp1.so 2>/dev/null || true
+
+$DNETWORK rm nos3_core 2>/dev/null || true
+$DNETWORK create \
+    --driver=bridge \
+    --subnet=192.168.41.0/24 \
+    --gateway=192.168.41.1 \
+    nos3_core
+
+
+echo "Launch GSW..."
+docker run -d --name cosmos_openc3-operator_1 \
+    --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+    -v "$GSW_DIR/config:/cosmos/config:ro" \
+    -v "$GSW_DIR:/cosmos" \
+    -v "$BASE_DIR/scripts:/scripts:ro" \
+    -v /tmp/nos3:/tmp/nos3 \
+    --network=nos3_core \
+    -w /cosmos/tools \
+    ballaerospace/cosmos:4.5.0 tail -f /dev/null
+
+sleep 5
+docker exec cosmos_openc3-operator_1 bash -c "apt update && apt install -y xvfb"
+docker exec -d cosmos_openc3-operator_1 bash -c "xvfb-run ruby CmdTlmServer /cosmos/config/tools/cmd_tlm_server/cmd_tlm_server.txt"
+
+CFG_FILE="-f nos3-simulator.xml"
+
+sleep 1
+docker run -dit --name nos_time_driver --network=nos3_core \
+    --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+    -v "$SIM_DIR:$SIM_DIR" -w "$SIM_BIN" $DBOX \
+    ./nos3-single-simulator -f nos3-simulator.xml time
+
+
+SATNUM=1
+for (( i=1; i<=$SATNUM; i++ )); do
+    SC_NUM="sc_$i"
+    SC_NET="nos3_${SC_NUM}"
+    CFG_FILE="-f nos3-simulator.xml"
+
+    $DNETWORK rm $SC_NET 2>/dev/null || true
+    $DNETWORK create $SC_NET
+
+    echo "$SC_NUM - Create spacecraft network..."
+    echo "$SC_NUM - Connect GSW to spacecraft network..."
+    $DNETWORK connect $SC_NET cosmos_openc3-operator_1 --alias cosmos --alias active-gs
+
+    echo "$SC_NUM - 42..."
+    rm -rf $USER_NOS3_DIR/42/NOS3InOut
+    cp -r $BASE_DIR/cfg/build/InOut $USER_NOS3_DIR/42/NOS3InOut
+    docker run -d --name ${SC_NUM}_42 --network=$SC_NET \
+        --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+        -e DISPLAY=$DISPLAY -v "$USER_NOS3_DIR:$USER_NOS3_DIR" \
+        -v /tmp/.X11-unix:/tmp/.X11-unix:ro -w "$USER_NOS3_DIR/42" $DBOX ./42 NOS3InOut
+
+    echo "$SC_NUM - Flight Software..."
+    docker run -dit --name ${SC_NUM}_nos_fsw -h nos_fsw --network=$SC_NET \
+        --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+        -v "$BASE_DIR:$BASE_DIR" -v "$FSW_DIR:$FSW_DIR" -v "$SCRIPT_DIR:$SCRIPT_DIR" \
+        -e USER=$(whoami) -e LD_LIBRARY_PATH=$FSW_DIR:/usr/lib:/usr/local/lib \
+        --sysctl fs.mqueue.msg_max=10000 --ulimit rtprio=99 --cap-add=sys_nice \
+        $DBOX bash -c "cd $FSW_DIR && exec ./core-cpu1 -R PO"
+
+    echo "$SC_NUM - CryptoLib..."
+    docker run -d --name ${SC_NUM}_cryptolib --network=$SC_NET \
+        --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+        --network-alias=cryptolib \
+        -v "$BASE_DIR:$BASE_DIR" -w "$BASE_DIR/gsw/build" $DBOX ./support/standalone
+
+    echo "$SC_NUM - Simulators..."
+    echo "$SC_NUM - NOS Engine Server..."
+    docker run -dit --name ${SC_NUM}_nos_engine_server -h nos_engine_server --network=$SC_NET \
+        --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+        -v "$SIM_DIR:$SIM_DIR" -w "$SIM_BIN" $DBOX \
+        /usr/bin/nos_engine_server_standalone -f $SIM_BIN/nos_engine_server_config.json
+
+    docker run -d --name ${SC_NUM}_truth42sim --network=$SC_NET \
+        --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+        -v "$SIM_DIR:$SIM_DIR" -w "$SIM_BIN" $DBOX \
+        ./nos3-single-simulator $CFG_FILE truth42sim
+
+    for sim in \
+        camsim generic_css_sim generic_eps_sim generic_fss_sim \
+        gps generic_imu_sim generic_mag_sim \
+        generic-reactionwheel-sim0 generic-reactionwheel-sim1 \
+        generic-reactionwheel-sim2 generic_radio_sim sample_sim \
+        generic_star_tracker_sim generic_thruster_sim generic_torquer_sim; do
+        docker run -d --name ${SC_NUM}_${sim} --network=$SC_NET \
+            --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+            -v "$SIM_DIR:$SIM_DIR" -w "$SIM_BIN" $DBOX \
+            ./nos3-single-simulator $CFG_FILE $sim
+    done
+
+    $DNETWORK connect --alias nos_time_driver $SC_NET nos_time_driver
+
+done
+
+
+# Final message
+echo "Docker launch script completed!  "
+echo "Beginning System Test Script!  "
+
+sleep 5
+/bin/bash scripts/system_tests.sh
+
diff --git a/scripts/gsw/system_test.rb b/scripts/gsw/system_test.rb
new file mode 100644
index 000000000..d24b5e21b
--- /dev/null
+++ b/scripts/gsw/system_test.rb
@@ -0,0 +1,26 @@
+require 'cosmos'
+require 'cosmos/script'
+ 
+include Cosmos::Script
+puts("Begin SAMPLE Test Case:")
+initial_command_count = tlm("SAMPLE SAMPLE_HK_TLM CMD_COUNT")
+puts("Inital Command Count: #{initial_command_count}")
+initial_error_count = tlm("SAMPLE SAMPLE_HK_TLM CMD_ERR_COUNT")
+puts("Initial Error Count: #{initial_error_count}")
+initial_device_error_count = tlm("SAMPLE SAMPLE_HK_TLM DEVICE_ERR_COUNT")
+puts("Inital Device Error Count: #{initial_device_error_count}")
+cmd("SAMPLE SAMPLE_NOOP_CC")
+wait_check("SAMPLE SAMPLE_HK_TLM CMD_COUNT > #{initial_command_count}", 30)
+wait_check("SAMPLE SAMPLE_HK_TLM CMD_ERR_COUNT == #{initial_error_count}", 30)
+wait_check("SAMPLE SAMPLE_HK_TLM DEVICE_ERR_COUNT == #{initial_device_error_count}", 30)
+puts("")
+puts("Final Values:")
+final_command_count = tlm("SAMPLE SAMPLE_HK_TLM CMD_COUNT")
+puts("Inital Command Count: #{final_command_count}")
+final_error_count = tlm("SAMPLE SAMPLE_HK_TLM CMD_ERR_COUNT")
+puts("Initial Error Count: #{final_error_count}")
+final_device_error_count = tlm("SAMPLE SAMPLE_HK_TLM DEVICE_ERR_COUNT")
+puts("Inital Device Error Count: #{final_device_error_count}")
+puts("")
+puts("SAMPLE TEST CASE COMPLETED SUCCESSFULLY!")
+puts("")
diff --git a/scripts/system_tests.sh b/scripts/system_tests.sh
new file mode 100644
index 000000000..10812337d
--- /dev/null
+++ b/scripts/system_tests.sh
@@ -0,0 +1,10 @@
+#!/bin/bash 
+set -e
+
+# Sleep for a moment to sensure the system is up and running
+sleep 10
+
+# Set of System Tests to be called
+
+#Sample
+docker exec -it cosmos_openc3-operator_1 ruby $SYSTEM_TEST_FILE_PATH
\ No newline at end of file


Reviews:
user: login: zlynch2
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-08T16:05:00Z
user: login: kevincbruce
body: Looks good to me! `make ci-launch` seemed to work. `make system-tests` had an issue when I ran it after `make ci-launch`, but after a `make clean`/`make` cycle, it worked perfectly. 

EDIT: Likely was that I needed to run `make stop` after `make ci-launch`
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-08T16:32:54Z
user: login: dacarter22
body: Ran successfully, needed to run from shared folder to work, cloned in folder had permissions issues
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-08T16:38:13Z
user: login: jlucas9
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-08T16:39:08Z
user: login: jlucas9
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-08T16:41:48Z
user: login: Isgaroth-the-Green
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-08T16:43:23Z
user: login: Isgaroth-the-Green
body: Other than making ci-launch.sh executable, I have no suggestions of changes.  Everything appears to work as intended.
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-08T16:47:01Z
user: login: kevincbruce
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-08T17:08:11Z

Review comments:
path: gsw/cosmos
user: login: jlucas9
body: Should this file be moved into the ./gsw folder so we don't have to update the cosmos submodule each time we want to add or modify the test list for a particular mission?
created_at: 2025-04-08T16:39:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,128 @@
+#!/bin/bash -i
+set -e
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source "$SCRIPT_DIR/env.sh"
+
+if [ ! -d $USER_NOS3_DIR ]; then
+    echo ""
+    echo "    Need to run make prep first!"
+    echo ""
+    exit 1
+fi
+
+if [ ! -d $BASE_DIR/cfg/build ]; then
+    echo ""
+    echo "    Need to run make config first!"
+    echo ""
+    exit 1
+fi
+
+mkdir -p $FSW_DIR/data/{cam,evs,hk,inst}
+mkdir -p /tmp/nos3/data/{cam,evs,hk,inst} /tmp/nos3/uplink
+cp $BASE_DIR/fsw/build/exe/cpu1/cf/cfe_es_startup.scr /tmp/nos3/uplink/tmp0.so 2>/dev/null || true
+cp $BASE_DIR/fsw/build/exe/cpu1/cf/sample.so /tmp/nos3/uplink/tmp1.so 2>/dev/null || true
+
+$DNETWORK rm nos3_core 2>/dev/null || true
+$DNETWORK create \
+    --driver=bridge \
+    --subnet=192.168.41.0/24 \
+    --gateway=192.168.41.1 \
+    nos3_core
+
+
+echo "Launch GSW..."
+docker run -d --name cosmos_openc3-operator_1 \
+    --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+    -v "$GSW_DIR/config:/cosmos/config:ro" \
+    -v "$GSW_DIR:/cosmos" \
+    -v "$BASE_DIR/scripts:/scripts:ro" \
+    -v /tmp/nos3:/tmp/nos3 \
+    --network=nos3_core \
+    -w /cosmos/tools \
+    ballaerospace/cosmos:4.5.0 tail -f /dev/null
+
+sleep 5
+docker exec cosmos_openc3-operator_1 bash -c "apt update && apt install -y xvfb"
+docker exec -d cosmos_openc3-operator_1 bash -c "xvfb-run ruby CmdTlmServer /cosmos/config/tools/cmd_tlm_server/cmd_tlm_server.txt"
+
+CFG_FILE="-f nos3-simulator.xml"
+
+sleep 1
+docker run -dit --name nos_time_driver --network=nos3_core \
+    --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+    -v "$SIM_DIR:$SIM_DIR" -w "$SIM_BIN" $DBOX \
+    ./nos3-single-simulator -f nos3-simulator.xml time
+
+
+SATNUM=1
+for (( i=1; i<=$SATNUM; i++ )); do
+    SC_NUM="sc_$i"
+    SC_NET="nos3_${SC_NUM}"
+    CFG_FILE="-f nos3-simulator.xml"
+
+    $DNETWORK rm $SC_NET 2>/dev/null || true
+    $DNETWORK create $SC_NET
+
+    echo "$SC_NUM - Create spacecraft network..."
+    echo "$SC_NUM - Connect GSW to spacecraft network..."
+    $DNETWORK connect $SC_NET cosmos_openc3-operator_1 --alias cosmos --alias active-gs
+
+    echo "$SC_NUM - 42..."
+    rm -rf $USER_NOS3_DIR/42/NOS3InOut
+    cp -r $BASE_DIR/cfg/build/InOut $USER_NOS3_DIR/42/NOS3InOut
+    docker run -d --name ${SC_NUM}_42 --network=$SC_NET \
+        --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+        -e DISPLAY=$DISPLAY -v "$USER_NOS3_DIR:$USER_NOS3_DIR" \
+        -v /tmp/.X11-unix:/tmp/.X11-unix:ro -w "$USER_NOS3_DIR/42" $DBOX ./42 NOS3InOut
+
+    echo "$SC_NUM - Flight Software..."
+    docker run -dit --name ${SC_NUM}_nos_fsw -h nos_fsw --network=$SC_NET \
+        --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+        -v "$BASE_DIR:$BASE_DIR" -v "$FSW_DIR:$FSW_DIR" -v "$SCRIPT_DIR:$SCRIPT_DIR" \
+        -e USER=$(whoami) -e LD_LIBRARY_PATH=$FSW_DIR:/usr/lib:/usr/local/lib \
+        --sysctl fs.mqueue.msg_max=10000 --ulimit rtprio=99 --cap-add=sys_nice \
+        $DBOX bash -c "cd $FSW_DIR && exec ./core-cpu1 -R PO"
+
+    echo "$SC_NUM - CryptoLib..."
+    docker run -d --name ${SC_NUM}_cryptolib --network=$SC_NET \
+        --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+        --network-alias=cryptolib \
+        -v "$BASE_DIR:$BASE_DIR" -w "$BASE_DIR/gsw/build" $DBOX ./support/standalone
+
+    echo "$SC_NUM - Simulators..."
+    echo "$SC_NUM - NOS Engine Server..."
+    docker run -dit --name ${SC_NUM}_nos_engine_server -h nos_engine_server --network=$SC_NET \
+        --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+        -v "$SIM_DIR:$SIM_DIR" -w "$SIM_BIN" $DBOX \
+        /usr/bin/nos_engine_server_standalone -f $SIM_BIN/nos_engine_server_config.json
+
+    docker run -d --name ${SC_NUM}_truth42sim --network=$SC_NET \
+        --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+        -v "$SIM_DIR:$SIM_DIR" -w "$SIM_BIN" $DBOX \
+        ./nos3-single-simulator $CFG_FILE truth42sim
+
+    for sim in \
+        camsim generic_css_sim generic_eps_sim generic_fss_sim \
+        gps generic_imu_sim generic_mag_sim \
+        generic-reactionwheel-sim0 generic-reactionwheel-sim1 \
+        generic-reactionwheel-sim2 generic_radio_sim sample_sim \
+        generic_star_tracker_sim generic_thruster_sim generic_torquer_sim; do
+        docker run -d --name ${SC_NUM}_${sim} --network=$SC_NET \
+            --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+            -v "$SIM_DIR:$SIM_DIR" -w "$SIM_BIN" $DBOX \
+            ./nos3-single-simulator $CFG_FILE $sim
+    done
+
+    $DNETWORK connect --alias nos_time_driver $SC_NET nos_time_driver
+
+done
+
+
+# Final message
+echo "Docker launch script completed!  "
+echo "Beginning System Test Script!  "
+
+sleep 5
+/bin/bash scripts/system_tests.sh
+
path: scripts/ci_launch.sh
user: login: jlucas9
body: Should we `make stop` automatically after this test completes?
created_at: 2025-04-08T16:41:48Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,128 @@
+#!/bin/bash -i
+set -e
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source "$SCRIPT_DIR/env.sh"
+
+if [ ! -d $USER_NOS3_DIR ]; then
+    echo ""
+    echo "    Need to run make prep first!"
+    echo ""
+    exit 1
+fi
+
+if [ ! -d $BASE_DIR/cfg/build ]; then
+    echo ""
+    echo "    Need to run make config first!"
+    echo ""
+    exit 1
+fi
+
+mkdir -p $FSW_DIR/data/{cam,evs,hk,inst}
+mkdir -p /tmp/nos3/data/{cam,evs,hk,inst} /tmp/nos3/uplink
+cp $BASE_DIR/fsw/build/exe/cpu1/cf/cfe_es_startup.scr /tmp/nos3/uplink/tmp0.so 2>/dev/null || true
+cp $BASE_DIR/fsw/build/exe/cpu1/cf/sample.so /tmp/nos3/uplink/tmp1.so 2>/dev/null || true
+
+$DNETWORK rm nos3_core 2>/dev/null || true
+$DNETWORK create \
+    --driver=bridge \
+    --subnet=192.168.41.0/24 \
+    --gateway=192.168.41.1 \
+    nos3_core
+
+
+echo "Launch GSW..."
+docker run -d --name cosmos_openc3-operator_1 \
+    --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+    -v "$GSW_DIR/config:/cosmos/config:ro" \
+    -v "$GSW_DIR:/cosmos" \
+    -v "$BASE_DIR/scripts:/scripts:ro" \
+    -v /tmp/nos3:/tmp/nos3 \
+    --network=nos3_core \
+    -w /cosmos/tools \
+    ballaerospace/cosmos:4.5.0 tail -f /dev/null
+
+sleep 5
+docker exec cosmos_openc3-operator_1 bash -c "apt update && apt install -y xvfb"
+docker exec -d cosmos_openc3-operator_1 bash -c "xvfb-run ruby CmdTlmServer /cosmos/config/tools/cmd_tlm_server/cmd_tlm_server.txt"
+
+CFG_FILE="-f nos3-simulator.xml"
+
+sleep 1
+docker run -dit --name nos_time_driver --network=nos3_core \
+    --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+    -v "$SIM_DIR:$SIM_DIR" -w "$SIM_BIN" $DBOX \
+    ./nos3-single-simulator -f nos3-simulator.xml time
+
+
+SATNUM=1
+for (( i=1; i<=$SATNUM; i++ )); do
+    SC_NUM="sc_$i"
+    SC_NET="nos3_${SC_NUM}"
+    CFG_FILE="-f nos3-simulator.xml"
+
+    $DNETWORK rm $SC_NET 2>/dev/null || true
+    $DNETWORK create $SC_NET
+
+    echo "$SC_NUM - Create spacecraft network..."
+    echo "$SC_NUM - Connect GSW to spacecraft network..."
+    $DNETWORK connect $SC_NET cosmos_openc3-operator_1 --alias cosmos --alias active-gs
+
+    echo "$SC_NUM - 42..."
+    rm -rf $USER_NOS3_DIR/42/NOS3InOut
+    cp -r $BASE_DIR/cfg/build/InOut $USER_NOS3_DIR/42/NOS3InOut
+    docker run -d --name ${SC_NUM}_42 --network=$SC_NET \
+        --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+        -e DISPLAY=$DISPLAY -v "$USER_NOS3_DIR:$USER_NOS3_DIR" \
+        -v /tmp/.X11-unix:/tmp/.X11-unix:ro -w "$USER_NOS3_DIR/42" $DBOX ./42 NOS3InOut
+
+    echo "$SC_NUM - Flight Software..."
+    docker run -dit --name ${SC_NUM}_nos_fsw -h nos_fsw --network=$SC_NET \
+        --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+        -v "$BASE_DIR:$BASE_DIR" -v "$FSW_DIR:$FSW_DIR" -v "$SCRIPT_DIR:$SCRIPT_DIR" \
+        -e USER=$(whoami) -e LD_LIBRARY_PATH=$FSW_DIR:/usr/lib:/usr/local/lib \
+        --sysctl fs.mqueue.msg_max=10000 --ulimit rtprio=99 --cap-add=sys_nice \
+        $DBOX bash -c "cd $FSW_DIR && exec ./core-cpu1 -R PO"
+
+    echo "$SC_NUM - CryptoLib..."
+    docker run -d --name ${SC_NUM}_cryptolib --network=$SC_NET \
+        --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+        --network-alias=cryptolib \
+        -v "$BASE_DIR:$BASE_DIR" -w "$BASE_DIR/gsw/build" $DBOX ./support/standalone
+
+    echo "$SC_NUM - Simulators..."
+    echo "$SC_NUM - NOS Engine Server..."
+    docker run -dit --name ${SC_NUM}_nos_engine_server -h nos_engine_server --network=$SC_NET \
+        --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+        -v "$SIM_DIR:$SIM_DIR" -w "$SIM_BIN" $DBOX \
+        /usr/bin/nos_engine_server_standalone -f $SIM_BIN/nos_engine_server_config.json
+
+    docker run -d --name ${SC_NUM}_truth42sim --network=$SC_NET \
+        --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+        -v "$SIM_DIR:$SIM_DIR" -w "$SIM_BIN" $DBOX \
+        ./nos3-single-simulator $CFG_FILE truth42sim
+
+    for sim in \
+        camsim generic_css_sim generic_eps_sim generic_fss_sim \
+        gps generic_imu_sim generic_mag_sim \
+        generic-reactionwheel-sim0 generic-reactionwheel-sim1 \
+        generic-reactionwheel-sim2 generic_radio_sim sample_sim \
+        generic_star_tracker_sim generic_thruster_sim generic_torquer_sim; do
+        docker run -d --name ${SC_NUM}_${sim} --network=$SC_NET \
+            --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+            -v "$SIM_DIR:$SIM_DIR" -w "$SIM_BIN" $DBOX \
+            ./nos3-single-simulator $CFG_FILE $sim
+    done
+
+    $DNETWORK connect --alias nos_time_driver $SC_NET nos_time_driver
+
+done
+
+
+# Final message
+echo "Docker launch script completed!  "
+echo "Beginning System Test Script!  "
+
+sleep 5
+/bin/bash scripts/system_tests.sh
+
path: scripts/ci_launch.sh
user: login: Isgaroth-the-Green
body: I think so; since nothing pops up when it is running, it is very easy to forget to do so.
created_at: 2025-04-08T16:43:23Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,128 @@
+#!/bin/bash -i
+set -e
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source "$SCRIPT_DIR/env.sh"
+
+if [ ! -d $USER_NOS3_DIR ]; then
+    echo ""
+    echo "    Need to run make prep first!"
+    echo ""
+    exit 1
+fi
+
+if [ ! -d $BASE_DIR/cfg/build ]; then
+    echo ""
+    echo "    Need to run make config first!"
+    echo ""
+    exit 1
+fi
+
+mkdir -p $FSW_DIR/data/{cam,evs,hk,inst}
+mkdir -p /tmp/nos3/data/{cam,evs,hk,inst} /tmp/nos3/uplink
+cp $BASE_DIR/fsw/build/exe/cpu1/cf/cfe_es_startup.scr /tmp/nos3/uplink/tmp0.so 2>/dev/null || true
+cp $BASE_DIR/fsw/build/exe/cpu1/cf/sample.so /tmp/nos3/uplink/tmp1.so 2>/dev/null || true
+
+$DNETWORK rm nos3_core 2>/dev/null || true
+$DNETWORK create \
+    --driver=bridge \
+    --subnet=192.168.41.0/24 \
+    --gateway=192.168.41.1 \
+    nos3_core
+
+
+echo "Launch GSW..."
+docker run -d --name cosmos_openc3-operator_1 \
+    --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+    -v "$GSW_DIR/config:/cosmos/config:ro" \
+    -v "$GSW_DIR:/cosmos" \
+    -v "$BASE_DIR/scripts:/scripts:ro" \
+    -v /tmp/nos3:/tmp/nos3 \
+    --network=nos3_core \
+    -w /cosmos/tools \
+    ballaerospace/cosmos:4.5.0 tail -f /dev/null
+
+sleep 5
+docker exec cosmos_openc3-operator_1 bash -c "apt update && apt install -y xvfb"
+docker exec -d cosmos_openc3-operator_1 bash -c "xvfb-run ruby CmdTlmServer /cosmos/config/tools/cmd_tlm_server/cmd_tlm_server.txt"
+
+CFG_FILE="-f nos3-simulator.xml"
+
+sleep 1
+docker run -dit --name nos_time_driver --network=nos3_core \
+    --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+    -v "$SIM_DIR:$SIM_DIR" -w "$SIM_BIN" $DBOX \
+    ./nos3-single-simulator -f nos3-simulator.xml time
+
+
+SATNUM=1
+for (( i=1; i<=$SATNUM; i++ )); do
+    SC_NUM="sc_$i"
+    SC_NET="nos3_${SC_NUM}"
+    CFG_FILE="-f nos3-simulator.xml"
+
+    $DNETWORK rm $SC_NET 2>/dev/null || true
+    $DNETWORK create $SC_NET
+
+    echo "$SC_NUM - Create spacecraft network..."
+    echo "$SC_NUM - Connect GSW to spacecraft network..."
+    $DNETWORK connect $SC_NET cosmos_openc3-operator_1 --alias cosmos --alias active-gs
+
+    echo "$SC_NUM - 42..."
+    rm -rf $USER_NOS3_DIR/42/NOS3InOut
+    cp -r $BASE_DIR/cfg/build/InOut $USER_NOS3_DIR/42/NOS3InOut
+    docker run -d --name ${SC_NUM}_42 --network=$SC_NET \
+        --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+        -e DISPLAY=$DISPLAY -v "$USER_NOS3_DIR:$USER_NOS3_DIR" \
+        -v /tmp/.X11-unix:/tmp/.X11-unix:ro -w "$USER_NOS3_DIR/42" $DBOX ./42 NOS3InOut
+
+    echo "$SC_NUM - Flight Software..."
+    docker run -dit --name ${SC_NUM}_nos_fsw -h nos_fsw --network=$SC_NET \
+        --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+        -v "$BASE_DIR:$BASE_DIR" -v "$FSW_DIR:$FSW_DIR" -v "$SCRIPT_DIR:$SCRIPT_DIR" \
+        -e USER=$(whoami) -e LD_LIBRARY_PATH=$FSW_DIR:/usr/lib:/usr/local/lib \
+        --sysctl fs.mqueue.msg_max=10000 --ulimit rtprio=99 --cap-add=sys_nice \
+        $DBOX bash -c "cd $FSW_DIR && exec ./core-cpu1 -R PO"
+
+    echo "$SC_NUM - CryptoLib..."
+    docker run -d --name ${SC_NUM}_cryptolib --network=$SC_NET \
+        --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+        --network-alias=cryptolib \
+        -v "$BASE_DIR:$BASE_DIR" -w "$BASE_DIR/gsw/build" $DBOX ./support/standalone
+
+    echo "$SC_NUM - Simulators..."
+    echo "$SC_NUM - NOS Engine Server..."
+    docker run -dit --name ${SC_NUM}_nos_engine_server -h nos_engine_server --network=$SC_NET \
+        --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+        -v "$SIM_DIR:$SIM_DIR" -w "$SIM_BIN" $DBOX \
+        /usr/bin/nos_engine_server_standalone -f $SIM_BIN/nos_engine_server_config.json
+
+    docker run -d --name ${SC_NUM}_truth42sim --network=$SC_NET \
+        --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+        -v "$SIM_DIR:$SIM_DIR" -w "$SIM_BIN" $DBOX \
+        ./nos3-single-simulator $CFG_FILE truth42sim
+
+    for sim in \
+        camsim generic_css_sim generic_eps_sim generic_fss_sim \
+        gps generic_imu_sim generic_mag_sim \
+        generic-reactionwheel-sim0 generic-reactionwheel-sim1 \
+        generic-reactionwheel-sim2 generic_radio_sim sample_sim \
+        generic_star_tracker_sim generic_thruster_sim generic_torquer_sim; do
+        docker run -d --name ${SC_NUM}_${sim} --network=$SC_NET \
+            --log-driver json-file --log-opt max-size=5m --log-opt max-file=3 \
+            -v "$SIM_DIR:$SIM_DIR" -w "$SIM_BIN" $DBOX \
+            ./nos3-single-simulator $CFG_FILE $sim
+    done
+
+    $DNETWORK connect --alias nos_time_driver $SC_NET nos_time_driver
+
+done
+
+
+# Final message
+echo "Docker launch script completed!  "
+echo "Beginning System Test Script!  "
+
+sleep 5
+/bin/bash scripts/system_tests.sh
+
path: scripts/ci_launch.sh
user: login: kevincbruce
body: I agree too. For automated tests, would make sense to auto-stop.
created_at: 2025-04-08T17:08:11Z
author_association: CONTRIBUTOR

PR comments:
user: login: codecov-commenter
created_at: 2025-04-03T14:07:59Z
body: ## [Codecov](https://app.codecov.io/gh/nasa/nos3/pull/609?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=nasa) Report
:white_check_mark: All modified and coverable lines are covered by tests.
:white_check_mark: Project coverage is 72.88%. Comparing base ([`901dbdc`](https://app.codecov.io/gh/nasa/nos3/commit/901dbdc170b6715134b579bd81c73a6c202c7093?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=nasa)) to head ([`7a063cc`](https://app.codecov.io/gh/nasa/nos3/commit/7a063cccab94d2f71d7e3ccb9129edf2fd7c74ef?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=nasa)).
:warning: Report is 633 commits behind head on dev.

<details><summary>Additional details and impacted files</summary>



```diff
@@            Coverage Diff             @@
##              dev     #609      +/-   ##
==========================================
- Coverage   72.90%   72.88%   -0.02%     
==========================================
  Files         747      747              
  Lines       91886    91894       +8     
  Branches     6968     6974       +6     
==========================================
- Hits        66986    66981       -5     
- Misses      24900    24913      +13     
```
</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/nasa/nos3/pull/609?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=nasa).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=nasa).
<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE
user: login: rjbrown2
created_at: 2025-04-07T19:26:55Z
body: This should be good to go.  There's still some work to be done as far as adding any other system tests to the list of tests script (a line per test file).  However everything should be functioning.

make ci-launch (will run everything headlessly locally)
make system-tests (will run with the GUI.  Will need to start Cosmos manually and quickly)
author_association: CONTRIBUTOR
user: login: zlynch2
created_at: 2025-04-08T16:04:51Z
body: tested, make ci-launch works after merging dev in. Approving
author_association: COLLABORATOR
user: login: jlucas9
created_at: 2025-04-08T16:37:53Z
body: Should we look to add the additional packages installed manually in the scripts here to the existing docker container in the next update?
author_association: CONTRIBUTOR
user: login: Isgaroth-the-Green
created_at: 2025-04-08T16:44:22Z
body: For whatever reason the ci-launch script did not have execute permissions when I cloned it; other than that, everything seems largely functional.

author_association: CONTRIBUTOR
user: login: dacarter22
created_at: 2025-04-08T16:48:13Z
body: > For whatever reason the ci-launch script did not have execute permissions when I cloned it; other than that, everything seems largely functional. You've got to run it from shared folder, I had the same issue on clone


author_association: CONTRIBUTOR