Repository: ORNL/DataFed (language: python)

Issue:
[Foxx, CI] - Foxx test bash script is failing
# Description

This foxx test script is failing when it is run.

core/database/tests/test_foxx.sh

# Acceptance

- [x] foxx_test.sh runs using DataFed installed dependencies as opposed to system ones and relies on centralized scripts in the /scripts folders for node version
1055 fix foxx setup script
#1055 


Discussion / hints:
Relates to #1031 


Fix patch:
diff --git a/.gitlab/end_to_end.yml b/.gitlab/end_to_end.yml
index ed52e289f..8b3fbd9b6 100644
--- a/.gitlab/end_to_end.yml
+++ b/.gitlab/end_to_end.yml
@@ -42,9 +42,10 @@ end-to-end-foxx-setup:
     - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}"
     - ./scripts/container_stop.sh -n "foxx-" -p
     - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1")
+    - CONTAINER_NAME="foxx-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}"
     - echo "#!/bin/bash" > "${RUN_FILE}"
     - echo "docker run -d \\" >> "${RUN_FILE}"
-    - echo "--name \"foxx-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> "${RUN_FILE}"
+    - echo "--name \"${CONTAINER_NAME}\" \\" >> "${RUN_FILE}"
     - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> "${RUN_FILE}"
     - echo "-e ENABLE_FOXX_TESTS=\"TRUE\" \\" >> "${RUN_FILE}"
     - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> "${RUN_FILE}"
@@ -63,10 +64,21 @@ end-to-end-foxx-setup:
       # be ephermal anyway, this is not the same for the other containers
     - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${PROJECT}/${COMPONENT}-${BRANCH_LOWER}:${CI_COMMIT_SHA}"
     - while [ ! -f "foxx_tmp/.foxx_is_installed" ]; do
-    -   echo "Waiting for foxx_tmp/.foxx_is_installed";
-    -   docker logs "foxx-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}";
-    -   sleep 10;
+      # Container is found to be running
+    -   if docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
+    -     echo "Waiting for foxx_tmp/.foxx_is_installed"
+    -     docker logs "foxx-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}";
+    -     sleep 10
+    -   else
+      # Container is not running get exit code
+    -     exit_code=$(docker inspect -f '{{.State.ExitCode}}' "$CONTAINER_NAME")
+    -     if [ "$exit_code" != "0" ]; then
+    -       docker logs "foxx-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}";
+    -       exit "$exit_code"
+    -     fi
+    -   fi 
     - done
+    - docker logs "foxx-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}";
     - ./scripts/ci_database_health_check.sh
   after_script:
     - rm -rf foxx_tmp
diff --git a/docker/entrypoint_foxx.sh b/docker/entrypoint_foxx.sh
index 366cc39d5..ea60c6bee 100755
--- a/docker/entrypoint_foxx.sh
+++ b/docker/entrypoint_foxx.sh
@@ -63,10 +63,18 @@ then
   sleep 5
   "${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake" --build build --target install
 
+  if [ "$ENABLE_FOXX_TESTS" == "TRUE" ]
+  then
+    "${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake" \
+      --build build \
+      --target test
+    EXIT_CODE="$?"
+    if [ "$EXIT_CODE" != "0" ]; then exit "$EXIT_CODE"; fi
+  fi
+
   # Create flag to indicate container has done its job  
   touch "$install_flag"
   chown -R "$UID":"$UID" "/tmp"
-
 else
   echo "$install_flag has been found skipping reinstall"
 fi
diff --git a/scripts/dependency_install_functions.sh b/scripts/dependency_install_functions.sh
index e16eacea0..f920191a4 100644
--- a/scripts/dependency_install_functions.sh
+++ b/scripts/dependency_install_functions.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-SCRIPT=$(realpath "$0")
+SCRIPT=$(realpath "$BASH_SOURCE[0]")
 SOURCE=$(dirname "$SCRIPT")
 source "${SOURCE}/dependency_versions.sh"
 PROJECT_ROOT=$(realpath "${SOURCE}/..")
@@ -384,12 +384,16 @@ install_node() {
 
     [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm
     nvm install "$DATAFED_NODE_VERSION"
+    nvm use "$DATAFED_NODE_VERSION"
     # Mark node as installed
     touch "${DATAFED_DEPENDENCIES_INSTALL_PATH}/.node_installed-${DATAFED_NODE_VERSION}"
     cd "$original_dir"
   else
     export NVM_DIR="${DATAFED_DEPENDENCIES_INSTALL_PATH}/nvm"
+    # Used by nvm 
+    export NODE_VERSION="$DATAFED_NODE_VERSION"
     [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm
+    nvm use "$DATAFED_NODE_VERSION"
   fi
 }
 
diff --git a/scripts/install_foxx.sh b/scripts/install_foxx.sh
index e755a3720..3807b4ef1 100755
--- a/scripts/install_foxx.sh
+++ b/scripts/install_foxx.sh
@@ -233,6 +233,7 @@ echo "$local_DATAFED_DATABASE_PASSWORD" > "${PATH_TO_PASSWD_FILE}"
   then
       INSTALL_API="TRUE"
     # Remove the api at this point
+    # WARNING Foxx and arangosh arguments differ --server is used for Foxx not --server.endpoint 
     "${FOXX_PREFIX}foxx" remove \
       "/api/${local_FOXX_MAJOR_API_VERSION}" \
       --server "http://${local_DATAFED_DATABASE_HOST}:${local_DATABASE_PORT}" \
@@ -244,6 +245,7 @@ echo "$local_DATAFED_DATABASE_PASSWORD" > "${PATH_TO_PASSWD_FILE}"
   echo "$RESULT"
   if [ "${INSTALL_API}" == "TRUE"  ]
   then
+    # WARNING Foxx and arangosh arguments differ --server is used for Foxx not --server.endpoint 
     "${FOXX_PREFIX}foxx" install \
       --server "http://${local_DATAFED_DATABASE_HOST}:${local_DATABASE_PORT}" \
       -u "${local_DATABASE_USER}" \
@@ -253,6 +255,7 @@ echo "$local_DATAFED_DATABASE_PASSWORD" > "${PATH_TO_PASSWD_FILE}"
       "${PROJECT_ROOT}/core/database/foxx/"
   else
     echo "DataFed Foxx Services have already been uploaded, replacing to ensure consisency"
+    # WARNING Foxx and arangosh arguments differ --server is used for Foxx not --server.endpoint 
     "${FOXX_PREFIX}foxx" replace \
       --server "http://${local_DATAFED_DATABASE_HOST}:${local_DATABASE_PORT}" \
       -u "${local_DATABASE_USER}" \


Test patch:
diff --git a/core/database/tests/test_foxx.sh b/core/database/tests/test_foxx.sh
index 950ad8a41..ccfc706e5 100755
--- a/core/database/tests/test_foxx.sh
+++ b/core/database/tests/test_foxx.sh
@@ -1,13 +1,14 @@
 #!/bin/bash
 
 # -e has been removed so that if an error occurs the PASSWORD File is deleted and not left lying around
-# -u has been removed because we are checking for possible non existent env variables
-set -f -o pipefail
+set -uf -o pipefail
 
-SCRIPT=$(realpath "$0")
+SCRIPT=$(realpath "$BASH_SOURCE[0]")
 SOURCE=$(dirname "$SCRIPT")
 PROJECT_ROOT=$(realpath ${SOURCE}/../../../)
 source ${PROJECT_ROOT}/config/datafed.sh
+source "${PROJECT_ROOT}/scripts/dependency_versions.sh"
+source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh"
 
 Help()
 {
@@ -32,14 +33,14 @@ Help()
 local_DATABASE_NAME="sdms"
 local_DATABASE_USER="root"
 
-if [ -z "${DATAFED_DATABASE_PASSWORD}" ]
+if [ -z "${DATAFED_DATABASE_PASSWORD:-}" ]
 then
   local_DATAFED_DATABASE_PASSWORD=""
 else
   local_DATAFED_DATABASE_PASSWORD=$(printenv DATAFED_DATABASE_PASSWORD)
 fi
 
-if [ -z "${FOXX_MAJOR_API_VERSION}" ]
+if [ -z "${FOXX_MAJOR_API_VERSION:-}" ]
 then
   local_FOXX_MAJOR_API_VERSION=$(cat ${PROJECT_ROOT}/cmake/Version.cmake | grep -o -P "(?<=FOXX_API_MAJOR).*(?=\))" | xargs )
 else
@@ -110,30 +111,28 @@ fi
 # The web deployment requires manual interaction, and I could not figure out the 
 # syntax for the REST http endpoints with curl so we are going to try the node module
 
-NODE_VERSION="v14.21.3"
-
-export NVM_DIR="$HOME/.nvm"
-[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm
-
-nvm use $NODE_VERSION
+# Will only install if not already present
+install_nvm
+install_node
 
 FOXX_PREFIX=""
-{
-	# Determine if exists globally first
-	which foxx
-} || {
-	FOXX_PREFIX="~/bin/"
-}
+if ! command -v foxx > /dev/null 2>&1; then
+      FOXX_PREFIX="${DATAFED_DEPENDENCIES_INSTALL_PATH}/npm/bin/"
+fi
 
 PATH_TO_PASSWD_FILE=${SOURCE}/database_temp.password
 if [ "$TEST_TO_RUN" == "all" ]
 then
+  # WARNING Foxx and arangosh arguments differ --server is used for Foxx not --server.endpoint 
   "${FOXX_PREFIX}foxx" test -u "${local_DATABASE_USER}" \
+   --server "tcp://${DATAFED_DATABASE_HOST}:8529" \
     -p "${PATH_TO_PASSWD_FILE}" \
     --database "${local_DATABASE_NAME}" \
     "/api/${local_FOXX_MAJOR_API_VERSION}" --reporter spec
 else
+  # WARNING Foxx and arangosh arguments differ --server is used for Foxx not --server.endpoint 
   "${FOXX_PREFIX}foxx" test -u "${local_DATABASE_USER}" \
+  --server "tcp://${DATAFED_DATABASE_HOST}:8529" \
     -p "${PATH_TO_PASSWD_FILE}" \
     --database "${local_DATABASE_NAME}" \
     "/api/${local_FOXX_MAJOR_API_VERSION}" "$TEST_TO_RUN" --reporter spec
diff --git a/core/database/tests/test_setup.sh b/core/database/tests/test_setup.sh
index fbb85a3c6..bbae7725b 100755
--- a/core/database/tests/test_setup.sh
+++ b/core/database/tests/test_setup.sh
@@ -1,13 +1,14 @@
 #!/bin/bash
 
 # -e has been removed so that if an error occurs the PASSWORD File is deleted and not left lying around
-# -u has been removed because we are checking for possible non existent env variables
-set -f -o pipefail
+set -uf -o pipefail
 
-SCRIPT=$(realpath "$0")
+SCRIPT=$(realpath "$BASH_SOURCE[0]")
 SOURCE=$(dirname "$SCRIPT")
 PROJECT_ROOT=$(realpath "${SOURCE}/../../../")
 source "${PROJECT_ROOT}/config/datafed.sh"
+source "${PROJECT_ROOT}/scripts/dependency_versions.sh"
+source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh"
 
 Help()
 {
@@ -31,21 +32,28 @@ Help()
 local_DATABASE_NAME="sdms"
 local_DATABASE_USER="root"
 
-if [ -z "${DATAFED_DATABASE_PASSWORD}" ]
+if [ -z "${DATAFED_DATABASE_HOST:-}" ]
+then
+  local_DATAFED_DATABASE_HOST="localhost"
+else
+  local_DATAFED_DATABASE_HOST=$(printenv DATAFED_DATABASE_HOST)
+fi
+
+if [ -z "${DATAFED_DATABASE_PASSWORD:-}" ]
 then
   local_DATAFED_DATABASE_PASSWORD=""
 else
   local_DATAFED_DATABASE_PASSWORD=$(printenv DATAFED_DATABASE_PASSWORD)
 fi
 
-if [ -z "${DATAFED_ZEROMQ_SYSTEM_SECRET}" ]
+if [ -z "${DATAFED_ZEROMQ_SYSTEM_SECRET:-}" ]
 then
   local_DATAFED_ZEROMQ_SYSTEM_SECRET=""
 else
   local_DATAFED_ZEROMQ_SYSTEM_SECRET=$(printenv DATAFED_ZEROMQ_SYSTEM_SECRET)
 fi
 
-if [ -z "${FOXX_MAJOR_API_VERSION}" ]
+if [ -z "${FOXX_MAJOR_API_VERSION:-}" ]
 then
   local_FOXX_MAJOR_API_VERSION=$(cat ${PROJECT_ROOT}/cmake/Version.cmake | grep -o -P "(?<=FOXX_API_MAJOR).*(?=\))" | xargs )
 else
@@ -117,17 +125,17 @@ fi
 
 # We are now going to initialize the DataFed database in Arango, but only if sdms database does
 # not exist
-output=$(curl --dump - --user $local_DATABASE_USER:$local_DATAFED_DATABASE_PASSWORD http://localhost:8529/_api/database/user)
+output=$(curl --dump - --user $local_DATABASE_USER:$local_DATAFED_DATABASE_PASSWORD http://${local_DATAFED_DATABASE_HOST}:8529/_api/database/user)
 
 if [[ "$output" =~ .*"sdms".* ]]; then
 	echo "SDMS already exists do nothing"
 else
 	echo "Creating SDMS"
-  arangosh  --server.password "${local_DATAFED_DATABASE_PASSWORD}" --server.username "${local_DATABASE_USER}" --javascript.execute "${PROJECT_ROOT}/core/database/foxx/db_create.js"
+  arangosh  --server.endpoint "tcp://${local_DATAFED_DATABASE_HOST}:8529"   --server.password "${local_DATAFED_DATABASE_PASSWORD}" --server.username "${local_DATABASE_USER}" --javascript.execute "${PROJECT_ROOT}/core/database/foxx/db_create.js"
   # Give time for the database to be created
   sleep 2
-  arangosh --server.password "${local_DATAFED_DATABASE_PASSWORD}" --server.username "${local_DATABASE_USER}" --javascript.execute-string 'db._useDatabase("sdms"); db.config.insert({"_key": "msg_daily", "msg" : "DataFed servers will be off-line for regular maintenance every Sunday night from 11:45 pm until 12:15 am EST Monday morning."}, {overwrite: true});'
-  arangosh --server.password "${local_DATAFED_DATABASE_PASSWORD}" --server.username "${local_DATABASE_USER}" --javascript.execute-string "db._useDatabase(\"sdms\"); db.config.insert({ \"_key\": \"system\", \"_id\": \"config/system\", \"secret\": \"${local_DATAFED_ZEROMQ_SYSTEM_SECRET}\"}, {overwrite: true } );"
+  arangosh  --server.endpoint "tcp://${local_DATAFED_DATABASE_HOST}:8529" --server.password "${local_DATAFED_DATABASE_PASSWORD}" --server.username "${local_DATABASE_USER}" --javascript.execute-string 'db._useDatabase("sdms"); db.config.insert({"_key": "msg_daily", "msg" : "DataFed servers will be off-line for regular maintenance every Sunday night from 11:45 pm until 12:15 am EST Monday morning."}, {overwrite: true});'
+  arangosh  --server.endpoint "tcp://${local_DATAFED_DATABASE_HOST}:8529" --server.password "${local_DATAFED_DATABASE_PASSWORD}" --server.username "${local_DATABASE_USER}" --javascript.execute-string "db._useDatabase(\"sdms\"); db.config.insert({ \"_key\": \"system\", \"_id\": \"config/system\", \"secret\": \"${local_DATAFED_ZEROMQ_SYSTEM_SECRET}\"}, {overwrite: true } );"
 fi
 
 # There are apparently 3 different ways to deploy Foxx microservices,
@@ -146,21 +154,23 @@ fi
 ## Get the size of the file in bytes
 #bytes=$(wc -c < datafed.zip)
 
-NODE_VERSION="v14.21.3"
-
-export NVM_DIR="$HOME/.nvm"
-[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm
-
-nvm install "$NODE_VERSION"
-nvm use "$NODE_VERSION"
+# Will only install if it is not already installed
+install_nvm
+install_node
 
 PATH_TO_PASSWD_FILE=${SOURCE}/database_temp.password
 # Install foxx service node module
 $NVM_DIR/nvm-exec npm install --global foxx-cli
 echo "$local_DATAFED_DATABASE_PASSWORD" > "${PATH_TO_PASSWD_FILE}"
 
+FOXX_PREFIX=""
+if ! command -v foxx > /dev/null 2>&1; then
+    FOXX_PREFIX="${DATAFED_DEPENDENCIES_INSTALL_PATH}/npm/bin/"
+fi
+
 # Check if database foxx services have already been installed
-existing_services=$(foxx list -a -u "$local_DATABASE_USER" -p "${PATH_TO_PASSWD_FILE}" --database "$local_DATABASE_NAME")
+# WARNING Foxx and arangosh arguments differ --server is used for Foxx not --server.endpoint
+existing_services=$("${FOXX_PREFIX}foxx" list -a -u "$local_DATABASE_USER" -p "${PATH_TO_PASSWD_FILE}"  --server "tcp://${local_DATAFED_DATABASE_HOST}:8529"  --database "$local_DATABASE_NAME")
 echo "existing services ${existing_services}"
 
 if [[ "$existing_services" =~ .*"DataFed".* ]]