Repository: ORNL/DataFed (language: python)

Issue:
Joshua s brown fix check ci infrastructure script
# Description

This pr fixes issues with the ci_pipeline script. That script is designed to check if a CI infrastructure pipeline is already running, it also checks to see if the infrastructure is setup already. If it is not setup it will return an exit code that indicates it needs to be triggered. 

Unfortunately, the repo token secret expired, the ci job was not however throwing an error, but instead continuing to trigger the downstream infrastructure setup pipeline. This is wasteful. Two items are fixed in this pr. 

1. A more meaningful error message was thrown
2. The job fails if the repo token is invalid

In addition this PR, addresses two other concerns. 

1. The api python tests were not respecting python env activation because they were hardcoded to use /bin/python meaning if the system path did not include the right python modules the code was crashing.
2. foxx is needed for the end_to_end tests and was missing from the python client as part of the installation, a script was created to address this.


Discussion / hints:

Fix patch:
diff --git a/.gitlab/end_to_end.yml b/.gitlab/end_to_end.yml
index 319c47f50..98d8e848f 100644
--- a/.gitlab/end_to_end.yml
+++ b/.gitlab/end_to_end.yml
@@ -336,6 +336,7 @@ end_to_end_client-test:
     - export DATAFED_REPO_FORM_PATH="$(pwd)/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.json"
     - env > env_file
     - echo "Testing"
+    - source /shared/install/python/datafed/bin/activate
     - ./scripts/generate_datafed.sh
     - >
       cmake -S. -B build
diff --git a/.gitlab/infrastructure.yml b/.gitlab/infrastructure.yml
index 81c616867..b22911396 100644
--- a/.gitlab/infrastructure.yml
+++ b/.gitlab/infrastructure.yml
@@ -15,8 +15,14 @@ check-ci-infrastructure:
       BUILD_INFRASTRUCTURE="FALSE"
       COMPUTE_INSTANCE_NAMES=("ci-datafed-arangodb" "ci-datafed-core" "ci-datafed-globus2" "ci-datafed-client")
       for INSTANCE_NAME in "${COMPUTE_INSTANCE_NAMES[@]}"; do
-        if ! ./scripts/ci_pipeline_setup.sh --compute-instance-name "$INSTANCE_NAME"; then
+        set +e
+        ./scripts/ci_pipeline_setup.sh --compute-instance-name "$INSTANCE_NAME"
+        EXIT_CODE="$?"
+        set -e
+        if [ "${EXIT_CODE}" -eq 1 ]; then
           BUILD_INFRASTRUCTURE="TRUE"
+        elif [ "${EXIT_CODE}" -eq 2 ]; then
+          exit 1
         fi
       done
       if [ "$BUILD_INFRASTRUCTURE" == "TRUE" ]
diff --git a/.gitlab/stage_provision_client.yml b/.gitlab/stage_provision_client.yml
index 74c86cf23..097a93250 100644
--- a/.gitlab/stage_provision_client.yml
+++ b/.gitlab/stage_provision_client.yml
@@ -15,3 +15,4 @@ provision-client:
   script:
     - ./scripts/generate_datafed.sh
     - ./scripts/install_client_dependencies.sh
+    - ./scripts/install_end_to_end_test_dependencies.sh
diff --git a/scripts/ci_pipeline_setup.sh b/scripts/ci_pipeline_setup.sh
index 6084a77d1..8621f1046 100755
--- a/scripts/ci_pipeline_setup.sh
+++ b/scripts/ci_pipeline_setup.sh
@@ -5,8 +5,8 @@ set -eu
 Help()
 {
   echo "$(basename $0) Will determine if a Open Stack VM exists if not it will"
-  echo " will exit with an error code. It requires that you "
-  echo "provide the Open Stack VM ID"
+  echo " will exit with an error code 1. If some other problem exists will exit"
+  echo " with error code 2. It requires that you provide the Open Stack VM ID"
   echo
   echo "Syntax: $(basename $0) [-h|i|s|c|a|n]"
   echo "options:"
@@ -63,7 +63,7 @@ COMPUTE_ID_PROVIDED="FALSE"
 
 VALID_ARGS=$(getopt -o hi:s:c:a:n: --long 'help',app-credential-id:,app-credential-secret:,compute-instance-id:,gitlab-api-token:,compute-instance-name: -- "$@")
 if [[ $? -ne 0 ]]; then
-      exit 1;
+      exit 2;
 fi
 eval set -- "$VALID_ARGS"
 while [ : ]; do
@@ -107,28 +107,28 @@ if [ -z "$local_OS_APP_ID" ]
 then
   echo "The open stack application credential id has not been defined this is"
   echo " a required parameter."
-  exit 1
+  exit 2
 fi
 
 if [ -z "$local_OS_APP_SECRET" ]
 then
   echo "The open stack application credential secret has not been defined this is"
   echo " a required parameter."
-  exit 1
+  exit 2
 fi
 
 if [[ -z "$COMPUTE_INSTANCE_ID" && -z "$COMPUTE_INSTANCE_NAME" ]]
 then
   echo "The open stack compute instance id or name has not been defined, at "
   echo "least one is required."
-  exit 1
+  exit 2
 fi
 
 if [ -z "$local_GITLAB_DATAFEDCI_REPO_API_TOKEN" ]
 then
   echo "The GitLab token for accessing the API of the DataFed ci repo is missing."
   echo "It is a required parameter."
-  exit 1
+  exit 2
 fi
 
 data=$(curl -s --retry 5 -i -X POST \
@@ -150,7 +150,7 @@ if [ "$error_code" == "6" ]
 then
   echo "Unable to connect to Open Stack API endpoints, make sure you are"
   echo "connected to the network"
-  exit 1
+  exit 2
 fi
 
 # Make sure jq is installed
@@ -158,7 +158,7 @@ jq_path=$(which jq || true)
 if [ -z "$jq_path" ]
 then
   echo "jq command not found exiting!"
-  exit 1
+  exit 2
 fi
 
 wait_for_running_infrastructure_pipelines_to_finish() {
@@ -168,13 +168,21 @@ wait_for_running_infrastructure_pipelines_to_finish() {
   then
     echo "No other running infrastructure provisioning pipelines detected!"
   fi
- 
+
+  if [[ "$all_other_pipelines" == *"invalid_token"* ]]
+  then
+    echo "Error detected with GITLAB_DATAFEDCI_REPO_API_TOKEN"
+    echo "$all_other_pipelines"
+    exit 2
+  fi
+
   local count=0
   while [ ! -z "$all_other_pipelines" ] 
   do
     echo "Attempt $count, Other running infrastructure provisioning pipelines detected... waiting for them to complete."
     echo
     echo "Running Pipelines Are:"
+    echo "$all_other_pipelines"
     echo "$all_other_pipelines" | jq '.id'
     sleep 30s
     count=$(($count + 1))
@@ -309,7 +317,7 @@ then
       if [ "$count" == "$MAX_COUNT" ]
       then
         echo "Exceeded time limit!"
-        exit 1
+        exit 2
       fi
     done
 
@@ -327,32 +335,32 @@ if [ ! -z "$pipeline_id" ]
 then
 
   count=0
-	KEEP_RUNNING="TRUE"	
-	while [ "$KEEP_RUNNING" == "TRUE" ]
-	do
-		pipeline_status=$(curl -s --header "PRIVATE-TOKEN: ${local_GITLAB_DATAFEDCI_REPO_API_TOKEN}" "https://code.ornl.gov/api/v4/projects/${GITLAB_PROJECT_ID}/pipelines/$pipeline_id" | jq .status | sed 's/\"//g')
-
-		printf "Attempt $count, Waiting for triggered infrastructure provisioning pipeline: ${pipeline_id} to complete ... "
-		if [ "$pipeline_status" == "failed" ]
-		then
-			echo "Infrastructure triggered pipeline has failed unable to execute CI. STATUS: $pipeline_status"
-			exit 1
-		elif [ "$pipeline_status" == "success" ]
-		then
-			echo "Infrastructure triggered pipeline has passed. STATUS: $pipeline_status"
-			exit 0
-		elif [ "$pipeline_status" == "canceled" ]
-		then
-			echo "Infrastructure triggered pipeline has failed unable to execute CI. STATUS: $pipeline_status"
-			exit 1
+  KEEP_RUNNING="TRUE"  
+  while [ "$KEEP_RUNNING" == "TRUE" ]
+  do
+    pipeline_status=$(curl -s --header "PRIVATE-TOKEN: ${local_GITLAB_DATAFEDCI_REPO_API_TOKEN}" "https://code.ornl.gov/api/v4/projects/${GITLAB_PROJECT_ID}/pipelines/$pipeline_id" | jq .status | sed 's/\"//g')
+
+    printf "Attempt $count, Waiting for triggered infrastructure provisioning pipeline: ${pipeline_id} to complete ... "
+    if [ "$pipeline_status" == "failed" ]
+    then
+      echo "Infrastructure triggered pipeline has failed unable to execute CI. STATUS: $pipeline_status"
+      exit 2
+    elif [ "$pipeline_status" == "success" ]
+    then
+      echo "Infrastructure triggered pipeline has passed. STATUS: $pipeline_status"
+      exit 0
+    elif [ "$pipeline_status" == "canceled" ]
+    then
+      echo "Infrastructure triggered pipeline has failed unable to execute CI. STATUS: $pipeline_status"
+      exit 2
     else
       echo "STATUS: $pipeline_status"
-		fi
-    			
-		sleep 30s
+    fi
+          
+    sleep 30s
 
     count=$(($count + 1))
-	done
+  done
 fi
 
 ################################################################################
diff --git a/scripts/dependency_install_functions.sh b/scripts/dependency_install_functions.sh
index 223aae09d..e16eacea0 100644
--- a/scripts/dependency_install_functions.sh
+++ b/scripts/dependency_install_functions.sh
@@ -416,6 +416,22 @@ install_foxx_cli() {
     export NVM_DIR="${DATAFED_DEPENDENCIES_INSTALL_PATH}/nvm"
     [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" # This loads nvm
     export NODE_VERSION="$DATAFED_NODE_VERSION"
+    
+    # check that foxx can be found
+    if [ ! -d "${DATAFED_DEPENDENCIES_INSTALL_PATH}/npm" ]
+    then
+	echo "Something went wrong Foxx is supposed to be installed i.e. "
+	echo "(${DATAFED_DEPENDENCIES_INSTALL_PATH}/.foxx_cli_installed) "
+	echo "exists. But there is no npm folder in: ${DATAFED_DEPENDENCIES_INSTALL_PATH}"
+	exit 1
+    fi
+    if [ ! -e "${DATAFED_DEPENDENCIES_INSTALL_PATH}/npm/bin/foxx" ]
+    then
+	echo "Something went wrong Foxx is supposed to be installed i.e. "
+	echo "(${DATAFED_DEPENDENCIES_INSTALL_PATH}/.foxx_cli_installed) "
+	echo "exists. But there is no foxx binary here: ${DATAFED_DEPENDENCIES_INSTALL_PATH}/npm/bin/foxx"
+	exit 1
+    fi
   fi
 }
 
@@ -534,6 +550,9 @@ install_dep_by_name() {
     "cmake")
       install_cmake
       ;;
+    "foxx")
+      install_foxx_cli
+      ;;
     "protobuf")
       install_protobuf
       ;;


Test patch:
diff --git a/scripts/install_end_to_end_test_dependencies.sh b/scripts/install_end_to_end_test_dependencies.sh
new file mode 100755
index 000000000..a4627aceb
--- /dev/null
+++ b/scripts/install_end_to_end_test_dependencies.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Exit on error
+set -e
+
+SCRIPT=$(realpath "$0")
+SOURCE=$(dirname "$SCRIPT")
+PROJECT_ROOT=$(realpath ${SOURCE}/..)
+
+source "${PROJECT_ROOT}/scripts/utils.sh"
+source "${PROJECT_ROOT}/scripts/dependency_install_functions.sh"
+
+packages=("libtool" "build-essential" "g++" "gcc" "make" "libboost-all-dev" "pkg-config" "autoconf" "automake" "unzip" "libcurl4-openssl-dev" "wget"
+	"rapidjson-dev" "libkrb5-dev" "git" "python3-pkg-resources" "python3-pip" "python3-venv" "libssl-dev")
+
+	
+pip_packages=("setuptools")
+# NOTE the order matters here
+externals=("cmake" "protobuf" "nvm" "node" "foxx")
+
+local_UNIFY=false
+
+if [ $# -eq 1 ]; then
+  case "$1" in
+    -h|--help)
+      # If -h or --help is provided, print help
+      echo "Usage: $0 [-h|--help] [unify]"
+      ;;
+    unify)
+      # If 'unify' is provided, print the packages
+      # The extra space is necessary to not conflict with the other install scripts
+      echo -n "${packages[@]} " >> "$apt_file_path"
+      echo -n "${externals[@]} " >> "$ext_file_path"
+      echo -n "${pip_packages[@]} " >> "$pip_file_path"
+      local_UNIFY=true
+      ;;
+    *)
+      echo "Invalid Argument"
+      ;;
+  esac
+fi
+
+if [[ $local_UNIFY = false ]]; then
+  sudo_command
+  "$SUDO_CMD" apt-get update
+  "$SUDO_CMD" dpkg --configure -a
+  "$SUDO_CMD" apt-get install -y "${packages[@]}"
+  init_python
+  source "${DATAFED_PYTHON_ENV}/bin/activate"
+  python3 -m pip install --upgrade pip
+  python3 -m pip install "${pip_packages[@]}"
+
+  for ext in "${externals[@]}"; do
+    install_dep_by_name "$ext"
+  done
+fi
diff --git a/tests/end-to-end/test_api_alloc.py b/tests/end-to-end/test_api_alloc.py
index 5b59659c4..89c9c4806 100755
--- a/tests/end-to-end/test_api_alloc.py
+++ b/tests/end-to-end/test_api_alloc.py
@@ -1,4 +1,6 @@
-#!/bin/python3
+#!/usr/bin/env python3
+# WARNING - to work with python environments we cannot use /bin/python3 or
+#           a hardcoded abs path.
 import json
 import os
 import sys
diff --git a/tests/end-to-end/test_api_collection.py b/tests/end-to-end/test_api_collection.py
index ba1fcf8ac..2118e149d 100755
--- a/tests/end-to-end/test_api_collection.py
+++ b/tests/end-to-end/test_api_collection.py
@@ -1,4 +1,6 @@
-#!/bin/python3
+#!/usr/bin/env python3
+# WARNING - to work with python environments we cannot use /bin/python3 or
+#           a hardcoded abs path.
 import json
 import os
 import sys
diff --git a/tests/end-to-end/test_api_context.py b/tests/end-to-end/test_api_context.py
index 47ff13cdc..b6a8dafc1 100755
--- a/tests/end-to-end/test_api_context.py
+++ b/tests/end-to-end/test_api_context.py
@@ -1,5 +1,6 @@
-#!/bin/python3
-# import json
+#!/usr/bin/env python3
+# WARNING - to work with python environments we cannot use /bin/python3 or
+#           a hardcoded abs path.
 import os
 import sys
 import unittest
diff --git a/tests/end-to-end/test_api_endpoint.py b/tests/end-to-end/test_api_endpoint.py
index 66b473390..4d97dd366 100755
--- a/tests/end-to-end/test_api_endpoint.py
+++ b/tests/end-to-end/test_api_endpoint.py
@@ -1,5 +1,6 @@
-#!/bin/python3
-# import json
+#!/usr/bin/env python3
+# WARNING - to work with python environments we cannot use /bin/python3 or
+#           a hardcoded abs path.
 import os
 import sys
 import unittest
diff --git a/tests/end-to-end/test_api_record.py b/tests/end-to-end/test_api_record.py
index 020fc5937..fb85d703f 100755
--- a/tests/end-to-end/test_api_record.py
+++ b/tests/end-to-end/test_api_record.py
@@ -1,4 +1,6 @@
-#!/bin/python3
+#!/usr/bin/env python3
+# WARNING - to work with python environments we cannot use /bin/python3 or
+#           a hardcoded abs path.
 import json
 import os
 import sys
diff --git a/tests/end-to-end/test_api_repo.py b/tests/end-to-end/test_api_repo.py
index 9f8989240..c74c5a402 100755
--- a/tests/end-to-end/test_api_repo.py
+++ b/tests/end-to-end/test_api_repo.py
@@ -1,4 +1,6 @@
-#!/bin/python3
+#!/usr/bin/env python3
+# WARNING - to work with python environments we cannot use /bin/python3 or
+#           a hardcoded abs path.
 import json
 import os
 import sys
diff --git a/tests/end-to-end/test_api_user_login.py b/tests/end-to-end/test_api_user_login.py
index 571c56f89..8867ecf89 100755
--- a/tests/end-to-end/test_api_user_login.py
+++ b/tests/end-to-end/test_api_user_login.py
@@ -1,5 +1,6 @@
-#!/bin/python3
-# import json
+#!/usr/bin/env python3
+# WARNING - to work with python environments we cannot use /bin/python3 or
+#           a hardcoded abs path.
 import os
 import sys
 import unittest