Added terraform skeleton code

accounts/hasaniqbal-ds.backend.conf

@@ -0,0 +1,6 @@
+    bucket         = "hasaniqbal-ds-terraform-state"
+    region         = "us-east-1"
+    encrypt        = true
+    kms_key_id     = "alias/terraform-state-bucket-key"
+    dynamodb_table = "hasaniqbal-ds-terraform-lock"
+    profile        = "ds"

accounts/hasaniqbal-ds.tfvars.json

@@ -0,0 +1,6 @@
+{
+  "aws_account_name": "hasaniqbal-ds",
+  "aws_account": "252286678527",
+  "aws_region": "us-east-1",
+  "aws_profile": "ds"
+}

deployments/terraform/.terraform-version

@@ -0,0 +1 @@
+1.9.5

deployments/terraform/.terraform.lock.hcl

@@ -0,0 +1,44 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/aws" {
+  version     = "5.64.0"
+  constraints = "~> 5.64.0"
+  hashes = [
+    "h1:YH4I78rsS9t+YoGMPNzrM53aWi0Rb9Nud16iusrSXMg=",
+    "zh:1d361f8062c68c9d5ac14b0aa8390709542129b8a9b258e61bbbabc706078b44",
+    "zh:39dcbf53e3896bdd77071384c8fad4a5862c222c73f3bcf356aca488101f22fd",
+    "zh:3fad63505f0c5b6f01cc9a6ef02b2226983b79424126a9caf6eb724f654299f4",
+    "zh:53a8b90d00829cc27e3171a13a8ff1404ee0ea018e73f31d3f916d246cc39613",
+    "zh:5734c25ef5a04b40f3c1ac5f817f11e42ee3328f74dbc141c0e64afbb0acc834",
+    "zh:66ea14dbd87f291ce4a877123363933d3ca4022f209f885807a6689c22c24e80",
+    "zh:68e79654ad0894a3d93134c3377748ace3058d5fad5ec09d1e9a8f8f9b8a47ea",
+    "zh:7b74259d0ceef0c49cea6bcd171df997b6bad141085bbadded15b440faeb0eee",
+    "zh:988ebfb5d115dc57070b5abf2e4200ad49cde535f27fd2ba5e34cf9ab336a57f",
+    "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
+    "zh:a0a2d4efe2835f0101a0a5024e044a3f28c00e10a8d87fce89c707ef6db75cea",
+    "zh:aecb3e4b9121771dee9cac7975bf5d0657b5f3e8b57788c455beaeb0f3c48d93",
+    "zh:d2d3393170b8ef761d3146f39f6788c4a3e876e6c5d4cedca4870c2680688ae6",
+    "zh:daba5a005c1baa4a5eefbfb86d43ccf880eb5b42e8136f0d932f55886d72bda0",
+    "zh:de16a6ff3baacdaf9609a0a89aa1913fc19cccaf5ee0fc1c49c5a075baa47c02",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/local" {
+  version = "2.5.1"
+  hashes = [
+    "h1:/GAVA/xheGQcbOZEq0qxANOg+KVLCA7Wv8qluxhTjhU=",
+    "zh:0af29ce2b7b5712319bf6424cb58d13b852bf9a777011a545fac99c7fdcdf561",
+    "zh:126063ea0d79dad1f68fa4e4d556793c0108ce278034f101d1dbbb2463924561",
+    "zh:196bfb49086f22fd4db46033e01655b0e5e036a5582d250412cc690fa7995de5",
+    "zh:37c92ec084d059d37d6cffdb683ccf68e3a5f8d2eb69dd73c8e43ad003ef8d24",
+    "zh:4269f01a98513651ad66763c16b268f4c2da76cc892ccfd54b401fff6cc11667",
+    "zh:51904350b9c728f963eef0c28f1d43e73d010333133eb7f30999a8fb6a0cc3d8",
+    "zh:73a66611359b83d0c3fcba2984610273f7954002febb8a57242bbb86d967b635",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:7ae387993a92bcc379063229b3cce8af7eaf082dd9306598fcd42352994d2de0",
+    "zh:9e0f365f807b088646db6e4a8d4b188129d9ebdbcf2568c8ab33bddd1b82c867",
+    "zh:b5263acbd8ae51c9cbffa79743fbcadcb7908057c87eb22fd9048268056efbc4",
+    "zh:dfcd88ac5f13c0d04e24be00b686d069b4879cc4add1b7b1a8ae545783d97520",
+  ]
+}

deployments/terraform/aws_dynamodb.tf

@@ -0,0 +1,31 @@
+# ##############################################################################
+# DynamoDB Table - Default
+# ##############################################################################
+resource "aws_dynamodb_table" "openfactcheck_default" {
+  name = "openfactcheck-db-${terraform.workspace}-${var.aws_region}"
+
+  billing_mode = "PAY_PER_REQUEST"
+  hash_key     = "PK"
+
+  global_secondary_index {
+    name            = "gs1"
+    hash_key        = "GS1PK"
+    projection_type = "ALL"
+  }
+
+  // ID
+  attribute {
+    name = "PK"
+    type = "S"
+  }
+
+  // Type
+  attribute {
+    name = "GS1PK"
+    type = "S"
+  }
+
+  tags = {
+    Name = "OpenFactCheck - DynamoDB - ${terraform.workspace} - ${var.aws_region}"
+  }
+}

deployments/terraform/main.tf

@@ -0,0 +1,93 @@
+# ##############################################################################
+# Snippet to require usage of a 'non-default' workspace.
+# ##############################################################################
+data "local_file" "is_default_workspace" {
+  count    = terraform.workspace == "default" ? 1 : 0
+  filename = "ERROR: Workspace does not match given environment name!"
+}
+
+# ##############################################################################
+# Backend
+# ##############################################################################
+terraform {
+  backend "s3" {
+    key                  = "openfactcheck/terraform.tfstat"
+    workspace_key_prefix = "openfactcheck"
+  }
+}
+
+# ##############################################################################
+# Terraform and Provider Requirements
+# ##############################################################################
+terraform {
+  required_version = "~>1.9.5"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~>5.64.0"
+    }
+  }
+}
+
+provider "aws" {
+  region              = var.aws_region
+  profile             = var.aws_profile
+  allowed_account_ids = [var.aws_account]
+
+  default_tags {
+    # tags holds the default tags applied to all resources.
+    # Individual resources should override Name and Function,
+    # (as well as anything else they find appropriate).
+    tags = {
+      Name          = "OpenFactCheck - ${terraform.workspace} - ${var.aws_region}"
+      Group         = "OpenFactCheck - ${terraform.workspace}"
+      Owner         = "OpenFactCheck Team"
+      Purpose       = "LLM Factuality Evaluation Framework - ${terraform.workspace} - ${var.aws_region}"
+      Function      = "App"
+      Environment   = terraform.workspace
+      Criticality   = "High"
+      TTL           = "2021-06-01"
+      MangedBy      = "terraform"
+      Business_Unit = "Infrastructure"
+      Region        = var.aws_region
+    }
+  }
+}
+
+data "aws_caller_identity" "current" {}
+
+# ##############################################################################
+# Variables
+# ##############################################################################
+variable "aws_account_name" {
+  type = string
+  validation {
+    condition     = length(var.aws_account_name) > 0
+    error_message = "Use the -var-file flag with the terraform command to specify the account configuration."
+  }
+}
+
+variable "aws_account" {
+  type = string
+  validation {
+    condition     = length(var.aws_account) > 0
+    error_message = "Use the -var-file flag with the terraform command to specify the account configuration."
+  }
+}
+
+variable "aws_region" {
+  type = string
+  validation {
+    condition     = length(var.aws_region) > 0
+    error_message = "Use the -var-file flag with the terraform command to specify the account configuration."
+  }
+}
+
+variable "aws_profile" {
+  type = string
+  validation {
+    condition     = length(var.aws_profile) > 0
+    error_message = "Use the -var-file flag with the terraform command to specify the account configuration."
+  }
+}

deployments/terraform/terraform-apply.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+source "${BASH_SOURCE%/*}/../../scripts/common.sh"
+
+if [ -z "$1" ] ; then
+    c_echo $RED "Need to provide the ENVIRONMENT as first argument. \n"
+    c_echo $YELLOW "Usage: $0 <environment> [auto]"
+    exit 1
+fi
+
+c_echo $GREEN "
+--------------------------------------------------------------------------------
+--                           Deploying Terraform                              --
+--------------------------------------------------------------------------------
+"
+
+INPUT_FALSE=""
+AUTO_APPROVE=""
+if [ "$2" = "auto" ] ; then
+    c_echo $YELLOW "Auto approval set..."
+    INPUT_FALSE="-input=false"
+    AUTO_APPROVE="-auto-approve"
+fi
+
+ENVIRONMENT=$1
+
+AWS_ACCOUNT_NAME=$(cat environments/${ENVIRONMENT}.tfvars.json | jq .aws_account_name | tr -d '"')
+
+if [ -z "$AWS_ACCOUNT_NAME" ] ; then
+    c_echo $RED "There was a problem loading Account Name configuration from file environments/${ENVIRONMENT}.tfvars.json"
+    exit 1
+fi
+
+echo "Account:"
+cat environments/${ENVIRONMENT}.tfvars.json 
+
+pushd deployments/terraform > /dev/null
+
+    terraform init -reconfigure -backend-config=../../accounts/${AWS_ACCOUNT_NAME}.backend.conf $INPUT_FALSE
+    if [ "$?" -ne "0" ]; then
+        echo "Error in initing the terraform!"
+        exit 1
+    fi
+
+    echo
+    terraform workspace select -or-create $ENVIRONMENT
+    if [ "$?" -ne "0" ]; then
+        echo "Error in selecting workspace!"
+        exit 1
+    fi
+
+    echo
+    echo "Workspaces:"
+    terraform workspace list
+    if [ "$?" -ne "0" ]; then
+        echo "Error in listing workspaces!"
+        exit 1
+    fi
+
+    terraform apply -var-file=../../environments/${ENVIRONMENT}.tfvars.json $INPUT_FALSE $AUTO_APPROVE
+    if [ "$?" -ne "0" ]; then
+        echo "Error in terraform apply!"
+        exit 1
+    fi
+
+popd > /dev/null

environments/integration.json

@@ -0,0 +1,6 @@
+{
+    "aws_account_name": "hasaniqbal-ds",
+    "aws_account": "252286678527",
+    "aws_region": "us-east-1",
+    "aws_profile": "ds"
+}

environments/production.json

@@ -0,0 +1,6 @@
+{
+    "aws_account_name": "hasaniqbal-ds",
+    "aws_account": "252286678527",
+    "aws_region": "us-east-1",
+    "aws_profile": "ds"
+}

scripts/cli.sh

@@ -11,4 +11,5 @@ source "${BASH_SOURCE%/*}/common.sh"
 
 # Executing Python script
 export PYTHONPATH="$PYTHONPATH:src/"
-python src/openfactcheck/cli.py "$@"
+export OPENFACTCHECK_VERBOSITY="info"
+python src/openfactcheck/cli.py "$@"

scripts/deploy.sh

@@ -0,0 +1,236 @@
+#!/bin/bash
+
+source "${BASH_SOURCE%/*}/common.sh"
+
+# Description:
+#   Deploys a deployment to an environment
+#
+# Usage:
+# ./scripts/deploy.sh --interactive
+# ./scripts/deploy.sh <environment>
+# ./scripts/deploy.sh --environment <environment>
+# ./scripts/deploy.sh --account <account> --deployment <deployment>
+# ./scripts/deploy.sh --help
+
+# Parse arguments and set variables using getops
+arg0=$(basename "$0" .sh)
+blnk=$(echo "$arg0" | sed 's/./ /g')
+
+# Print usage information
+usage_info()
+{
+    c_echo $YELLOW "Usage:"
+    c_echo $YELLOW "  $arg0 <environment> [--auto]"
+    c_echo $YELLOW "  $arg0 --interactive [--auto]"
+    c_echo $YELLOW "  $arg0 --environment <environment> [--auto]"
+    c_echo $YELLOW "  $arg0 --account <account> --deployment <deployment> [--auto]"
+}
+
+# Print usage in case of bad arguments
+usage()
+{
+    #exec 1>2   # Send standard output to standard error
+    usage_info
+    exit 1
+}
+
+# Print error message and exit
+error()
+{
+    echo "$arg0: $*" >&2
+    exit 1
+}
+
+# Confirmation message, take two arguments
+confirmation()
+{
+    c_echo $YELLOW "
+--------------------------------------------------------------------------------
+--                                Confirmation                                --
+--------------------------------------------------------------------------------
+"
+    c_echo $YELLOW "Are you sure you want to deploy $1 to $2? (y/n) (Default: y)"
+    read -r CONFIRMATION
+    if [[ -z $CONFIRMATION ]]; then
+        CONFIRMATION="y"
+    fi
+    if [[ $CONFIRMATION != "y" ]]; then
+        c_echo $RED "Exiting..."
+        exit 1
+    fi
+}
+
+# Print help message
+help()
+{
+    usage_info
+    echo
+    c_echo $YELLOW "Options:"
+    echo "  {-i|--interactive}               -- Interactive mode"
+    echo "  {-e|--environment} <environment> -- Environment to deploy to"
+    echo "  {-a|--account} <account>         -- Account to deploy to"
+    echo "  {-d|--deployment} <deployment>   -- Deployment to deploy"
+    echo "  {-y|--auto}                      -- Auto approve"
+    echo "  {-h|--help}                      -- Display this help and exit"
+    echo
+    echo
+    c_echo $YELLOW "Available Deployments:"
+    ls -d deployments/* | grep -v backend | sed 's/deployments\///g' | sort
+    echo
+    c_echo $YELLOW "Available Environments:"
+    ls -d environments/* | grep -v backend | sed 's/.tfvars.json//g' | sed 's/environments\///g'
+    echo
+    c_echo $YELLOW "Available Accounts:"
+    ls -d accounts/* | grep -v backend | sed 's/.tfvars.json//g' | sed 's/accounts\///g'
+    exit 0
+}
+
+# Parse command line flags using getopts
+flags()
+{
+    # No Arguments
+    [ $# -eq 0 ] && usage
+
+    # if argument is not a flag, then modify command with environment
+    if [ "${1:0:1}" != "-" ]; then
+        ENVIRONMENT=$1
+        shift
+        set -- "$@" "-e" "$ENVIRONMENT"
+    fi
+    
+    # Parse Arguments
+    while test $# -gt 0
+    do
+        case "$1" in
+        (-i|--interactive)
+            export INTERACTIVE=true
+            shift;;
+        (-e|--environment)
+            shift
+            [ $# = 0 ] && error "No environment specified"
+            export ENVIRONMENT="$1"
+
+            # Ensure that environment is valid
+            if [ ! -f "environments/$ENVIRONMENT.tfvars.json" ]; then
+                error "Invalid environment $ENVIRONMENT"
+            fi
+            shift;;
+        (-a|--account)
+            shift
+            [ $# = 0 ] && error "No account specified"
+            export ACCOUNT="$1"
+
+            # Ensure that account is valid
+            if [ ! -f "accounts/$ACCOUNT.tfvars.json" ]; then
+                error "Invalid account $ACCOUNT"
+            fi
+            shift;;
+        (-d|--deployment)
+            shift
+            [ $# = 0 ] && error "No deployment specified"
+            export DEPLOYMENT="$1"
+
+            # Ensure that deployment is valid
+            if [ ! -d "deployments/$DEPLOYMENT" ]; then
+                error "Invalid deployment $DEPLOYMENT"
+            fi
+            shift;;
+        (-y|--auto)
+            export AUTO_APPROVE="auto"
+            shift;;
+        (-h|--help)
+            help;;
+        (-*)
+            error "Unknown option $1";;
+        esac
+    done
+
+    # Ensure that account, deployment and environment flags are not passed with interactive flag
+    if [ -n "$INTERACTIVE" ] && ([ -n "$ACCOUNT" ] || [ -n "$DEPLOYMENT" ] || [ -n "$ENVIRONMENT" ]); then
+        error "--interactive flag must be passed alone"
+    fi
+
+    # Ensure that environment flag is not passed with account or deployment flags
+    if [ -n "$ENVIRONMENT" ] && ([ -n "$ACCOUNT" ] || [ -n "$DEPLOYMENT" ]); then
+        error "--environment flag cannot be passed with --account or --deployment flags"
+    fi
+
+    # Ensure that account and deployment flags are passed together
+    if ([ -n "$ACCOUNT" ] && [ -z "$DEPLOYMENT" ]) || ([ -z "$ACCOUNT" ] && [ -n "$DEPLOYMENT" ]); then
+        error "--account and --deployment flags must be passed together"
+    fi
+
+    # Make environment terraform if only environment is passed
+    if [ -n "$ENVIRONMENT" ] && [ -z "$ACCOUNT" ] && [ -z "$DEPLOYMENT" ]; then
+        export DEPLOYMENT="terraform"
+    fi
+}
+
+flags "$@"
+
+# Confguring interactive mode
+interactive()
+{
+    # Get all deployments except backend and list terraform at the top
+    DEPLOYMENTS=$(ls -d deployments/* | grep -v backend | sed 's/deployments\///g' | sort)
+
+    # Ask for deployment
+    c_echo $GREEN "Which deployment would you like to deploy?"
+    select DEPLOYMENT in $DEPLOYMENTS; do test -n "$DEPLOYMENT" && break; echo ">>> Invalid Selection"; done
+
+    # Ask for environment if deployment is terraform
+    if [[ $DEPLOYMENT == *"terraform"* ]]; then
+        ENVIRONMENTS=$(ls -d environments/* | sed 's/.tfvars.json//g' | sed 's/environments\///g')
+        c_echo $GREEN "On which environment would you like to deploy?"
+        select ENVIRONMENT in $ENVIRONMENTS; do test -n "$ENVIRONMENT" && break; echo ">>> Invalid Selection"; done
+
+        # Ask for confirmation and default to yes
+        confirmation $DEPLOYMENT $ENVIRONMENT
+    fi
+
+    # Ask for account if deployment is not terraform
+    if [[ $DEPLOYMENT != *"terraform"* ]]; then
+        ACCOUNTS=$(ls -d accounts/* | grep -v backend | sed 's/.tfvars.json//g' | sed 's/accounts\///g')
+        c_echo $GREEN "On which account would you like to deploy?"
+        select ACCOUNT in $ACCOUNTS; do test -n "$ACCOUNT" && break; echo ">>> Invalid Selection"; done
+
+        # Ask for confirmation and default to yes
+        confirmation $DEPLOYMENT $ACCOUNT
+    fi
+}
+
+# If --interactive is passed, run interactive mode
+if [ -n "$INTERACTIVE" ]; then
+    c_echo $YELLOW "Running in interactive mode..."
+    interactive
+fi
+
+# Do the build and deployment only if the deployment is terraform
+if [[ $DEPLOYMENT == *"terraform"* ]]; then
+    echo
+    c_echo $GREEN "Deploying to ${ENVIRONMENT}"
+
+    deployments/terraform/terraform-apply.sh $ENVIRONMENT $AUTO_APPROVE
+    if [ "$?" -ne "0" ]; then
+        c_echo $RED "Deployment failed"
+        exit 1
+    fi
+    echo
+    c_echo $GREEN "Deployment succeeded"
+fi
+
+# Do the the deployment only if the deployment is not terraform
+if [[ $DEPLOYMENT != *"terraform"* ]]; then
+    echo
+    c_echo $GREEN "Deploying ${DEPLOYMENT} to ${ACCOUNT}"
+
+    deployments/${DEPLOYMENT}/${DEPLOYMENT}-apply.sh $ACCOUNT $AUTO_APPROVE
+    if [ "$?" -ne "0" ]; then
+        c_echo $RED "Deployment failed"
+        exit 1
+    fi
+    echo
+    c_echo $GREEN "Deployment succeeded"
+fi
+
+

src/openfactcheck/utils/logging.py

@@ -29,7 +29,7 @@ log_levels = {
     "critical": CRITICAL,
 }
 
-_default_log_level = logging.INFO
+_default_log_level = logging.WARNING
 
 
 class CustomStreamLoggingFormatter(logging.Formatter):