Repository: ModioAB/caramel (language: python)

Issue:
Add launcher that use env-vars for conf 
Adds the POSIX-script caramel_launcher.sh that just starts caramel using pserve with the addition that it reads the environment for server configuration, using caramel_launcher.ini as a bypass for the need to set server configuration in an ini-file. Current env-vars are:

CARAMEL_CA_CERT : a path to a ca cert to use, default ${BASEDIR}/example/caramel.ca.cert
CARAMEL_CA_KEY : a path to a ca key to use, default ${BASEDIR}/example/caramel.ca.key
CARAMEL_PORT : port number for caramel server to use, default 6543
CARAMEL_HOST : address for caramel server to use, defualt 127.0.0.1
CARAMEL_LIFE_SHORT : the short lifetime for CSRs, default 48
CARAMEL_LIFE_LONG : the long lifetime for CSRs, default 720
CARAMEL_DBURL : URI to database to use, default sqlite:///${BASEDIR}/caramel.sqlite
CARAMEL_LOG_LEVEL : what level the root logger should be at, default INFO

~~Based on branch #66~~, related to issue #55


Discussion / hints:

Fix patch:
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 171d14e..6a219e5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -30,6 +30,7 @@ caramel:test:
     - pip3 install .
   script:
     - python3 -m unittest discover
+    - make systest
 
 rebase:test:
   extends: .rebase
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..f9c7ea4
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,156 @@
+#Python3 and virtual environment
+VENV := $(shell mktemp -d)
+PYTHON3 := $(VENV)/bin/python3
+
+# PIDs of background processes
+SERVER_PID := $(VENV)/server.pid
+AUTOSIGN_PID := $(VENV)/autosign.pid
+
+# Database and CA cert and key for caramel server to use
+DB_FILE := $(VENV)/caramel.sqlite
+CA_CERT := $(VENV)/caramel.ca.cert
+CA_KEY := $(VENV)/caramel.ca.key
+
+# client.crt will be generated if the server correctly gives our stored CSR back
+CLIENT_CERT := $(VENV)/client.crt
+
+# If caramel_tool exists in the venv caramel has been installed
+CARAMEL_TOOL := $(VENV)/bin/caramel_tool
+
+# Terminal formatting
+BOLD := tput bold
+PASS := $(BOLD); tput setaf 2
+FAIL := $(BOLD); tput setaf 1
+LINE := echo "---------------------------------------"
+RESET_TERM := tput op; tput sgr0
+BLR := $(BOLD); $(LINE); $(RESET_TERM) #Bold Line, Reset formatting
+
+#Check for python3 install and virtual environment
+$(PYTHON3):
+	@if [ -z python3 ]; then \
+		$(FAIL);\
+		echo "Python 3 could not be found.";\
+		$(RESET_TERM);\
+		exit 2; \
+	fi
+	@$(BOLD); echo "Create a new venv for testing at $(VENV)";\
+	$(BLR);
+	python3 -m venv $(VENV)
+	@$(BLR)
+
+
+#Install the project via setup.py
+.PHONEY: venv-install
+venv-install: $(CARAMEL_TOOL)
+$(CARAMEL_TOOL): $(PYTHON3) setup.py
+	@$(BOLD); echo "Install caramel and its dependencies in venv: $(VENV)";\
+	$(BLR);
+	$(VENV)/bin/pip3 install -e .
+	@$(BLR)
+
+# Create a sqlite-db configured for use with caramel
+$(DB_FILE): $(CARAMEL_TOOL)
+	@$(BOLD); echo "Create a new DB at $(DB_FILE)";\
+	$(BLR);
+	$(VENV)/bin/caramel_initialize_db
+	@$(BLR)
+
+# Generate a new CA cert and key pair
+$(CA_CERT): $(CARAMEL_TOOL)
+	@$(BOLD); echo "Generate new CA cert and key with tests/ca_test_input.txt";\
+	$(BLR)
+	$(VENV)/bin/caramel_ca < tests/ca_test_input.txt
+	@$(BLR)
+
+# Start caramel using pserve in the background, save PID to SERVER_PID
+$(SERVER_PID): $(CA_CERT) $(DB_FILE)
+	@ $(BOLD); echo "Start new caramel server in the background, sleep 2s to \
+	give it time to start";\
+	$(BLR)
+	setsid $(VENV)/bin/pserve tests/caramel_launcher.ini \
+		ca_cert=$(CARAMEL_CA_CERT) \
+		ca_key=$(CARAMEL_CA_KEY) \
+		http_port=6543 \
+		http_host=127.0.0.1 \
+		life_short=48 \
+		life_long=720 \
+		dburl=$(CARAMEL_DBURL) \
+		log_level=INFO\
+		>/dev/null 2>&1 < /dev/null & \
+	echo $$! > $(SERVER_PID)
+	sleep 2s
+	@$(BLR)
+
+# Start caramel_autosign in the background, save PID to AUTOSIGN_PID
+$(AUTOSIGN_PID): $(CARAMEL_TOOL) $(CA_CERT) $(DB_FILE)
+	@$(BOLD);echo "Start new caramel_autosign in the background";\
+	$(BLR)
+	setsid $(VENV)/bin/caramel_autosign >/dev/null 2>&1 < /dev/null &\
+	echo $$! > $(AUTOSIGN_PID)
+	@$(BLR)
+
+# Try to upload a CSR to a caramel server and then confirm our CSR was stored
+$(CLIENT_CERT): $(SERVER_PID) $(AUTOSIGN_PID)
+	@ $(BOLD); echo "Use client-example.sh to upload our CSR, wait for it to \
+	get processed, then call it again to confirm the server stored our CSR";\
+	$(BLR)
+	chmod +x client-example.sh
+	./client-example.sh $(VENV)
+	sleep 2s
+	./client-example.sh $(VENV)
+	@$(BLR)
+
+# Basic tests that caramel can be installed and run with test data,
+# using environment variables for config
+.PHONEY: systest
+systest: export CARAMEL_INI = development.ini
+systest: export CARAMEL_DBURL = sqlite:///$(DB_FILE)
+systest: export CARAMEL_CA_CERT = $(CA_CERT)
+systest: export CARAMEL_CA_KEY = $(CA_KEY)
+systest: export CARAMEL_CA = 127.0.0.1:6543
+systest: export CARAMEL_LOG_LEVEL = 0
+systest: $(CLIENT_CERT)
+	@kill $(shell cat $(SERVER_PID));\
+	if [ $$? -eq 0 ]; then \
+		$(PASS); $(LINE);\
+		echo "Caramel server started and terminated successfully";\
+	else \
+		$(FAIL); $(LINE);\
+		echo "System test failed: Caramel server exited before termination with\
+		 exit code: $$?";\
+		exit 1;\
+	fi;
+	@$(BLR)
+
+	@kill $(shell cat $(AUTOSIGN_PID));\
+	if [ $$? -eq 0 ]; then \
+		$(PASS); $(LINE);\
+		echo "Autosign server started and terminated successfully";\
+	else \
+		$(FAIL); $(LINE);\
+		echo "System test failed: Autosign server exited before terminated with\
+		 exit code: $$?";\
+		exit 1;\
+	fi;
+	@$(BLR)
+
+	@if [ -f $(CLIENT_CERT) ]; then \
+		$(PASS); $(LINE);\
+		echo "System test passed: Caramel successfully registered our CSR";\
+	else \
+		$(FAIL); $(LINE);\
+		echo "System test failed: Something went wrong when communicating with \
+		the server";\
+		exit 1;\
+	fi;\
+	$(BLR)
+
+
+# Removes the virtual environment created via this makefile,
+# NOTE: since venv is a temporary directory, this will only remove files if
+# 		run after a target, e.g. "make systest clean"
+.PHONEY: clean
+clean:
+	@echo "Removing local test virtual environment"; $(BLR)
+	rm -rf $(VENV)
+	@$(BLR)
diff --git a/client-example.sh b/client-example.sh
index a666f83..15cc328 100755
--- a/client-example.sh
+++ b/client-example.sh
@@ -1,14 +1,15 @@
 #! /bin/sh
-CA_CERT=/etc/pki/tls/certs/ca.example.com.crt
-MY_CRT=/etc/pki/tls/certs/client.example.com.crt
-MY_CSR=/etc/pki/tls/certs/client.example.com.csr
-MY_KEY=/etc/pki/tls/private/client.example.com.key
+CA_DIR="$1"
+CA_CERT="$CA_DIR/caramel.ca.cert"
+MY_CRT="$CA_DIR/client.crt"
+MY_CSR="$CA_DIR/client.csr"
+MY_KEY="$CA_DIR/client.key"
 
-CLIENTID=`cat /etc/machine-id`   # For example
-CLIENTID=`sed -e 's/://g' /sys/class/net/eth0/address`
+CLIENTID=$(cat /etc/machine-id)               # For example machine ID or MAC-address
+CLIENTID=f11f05a9-773e-4b7e-b985-f5eb8fe18852 # or UUID https://xkcd.com/221/
 
-SUBJECT="/O=ExampleInc/OU=ExampleCom/CN=$CLIENTID"
-POST_URL=https://caramel.example.com/ra
+SUBJECT="/C=SE/ST=Östergötland/L=Linköping/O=Muppar AB/OU=Muppar Teknik/CN=$CLIENTID"
+POST_URL=http://127.0.0.1:6543/
 
 echo $SUBJECT
 
@@ -28,7 +29,6 @@ CSRSUM=$(sha256sum $MY_CSR |cut -f1 -d" ")
 
 CURL_OPTS="--silent --show-error --remote-time --connect-timeout 300 --max-time 600 --cacert $CA_CERT"
 
-
 # The logic here is fun.
 # If I don't have a cert, we try to download it.
 #   if we get 404 from downloading it, we upload the CSR.
@@ -60,11 +60,11 @@ fi
 #  if 404
 #       upload CSR
 
-STATUS=$(curl ${CURL_OPTS}  -w '%{http_code}' --url ${POST_URL}/${CSRSUM} -o ${CSRSUM})
+STATUS=$(curl ${CURL_OPTS} -w '%{http_code}' --url ${POST_URL}/"${CSRSUM}" -o "${CA_DIR}/${CSRSUM}")
 
 if [ $STATUS -eq 200 ];
 then
-    mv $CSRSUM $MY_CRT
+    mv $CA_DIR/$CSRSUM $MY_CRT
 fi
 
 if [ $STATUS -eq 202 -o $STATUS -eq 304 ];
@@ -76,6 +76,7 @@ fi
 
 if [ $STATUS -eq 404 ];
 then
-    rm -f $CSRSUM
-    curl ${CURL_OPTS} --data-binary @$MY_CSR ${POST_URL}/${CSRSUM}
+  rm -f "$CA_DIR/$CSRSUM"
+  curl ${CURL_OPTS} --data-binary @$MY_CSR ${POST_URL}/${CSRSUM}
+  echo "" # ugly "hack" to guarantee new line before next command
 fi


Test patch:
diff --git a/tests/ca_test_input.txt b/tests/ca_test_input.txt
new file mode 100644
index 0000000..996c504
--- /dev/null
+++ b/tests/ca_test_input.txt
@@ -0,0 +1,5 @@
+SE
+Östergötland
+Linköping
+Muppar AB
+Muppar Teknik
\ No newline at end of file
diff --git a/tests/caramel_launcher.ini b/tests/caramel_launcher.ini
new file mode 100644
index 0000000..381b323
--- /dev/null
+++ b/tests/caramel_launcher.ini
@@ -0,0 +1,80 @@
+###
+# app configuration
+# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html
+###
+[app:main]
+use = egg:caramel
+
+# Point the following two settings on where you want your CA certificate & Key to live
+ca.cert = %(ca_cert)s
+ca.key = %(ca_key)s
+
+# This causes all certs to be backdated to the age of the start cert.
+# This is an ugly workaround for our embedded systems that lack RTC.
+backdate = False
+# Default to 48 hour certs
+lifetime.short = %(life_short)s
+# Long term certs are for 30 days
+lifetime.long = %(life_long)s
+
+
+# Change this to match your database
+# http://docs.sqlalchemy.org/en/rel_0_9/core/engines.html#database-urls
+sqlalchemy.url = %(dburl)s
+
+
+pyramid.reload_templates = true
+pyramid.debug_authorization = false
+pyramid.debug_notfound = false
+pyramid.debug_routematch = false
+pyramid.default_locale_name = en
+pyramid.includes =
+    pyramid_tm
+
+###
+# wsgi server configuration
+###
+[server:main]
+use = egg:waitress#main
+host = %(http_host)s
+port = %(http_port)s
+
+###
+# logging configuration
+# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html
+###
+
+[loggers]
+keys = root, caramel, sqlalchemy
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = %(log_level)s
+handlers = console
+
+[logger_caramel]
+level = DEBUG
+handlers =
+qualname = caramel
+
+[logger_sqlalchemy]
+level = INFO
+handlers =
+qualname = sqlalchemy.engine
+# "level = INFO" logs SQL queries.
+# "level = DEBUG" logs SQL queries and results.
+# "level = WARN" logs neither.  (Recommended for production systems.)
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s
diff --git a/tests/caramel_launcher.sh b/tests/caramel_launcher.sh
new file mode 100755
index 0000000..36fbbd9
--- /dev/null
+++ b/tests/caramel_launcher.sh
@@ -0,0 +1,12 @@
+#! /bin/sh
+## Launch caramel with pserve as a Pyramid WSGI app using environment variables
+## for configuration
+pserve caramel_launcher.ini \
+  ca_cert="${CARAMEL_CA_CERT:-${BASEDIR}/example/caramel.ca.cert}" \
+  ca_key="${CARAMEL_CA_KEY:-${BASEDIR}/example/caramel.ca.key}" \
+  http_port="${CARAMEL_PORT:-6543}" \
+  http_host="${CARAMEL_HOST:-127.0.0.1}" \
+  life_short="${CARAMEL_LIFE_SHORT:-48}" \
+  life_long="${CARAMEL_LIFE_LONG:-720}" \
+  dburl="${CARAMEL_DBURL:-sqlite:///${BASEDIR}/caramel.sqlite}" \
+  log_level="${CARAMEL_LOG_LEVEL:-INFO}"