Gum script for Koboldcpp

Scripts/kobold-server.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/bash
+
+
+# requires the gum package
+# Uses conda & koboldcpp built from source
+# Assumes you are using nvidia
+
+# activates conda env named kobold
+eval "$(conda shell.bash hook)"
+conda activate kobold
+python -V
+
+# USER variables 
+# CHANGE
+KOBOLD_DIR=~/Git/koboldcpp/
+MODEL_FOLDER_DIR=~/Downloads/GGUF
+API_PORT=8001
+API_HOST=192.168.1.20
+BLAS_THREADS=16
+THREADS=8
+
+# CD to koboldcpp dir
+cd $KOBOLD_DIR
+
+# lists models
+echo "Select Model"
+MODEL=$(gum choose $(ls $MODEL_FOLDER_DIR))
+# uncomment if you use sharded models. It will take the first file in a dir and load it.
+#SHARDED_MODEL=$(ls -p $MODEL_FOLDER_DIR/$MODEL | grep -v / | head -1)
+#MODEL=$MODEL/$SHARDED_MODEL
+echo "$MODEL has been selected"
+
+echo "Layers to Offload"
+LAYERS=$(gum input --placeholder "99")
+echo "$LAYERS layers have been offloaded"
+
+echo "Context Size"
+CONTEXT=$(gum choose "4096" "8192" "12288" "16384" "32768")
+echo "Using a context size of $CONTEXT"
+
+# combined user flags
+USER_FLAGS="--host $API_HOST --port $API_PORT --flashattention --blasbatchsize 2048 --threads $THREADS --blasthreads $BLAS_THREADS --usecublas normal --contextsize $CONTEXT --gpulayers $LAYERS --model $MODEL_FOLDER_DIR/$MODEL"
+
+# runs koboldcpp
+python koboldcpp.py --skiplauncher $USER_FLAGS