Repository: moyix/fauxpilot
File: setup.sh (Shell)

Code before:
read -rp "External port for the API [5000]: " API_EXTERNAL_PORT
API_EXTERNAL_PORT=${API_EXTERNAL_PORT:-5000}

read -rp "Address for Triton [triton]: " TRITON_HOST
TRITON_HOST=${TRITON_HOST:-triton}

read -rp "Port of Triton host [8001]: " TRITON_PORT
TRITON_PORT=${TRITON_PORT:-8001}

# Read model directory
read -rp "Where do you want to save the model [$(pwd)/models]? " MODEL_DIR
if [ -z "$MODEL_DIR" ]; then
    MODEL_DIR="$(pwd)/models"
else
    MODEL_DIR="$(readlink -m "${MODEL_DIR}")"
fi

# Write .env
echo "MODEL=${MODEL}" > .env
echo "NUM_GPUS=${NUM_GPUS}" >> .env
echo "MODEL_DIR=${MODEL_DIR}/${MODEL}-${NUM_GPUS}gpu" >> .env
echo "API_EXTERNAL_PORT=${API_EXTERNAL_PORT}" >> .env
echo "TRITON_HOST=${TRITON_HOST}" >> .env
echo "TRITON_PORT=${TRITON_PORT}" >> .env
GPUS="$(seq -s ','  -t '\n' "${NUM_GPUS}")"
echo "GPUS=${GPUS%?}" >> .env

if [ -d "$MODEL_DIR"/"${MODEL}"-"${NUM_GPUS}"gpu ]; then
    echo "Converted model for ${MODEL}-${NUM_GPUS}gpu already exists."
    read -rp "Do you want to re-use it? y/n: " REUSE_CHOICE
    if [[ ${REUSE_CHOICE:-y} =~ ^[Yy]$ ]]
    then
        echo "Re-using model"
    else
      # Create model directory
      mkdir -p "${MODEL_DIR}"

      if [ "$NUM_GPUS" -le 2 ]; then
        echo "Downloading the model from HuggingFace, this will take a while..."
        SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
        DEST="${MODEL}-${NUM_GPUS}gpu"
        ARCHIVE="${MODEL_DIR}/${DEST}.tar.zst"
        cp -r "$SCRIPT_DIR"/converter/models/"$DEST" "${MODEL_DIR}"
        curl -L "https://huggingface.co/moyix/${MODEL}-gptj/resolve/main/${MODEL}-${NUM_GPUS}gpu.tar.zst" \
            -o "$ARCHIVE"
        zstd -dc "$ARCHIVE" | tar -xf - -C "${MODEL_DIR}"
        rm -f "$ARCHIVE"
      else
        echo "Downloading and converting the model, this will take a while..."
        docker run --rm -v "${MODEL_DIR}":/models -e MODEL=${MODEL} -e NUM_GPUS="${NUM_GPUS}" moyix/model_converter:latest
      fi

Review comment (suggestion):
seq works differently on Mac (BSD sed) and Linux (GNU sed). GNU sed's `-s` inserts the delimiter between each pair of items, whereas BSD's adds the delimiter after each item, which is why you need the `${GPUS%?}` – but on Linux this will delete the last character even when it's not a `,`.

Also, on both OSes, `seq` starts at 1 and goes to the endpoint, inclusive. But CUDA_VISIBLE_DEVICES is zero-indexed, so we instead want `seq 0 $((NUM_GPUS-1))`.

I think the following resolves all of these differences and works on both:

```
GPUS=$(seq 0 $(( NUM_GPUS - 1)) | paste -s -d ',' -)
```

Code after:
read -rp "External port for the API [5000]: " API_EXTERNAL_PORT
API_EXTERNAL_PORT=${API_EXTERNAL_PORT:-5000}

read -rp "Address for Triton [triton]: " TRITON_HOST
TRITON_HOST=${TRITON_HOST:-triton}

read -rp "Port of Triton host [8001]: " TRITON_PORT
TRITON_PORT=${TRITON_PORT:-8001}

# Read model directory
read -rp "Where do you want to save the model [$(pwd)/models]? " MODEL_DIR
if [ -z "$MODEL_DIR" ]; then
    MODEL_DIR="$(pwd)/models"
else
    MODEL_DIR="$(readlink -m "${MODEL_DIR}")"
fi

# Write .env
echo "MODEL=${MODEL}" > .env
echo "NUM_GPUS=${NUM_GPUS}" >> .env
echo "MODEL_DIR=${MODEL_DIR}/${MODEL}-${NUM_GPUS}gpu" >> .env
echo "API_EXTERNAL_PORT=${API_EXTERNAL_PORT}" >> .env
echo "TRITON_HOST=${TRITON_HOST}" >> .env
echo "TRITON_PORT=${TRITON_PORT}" >> .env
echo "GPUS=$(seq 0 $(( NUM_GPUS - 1)) | paste -s -d ',' -)" >> .env

if (test -d "$MODEL_DIR"/"${MODEL}"-"${NUM_GPUS}"gpu ); then
  echo "$MODEL_DIR"/"${MODEL}"-"${NUM_GPUS}"gpu
  echo "Converted model for ${MODEL}-${NUM_GPUS}gpu already exists."
  read -rp "Do you want to re-use it? y/n: " REUSE_CHOICE
  if [[ ${REUSE_CHOICE:-y} =~ ^[Yy]$ ]]
  then
    DOWNLOAD_MODEL=n
    echo "Re-using model"
  else
    DOWNLOAD_MODEL=y
    rm -rf "$MODEL_DIR"/"${MODEL}"-"${NUM_GPUS}"gpu
  fi
else
  DOWNLOAD_MODEL=y
fi

if [[ ${DOWNLOAD_MODEL:-y} =~ ^[Yy]$ ]]
then
  # Create model directory
  mkdir -p "${MODEL_DIR}"
  if [ "$NUM_GPUS" -le 2 ]; then
    echo "Downloading the model from HuggingFace, this will take a while..."
    SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
    DEST="${MODEL}-${NUM_GPUS}gpu"
    ARCHIVE="${MODEL_DIR}/${DEST}.tar.zst"