wip: Move to Fly app named medicode and attempt GPU

Dockerfile

@@ -26,6 +26,15 @@ RUN apt-get update -y && apt-get install -y build-essential git curl \
     libportaudio2 libportaudiocpp0 \
     && apt-get clean && rm -f /var/lib/apt/lists/*_*
 
+# Add the repository for the Nvidia CUDA
+# Import the Nvidia repository GPG key
+RUN apt update -q && apt install -y ca-certificates wget && \
+    wget -qO /cuda-keyring.deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb && \
+    dpkg -i /cuda-keyring.deb && apt update -q
+
+# Install nvidia GPU support
+RUN apt-get install -y cuda-nvcc-12-2 libcublas-12-2 libcudnn8
+
 # prepare build dir
 WORKDIR /app
 
@@ -36,6 +45,11 @@ RUN mix local.hex --force && \
 # set build ENV
 ENV MIX_ENV="prod"
 
+# bumblebee - XLA_TARGET must exist before deps pulled
+ENV XLA_TARGET="cuda120"
+ENV BUMBLEBEE_CACHE_DIR="/data/cache/bumblebee"
+ENV XLA_CACHE_DIR="/data/cache/xla"
+
 # install mix dependencies
 COPY mix.exs mix.lock ./
 RUN mix deps.get --only $MIX_ENV
@@ -95,6 +109,14 @@ RUN curl -LO https://dl.google.com/linux/direct/google-chrome-stable_current_amd
 RUN apt-get install -y ./google-chrome-stable_current_amd64.deb \
   && apt-get clean && rm -f ./google-chrome-stable_current_amd64.deb
 
+# Nvidia stuff
+RUN apt update -q && apt install -y ca-certificates wget && \
+    wget -qO /cuda-keyring.deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb && \
+    dpkg -i /cuda-keyring.deb && apt update -q
+
+# Nvidia support in runtime layer
+RUN apt-get install -y --no-install-recommends cuda-nvcc-12-2 libcublas-12-2 libcudnn8
+
 # Set the locale
 RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
 
@@ -108,6 +130,18 @@ RUN chown nobody /app
 # set runner ENV
 ENV MIX_ENV="prod"
 
+# Bumblebee
+ENV XLA_TARGET="cuda120"
+ENV BUMBLEBEE_CACHE_DIR="/data/cache/bumblebee"
+ENV XLA_CACHE_DIR="/data/cache/xla"
+
+# ENV CUDA_HOME="/usr/local/cuda"
+# ENV PATH="${CUDA_HOME}/bin:${PATH}"
+# ENV LD_LIBRARY_PATH="${CUDA_HOME}/lib64:$LD_LIBRARY_PATH"
+
+RUN echo "export PATH=/usr/local/cuda/bin:${PATH}" >> /root/.bashrc
+RUN echo "export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH" >> /root/.bashrc
+
 # Only copy the final release from the build stage
 COPY --from=builder --chown=nobody:root /app/_build/${MIX_ENV}/rel/medicode ./
 

README.md

@@ -39,7 +39,7 @@ In addition to connecting to the deployed application via `iex`, Livebook suppor
 2. Install and start Livebook: [Livebook.dev](https://livebook.dev/)
 3. [Connecting Livebook to a Production App](https://fly.io/docs/elixir/advanced-guides/connect-livebook-to-your-app/) requires a node name and cookie value:
 
-- Node name: `medical-transcription-cpu@myipfromfly` ("myipfromfly" can be retrieved with `fly ips private --app medical-transcription-cpu`)
+- Node name: `medicode@myipfromfly` ("myipfromfly" can be retrieved with `fly ips private --app medical-transcription-cpu`)
 - Cookie value: `0gfxcPtwryKxI2O1N0eFAg9p4MJGC-oUGShgj_wgvNEGiba5EDEJFA==` (this value is set in `fly.toml`)
 
 ## Run in Docker

config/prod.exs

@@ -13,6 +13,10 @@ config :logger, level: :info
 # NOTE: This path is the default from `apt-get install google-chrome-stable_current_amd64.deb` as found in Dockerfile
 config :medicode, chrome_executable: "/usr/bin/google-chrome"
 
+# For prod, disable the in-place console progress bar. It breaks the IO
+# output in production environments causing the server to crash.
+config :bumblebee, progress_bar_enabled: false
+
 config :sentry,
   dsn:
     "https://93626ebe7f1cacb3db3d824f1ef9c770@o4506193464459264.ingest.us.sentry.io/4506876998123520",

config/runtime.exs

@@ -50,7 +50,7 @@ if config_env() == :prod do
       """
 
   host = System.get_env("PHX_HOST") || "example.com"
-  port = String.to_integer(System.get_env("PORT") || "4000")
+  port = String.to_integer(System.fetch_env!("PORT"))
 
   config :medicode, :dns_cluster_query, System.get_env("DNS_CLUSTER_QUERY")
 
@@ -76,7 +76,7 @@ if config_env() == :prod do
     config :flame, :backend, FLAME.FlyBackend
 
     config :flame, FLAME.FlyBackend,
-      token: System.fetch_env!("MEDICODE_API_TOKEN"),
+      token: System.get_env("MEDICODE_API_TOKEN", "MEDICODE_API_TOKEN is currently not in use"),
       gpu_kind: "a100-pcie-40gb",
       cpu_kind: "performance",
       cpus: 8,

fly.toml

@@ -1,17 +1,11 @@
-# fly.toml app configuration file generated for medical-transcription-cpu on 2024-01-22T15:15:29-06:00
+# fly.toml app configuration file generated for medicode on 2024-04-02T09:09:18-05:00
 #
 # See https://fly.io/docs/reference/configuration/ for information about how to use this file.
 #
 
-app = "medical-transcription-cpu"
-primary_region = "ord"
-kill_signal = "SIGTERM"
-
-# TODO: Enable the following line and install the necessary CUDA packages in the Dockerfile for GPU usage.
-# The volumes will need to be recreated as well to match the GPU kind.
-# See https://fly.io/docs/gpus/gpu-quickstart/
-# and https://fly.io/docs/gpus/getting-started-gpus/
-# vm.size = "a100-40gb"
+app = 'medicode'
+primary_region = 'ord'
+kill_signal = 'SIGTERM'
 
 [build]
 
@@ -21,22 +15,22 @@ kill_signal = "SIGTERM"
 [env]
   AUDIO_TAGGER_CACHE_DIR = '/data/cache/audio_tagger'
   BUMBLEBEE_CACHE_DIR = '/data/cache/bumblebee'
-  # ELIXIR_ERL_OPTIONS = '-proto_dist inet6_tcp'
-  PHX_HOST = 'medical-transcription-cpu.fly.dev'
+  PHX_HOST = 'medicode.fly.dev'
   PORT = '8080'
   RELEASE_COOKIE = '0gfxcPtwryKxI2O1N0eFAg9p4MJGC-oUGShgj_wgvNEGiba5EDEJFA=='
   XLA_CACHE_DIR = '/data/cache/xla'
+  CUDA_HOME = '/usr/local/cuda'
 
 [[mounts]]
   source = 'data'
   destination = '/data'
-  initial_size = '40gb'
+  initial_size = '100gb'
 
 [http_service]
   internal_port = 8080
   force_https = true
   auto_stop_machines = true
-  auto_start_machines = true
+  auto_start_machines = false
   min_machines_running = 0
   processes = ['app']
 
@@ -46,6 +40,6 @@ kill_signal = "SIGTERM"
     soft_limit = 1000
 
 [[vm]]
-  memory = '8gb'
-  cpu_kind = 'shared'
+  memory = '16gb'
+  cpu_kind = 'performance'
   cpus = 4

lib/medicode/application.ex

@@ -17,7 +17,7 @@ defmodule Medicode.Application do
         Medicode.Repo,
         {DNSCluster, query: Application.get_env(:medicode, :dns_cluster_query) || :ignore},
         {Phoenix.PubSub, name: :medicode_pubsub},
-        {FLAME.Pool, name: Medicode.OfflineServing, min: 0, max: 1},
+        # {FLAME.Pool, name: Medicode.OfflineServing, min: 0, max: 1},
         Medicode.ServingSupervisor,
         {
           Medicode.TranscriptionSupervisor,