feat: Create mix task to precompute label vectors

README.md

@@ -12,6 +12,7 @@
 To start your Phoenix server:
 
   * Run `mix setup` to install and setup dependencies
+  * Run `mix build_code_vectors` to download the ICD-9 codelist, precompute vectors, and store the results in the database.
   * Start Phoenix endpoint with `mix phx.server` or inside IEx with `iex -S mix phx.server`
 
 Now you can visit [`localhost:4000`](http://localhost:4000) from your browser.

lib/mix/build_code_vectors.ex

@@ -0,0 +1,84 @@
+defmodule Mix.Tasks.BuildCodeVectors do
+  @moduledoc "Populate database with vector embeddings from downloaded ICD-9 code list"
+
+  use Mix.Task
+  alias MedicalTranscription.CodeVector
+
+  @shortdoc "Downloads the ICD-9 codelist, calculates vector embeddings for each, and adds them to the database"
+  def run(_args) do
+    Mix.Task.run("app.start")
+    Logger.configure(level: :info)
+
+    if File.exists?(csv_file()) do
+      IO.puts("CSV file found. Precomputing vectors...")
+    else
+      IO.puts("CSV file not found. Downloading and preparing...")
+      AudioTagger.SampleData.get_icd9_code_list_csv()
+
+      IO.puts("Precomputing vectors...")
+    end
+
+    precompute_vectors()
+
+    :ok
+  end
+
+  defp precompute_vectors() do
+    time_start = System.monotonic_time()
+    df = load_dataframe_from_csv()
+    model_tuple = AudioTagger.Classifier.SemanticSearch.prepare_model()
+
+    num_rows = Explorer.DataFrame.n_rows(df)
+    ProgressBar.render(0, num_rows, suffix: :count)
+
+    df
+    |> Explorer.DataFrame.to_rows_stream()
+    |> Stream.filter(fn %{"code" => code} -> String.length(code) > 0 end)
+    |> Stream.with_index()
+    |> Enum.each(fn {%{"code" => code, "long_description" => description}, index} ->
+      if !CodeVector.exists_for_code?(code) do
+        compute_vector_for_code(model_tuple, code, description)
+      end
+
+      ProgressBar.render(index + 1, num_rows, suffix: :count)
+    end)
+
+    ProgressBar.render(num_rows, num_rows, suffix: :count)
+    time_end = System.monotonic_time()
+
+    IO.puts(
+      "Finished in #{System.convert_time_unit(time_end - time_start, :native, :millisecond)}ms"
+    )
+  end
+
+  defp load_dataframe_from_csv() do
+    {:ok, df} =
+      Explorer.DataFrame.from_csv(
+        csv_file(),
+        dtypes: [
+          {"code", :string},
+          {"long_description", :string}
+        ]
+      )
+
+    df
+  end
+
+  defp compute_vector_for_code({model_info, tokenizer}, code, description) do
+    vector =
+      AudioTagger.Vectors.embed_with_model(model_info, tokenizer, [description])
+
+    vector_for_db = Nx.to_flat_list(vector.pooled_state)
+
+    CodeVector.insert_vector(%{
+      code: code,
+      description: description,
+      description_vector: vector_for_db
+    })
+  end
+
+  defp csv_file() do
+    AudioTagger.SampleData.cache_dir()
+    |> Path.join("icd9_codelist.csv")
+  end
+end

mix.exs

@@ -54,6 +54,7 @@ defmodule MedicalTranscription.MixProject do
       {:plug_cowboy, "~> 2.5"},
       {:credo, "~> 1.7.3"},
       {:audio_tagger, git: "https://github.com/headwayio/audio_tagger.git"},
+      {:progress_bar, "~> 3.0"},
       {:membrane_core, "~> 1.0"},
       {:membrane_raw_audio_format, "~> 0.12.0"}
       # {:membrane_portaudio_plugin, "~> 0.18.0"},

mix.lock

@@ -1,5 +1,5 @@
 %{
-  "audio_tagger": {:git, "https://github.com/headwayio/audio_tagger.git", "b960515109d6249a792f37aa07452373d4d8bdd1", []},
+  "audio_tagger": {:git, "https://github.com/headwayio/audio_tagger.git", "dc02cf6990ab5fef0100269c03faa6e2dbdbaea7", []},
   "aws_signature": {:hex, :aws_signature, "0.3.1", "67f369094cbd55ffa2bbd8cc713ede14b195fcfb45c86665cd7c5ad010276148", [:rebar3], [], "hexpm", "50fc4dc1d1f7c2d0a8c63f455b3c66ecd74c1cf4c915c768a636f9227704a674"},
   "axon": {:hex, :axon, "0.6.0", "fd7560079581e4cedebaf0cd5f741d6ac3516d06f204ebaf1283b1093bf66ff6", [:mix], [{:kino, "~> 0.7", [hex: :kino, repo: "hexpm", optional: true]}, {:kino_vega_lite, "~> 0.1.7", [hex: :kino_vega_lite, repo: "hexpm", optional: true]}, {:nx, "~> 0.6.0", [hex: :nx, repo: "hexpm", optional: false]}, {:polaris, "~> 0.1", [hex: :polaris, repo: "hexpm", optional: false]}, {:table_rex, "~> 3.1.1", [hex: :table_rex, repo: "hexpm", optional: true]}], "hexpm", "204e7aeb50d231a30b25456adf17bfbaae33fe7c085e03793357ac3bf62fd853"},
   "bimap": {:hex, :bimap, "1.3.0", "3ea4832e58dc83a9b5b407c6731e7bae87458aa618e6d11d8e12114a17afa4b3", [:mix], [], "hexpm", "bf5a2b078528465aa705f405a5c638becd63e41d280ada41e0f77e6d255a10b4"},