chore: Move audio recording state to sidebar button

assets/js/app.js

@@ -52,6 +52,42 @@ Hooks.ContentEditor = {
   },
 };
 
+let audioContext = null;
+let pcmNode = null;
+
+Hooks.AudioRecorder = {
+  mounted() {
+    this.handleEvent("start_audio_recording", async () => {
+      const audio = this.el;
+      const context = new AudioContext({ sampleRate: 16_000 });
+
+      await context.audioWorklet.addModule("/assets/pcm-processor.js");
+
+      const node = new AudioWorkletNode(context, "pcm-processor", {
+        processorOptions: { chunkSize: 16_000 },
+      });
+
+      node.port.onmessage = (event) => {
+        const audioData = new Float32Array(event.data);
+        this.pushEvent("audio_chunk", { data: event.data });
+      };
+
+      audioContext = context;
+      pcmNode = node;
+
+      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+      const source = audioContext.createMediaStreamSource(stream);
+
+      source.connect(pcmNode);
+      pcmNode.connect(audioContext.destination); // Necessary but doesn't output sound to speakers
+    });
+
+    this.handleEvent("stop_audio_recording", async () => {
+      audioContext.close();
+    });
+  },
+};
+
 let csrfToken = document
   .querySelector("meta[name='csrf-token']")
   .getAttribute("content");

assets/js/pcm-processor.js

@@ -0,0 +1,24 @@
+class PCMProcessor extends AudioWorkletProcessor {
+  constructor(options) {
+    super();
+    this.sampleBuffer = [];
+    this.chunkSize = options.processorOptions.chunkSize || 16_000;
+  }
+
+  process(inputs) {
+    const input = inputs[0];
+    if (input.length > 0) {
+      const inputData = input[0];
+      for (let i = 0; i < inputData.length; ++i) {
+        this.sampleBuffer.push(inputData[i]);
+        if (this.sampleBuffer.length >= this.chunkSize) {
+          this.port.postMessage(this.sampleBuffer.slice(0, this.chunkSize));
+          this.sampleBuffer = this.sampleBuffer.slice(this.chunkSize);
+        }
+      }
+    }
+    return true;
+  }
+}
+
+registerProcessor("pcm-processor", PCMProcessor);

config/config.exs

@@ -44,7 +44,7 @@ config :esbuild,
   version: "0.17.11",
   default: [
     args:
-      ~w(js/app.js js/storybook.js --bundle --target=es2017 --outdir=../priv/static/assets --external:/fonts/* --external:/images/*),
+      ~w(js/app.js js/storybook.js js/pcm-processor.js --bundle --target=es2017 --outdir=../priv/static/assets --external:/fonts/* --external:/images/*),
     cd: Path.expand("../assets", __DIR__),
     env: %{"NODE_PATH" => Path.expand("../deps", __DIR__)}
   ]

lib/medicode/application.ex

@@ -23,7 +23,6 @@ defmodule Medicode.Application do
         transcription_spec(),
         token_classification_spec(),
         text_embedding_spec(),
-        {Registry, keys: :unique, name: :transcription_registry},
         {
           Medicode.TranscriptionSupervisor,
           strategy: :one_for_one, max_restarts: 1

lib/medicode/transcription_server.ex

@@ -7,10 +7,13 @@ defmodule Medicode.TranscriptionServer do
   alias Medicode.Transcriptions
   alias Medicode.Transcriptions.Transcription
 
-  @registry :transcription_registry
-
-  def start_link(%{transcription: transcription, name: name}) do
-    GenServer.start_link(__MODULE__, {:transcription, transcription}, name: via_tuple(name))
+  def start_link(%{
+        transcription: transcription,
+        transcription_id: transcription_id
+      }) do
+    GenServer.start_link(__MODULE__, %{transcription: transcription},
+      name: via_tuple(transcription_id)
+    )
   end
 
   @doc """
@@ -38,7 +41,19 @@ defmodule Medicode.TranscriptionServer do
   end
 
   @impl GenServer
-  def handle_continue(:start, {:transcription, transcription}) do
+  def handle_continue(:start, %{transcription: %{status: :recording}} = state) do
+    %{transcription: transcription} = state
+
+    Phoenix.PubSub.broadcast(
+      :medicode_pubsub,
+      "transcriptions:#{transcription.id}",
+      {:transcription_started, transcription.id}
+    )
+
+    {:noreply, %{transcription: transcription}}
+  end
+
+  def handle_continue(:start, %{transcription: transcription}) do
     {:ok, transcription} =
       Transcriptions.update_transcription(transcription, %{status: :transcribing})
 
@@ -50,12 +65,12 @@ defmodule Medicode.TranscriptionServer do
 
     stream_transcription_and_search(transcription.filename)
 
-    {:noreply, {:transcription, transcription}}
+    {:noreply, %{transcription: transcription}}
   end
 
   @impl GenServer
   def handle_info({:chunk, result}, state) do
-    {:transcription, transcription} = state
+    %{transcription: transcription} = state
 
     %Transcription{id: id} = transcription
 
@@ -85,7 +100,7 @@ defmodule Medicode.TranscriptionServer do
 
   @impl GenServer
   def terminate(reason, state) do
-    {:transcription, transcription} = state
+    %{transcription: transcription} = state
 
     {:ok, transcription} =
       Transcriptions.update_transcription(transcription, %{status: :finished})
@@ -101,6 +116,40 @@ defmodule Medicode.TranscriptionServer do
     reason
   end
 
+  @impl GenServer
+  # def handle_call({:recording, %{data: data}}, _from, state) do
+  def handle_cast({:recording, %{data: data}}, state) do
+    tensor =
+      Nx.tensor(data)
+      |> Nx.stack()
+      |> Nx.reshape({:auto, 1})
+      |> Nx.mean(axes: [1])
+
+    # audio transcription + semantic search
+    Medicode.TranscriptionServing
+    |> Nx.Serving.batched_run(tensor)
+    |> Enum.each(fn chunk ->
+      dbg(chunk)
+
+      result = %{
+        start_mark: format_timestamp(chunk.start_timestamp_seconds),
+        end_mark: format_timestamp(chunk.end_timestamp_seconds),
+        text: chunk.text
+      }
+
+      send(self(), {:chunk, result})
+    end)
+
+    {:noreply, state}
+  end
+
+  def process_recording_data(transcription_id, data) do
+    GenServer.cast(
+      {:via, :gproc, {:n, :l, {:transcription, transcription_id}}},
+      {:recording, %{data: data}}
+    )
+  end
+
   # Ideas for future exploration:
   # - A potential improvement would be to not code each chunk of transcribed audio separately, but to instead gather
   #   complete sentences based on punctuation. We may want to suggest codes for the entire audio as a single piece as
@@ -128,6 +177,8 @@ defmodule Medicode.TranscriptionServer do
     seconds |> round() |> Time.from_seconds_after_midnight() |> Time.to_string()
   end
 
-  defp via_tuple(name),
-    do: {:via, Registry, {@registry, name}}
+  defp via_tuple(transcription_id),
+    # NOTE: gproc requires keys that are a tuple of {type, scope, key}
+    # See https://www.brianstorti.com/process-registry-in-elixir
+    do: {:via, :gproc, {:n, :l, {:transcription, transcription_id}}}
 end

lib/medicode/transcription_supervisor.ex

@@ -17,7 +17,7 @@ defmodule Medicode.TranscriptionSupervisor do
   def start_transcription(transcription) do
     spec = {
       Medicode.TranscriptionServer,
-      %{transcription: transcription, name: "transcription:#{transcription.id}"}
+      %{transcription: transcription, transcription_id: transcription.id}
     }
 
     DynamicSupervisor.start_child(__MODULE__, spec)

lib/medicode/transcriptions/transcription.ex

@@ -13,7 +13,7 @@ defmodule Medicode.Transcriptions.Transcription do
     field :recording_length_in_seconds, :integer
 
     field :status, Ecto.Enum,
-      values: [new: 0, waiting: 1, transcribing: 2, finished: 3, failed: 4]
+      values: [new: 0, waiting: 1, transcribing: 2, finished: 3, failed: 4, recording: 5]
 
     belongs_to :user, Medicode.Accounts.User
 

lib/medicode_web/components/components.ex

@@ -102,8 +102,6 @@ defmodule MedicodeWeb.Components do
           class="mr-[17px]"
         />
 
-        <.record_button_in_heading status={@transcription.status} />
-
         <div class="px-[14px] py-3 flex items-center gap-3 bg-brand rounded-lg text-white mr-4 overflow-hidden">
           <img src={~p"/images/document.svg"} width="20" />
           <p
@@ -165,33 +163,6 @@ defmodule MedicodeWeb.Components do
     """
   end
 
-  # Renders the record button within the result_heading component
-  defp record_button_in_heading(assigns) when assigns.status == :streaming_audio do
-    ~H"""
-    <button
-      disabled
-      phx-click="toggle_recording"
-      title="Not available"
-      class="cursor-not-allowed mr-6 px-4 py-3 bg-red-200 rounded-lg"
-    >
-      <.icon name="hero-microphone" class="animate-pulse" />
-    </button>
-    """
-  end
-
-  defp record_button_in_heading(assigns) do
-    ~H"""
-    <button
-      disabled
-      phx-click="toggle_recording"
-      title="Not available"
-      class="cursor-not-allowed mr-6 px-4 py-3 bg-emerald-200 rounded-lg"
-    >
-      <.icon name="hero-microphone" />
-    </button>
-    """
-  end
-
   # Formats keywords for display in the result_heading component
   defp format_keywords(keyword_predictions) do
     keyword_predictions

lib/medicode_web/components/sidebar_component.ex

@@ -22,6 +22,7 @@ defmodule MedicodeWeb.Components.SidebarComponent do
 
     socket =
       socket
+      |> assign(:status, nil)
       |> assign(:current_user, assigns.current_user)
       |> assign(:transcriptions, list_transcriptions(assigns.current_user))
 
@@ -49,14 +50,18 @@ defmodule MedicodeWeb.Components.SidebarComponent do
             <.icon name="hero-document-plus" />
             <span>Upload Audio File</span>
           </.link>
-          <.link
+          <button
+            id="audio-recorder-button"
+            phx-hook="AudioRecorder"
+            phx-click="record_transcription"
+            phx-target={@myself}
             navigate={~p"/transcriptions/new"}
             class="text-[0.8125rem] flex justify-center gap-2 w-full text-white font-semibold hover:text-slate-300 px-3 py-2 bg-emerald-600 rounded-lg"
           >
-            <.icon name="hero-microphone" />
+            <.icon :if={@status == :streaming_audio} name="hero-speaker-wave" />
+            <.icon :if={@status != :streaming_audio} name="hero-microphone" />
             <span>Record Patient Notes</span>
-          </.link>
-          <p class="text-xs leading-normal tracking-[0.2em] font-semibold uppercase">Today</p>
+          </button>
           <div class="flex flex-col gap-4">
             <.link
               :for={transcription <- @transcriptions}
@@ -119,6 +124,31 @@ defmodule MedicodeWeb.Components.SidebarComponent do
     """
   end
 
+  @impl Phoenix.LiveView
+  def handle_event("record_transcription", _params, socket) do
+    socket =
+      if socket.assigns.status == :streaming_audio do
+        socket
+        |> push_event("stop_audio_recording", %{})
+        |> assign(:status, nil)
+      else
+        {:ok, new_transcription} =
+          Transcriptions.create_transcription(%{
+            user_id: socket.assigns.current_user.id,
+            filename: "Untitled",
+            status: :recording
+          })
+
+        Medicode.TranscriptionSupervisor.start_transcription(new_transcription)
+
+        socket
+        |> assign(:status, :streaming_audio)
+        |> push_patch(to: ~p"/transcriptions/#{new_transcription.id}")
+      end
+
+    {:noreply, socket}
+  end
+
   @impl Phoenix.LiveView
   def handle_info({:transcription_created, _transcription_id}, socket) do
     transcriptions = list_transcriptions(socket.assigns.current_user)

lib/medicode_web/components/transcription_text_component.ex

@@ -45,7 +45,8 @@ defmodule MedicodeWeb.Components.TranscriptionTextComponent do
         # NOTE: This assumes that if a process matches by transcription chunk ID that
         # the chunk is in the process of loading. Specific state inspection could happen
         # with the :sys module (ie :sys.get_state/1)
-        case Registry.lookup(:transcription_registry, "transcription_chunk:#{assigns.chunk.id}") do
+        # case Registry.lookup({:via, :gproc, {:n, :l, {:transcription, transcription_id}}}) do
+        case [] do
           [] -> false
           [{_pid, _}] -> true
         end

lib/medicode_web/live/transcriptions_live/show.ex

@@ -1,22 +1,36 @@
 defmodule MedicodeWeb.TranscriptionsLive.Show do
   use MedicodeWeb, :live_view
 
-  alias Medicode.Audio.RecordingPipeline
   alias Medicode.Transcriptions
 
   alias MedicodeWeb.Components.TranscriptionTextComponent
   alias MedicodeWeb.Components.TranscriptionTextCodingsComponent
 
   @impl Phoenix.LiveView
-  def mount(params, session, socket) do
-    transcription = get_transcription(params["id"])
-
+  def mount(_params, session, socket) do
     timezone = get_connect_params(socket)["timezone"] || "Etc/UTC"
 
+    socket =
+      socket
+      |> assign(:status, nil)
+      |> assign(:current_user, Medicode.Accounts.get_user!(session["current_user"].id))
+      |> assign(:timezone, timezone)
+      |> stream_configure(:chunk_ids, dom_id: &"transcription-chunk-#{&1.id}")
+
+    {:ok, socket}
+  end
+
+  @impl Phoenix.LiveView
+  def handle_params(params, _url, socket) do
+    if connected?(socket) do
+      Phoenix.PubSub.subscribe(:medicode_pubsub, "medicode:#{socket.assigns.current_user.id}")
+    end
+
+    transcription = get_transcription(params["id"])
+
     if is_nil(transcription), do: raise(Medicode.Fallback)
 
     if connected?(socket) do
-      Phoenix.PubSub.subscribe(:medicode_pubsub, "medicode:#{session["current_user"].id}")
       Phoenix.PubSub.subscribe(:medicode_pubsub, "transcriptions:#{transcription.id}")
     end
 
@@ -26,24 +40,24 @@ defmodule MedicodeWeb.TranscriptionsLive.Show do
     finalized_codes = Transcriptions.list_transcription_finalized_codes(transcription.id)
 
     initial_state = %{
-      current_recording_id: 0,
-      uploaded_file_name: nil,
-      status: :pending,
-      audio_pipeline: nil,
       summary_keywords: summary_keywords,
       transcription: transcription,
-      transcriptions: list_transcriptions(session["current_user"]),
-      finalized_codes: finalized_codes,
-      timezone: timezone
+      finalized_codes: finalized_codes
     }
 
     socket =
       socket
       |> assign(initial_state)
-      |> stream_configure(:chunk_ids, dom_id: &"transcription-chunk-#{&1.id}")
-      |> stream(:chunk_ids, transcription_chunk_ids)
+      |> stream(:chunk_ids, transcription_chunk_ids, reset: true)
 
-    {:ok, socket}
+    socket =
+      if transcription.status == :recording do
+        push_event(socket, "start_audio_recording", %{})
+      else
+        socket
+      end
+
+    {:noreply, socket}
   end
 
   @impl Phoenix.LiveView
@@ -59,6 +73,7 @@ defmodule MedicodeWeb.TranscriptionsLive.Show do
       <main class="flex-1 pl-16 pr-16 pt-[25px]">
         <div class="flex flex-col h-full mx-auto max-w-5xl">
           <.result_heading
+            :if={@transcription}
             id="transcription-name"
             target={self()}
             transcription={@transcription}
@@ -68,7 +83,7 @@ defmodule MedicodeWeb.TranscriptionsLive.Show do
           />
 
           <img
-            :if={@transcription.status == :transcribing}
+            :if={@transcription && @transcription.status == :transcribing}
             src={~p"/images/loading.svg"}
             width="36"
             class="m-8 animate-spin"
@@ -112,23 +127,39 @@ defmodule MedicodeWeb.TranscriptionsLive.Show do
 
   def handle_event("toggle_recording", _params, socket) do
     socket =
-      if is_nil(socket.assigns.audio_pipeline) do
-        pipeline = RecordingPipeline.start_pipeline(self())
-
+      if socket.assigns.status == :streaming_audio do
         socket
-        |> assign(:status, :streaming_audio)
-        |> assign(:audio_pipeline, pipeline)
+        |> push_event("stop_audio_recording", %{})
+        |> assign(:status, nil)
       else
-        RecordingPipeline.stop_pipeline(socket.assigns.audio_pipeline)
+        {:ok, new_transcription} =
+          Transcriptions.create_transcription(%{
+            user_id: socket.assigns.current_user.id,
+            filename: "Untitled"
+          })
+
+        Medicode.TranscriptionSupervisor.start_transcription(new_transcription, true)
 
         socket
-        |> assign(:status, :success)
-        |> assign(:audio_pipeline, nil)
+        |> assign(:status, :streaming_audio)
+        |> push_event("start_audio_recording", %{})
+        |> push_patch(to: "/transcriptions/#{new_transcription.id}")
+
+        #
+        # socket
+        # |> assign(:transcription, transcription)
+        # |> assign(:summary_keywords, [])
+        # |> stream(:chunk_ids, [], reset: true)
       end
 
     {:noreply, socket}
   end
 
+  def handle_event("audio_chunk", %{"data" => data}, socket) do
+    Medicode.TranscriptionServer.process_recording_data(socket.assigns.transcription.id, data)
+    {:noreply, socket}
+  end
+
   def handle_event("rename_transcription", %{"value" => new_name}, socket) do
     new_name_trimmed = String.trim(new_name)
 
@@ -150,17 +181,7 @@ defmodule MedicodeWeb.TranscriptionsLive.Show do
     end
   end
 
-  @impl true
-  def handle_info({:transcription_created, _transcription_id}, socket) do
-    transcriptions = list_transcriptions(socket.assigns.current_user)
-
-    socket =
-      socket
-      |> assign(:transcriptions, transcriptions)
-
-    {:noreply, socket}
-  end
-
+  @impl Phoenix.LiveView
   def handle_info({:transcription_updated, %{id: chunk_id}}, socket) do
     transcription = get_transcription(socket.assigns.transcription.id)
 

lib/medicode_web/live/user_settings_live.ex

@@ -8,9 +8,10 @@ defmodule MedicodeWeb.UserSettingsLive do
   def render(assigns) do
     ~H"""
     <div class="min-h-[calc(100vh-56px)] flex gap-5">
-      <MedicodeWeb.Components.SidebarComponent.sidebar
+      <.live_component
+        id="sidebar"
+        module={MedicodeWeb.Components.SidebarComponent}
         current_user={@current_user}
-        transcriptions={@transcriptions}
       />
 
       <div class="w-full flex justify-center">

mix.exs

@@ -73,7 +73,8 @@ defmodule Medicode.MixProject do
       {:sentry, "~> 8.0"},
       {:ecto_psql_extras, "~> 0.6"},
       {:circular_buffer, "~> 0.4.0"},
-      {:tzdata, "~> 1.1"}
+      {:tzdata, "~> 1.1"},
+      {:gproc, "~> 1.0.0"}
       # {:membrane_portaudio_plugin, "~> 0.18.0"}
     ]
   end

mix.lock

@@ -40,6 +40,7 @@
   "floki": {:hex, :floki, "0.35.2", "87f8c75ed8654b9635b311774308b2760b47e9a579dabf2e4d5f1e1d42c39e0b", [:mix], [], "hexpm", "6b05289a8e9eac475f644f09c2e4ba7e19201fd002b89c28c1293e7bd16773d9"},
   "fss": {:hex, :fss, "0.1.1", "9db2344dbbb5d555ce442ac7c2f82dd975b605b50d169314a20f08ed21e08642", [:mix], [], "hexpm", "78ad5955c7919c3764065b21144913df7515d52e228c09427a004afe9c1a16b0"},
   "gettext": {:hex, :gettext, "0.24.0", "6f4d90ac5f3111673cbefc4ebee96fe5f37a114861ab8c7b7d5b30a1108ce6d8", [:mix], [{:expo, "~> 0.5.1", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "bdf75cdfcbe9e4622dd18e034b227d77dd17f0f133853a1c73b97b3d6c770e8b"},
+  "gproc": {:hex, :gproc, "1.0.0", "aa9ec57f6c9ff065b16d96924168d7c7157cd1fd457680efe4b1274f456fa500", [:rebar3], [], "hexpm", "109f253c2787de8a371a51179d4973230cbec6239ee673fa12216a5ce7e4f902"},
   "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"},
   "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"},
   "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},