app/streamlitapp.py

from __future__ import annotations

# Import all of the dependencies
from pathlib import Path
import subprocess
import tempfile

import imageio
import streamlit as st
import tensorflow as tf

from modelutil import load_model
from utils import load_data, num_to_char

# Set the layout to the streamlit app as wide
st.set_page_config(layout='wide')

# Setup the sidebar
with st.sidebar:
    st.image('https://www.onepointltd.com/wp-content/uploads/2020/03/inno2.png')
    st.title('NeuroSync Lipscape')
    st.info('This application is originally developed from the Lip-Reader deep learning model.')

st.title('NeuroSync Lipscape Full Stack App')

BASE_DIR = Path(__file__).resolve().parent
DATA_DIR = BASE_DIR / 'data' / 's1'

# Generating a list of options or videos
options = sorted([item.name for item in DATA_DIR.glob('*.mpg')])
selected_video = st.selectbox('Choose video', options)

# Generate two columns
col1, col2 = st.columns(2)

if options:
    # Rendering the video
    with col1:
        st.info('The video below displays the converted video in mp4 format')
        file_path = DATA_DIR / selected_video
        if not file_path.exists():
            st.error(f"Video not found: {file_path}")
        else:
            output_path = None
            try:
                with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as output_file:
                    output_path = Path(output_file.name)
                subprocess.run(
                    [
                        "ffmpeg",
                        "-i",
                        str(file_path),
                        "-vcodec",
                        "libx264",
                        str(output_path),
                        "-y",
                    ],
                    check=True,
                    capture_output=True,
                    text=True,
                )
                with output_path.open('rb') as video:
                    video_bytes = video.read()
                st.video(video_bytes)
            except subprocess.CalledProcessError as exc:
                st.error("ffmpeg failed to convert the selected video.")
                st.code(exc.stderr or "No ffmpeg error output captured.")
            finally:
                if output_path and output_path.exists():
                    output_path.unlink()

    with col2:
        if file_path.exists():
            st.info('This is all the machine learning model sees when making a prediction')
            video, _annotations = load_data(tf.convert_to_tensor(str(file_path)))
            gif_path = None
            try:
                with tempfile.NamedTemporaryFile(suffix=".gif", delete=False) as gif_file:
                    gif_path = Path(gif_file.name)
                imageio.mimsave(str(gif_path), video, fps=10)
                st.image(str(gif_path), width=400)
            finally:
                if gif_path and gif_path.exists():
                    gif_path.unlink()

            st.info('This is the output of the machine learning model as tokens')
            model = load_model()
            yhat = model.predict(tf.expand_dims(video, axis=0))
            decoder = tf.keras.backend.ctc_decode(yhat, [75], greedy=True)[0][0].numpy()
            st.text(decoder)

            # Convert prediction to text
            st.info('Decode the raw tokens into words')
            converted_prediction = tf.strings.reduce_join(num_to_char(decoder)).numpy().decode('utf-8')
            st.text(converted_prediction)
else:
    st.warning(f"No videos were found in {DATA_DIR}.")