add app

src/app.py

@@ -0,0 +1,109 @@
+import streamlit as st
+import pandas as pd
+from utils import extract_from_url, get_model, calculate_memory
+import plotly.express as px
+import numpy as np
+
+st.set_page_config(page_title='Can you run it? LLM GPU check', layout="wide", initial_sidebar_state="expanded")
+
+st.title("Can you run it? LLM GPU check")
+
+percentage_width_main = 80
+st.markdown(
+        f"""<style>
+        .appview-container .main .block-container{{
+        max-width: {percentage_width_main}%;}}
+        </style>
+        """,
+        unsafe_allow_html=True,
+    )
+@st.cache_resource
+def get_gpu_specs():
+    return pd.read_csv("data/gpu_specs.csv")
+
+
+
+def get_name(index):
+    row = gpu_specs.iloc[index]
+    return f"{row['Product Name']} ({row['RAM (GB)']} GB, {row['Year']})"
+
+def create_plot(memory_table, y, title, container):
+    fig = px.bar(memory_table, x=memory_table.index, y=y, color_continuous_scale="RdBu_r")
+    fig.update_layout(yaxis_title="Number of GPUs", title=dict(text=title, font=dict(size=25)))
+    fig.update_coloraxes(showscale=False)
+    
+    container.plotly_chart(fig, use_container_width=True)
+
+gpu_specs = get_gpu_specs()
+
+access_token = st.sidebar.text_input("Access token")
+model_name = st.sidebar.text_input("Model name", value="mistralai/Mistral-7B-v0.1")
+if not model_name:
+    st.info("Please enter a model name")
+    st.stop()
+
+
+
+model_name = extract_from_url(model_name)
+if model_name not in st.session_state:
+    model = get_model(model_name, library="transformers", access_token=access_token)
+    st.session_state[model_name] = (model, calculate_memory(model, ["float32", "float16/bfloat16", "int8", "int4"]))
+
+
+gpu_vendor = st.sidebar.selectbox("GPU Vendor", ["NVIDIA", "AMD", "Intel"])
+year = st.sidebar.selectbox("Filter by Release Year", list(range(2014, 2024))[::-1], index=None)
+gpu_info = gpu_specs[gpu_specs['Vendor'] == gpu_vendor].sort_values('RAM (GB)', ascending=False)
+if year:
+    gpu_info = gpu_info[gpu_info['Year'] == year]
+
+min_ram = gpu_info['RAM (GB)'].min()
+max_ram = gpu_info['RAM (GB)'].max()
+ram = st.sidebar.slider("Filter by RAM (GB)", min_ram, max_ram, (min_ram, max_ram), step=0.5)
+gpu_info = gpu_info[gpu_info["RAM (GB)"].between(*ram)] 
+gpu = st.sidebar.selectbox("GPU", gpu_info['Product Name'].index.tolist(), index=21, format_func=lambda x : gpu_specs.iloc[x]['Product Name'])
+
+gpu_spec = gpu_specs.iloc[gpu]
+gpu_spec.name = 'INFO'
+
+lora_pct = st.sidebar.slider("LoRa % trainable parameters", 0.1, 100.0, 2.0, step=0.1)
+
+st.sidebar.dataframe(gpu_spec.T)
+
+memory_table = pd.DataFrame(st.session_state[model_name][1]).set_index('dtype')
+memory_table['LoRA Fine-Tunning (GB)'] = (memory_table["Total Size (GB)"] + 
+                                          (memory_table["Parameters (Billion)"]* lora_pct/100 * (16/8)*4)) * 1.2
+
+_, col, _ = st.columns([1,3,1])
+with col.expander("Information", expanded=True):
+    st.markdown("""- GPU information comes from [TechPowerUp GPU Specs](https://www.techpowerup.com/gpu-specs/)
+- Mainly based on [Model Memory Calculator by hf-accelerate](https://huggingface.co/spaces/hf-accelerate/model-memory-usage)
+    using `transformers` library
+- Inference is calculated following [EleutherAI Transformer Math 101](https://blog.eleuther.ai/transformer-math/),
+    where is estimated as """)
+    
+    st.latex(r"""\text{Memory}_\text{Inference} \approx \text{Model Size} \times 1.2""")
+    st.markdown("""- For LoRa Fine-tunning, I'm asuming a **16-bit** dtype of trainable parameters. The formula (in terms of GB) is""")
+    st.latex(r"\text{Memory}_\text{LoRa} \approx \text{Model Size} + \left(\text{ \# trainable Params}_\text{Billions}\times\frac{16}{8} \times 4\right) \times 1.2")
+    st.markdown("- You can understand `int4` as models in `GPTQ-4bit`, `AWQ-4bit` or `Q4_0 GGUF/GGML` formats")
+
+    
+_memory_table = memory_table.copy()
+memory_table = memory_table.round(2).T
+_memory_table /= gpu_spec['RAM (GB)']
+_memory_table = _memory_table.apply(np.ceil).astype(int).drop(columns=['Parameters (Billion)', 'Total Size (GB)'])
+_memory_table.columns = ['Inference', 'Full Training Adam', 'LoRa Fine-tuning']
+_memory_table = _memory_table.stack().reset_index()
+_memory_table.columns = ['dtype', 'Variable', 'Number of GPUs']
+
+col1, col2 = st.columns([1,1.3])
+with col1:
+    st.write(f"####  [{model_name}](https://huggingface.co/{model_name}) ({memory_table.iloc[3,0]:.1f}B)")
+    st.write(memory_table.iloc[[0, 1, 2, 4]])
+with col2:
+    num_colors= 4
+    colors = [px.colors.sequential.RdBu[int(i*(len(px.colors.sequential.RdBu)-1)/(num_colors-1))] for i in range(num_colors)]
+    fig = px.bar(_memory_table, x='Variable', y='Number of GPUs', color='dtype', barmode='group', color_discrete_sequence=colors)
+    fig.update_layout(title=dict(text=f"Number of GPUs required for<br> {get_name(gpu)}", font=dict(size=25))
+                    , xaxis_tickfont_size=14, yaxis_tickfont_size=16, yaxis_dtick='1')
+    st.plotly_chart(fig, use_container_width=True)
+