Update app.py

app.py

@@ -8,6 +8,7 @@ import torch
 import gradio as gr
 import pandas as pd
 import yfinance as yf
+from pynvml import *
 from peft import PeftModel
 from collections import defaultdict
 from datetime import date, datetime, timedelta
@@ -31,8 +32,11 @@ model = PeftModel.from_pretrained(
     'FinGPT/fingpt-forecaster_dow30_llama2-7b_lora',
     offload_folder="offload/"
 )
+model = model.half()
 model = model.eval()
 
+print_gpu_utilization()
+
 tokenizer = AutoTokenizer.from_pretrained(
     'meta-llama/Llama-2-7b-chat-hf',
     token=access_token
@@ -47,6 +51,13 @@ SYSTEM_PROMPT = "You are a seasoned stock market analyst. Your task is to list t
     "Your answer format should be as follows:\n\n[Positive Developments]:\n1. ...\n\n[Potential Concerns]:\n1. ...\n\n[Prediction & Analysis]\nPrediction: ...\nAnalysis: ..."
 
 
+def print_gpu_utilization():
+    
+    nvmlInit()
+    handle = nvmlDeviceGetHandleByIndex(0)
+    info = nvmlDeviceGetMemoryInfo(handle)
+    print(f"GPU memory occupied: {info.used//1024**2} MB.")
+
 
 def get_curday():
     
@@ -229,8 +240,8 @@ def construct_prompt(ticker, curday, n_weeks, use_basics):
 
 def predict(ticker, date, n_weeks, use_basics):
 
-    torch.cuda.empty_cache()
-    
+    print_gpu_utilization()
+
     info, prompt = construct_prompt(ticker, date, n_weeks, use_basics)
       
     inputs = tokenizer(