handling local env; added state-management

.gitignore

@@ -1,2 +1,3 @@
+.env
 tmp.ipynb
 __pycache__

README.md

@@ -4,7 +4,7 @@ emoji: 🎭
 colorFrom: blue
 colorTo: yellow
 sdk: streamlit
-python_version: 3.8.16
+python_version: 3.10.6
 sdk_version: 1.17.0
 app_file: app.py
 pinned: false

app.py

@@ -5,10 +5,12 @@ import streamlit as st
 import pandas as pd
 from transformers import AutoTokenizer, AutoModelForSequenceClassification
 from transformers import pipeline
+from dotenv import load_dotenv
 from plotly.subplots import make_subplots
 import plotly.graph_objects as go
 import plotly.express as px
 
+load_dotenv()
 
 def z_score(y, mean=.04853076, sd=.9409466):
     return (y - mean) / sd
@@ -90,35 +92,37 @@ covariate_columns = {
         'rater_group': 'Rater Group',
     }
 
-df = (
-    pd
-    .read_feather(path='data.feather').query('partition == "test" | partition == "dev"')
-    .melt(
-        value_vars=['sentiment_model', 'desirability_model'],
-        var_name='x_group',
-        value_name='x',
-        id_vars=['mean_z', 'text', 'content_domain', 'language', 'rater_group', 'study', 'instrument']
+if 'df' not in st.session_state:
+    st.session_state.df = (
+        pd
+        .read_feather(path='data.feather').query('partition == "test" | partition == "dev"')
+        .melt(
+            value_vars=['sentiment_model', 'desirability_model'],
+            var_name='x_group',
+            value_name='x',
+            id_vars=['mean_z', 'text', 'content_domain', 'language', 'rater_group', 'study', 'instrument']
+            )
+        .replace(
+            to_replace={
+                'en': 'English',
+                'de': 'German',
+                'other': 'Other',
+                'personality': 'Personality',
+                'laypeople': 'Laypeople',
+                'students': 'Students',
+                'sentiment_model': 'Sentiment Model',
+                'desirability_model': 'Desirability Model'
+            }    
+        )
+        .rename(columns=covariate_columns)
+        .rename(
+            columns={
+                'mean_z': 'Human-ratings',
+                'x': 'Machine-ratings',
+            }
         )
-    .replace(
-        to_replace={
-            'en': 'English',
-            'de': 'German',
-            'other': 'Other',
-            'personality': 'Personality',
-            'laypeople': 'Laypeople',
-            'students': 'Students',
-            'sentiment_model': 'Sentiment Model',
-            'desirability_model': 'Desirability Model'
-        }    
-    )
-    .rename(columns=covariate_columns)
-    .rename(
-        columns={
-            'mean_z': 'Human-ratings',
-            'x': 'Machine-ratings',
-        }
     )
-)
+
 
 st.markdown("""
     # NLP for Item Desirability Ratings
@@ -145,19 +149,19 @@ with st.spinner('Processing...'):
     if os.environ.get('item-desirability'):
         model_path = 'magnolia-psychometrics/item-desirability'
     else:
-        model_path = '/nlp/nlp/models/finetuned/twitter-xlm-roberta-base-regressive-desirability-ft-4'   
+        model_path = os.getenv('model_path')
 
     auth_token = os.environ.get('item-desirability') or True
 
-    if 'tokenizer' not in globals():
-        tokenizer = AutoTokenizer.from_pretrained(
+    if 'tokenizer' not in st.session_state:
+        st.session_state.tokenizer = AutoTokenizer.from_pretrained(            
             pretrained_model_name_or_path=model_path,
             use_fast=True,
             use_auth_token=auth_token
         )
 
-    if 'model' not in globals():
-        model = AutoModelForSequenceClassification.from_pretrained(
+    if 'model' not in st.session_state:
+        st.session_state.model = AutoModelForSequenceClassification.from_pretrained(
             pretrained_model_name_or_path=model_path, 
             num_labels=1, 
             ignore_mismatched_sizes=True,
@@ -165,9 +169,15 @@ with st.spinner('Processing...'):
         )
 
     ## sentiment model
-    if 'classifier' not in globals():
-        sentiment_model = 'cardiffnlp/twitter-xlm-roberta-base-sentiment'
-        classifier = pipeline("sentiment-analysis", model=sentiment_model, tokenizer=sentiment_model, use_fast=False, top_k=3)
+    if 'classifier' not in st.session_state:
+        st.session_state.sentiment_model = 'cardiffnlp/twitter-xlm-roberta-base-sentiment'
+        st.session_state.classifier = pipeline(
+            task='sentiment-analysis',
+            model=st.session_state.sentiment_model,
+            tokenizer=st.session_state.sentiment_model,
+            use_fast=False,
+            top_k=3
+        )
 
     input_text = st.text_input(
         label='Estimate item desirability:',
@@ -177,14 +187,14 @@ with st.spinner('Processing...'):
 
     if input_text:    
         
-        classifier_output = classifier(input_text)
+        classifier_output = st.session_state.classifier(input_text)
         classifier_output_dict = {x['label']: x['score'] for x in classifier_output[0]}
         classifier_score = classifier_output_dict['positive'] - classifier_output_dict['negative']
 
-        inputs = tokenizer(input_text, padding=True, return_tensors='pt')
+        inputs = st.session_state.tokenizer(text=input_text, padding=True, return_tensors='pt')
 
         with torch.no_grad():
-            score = model(**inputs).logits.squeeze().tolist()
+            score = st.session_state.model(**inputs).logits.squeeze().tolist()
             z = z_score(score)
 
         p1 = indicator_plot(
@@ -231,6 +241,6 @@ if show_covariates:
 else:
     option = None
 
-plot = scatter_plot(df, option)
+plot = scatter_plot(st.session_state.df, option)
 
 st.plotly_chart(plot, theme=None, use_container_width=True)

requirements.txt

@@ -3,4 +3,5 @@ transformers
 plotly
 dash
 statsmodels
-sentencepiece
+sentencepiece
+python-dotenv