update

.gitattributes

@@ -29,3 +29,4 @@ models/bertopic_model_tokyo_olympics_tweets filter=lfs diff=lfs merge=lfs -text
 models/bertopic_model_tokyo_olympics_tweets_unclean filter=lfs diff=lfs merge=lfs -text
 models/distilbart-mnli-12-1/flax_model.msgpack filter=lfs diff=lfs merge=lfs -text
 models/distilbart-mnli-12-1/pytorch_model.bin filter=lfs diff=lfs merge=lfs -text
+models/distilbert-base-uncased-finetuned-sst-2-english/pytorch_model.bin filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,3 +1,8 @@
+# custom
+survey_analytics.ipynb
+embeddings_unclean.pickle
+embeddings.pickle
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -127,3 +132,5 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+survey_analytics.ipynb
+survey_analytics.ipynb

app.py

@@ -22,14 +22,14 @@ from bertopic import BERTopic
 # custom
 import survey_analytics_library as LIB
 
-st.set_page_config(layout='wide')
+# st.set_page_config(layout='wide')
 
 # define data file path
 data_path = 'data' + os.sep
 # define model file path
 model_path = 'models' + os.sep
 
-# load all data and models
+# load and cache all data and models to improve app performance
 @st.cache
 def read_survey_data():
     data_survey = pd.read_csv(data_path+'bfi_sample_answers.csv')
@@ -77,7 +77,7 @@ st.write('''
 st.write('\n')
 st.write('\n')
 
-# copy daya
+# copy data
 df_factor_analysis = data_survey.copy()
 
 st.subheader('Sample Survey Data')
@@ -87,7 +87,7 @@ st.write('''
     ''')
 
 # split page into two columns
-# display survey questions and responses as dataframes
+# display survey questions and answers as dataframes side by side
 col1, col2 = st.columns(2)
 with col1:
     st.write('Survey Questions')
@@ -106,16 +106,18 @@ st.write('''
 
 # interactive button to run statistical test to determine suitability for factor analysis
 if st.button('Run Tests'):
-    # Test with the null hypothesis that the correlation matrix is an identity matrix
+    # test with the null hypothesis that the correlation matrix is an identity matrix
     bartlett_sphericity_stat, p_value = calculate_bartlett_sphericity(x=df_factor_analysis)
-    # Test how predictable of a variable by others
+    # test how predictable of a variable by others
     kmo_per_variable, kmo_total = calculate_kmo(x=df_factor_analysis)
+    # print test results
     st.write(f'''
         The P Value from Bartlett\'s Test (suitability is less than 0.05): **{round(p_value, 2)}**  
         The Value from KMO Test (suitability is more than 0.60): **{round(kmo_total, 2)}**  
         ''')
+    # set default status to 'Failed'
     fa_stat_test = 'Failed'
-
+    # check if data passes both tests
     if p_value < 0.05 and kmo_total >= 0.6:
         fa_stat_test = 'Passed'
 
@@ -160,9 +162,13 @@ st.write(f'''
     Kaiser criterion is one of many guides to determine the number of factors, ultimately the decision on the number of factors to use is best decided by the user based on their use case.  
     ''')
 
+# interactive form for user to enter different number of factors for analysis
 with st.form('num_factor_form'):
+    # define number input 
     user_num_factors = st.number_input('Enter desired number of factors:', min_value=1, max_value=10, value=6)
+    # set factors to user input
     optimal_factors = user_num_factors
+    # submit button for form to rerun app when user is ready
     submit = st.form_submit_button('Run Factor Analysis')
 
 st.write('\n')
@@ -175,14 +181,16 @@ fa.fit(df_factor_analysis)
 # generate factor loadings
 loads_df = pd.DataFrame(fa.loadings_, index=df_factor_analysis.columns)
 
-transformed_df = fa.fit_transform(df_factor_analysis)
-transformed_df = pd.DataFrame(transformed_df)
-transformed_df.columns = ['factor_'+str(col) for col in list(transformed_df)]
-
-responder_factors = transformed_df.copy()
+# fit and transform data
+responder_factors = fa.fit_transform(df_factor_analysis)
+# store results as df
+responder_factors = pd.DataFrame(responder_factors)
+# rename columns to 'factor_n'
+responder_factors.columns = ['factor_'+str(col) for col in list(responder_factors)]
+# use the max loading across all factors to determine a responder's cluster
 responder_factors['cluster'] = responder_factors.apply(lambda s: s.argmax(), axis=1)
 
-# list of factor columns
+# define list of factor columns
 list_of_factor_cols = [col for col in responder_factors.columns if 'factor_' in col]
 st.subheader('Fator Analysis Results')
 st.write('''
@@ -193,30 +201,36 @@ st.write('''
 st.dataframe(responder_factors.style.highlight_max(axis=1, subset=list_of_factor_cols, props='color:white; background-color:green;').format(precision=2))
 st.write('\n')
 
+# count number of responders in each cluster
 fa_clusters = df_factor_analysis.copy().reset_index(drop=True)
 fa_clusters['cluster'] = responder_factors['cluster']
+cluster_counts = fa_clusters['cluster'].value_counts().reset_index()
+cluster_counts = cluster_counts.rename(columns={'index':'Cluster', 'cluster':'Count'})
+
+# calculate z-scores for each cluster
 fa_z_scores = df_factor_analysis.copy().reset_index(drop=True)
 fa_z_scores = fa_z_scores.apply(zscore)
 fa_z_scores['cluster'] = responder_factors['cluster']
 fa_z_scores = fa_z_scores.groupby('cluster').mean().reset_index()
 fa_z_scores = fa_z_scores.apply(lambda x: round(x, 2))
 
-cm = sns.light_palette('green', as_cmap=True)
-list_of_question_cols = list(fa_z_scores.iloc[:,1:])
 st.write('''
     Aggregating the scores of the clusters gives us detail insights to the personality traits of the responders.  
     The scores here have been normalised to Z-scores, a measure of how many standard deviations (SD) is the score away from the mean.  
     E.g. A Z-score of 0 indicates the score is identical to the mean, while a Z-score of 1 indicates the score is 1 SD away from the mean.  
     ''')
+# define colour map for highlighting cells
+cm = sns.light_palette('green', as_cmap=True)
+# define list of question columns
+list_of_question_cols = list(fa_z_scores.iloc[:,1:])
+# display z-scores of clusters with conditional formatting
 st.dataframe(fa_z_scores.style.background_gradient(cmap=cm, subset=list_of_question_cols).format(precision=2))
 st.write('\n')
 
-cluster_counts = fa_clusters['cluster'].value_counts().reset_index()
-cluster_counts = cluster_counts.rename(columns={'index':'Cluster', 'cluster':'Count'})
-
 st.write('''
     Lastly, we can visualise the distribution of responders in each cluster.  
     ''')
+# plot percentage of responders in each cluster
 fig = px.pie(
     cluster_counts,
     values='Count', 
@@ -243,9 +257,12 @@ st.write('''
     ''')
 st.write('\n')
 
-st.write('''
-    Here we have 10,000 tweets from the Tokyo Olympics, going through them manually and coming up with topics would not be practical.  
+st.write(f'''
+    Here we have {len(tokyo):,} tweets from the Tokyo Olympics, going through them manually and coming up with topics would not be practical.  
     ''')
+# rename column
+tokyo = tokyo.rename(columns={'text':'Tweet'})
+# display raw tweets
 st.dataframe(tokyo)
 st.write('\n')
 st.write('\n')
@@ -255,6 +272,7 @@ st.write('''
     ''')
 st.write('\n')
 
+# plot topics using unclean data
 fig = LIB.visualize_barchart_titles(
     topic_model=topic_model_unclean,
     subplot_titles=None,
@@ -265,23 +283,25 @@ fig = LIB.visualize_barchart_titles(
 st.plotly_chart(fig, use_container_width=True)
 
 st.write('''
-    From the chart above, we can see that 'Topic 1' and 'Topic 3' have some words that are not as meaningful.  
-    For 'Topic 1', we already know that the tweets are about the Tokyo 2020 Olympics, having a topic for that isn't helpful.  
-    'Tokyo', '2020', etc., we refer to these as *stopwords*, and lets remove them and regenerate the topics.  
+    From the chart above, we can see that 'Topic 0' and 'Topic 5' have some words that are not as meaningful.  
+    For 'Topic 0', we already know that the tweets are about the Tokyo 2020 Olympics, having a topic for that isn't helpful.  
+    'Tokyo', '2020', 'Olympics', etc., we refer to these as *stopwords*, and lets remove them and regenerate the topics.  
     ''')
 st.write('\n')
 
+# define manually created topic labels
 labelled_topics = [
-    'Mirabai Chanu (Indian Weightlifter)',
-    'Hockey',
-    'Barbra Banda (Zambian Football Player)',
+    'Barbra Banda (Zambian Footballer)',
+    'Indian Pride',
     'Sutirtha Mukherjee (Indian Table Tennis Player)',
-    'Vikas Krishan (Indian Boxer)',
+    'Mirabai Chanu (Indian Weightlifter)',
     'Road Race',
-    'Brendon Smith (Australian Swimmer)',
+    'Japan Volleyball',
     'Sam Kerr (Australian Footballer)',
+    'Vikas Krishan (Indian Boxer)',
     ]
 
+# plot topics using clean data with stopwords removed
 fig = LIB.visualize_barchart_titles(
     topic_model=topic_model,
     subplot_titles=labelled_topics,
@@ -290,6 +310,7 @@ fig = LIB.visualize_barchart_titles(
     height=300
 )
 st.plotly_chart(fig, use_container_width=True)
+
 st.write('''
     Now we can see that the topics have improved.  
     We can make use of the top words in each topic to come up with a meaningful name.  
@@ -297,37 +318,57 @@ st.write('''
 st.write('\n')
 st.write('\n')
 
+# store topic info as dataframe
 topics_df = topic_model.get_topic_info()
 
 st.write(f'''
     Next, we can also review the total number of topics and how many tweets are in each topic, to give us a sense of importance or priority.  
     There are a total of **{len(topics_df)-1}** topics, and the larget topic contains **{topics_df['Count'][1]}** tweets.  
-    {topics_df['Count'][0]} tweets have also been assigned as Topic -1 or outliers.  
-    These tweets are more unique and there are enough of them to form a topic.   
+    {topics_df['Count'][0]} tweets have also been assigned as Topic -1 or outliers. These tweets are unique compared to the others and there aren't enough of them to form a topic.   
+    If there are too many or too few topics, there is also the option to further tune the model to refine the results. 
     ''')
+# display topic info
 st.dataframe(topics_df)
 st.write('\n')
 
 st.write('''
-    As there are many topics generated, we can also visualise how closely related they are to one another.  
-    Depending on the business case, we may want to merge these topics together or keep them separate.  
-    If there are too many or too few topics, there is also the option to tune the parameters of the model to refine the results.  
-    ''')
-fig = topic_model.visualize_topics()
-st.plotly_chart(fig, use_container_width=True)
-st.write('\n')
+    One point to also note is that the machine is not only picking out keywords in a tweet to determine its topic.  
+    The model has an understanding of the relationship between words, e.g. 'Andy Murray' is related to 'tennis'.  
+    For example:  
+    *'Cilic vs Menezes, after more than 3 hours and millions of unconverted match points, is one of the worst quality ten…'*  
+    This tweet is in the Topic 9 - Tennis without the word 'tennis' in it.  
 
-st.write('''
-    Lastly, we can inspect the individual tweets within each topic.  
+    Here we can inspect the individual tweets within each topic.  
     ''')
 
+# define the first and last topic number
+first_topic = topics_df['Topic'].iloc[0]
+last_topic = topics_df['Topic'].iloc[-1]
+
+# interative form for user to select a topic and inspect its top words and tweets
 with st.form('inspect_tweets'):
-    inspect_topic = st.number_input('Enter Topic (from -1 to 63) to Inspect:', min_value=-1, max_value=63, value=8)
+    inspect_topic = st.number_input(f'Enter Topic (from {first_topic} to {last_topic}) to Inspect:', min_value=first_topic, max_value=last_topic, value=8)
     submit = st.form_submit_button('Inspect Topic')
 
+# get top five words from list of tuples
 inspect_topic_words = [i[0] for i in topic_model.get_topic(inspect_topic)[:5]]
 
 st.write(f'''
     The top five words for Topic {inspect_topic} are: {inspect_topic_words}
     ''')
+# display tweets from selected topic
 st.dataframe(topic_results.loc[(topic_results['Topic'] == inspect_topic)])
+st.markdown('''---''')
+
+
+
+
+
+
+st.header('Classifiying Text Responses and Sentiment Analysis')
+st.write('''
+    With survey responses, sometimes as a business user, we already have an general idea of what responders are talking about and we want to categorise or classify the responses accordingly.  
+    E.g. 
+
+    ''')
+st.write('\n')

models/bertopic_model_tokyo_olympics_tweets

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2c6e3d236886eee8537b9259d44805006961088fe470253f4ab10f55836f87fa
-size 162358999
+oid sha256:875ef8aa63fb501456e693214267492335a90db95b1e9cae092fd57c8ca787db
+size 71005660

models/bertopic_model_tokyo_olympics_tweets_unclean

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:87c601adcf7e65d38fda982061a5c2e31e752126f9dd66f7abda6734a4ee2c84
-size 163381647
+oid sha256:0de856ed231c12e7baeaff15eb3159e1a5ef7c5512b459f915f46712f6d203a3
+size 71961846

models/distilbart-mnli-12-1/README.md

@@ -0,0 +1,59 @@
+---
+datasets:
+- mnli
+tags:
+- distilbart
+- distilbart-mnli
+pipeline_tag: zero-shot-classification
+---
+
+# DistilBart-MNLI
+
+distilbart-mnli is the distilled version of bart-large-mnli created using the **No Teacher Distillation** technique proposed for BART summarisation by Huggingface, [here](https://github.com/huggingface/transformers/tree/master/examples/seq2seq#distilbart).
+
+We just copy alternating layers from `bart-large-mnli` and finetune more on the same data. 
+
+
+|                                                                                      | matched acc | mismatched acc |
+| ------------------------------------------------------------------------------------ | ----------- | -------------- |
+| [bart-large-mnli](https://huggingface.co/facebook/bart-large-mnli) (baseline, 12-12) | 89.9        | 90.01          |
+| [distilbart-mnli-12-1](https://huggingface.co/valhalla/distilbart-mnli-12-1)         | 87.08       | 87.5           |
+| [distilbart-mnli-12-3](https://huggingface.co/valhalla/distilbart-mnli-12-3)         | 88.1        | 88.19          |
+| [distilbart-mnli-12-6](https://huggingface.co/valhalla/distilbart-mnli-12-6)         | 89.19       | 89.01          |
+| [distilbart-mnli-12-9](https://huggingface.co/valhalla/distilbart-mnli-12-9)         | 89.56       | 89.52          |
+
+
+This is a very simple and effective technique, as we can see the performance drop is very little.
+
+Detailed performace trade-offs will be posted in this [sheet](https://docs.google.com/spreadsheets/d/1dQeUvAKpScLuhDV1afaPJRRAE55s2LpIzDVA5xfqxvk/edit?usp=sharing).
+
+
+## Fine-tuning
+If you want to train these models yourself, clone the [distillbart-mnli repo](https://github.com/patil-suraj/distillbart-mnli) and follow the steps below
+
+Clone and install transformers from source
+```bash
+git clone https://github.com/huggingface/transformers.git
+pip install -qqq -U ./transformers
+```
+
+Download MNLI data
+```bash
+python transformers/utils/download_glue_data.py --data_dir glue_data --tasks MNLI
+```
+
+Create student model
+```bash
+python create_student.py \
+  --teacher_model_name_or_path facebook/bart-large-mnli \
+  --student_encoder_layers 12 \
+  --student_decoder_layers 6 \
+  --save_path student-bart-mnli-12-6 \
+```
+
+Start fine-tuning
+```bash
+python run_glue.py args.json
+```
+
+You can find the logs of these trained models in this [wandb project](https://wandb.ai/psuraj/distilbart-mnli).

models/distilbart-mnli-12-1/config.json

@@ -0,0 +1,56 @@
+{
+  "_num_labels": 3,
+  "activation_dropout": 0.0,
+  "activation_function": "gelu",
+  "add_bias_logits": false,
+  "add_final_layer_norm": false,
+  "architectures": [
+    "BartForSequenceClassification"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 0,
+  "classif_dropout": 0.0,
+  "classifier_dropout": 0.0,
+  "d_model": 1024,
+  "decoder_attention_heads": 16,
+  "decoder_ffn_dim": 4096,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 1,
+  "decoder_start_token_id": 2,
+  "dropout": 0.1,
+  "encoder_attention_heads": 16,
+  "encoder_ffn_dim": 4096,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 12,
+  "eos_token_id": 2,
+  "extra_pos_embeddings": 2,
+  "finetuning_task": "mnli",
+  "force_bos_token_to_be_generated": false,
+  "forced_eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "id2label": {
+    "0": "contradiction",
+    "1": "neutral",
+    "2": "entailment"
+  },
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "label2id": {
+    "contradiction": 0,
+    "entailment": 2,
+    "neutral": 1
+  },
+  "max_position_embeddings": 1024,
+  "model_type": "bart",
+  "normalize_before": false,
+  "normalize_embedding": true,
+  "num_hidden_layers": 12,
+  "output_past": false,
+  "pad_token_id": 1,
+  "scale_embedding": false,
+  "static_position_embeddings": false,
+  "total_flos": 153130534133111808,
+  "transformers_version": "4.7.0.dev0",
+  "use_cache": true,
+  "vocab_size": 50265
+}

models/distilbart-mnli-12-1/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa79ff59084a5036b07a9cffeaa1b1b7c1aa5edeb1885416a734c001a09aa046
+size 890410947

models/distilbart-mnli-12-1/special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "sep_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "cls_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true}}

models/distilbart-mnli-12-1/tokenizer_config.json

@@ -0,0 +1 @@
+{"model_max_length": 1024}

models/distilbert-base-uncased-finetuned-sst-2-english/README.md

@@ -0,0 +1,31 @@
+---
+language: en
+license: apache-2.0
+datasets:
+- sst-2
+---
+
+# DistilBERT base uncased finetuned SST-2
+
+This model is a fine-tune checkpoint of [DistilBERT-base-uncased](https://huggingface.co/distilbert-base-uncased), fine-tuned on SST-2.
+This model reaches an accuracy of 91.3 on the dev set (for comparison, Bert bert-base-uncased version reaches an accuracy of 92.7).
+
+For more details about DistilBERT, we encourage users to check out [this model card](https://huggingface.co/distilbert-base-uncased).
+
+# Fine-tuning hyper-parameters
+
+- learning_rate = 1e-5
+- batch_size = 32
+- warmup = 600
+- max_seq_length = 128
+- num_train_epochs = 3.0
+
+# Bias
+
+Based on a few experimentations, we observed that this model could produce biased predictions that target underrepresented populations.
+
+For instance, for sentences like `This film was filmed in COUNTRY`, this binary classification model will give radically different probabilities for the positive label depending on the country (0.89 if the country is France, but 0.08 if the country is Afghanistan) when nothing in the input indicates such a strong semantic shift. In this [colab](https://colab.research.google.com/gist/ageron/fb2f64fb145b4bc7c49efc97e5f114d3/biasmap.ipynb), [Aurélien Géron](https://twitter.com/aureliengeron) made an interesting map plotting these probabilities for each country.
+
+<img src="https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english/resolve/main/map.jpeg" alt="Map of positive probabilities per country." width="500"/>
+
+We strongly advise users to thoroughly probe these aspects on their use-cases in order to evaluate the risks of this model. We recommend looking at the following bias evaluation datasets as a place to start: [WinoBias](https://huggingface.co/datasets/wino_bias), [WinoGender](https://huggingface.co/datasets/super_glue), [Stereoset](https://huggingface.co/datasets/stereoset).

models/distilbert-base-uncased-finetuned-sst-2-english/config.json

@@ -0,0 +1,31 @@
+{
+  "activation": "gelu",
+  "architectures": [
+    "DistilBertForSequenceClassification"
+  ],
+  "attention_dropout": 0.1,
+  "dim": 768,
+  "dropout": 0.1,
+  "finetuning_task": "sst-2",
+  "hidden_dim": 3072,
+  "id2label": {
+    "0": "NEGATIVE",
+    "1": "POSITIVE"
+  },
+  "initializer_range": 0.02,
+  "label2id": {
+    "NEGATIVE": 0,
+    "POSITIVE": 1
+  },
+  "max_position_embeddings": 512,
+  "model_type": "distilbert",
+  "n_heads": 12,
+  "n_layers": 6,
+  "output_past": true,
+  "pad_token_id": 0,
+  "qa_dropout": 0.1,
+  "seq_classif_dropout": 0.2,
+  "sinusoidal_pos_embds": false,
+  "tie_weights_": true,
+  "vocab_size": 30522
+}

models/distilbert-base-uncased-finetuned-sst-2-english/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60554cbd7781b09d87f1ececbea8c064b94e49a7f03fd88e8775bfe6cc3d9f88
+size 267844284

models/distilbert-base-uncased-finetuned-sst-2-english/tokenizer_config.json

@@ -0,0 +1 @@
+{"model_max_length": 512, "do_lower_case": true}