first commit

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

Home.py

@@ -0,0 +1,43 @@
+import streamlit as st
+#import pickle
+import s3fs
+from gensim.models import KeyedVectors
+
+st.title('Historical Word Embeddings')
+
+st.write("Welcome!")
+st.write("This is an interactive web app that allows users to explore how the meaning of words change over time. Use the sidebar on the left to navigate.")
+st.write("Please note: The app is still under development and things might not always work properly.")
+st.write("Creator: Simon Walo")
+st.write("Data source: https://nlp.stanford.edu/projects/histwords/ (All English (1800s-1990s))")
+st.write("Please wait while the data is loading:")
+
+# Create connection object.
+# `anon=False` means not anonymous, i.e. it uses access keys to pull data.
+
+fs = s3fs.S3FileSystem(anon=False)
+fs.ls('bricktamlandstreamlitbucket')
+
+def read_file(filename):
+    with fs.open(filename) as f:
+        return f.read()
+
+
+#@st.cache(allow_output_mutation = True)
+@st.experimental_memo
+def load_data():
+    models_all = {
+        1810: KeyedVectors.load('./data/vectors1810.kv'),
+        1840: KeyedVectors.load('./data/vectors1840.kv'),
+        1870: KeyedVectors.load('./data/vectors1870.kv'),
+        1900: KeyedVectors.load('./data/vectors1900.kv'),
+        1930: KeyedVectors.load('./data/vectors1930.kv'),
+        1960: KeyedVectors.load('./data/vectors1960.kv'),
+        1990: KeyedVectors.load('./data/vectors1990.kv')
+    }
+    return models_all
+
+if 'models_all' not in st.session_state:
+    st.session_state['models_all'] = load_data()
+
+st.write("Data loaded!")

pages/2_Most Similar Terms.py

@@ -0,0 +1,46 @@
+import streamlit as st
+import numpy as np
+import pandas as pd
+
+st.subheader('Most similar terms')
+
+keyword = st.text_input("Input term", "gay", key="simkey")
+keyword = keyword.lower()
+
+
+def similarterms():
+
+    if keyword not in st.session_state['models_all'][1810]:
+        st.write('Keyword not found in data. Please check for spelling errors.')
+        return
+
+    years=[]
+    simterms=[]
+
+    for year, model in st.session_state['models_all'].items():
+        if model[keyword].all() == st.session_state['models_all'][1810]['biology'].all():
+            st.write('Keyword not available for ', year)
+        else:
+            years.append(year)
+            simterms.append(model.most_similar(keyword))
+
+    simterms2 = []
+    for x in simterms:
+        for y in x:
+            simterms2.append(y[0])
+
+    simterms3 = np.array_split(simterms2, len(st.session_state['models_all']))
+
+    simterms4 = []
+    for array in simterms3:
+        simterms4.append(list(array))
+
+    simterms5 = []
+    for x in simterms4:
+        simterms5.append((', '.join(x)))
+
+    simtermstable = pd.DataFrame(zip(years, simterms5))
+    simtermstable.columns = ["year", "terms"]
+    st.table(simtermstable)
+
+similarterms()

pages/3_Semantic Change.py

@@ -0,0 +1,81 @@
+import streamlit as st
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.decomposition import PCA
+from adjustText import adjust_text
+
+st.subheader('Semantic Change')
+
+keyword = st.text_input("Input term", "gay", key="semkey")
+keyword = keyword.lower()
+
+def semchange(keyword):
+
+    if keyword not in st.session_state['models_all'][1810]:
+        st.write('Keyword not found in data. Please check for spelling errors.')
+        return
+
+    # get list of all similar words from different periods
+
+    sim_words = []
+
+    for year, model in st.session_state['models_all'].items():
+        if year in range(1810, 2000, 60):
+            if model[keyword].all() == st.session_state['models_all'][1810]['biology'].all():
+                st.write('Keyword not available for ', year)
+            if model[keyword].all() != st.session_state['models_all'][1810]['biology'].all():
+                tempsim = model.most_similar(keyword, topn=7)
+                for term, vector in tempsim:
+                    sim_words.append(term)
+
+    sim_words = list(set(sim_words))
+
+    # get vectors of similar words in most recent embedding (1990)
+    sim_vectors1990 = np.array([st.session_state['models_all'][1990][w] for w in sim_words])
+
+    # get vectors of keyword in all periods
+
+    keyword_vectors = np.zeros(shape=(0,300))
+
+    for year, model in st.session_state['models_all'].items():
+        if year in range(1810, 2000, 60):
+            if model[keyword].all() != st.session_state['models_all'][1810]['biology'].all():
+                temp_keyword_vector = np.array([model[keyword]])
+                keyword_vectors = np.append(keyword_vectors, temp_keyword_vector, axis=0)
+
+    # add keyword vectors from all periods to vectors of similar words 1990
+
+    allvectors = np.append(sim_vectors1990, keyword_vectors, axis=0)
+
+    # "train" PCA model with only similar words
+    pca = PCA(n_components=2)
+    pca.fit(sim_vectors1990)
+    two_dim = pca.transform(allvectors)
+
+    # get labels
+    labels = sim_words
+    for year, model in st.session_state['models_all'].items():
+        if year in range(1810, 2000, 60):
+            if model[keyword].all() != st.session_state['models_all'][1810]['biology'].all():
+                labels.append(keyword + str(year))
+
+    #plot results
+
+    fig, ax = plt.subplots()
+    ax.scatter(two_dim[:, 0], two_dim[:, 1])
+
+    texts = [ax.text(x=two_dim[i, 0], y=two_dim[i, 1], s=labels[i]) for i in range(len(sim_words))]
+    adjust_text(texts)
+
+    #plot arrow between keywords
+
+    for i in range(-2, -(len(keyword_vectors)+1), -1):
+        ax.arrow(two_dim[i,0], two_dim[i,1],
+                  two_dim[i+1, 0] - two_dim[i,0], two_dim[i+1, 1] - two_dim[i,1],
+                  head_width=0.03, length_includes_head=True)
+
+    st.pyplot(fig)
+    fig.clear()
+    plt.close(fig)
+
+semchange(keyword)

pages/4_Word Similarity.py

@@ -0,0 +1,112 @@
+import streamlit as st
+import matplotlib.pyplot as plt
+import pandas as pd
+import numpy as np
+from scipy.interpolate import interp1d
+
+st.subheader('Word Similarity')
+
+col1, col2 = st.columns(2)
+
+with col1:
+    keyword1 = st.text_input("Input term A1", "work", key="word1")
+    keyword1 = keyword1.lower()
+
+    keyword3 = st.text_input("Input term B1", "test", key="word3")
+    keyword3 = keyword3.lower()
+
+with col2:
+    keyword2 = st.text_input("Input term A2", "hard", key="word2")
+    keyword2 = keyword2.lower()
+
+    keyword4 = st.text_input("Input term B2", "hello", key="word4")
+    keyword4 = keyword4.lower()
+
+def distchange(keyword1, keyword2):
+
+    if keyword1 not in st.session_state['models_all'][1810]:
+        st.write('Input term A1 not found in data. Please check for spelling errors.')
+        return
+    if keyword2 not in st.session_state['models_all'][1810]:
+        st.write('Input term A2 not found in data. Please check for spelling errors.')
+        return
+    if keyword3 not in st.session_state['models_all'][1810]:
+        st.write('Input term B1 not found in data. Please check for spelling errors.')
+        return
+    if keyword4 not in st.session_state['models_all'][1810]:
+        st.write('Input term B2 not found in data. Please check for spelling errors.')
+        return
+
+
+    d1 = []
+    d2 = []
+
+    for year, model in st.session_state['models_all'].items():
+        if year in range(1810, 2000, 30):
+            if model[keyword1].all() == st.session_state['models_all'][1810]['biology'].all():
+                st.write('Keyword ', keyword1, ' not available for ', year)
+            if model[keyword2].all() == st.session_state['models_all'][1810]['biology'].all():
+                st.write('Keyword ', keyword2, ' not available for ', year)
+            else:
+                d1.append(
+                    {
+                        "year": year,
+                        "similarity": model.n_similarity([keyword1], [keyword2])
+                    }
+                )
+
+    for year, model in st.session_state['models_all'].items():
+        if year in range(1810, 2000, 30):
+            if model[keyword1].all() == st.session_state['models_all'][1810]['biology'].all():
+                st.write('Keyword ', keyword3, ' not available for ', year)
+            if model[keyword2].all() == st.session_state['models_all'][1810]['biology'].all():
+                st.write('Keyword ', keyword4, ' not available for ', year)
+            else:
+                d2.append(
+                    {
+                        "year": year,
+                        "similarity": model.n_similarity([keyword3], [keyword4])
+                    }
+                )
+
+    data1 = pd.DataFrame(d1)
+    data2 = pd.DataFrame(d2)
+
+
+    # the trendline
+    x1 = data1['year'].tolist()
+    x2 = data2['year'].tolist()
+
+    y1 = data1['similarity'].tolist()
+    y2 = data2['similarity'].tolist()
+
+
+    if len(x1) < 4 or len(x2) < 4:
+        st.write('Not enough data points. Please try other keywords.')
+
+    else:
+
+        fun1 = interp1d(x1, y1, kind='cubic')
+        fun2 = interp1d(x2, y2, kind='cubic')
+
+
+        x1new = np.linspace(x1[0], 1990, 100)
+        x2new = np.linspace(x2[0], 1990, 100)
+
+
+        fig, ax = plt.subplots()
+        ax.plot(x1new, fun1(x1new), '-', label=(keyword1, keyword2))
+        ax.plot(x1, y1, 'o')
+        ax.plot(x2new, fun2(x2new), '-', label=(keyword3, keyword4))
+        ax.plot(x2, y2, 'o')
+        ax.legend()
+        ax.set_xticks(range(1810, 2000, 30))
+
+        # show plot
+        plt.xlabel("Year")
+        plt.ylabel("Cosine Similarity")
+        st.pyplot(fig)
+        fig.clear()
+        plt.close(fig)
+
+distchange(keyword1, keyword2)

requirements.txt

@@ -0,0 +1,9 @@
+streamlit~=1.13.0
+gensim~=4.1.2
+pandas~=1.4.2
+matplotlib~=3.5.1
+numpy~=1.22.3
+scikit-learn~=1.0.2
+adjusttext~=0.7.3
+s3fs~=2022.5.0
+scipy~=1.8.1