import streamlit as st import pandas as pd import re import nltk from PIL import Image import os import numpy as np import seaborn as sns from wordcloud import WordCloud, STOPWORDS from nltk.corpus import stopwords import datasets from datasets import load_dataset import matplotlib.pyplot as plt import sklearn from sklearn.preprocessing import LabelEncoder sns.set_palette("RdBu") # loading dataset dataset = load_dataset("merve/poetry", streaming=True) df = pd.DataFrame.from_dict(dataset["train"]) d = os.path.dirname(__file__) if "__file__" in locals() else os.getcwd() nltk.download("stopwords") stop = stopwords.words('english') # standardizing dataset by removing special characters and lowercasing def standardize(text, remove_digits=True): text=re.sub('[^a-zA-Z\d\s]', '',text) text = text.lower() return text st.set_option('deprecation.showPyplotGlobalUse', False) st.write("Poetry dataset, content column cleaned from special characters and lowercased") df.content = df.content.apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)])) df.content=df.content.apply(standardize) st.dataframe(df) st.subheader("Visualization on dataset statistics") st.write("Number of poems written in each type") sns.catplot(x="type", data=df, kind="count") plt.xticks(rotation=0) st.pyplot() st.write("Number of poems for each age") sns.catplot(x="age", data=df, kind="count") plt.xticks(rotation=0) st.pyplot() st.write("Number of poems for each author") sns.catplot(x="author", data=df, kind="count", aspect = 4) plt.xticks(rotation=90) st.pyplot() # distributions of poem types according to ages and authors st.write("Distributions of poem types according to ages and authors, \\nseems that folks in renaissance loved the love themed poems \\nand nature themed poems became popular later") le = LabelEncoder() df.author = le.fit_transform(df.author) sns.catplot(x="age", y="author",hue="type", data=df) st.pyplot() #words = df.content.str.split(expand=True).unstack().value_counts() # most appearing words other than stop words words = df.content.str.split(expand=True).unstack().value_counts() renaissance = df.content.loc[df.age == "Renaissance"].str.split(expand=True).unstack().value_counts() modern = df.content.loc[df.age == "Modern"].str.split(expand=True).unstack().value_counts() st.subheader("Visualizing content") mask = np.array(Image.open(os.path.join(d, "poet.png"))) import matplotlib.pyplot as plt def word_cloud(content, title): wc = WordCloud(background_color="white", max_words=200,contour_width=3, stopwords=STOPWORDS, max_font_size=50) wc.generate(" ".join(content.index.values)) fig = plt.figure(figsize=(10, 10)) plt.title(title, fontsize=20) plt.imshow(wc.recolor(colormap='magma', random_state=42), cmap=plt.cm.gray, interpolation = "bilinear", alpha=0.98) plt.axis('off') st.pyplot() st.subheader("Most appearing words excluding stopwords in poems according to ages") word_cloud(modern, "Word Cloud of Modern Poems") word_cloud(renaissance, "Word Cloud Renaissance Poems") # most appearing words including stopwords st.write("Most appearing words including stopwords") st.bar_chart(words[0:50]) st.set_option('deprecation.showPyplotGlobalUse', False)