from pathlib import Path import streamlit as st import streamlit.components.v1 as components import plotly.figure_factory as ff import numpy as np import pandas as pd import requests import plotly.express as px from PIL import Image #from streamlit_plotly_events import plotly_events def app(): st.markdown('### Themes of AI-generated images on Twitter') st.markdown(""" To investigate the themes and the corresponding longitudinal changes for images in the TWIGMA dataset, we applied k-means clustering to the TWIGMA data with k = 10 and plotted the change of cluster membership over time (see Figure 2). We note substantial changes in cluster memberships: clusters 1, 3, and 4 have experienced a steady decline over time, while clusters 8 and 9 showed a consistent increase. Due to the lack of direct access to the prompts used for generating every image in TWIGMA, as well as the distance between the input prompt and the resulting output, we used BLIP (Li et al. 2022) to caption the images in TWIGMA. Prominent themes we observed include painting, woman, man, and hair, aligning with the known interest of users in generating detailed human portraits in various styles using text-to-image models. """) img_2d_embed = pd.read_csv("./resources/csv_files/twigma_release_sampled.csv") img_2d_embed = img_2d_embed.sample(frac=0.1, random_state=2023) # txt_2d_embed = pd.read_csv('data/txt_2d_embedding.csv', index_col=0) # txt_2d_embed = txt_2d_embed.sample(frac=0.1, random_state=0) col1, col2 = st.columns([1,1]) with col1: fig1 = ff.create_2d_density( x=img_2d_embed['UMAP_dim_1'], y=img_2d_embed['UMAP_dim_2'], #colors=img_2d_embed['tag'], colorscale='Greens', # set the color map height=600, # set height of the figure width=600, # set width of the figure title='Figure 1: Image embedding visualized using 2D UMAP' ) #selected_points = plotly_events(fig1, click_event=True, hover_event=True) st.plotly_chart(fig1, use_container_width=True) with col2: fig4 = Image.open('./resources/4x/change_time.png') #selected_points = plotly_events(fig1, click_event=True, hover_event=True) st.image(fig4, output_format='png', caption='Figure 2: Composition of image clusters (estimated using k-means clustering with k = 10) \ over time. We observe notable changes in cluster membership and underlying themes of \ TWIGMA images from 2021 to 2023.') st.markdown('#### Visualizing the concept of each cluster') st.markdown(""" We further visualized each cluster along with the most frequent topics derived using BLIP captions in Figure 3, with emojis representing the relative trends over time. Our findings indicate a shift in preferences for image topics among Twitter users. There is a growing interest in sharing artistically sophisticated or distinct content, such as intricate human portraits, while interest in simpler themes such as natural scenes has declined. In addition, clusters 5 and 8 notably contain a substantial number of images with increas- ing popularity but also significant amounts of NSFW, porno- graphic, and nude content. This observation aligns with recent studies that highlight the rapid growth of online communities focused on generating these models, which are relatively less regulated """) col1, col2, col3 = st.columns([1,6,1]) with col2: fig2 = Image.open('./resources/4x/cluster_change.png') st.image(fig2, caption='Figure 3: Randomly sampled images from the 10 clusters \ identified using k-means clustering. Clusters 5 and 8 predominantly contain NSFW \ photos and have been pixelated accordingly. Cluster annotations are \ provided using the most frequent words in associated BLIP-inferred captions, \ with up arrow, down arrow, and right arrow emojis indicating increasing, decreasing, or unchanged trends over time, respectively.', output_format='png') img_2d_embed_small = img_2d_embed.sample(frac=0.3, random_state=2023) # Define the desired order of the cluster labels cluster_order = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] img_2d_embed_small = img_2d_embed_small.sort_values(['label_10_clusters']) col1, col2, col3 = st.columns([1,6,1]) with col2: fig3 = px.density_contour( img_2d_embed_small, x='UMAP_dim_1', y='UMAP_dim_2', facet_col_wrap=5, color = 'label_10_clusters', facet_col='label_10_clusters', color_discrete_sequence=px.colors.qualitative.Safe, title='Figure 4: Image embedding visualized in 2D UMAP by estimated clusters') st.plotly_chart(fig3, use_container_width=True) # Rename the x and y axes # fig3.update_xaxes(title='UMAP 1') # fig3.update_yaxes(title='UMAP 2') # st.plotly_chart(fig3)