########################################################
# In this lab we'll see various examples of network visualization, 
# using different design idioms and layout options,
# including adjacency matrix, dendrogram, circular tree and treemap 
########################################################

# first set the working directory on your computer
setwd('~/Dropbox/IS teaching/IS4335 data_vis/lab/R_codes')

# load the libraries
library(igraph)
library(ggplot2)
library(reshape2)


# load the network data
load("~/Dropbox/IS teaching/IS4335 data_vis/2024_files/NetData/data/kracknets.rda")

krack_full_data <- cbind(advice_data_frame, 
                         friendship_tie = friendship_data_frame$friendship_tie, 
                         reports_to_tie = reports_to_data_frame$reports_to_tie)

krack_full_nonzero_edges <- subset(krack_full_data_frame, 
                                   (advice_tie > 0 | friendship_tie > 0 | reports_to_tie > 0))

attributes2 <- cbind(id = 1:nrow(attributes), attributes)  # bind node id with attribute columns
krack_full <- graph_from_data_frame(d = krack_full_nonzero_edges, directed = F,
                                    vertices = attributes2)

##############################################################################
# 1. network properties
##############################################################################
# last time in the lecture we have discussed network properties, 
# such as degree distribution, diameter/path length, and clustering coefficient

# Many initial analyses of networks begin with degree distributions & distance, 
# and then move towards global summary statistics of the network. 
#
# As a reminder, entering a question mark followed by a function 
# name (e.g., ?graph.density) pulls up the help file for that function.
# This can be helpful to understand how, exactly, stats are calculated.

# let's first see an example of degree distribution
# in network language, degree is the number of connections that a node has
deg_full <- degree(krack_full, mode="all") 
is.directed(krack_full)
# if the graph is directed, we can check indegree and outdegree distributions separately
# but since the graph is undirected, there is no need for that
deg_full
# the degree distribution shows how many ties that each node has

mean(deg_full)  #check average degree
sd(deg_full)    #check standard deviation

# network diameter and average distance
diameter(krack_full)
mean_distance(krack_full)
# diameter and average distance is very small given everyone is connected. 

# let's compare the full network to friendship and reports-to networks
krack_friendship_only <- delete_edges(krack_full, E(krack_full)[edge_attr(krack_full, name = "friendship_tie") == 0])
krack_reports_to_only <- delete_edges(krack_full, E(krack_full)[edge_attr(krack_full, name = "reports_to_tie") == 0])

diameter(krack_friendship_only)
diameter(krack_reports_to_only)
# we see the diameters of the latter graphs are longer, since there are fewer ties
# connecting people. however, everyone in the company is related to each other in someway
# so the overall diameter and distance are very small

# Density: the portion of potential connections in a network that are realized
graph.density(krack_friendship_only)
graph.density(krack_reports_to_only)

# Transitivity: if a and b are connected, b and c are connected, 
# what's the likelihood that a and c are also connected?
transitivity(krack_full)
transitivity(krack_friendship_only)
transitivity(krack_reports_to_only)

##############################################################################
# 2. network visualization examples: graph layout and adjacency matrix
##############################################################################
# last time we saw visualization of network connections 
# the Fruchterman and Reingold's layout in week 8 lab uses a force-directed placement algorithm.
# let's play with other layout functions
# first Reingold-Tilford tree
plot(krack_full, vertex.color=dept_vertex_colors, layout = layout_as_tree(krack_full))

# alternatively, you can first calculate the layout and then pass it into the plot  
layout <- layout_as_tree(krack_full)  
plot(krack_full, vertex.color=dept_vertex_colors, layout = layout)

# circle
layout <- layout_in_circle(krack_full)
plot(krack_full, vertex.color=dept_vertex_colors, layout = layout)

# star
layout <- layout_as_star(krack_full)
plot(krack_full, vertex.color=dept_vertex_colors, layout = layout)


##############################################################################
# 3. community structure + adjacency matrix
##############################################################################

# the friendship pattern is unclear and it's better to cluster nodes by community memberships 
# ties within communities are denser since nodes have closer relationships.

# there are several ways to create communities, a common method is to use edge betweenness:
# The betweenness score of an edge measures the number of
# shortest paths from one node to another that go through it. 
# The idea of the edge-betweenness based community structure
# is that it is likely that edges connecting separate
# cluster have high edge-betweenness, as all the shortest paths
# from one cluster to another must traverse through them. So if we
# iteratively remove the edge with the highest betweenness
# score we will get a hierarchical map of the communities in the graph. 

# The following function performs community detection using the edge betweenness algorithm
friend_comm <- cluster_edge_betweenness(krack_friendship_only, weights = NULL, directed=F)
friend_comm

# community membership of each node can be queried as follows:
friend_comm$membership

# Visualize the network with community memberships
plot(friend_comm, krack_friendship_only) 
# communities are colored in different shades, and nodes within the same communities are clustered together

# This process also lends itself to visualization as a dendrogram.
# The y-axis reflects the distance metric used by the edge betweenness algorithm
plot(as.dendrogram(friend_comm))
# the tree branches show membership hierarchies. nodes belonging to the same community share common roots
# the y-axis is a distance measure showing how far apart are nodes


# now we can rearrange nodes by their community memberships
# first create a data frame that records node id and communities
# and reorder node id's by membership affiliation
membership <- data.frame(cbind(friend_comm$membership, friend_comm$names))
names(membership) <- c('group','id')
class(membership$group). # string var
membership$group <- as.numeric(membership$group)
membership <- membership[order(membership$group),] # sort data frame by the group column


########################################################################
# 4.  Adjacency matrix plots
########################################################################

# the previous graphs all show node-link diagrams. but how do we visualize networks as adjacency matrix? 
# let's first get the adjacency matrix of krack_full using friendship ties only:
krack_friend <- as.matrix(as_adjacency_matrix(krack_full, attr='friendship_tie'))
krack_friend
# some of the edges are redundant since there are multiple tie types, so the matrix cell values could be > 1
# we want to remove the redundant ties by converting matrix to binary 0/1 values

krack_friend[krack_friend>1] <- 1  # convert matrix to binary 

# we'll use the ggplot2 library to plot the adjacency matrix. but first we need to convert the matrix into a data frame.
# this could be done using the melt function from reshape2:
krack_friend_long <-melt(krack_friend)
head(krack_friend_long)
names(krack_friend_long) <- c('from','to','tie')

# convert first two columns to factors, since id's are discrete categories
krack_friend_long$from <- factor(krack_friend_long$from)
krack_friend_long$to <- factor(krack_friend_long$to)

# Create the adjacency matrix plot using ggplot2
# we plot the matrix using color tiles, each tile represent the friendship relations btw two nodes.
# nodes with friendship ties are colored as red, otherwise left blank
# baseline plot
ggplot(krack_friend_long[krack_friend_long$tie>0,], aes(x = from, y = to, fill = factor(tie))) +
  geom_raster() +  # create rectangular tiles when all the tiles are the same size. can also use geom_tile()
  labs(x='', y='', fill='Friend') +
  scale_fill_manual(values=c('red','grey90'), # manually define colors that correspond to the levels in fill
                    breaks=c(1,0),            # how to order legend for fill
                    labels=c('yes','no')) +   # legend label. here we only have "yes"
  theme_bw() +
  theme(aspect.ratio = 1)                     # Force the plot into a square aspect ratio


# how do we reorder the graph based on community affiliations?
# recall we have membership affiliations from before:
membership

# Reorder edge list "from" and "to" factor levels based on the ids ordered by membership affiliations
# in factor(), arg "levels" can specify how we want the factor levels to be ordered
krack_friend_long$from <- factor(krack_friend_long$from, levels = membership$id)
krack_friend_long$to <- factor(krack_friend_long$to, levels = membership$id)

# include membership info of from and to nodes using join function.
# first change the names of the membership columns to be matched with plot_data

names(membership) <- c('from_group','from')  # memebership of ego
plot_data <- krack_friend_long %>% plyr::join(membership, by='from')  # plyr::join calls the join function from plyr library

names(membership) <- c('to_group','to')   # memebership of alter
plot_data <- plot_data %>% plyr::join(membership, by='to')
names(plot_data)  # membership info added

# adding membership affiliations of nodes allow us to color in-group and out-group ties separately
# here we want to color in-group ties by communities, and out-group ties as grey 
# how do we do that?
# first we identify ties between members of the same community for in-group ties: from_group = to_group
# if from_group != to_group, then the tie is an out-group tie
# the info is stored in memb_tie variable
plot_data$memb_tie <- 0  # default is out-group
plot_data$memb_tie[plot_data$from_group==plot_data$to_group] <- 1  # mark out within-group ties
plot_data$memb_tie[plot_data$memb_tie==1] <- plot_data$from_group[plot_data$memb_tie==1] # replace the marker with group info
plot_data$memb_tie

# Now run the ggplot code again, include ties with non-zero values and color ties based on membership
# within-group ties have different colors, while between-group ties are all grey
# improved plot
ggplot(plot_data[plot_data$tie>0,], aes(x = from, y = to, fill = factor(memb_tie))) +
  geom_raster() +  
  labs(x='', y='', fill='Within-community ties') +
  scale_fill_manual(values = c('grey','magenta','darkgreen','royalblue')) +  # manually set community colors
  scale_x_discrete(drop = FALSE) +
  scale_y_discrete(drop = FALSE) +
  theme_bw() +
  theme(aspect.ratio = 1)    

# nodes of the same communities are placed close to each other, 
# and we can now see some group structure from this adjacency matrix plot.
# between-group ties are colored as grey, and within-group ties are colored according to membership affiliations
# the advantage of adjacency matrix representation is that:
# a) scalable, less messy than node-link diagrams when the number of nodes is large
# b) community structures are easier to spot


########################################################################
# 5. radial tree & treemaps 
########################################################################

# A treemap is a space-filling visualization of hierarchical structures. The treemap package offers great flexibility to draw it
install.packages('treemap')
library(treemap)
library(dplyr)

# radial tree:
# use the treegraph function in the treemap package. 
# index: vector of column names that specify the aggregation indices. It could
# contain only one column name, which results in a treemap without hierarchy.
# If multiple column names are provided, the first name is the highest aggregation level, 
# the second name the second-highest aggregation level, and so on.
treegraph(krack_friend_long[krack_friend_long$tie==1,], index = c('from','to'), directed=F, show.labels=T)

# here we map out the friendship connections among nodes using hierarchical tree structure.
# there's an artificial root in the center, and each node in the second layer corresponds to a node in the "from" column
# the third layer consists of nodes in the "to" column connected to the source node in the "from" column
# calculate degrees of source nodes by counting number of connections
krack_friend_degree <- 
  krack_friend_long %>% 
  group_by(from) %>% 
  summarise(degree = sum(tie))
  
krack_treemap <- plyr::join(krack_friend_long, krack_friend_degree, by='from')

# index treemap: colors are determined by the index argument
# different branches in the hierarchical tree get different colors. 
# in this case, colors are defined by the source nodes
# size of the rectangles are defined by the vSize argument, 
# the higher the degree count, the bigger the rectangle size
treemap(krack_treemap[krack_treemap$tie==1,],
        index = c('from','to'),
        vSize = 'degree',
        type="index")


########################################################################
# 6. week 9 assignment: due on Sunday midnight
########################################################################

# Task 1. visualize all network ties in krack_full using the adjacency matrix.
# color edges according to friendship, advice, and reports-to ties

# Task 2. cluster nodes by community membership using the friendship network, and replot the adjacency matrix
# with nodes belonging to the same communities placed close to each other.
# construct a new variable, tie strength, which is the combination of friendship, advice, and reports-to ties
# this counts the types of relationships between two nodes

# Task 3. use radial tree to visualize the reports-to network, and a treemap to show the advice network.
# in the treemap, set size proportional to ego's degrees from the advice and reports-to networks

help  me to do the task and write all R code