app.py

import streamlit as st
import pandas as pd
from wordcloud import WordCloud
import matplotlib.pyplot as plt
import plotly.express as px
import re
from collections import Counter
from itertools import chain

# Set page configuration
st.set_page_config(page_title="Advanced Log Analytics", page_icon="🔍")

# Title of the app
st.title("Interactive Log Analytics with N-gram Keyword Extraction")

# Function to read log files with error handling for invalid UTF-8 sequences
def read_log_file(file):
    try:
        # Attempt to read the file with error handling
        return file.read().decode('utf-8', errors='replace')
    except Exception as e:
        st.error(f'Error reading file: {str(e)}')
        st.stop()

# Function to manually extract n-grams from log lines
STOPWORDS = set(['the', 'is', 'in', 'and', 'to', 'a', 'of', 'for', 'with', 'on', 'this', 'as', 'that', 'by', 'from', 'at', 'are', 'it', 'was', 'an', 'be', 'will', 'or', 'but', 'not'])

def extract_ngrams(text, n=2):
    # Tokenize and filter tokens
    tokens = re.findall(r'\b\w+\b', text.lower())
    tokens_filtered = [token for token in tokens if token not in STOPWORDS]
    # Generate n-grams
    ngrams = zip(*[tokens_filtered[i:] for i in range(n)])
    return [' '.join(ngram) for ngram in ngrams]

# File uploader for log files
uploaded_file = st.file_uploader("Upload your log file", type=["txt", "log"])

if uploaded_file is not None:
    # Read the log contents
    log_data = read_log_file(uploaded_file)

    st.write("### Log File Content Preview")
    st.text(log_data[:1000])  # Display the first 1000 characters

    # Preprocess log data (simple split by lines)
    log_lines = log_data.splitlines()

    # Slider to select number of n-grams (1 for unigrams, 2 for bigrams, etc.)
    n_value = st.slider("Select number of words in N-grams", min_value=1, max_value=5, value=2)

    # Extract n-grams from each log line based on slider value
    ngrams_from_logs = [extract_ngrams(line, n_value) for line in log_lines]

    # Flatten the list of n-grams and count their frequencies
    all_ngrams = list(chain.from_iterable(ngrams_from_logs))
    ngram_frequencies = Counter(all_ngrams)

    # Convert to DataFrame for viewing and sort by frequency
    ngram_df = pd.DataFrame(ngram_frequencies.items(), columns=['N-gram', 'Frequency'])
    
    # Dropdown to select top or bottom N n-grams for visualization
    top_bottom_choice = st.selectbox("Select visualization type", ["Top N", "Bottom N"])
    
    # Slider to select how many N-grams to display
    num_ngrams_to_display = st.slider("Select number of N-grams to display", min_value=1, max_value=50, value=20)

    if top_bottom_choice == "Top N":
        selected_ngrams_df = ngram_df.sort_values(by='Frequency', ascending=False).head(num_ngrams_to_display)
    else:
        selected_ngrams_df = ngram_df.sort_values(by='Frequency').head(num_ngrams_to_display)

    # Display selected N-grams with frequencies
    st.write(f"### {top_bottom_choice} {num_ngrams_to_display} N-grams")
    st.dataframe(selected_ngrams_df)

    # Generate and display a word cloud of selected N-grams
    st.write("### N-gram Frequency Word Cloud")
    wordcloud = WordCloud(width=800, height=400, background_color='white').generate_from_frequencies(
        {row['N-gram']: row['Frequency'] for _, row in selected_ngrams_df.iterrows()}
    )

    plt.figure(figsize=(10, 5))
    plt.imshow(wordcloud, interpolation='bilinear')
    plt.axis('off')
    st.pyplot(plt)

    # Prepare data for heatmap: count occurrences of each selected n-gram per line number
    heatmap_data = pd.DataFrame([(ng, i) for i, ngs in enumerate(ngrams_from_logs) for ng in ngs],
                                columns=['N-gram', 'LineNumber'])
    
    heatmap_data = heatmap_data[heatmap_data['N-gram'].isin(selected_ngrams_df['N-gram'])]
    
    heatmap_matrix = pd.crosstab(index=heatmap_data['LineNumber'], columns=heatmap_data['N-gram'])

    # Create a heatmap using Plotly
    st.write("### Heatmap of Line Number and Selected N-grams")
    
    fig = px.imshow(heatmap_matrix.T, 
                    labels=dict(x="Line Number", y="N-grams", color="Frequency"),
                    x=heatmap_matrix.index,
                    y=heatmap_matrix.columns,
                    aspect="auto",
                    color_continuous_scale='YlGnBu')

    fig.update_layout(title='Heatmap of Line Number vs Selected N-grams',
                      xaxis_title='Line Number',
                      yaxis_title='N-grams')

    st.plotly_chart(fig)

    # Dropdown to filter logs based on selected N-grams
    filter_ngram_choice = st.selectbox("Filter logs by N-gram", options=['None'] + list(selected_ngrams_df['N-gram']))
    
    if filter_ngram_choice != 'None':
        filtered_logs = [line for line, ngrams in zip(log_lines, ngrams_from_logs) if filter_ngram_choice in ngrams]
        st.write(f"### Logs containing '{filter_ngram_choice}'")
        st.dataframe(pd.DataFrame(filtered_logs, columns=["Log Line"]))

# Sidebar information
st.sidebar.title("About")
st.sidebar.info("""
This tool uses manual extraction of n-grams from log files to identify trends, anomalies, 
and potential issues through interactive visualization and analysis.
""")