app.py

# -*- coding: utf-8 -*-
"""GradioInterface_v2.ipynb
Automatically generated by Colaboratory.
"""

# Commented out IPython magic to ensure Python compatibility.
# # Capture to supress the download ouput
# %%capture
# !pip install gradio
# !pip install pandas
# !pip install transformers
# !pip install parsezeeklogs
# !pip install elasticsearch

# Define imports for model use
import torch
from transformers import pipeline
from parsezeeklogs import ParseZeekLogs
from transformers import BertTokenizer
import gradio as gr
import pandas as pd

# Define model
pipe = pipeline(model="19kmunz/IoT-23-BERT-Network-Logs-Classification", tokenizer=BertTokenizer.from_pretrained("bert-base-cased"))

# Define string constants
LOG = "conn.log Output"
HEADER_TABLE = "Headers Table"
SENTENCES = "Sentences"
OUT = "out"
INPUT_TYPES = [LOG, HEADER_TABLE, SENTENCES]
STEPS = [HEADER_TABLE, SENTENCES]
HEADERS=['id.resp_p', 'proto', 'conn_state', 'orig_pkts', 'orig_ip_bytes', 'resp_ip_bytes']

# Define sentence-ization functions
# Dictionary of feature names to use in the make sentence function
feature_names = {'id.resp_p':'response port',
                 'proto':'transport protocol',
                 'orig_pkts':'number of packets sent by the origin',
                 'conn_state':'connection state',
                 'orig_ip_bytes':'number of IP level bytes sent by the originator',
                 'resp_ip_bytes':'number of IP level bytes sent by the responder'}

# Function to make sentences out of the data
def make_sentence(row):
  sentences = {}
  for feature in row.keys():
    if feature == 'label' or feature == "#":
      sentences[feature] = row[feature]
    else:
      sentences[feature] = feature_names[feature] + " is " + str(row[feature]) + "."
  return sentences

# Take all sentence observations and make them into paragraph inputs
def make_paragraphs(ser):
  paragraphs_list = []
  for index,obs in ser.items():
    new_para = obs['id.resp_p'] + " " + obs['proto'] + " " + obs['conn_state'] + " " + obs['orig_pkts'] + " " + obs['orig_ip_bytes'] + " " + obs['resp_ip_bytes']
    paragraphs_list.append(new_para)
  return pd.Series(paragraphs_list, name="Sentences", index=ser.index).to_frame()

# Define prediction Functions For Different Settings
def predictFromSentences(sentenceTable):
  output = pipe(sentenceTable[SENTENCES].tolist()) # This does the prediction!
  return { OUT: pd.DataFrame({"Output": ["Malicious" if pred['label'] == "LABEL_0" else "Benign" for pred in output] }) }

def predictFromHeaderTable(headerTable):
  sentences = headerTable.apply(make_sentence, axis=1);
  paragraphs = make_paragraphs(sentences)
  return {
      SENTENCES: paragraphs,
      OUT: predictFromSentences(paragraphs)[OUT]
  }

def predictFromFileUpload(fileUpload):
  if(fileUpload is None):
      raise gr.Error("No file uploaded")
  fileType = fileUpload.split('.')[-1]
  if(fileType == 'csv'):
    dataFrame = pd.read_csv(fileUpload, usecols=HEADERS)
  elif(fileType == 'log' or fileType == 'labeled'):
    with open('out.csv',"w") as outfile:
      for log_record in ParseZeekLogs(fileUpload, output_format="csv", safe_headers=False, fields=HEADERS):
          if log_record is not None:
              outfile.write(log_record + "\n")
    dataFrame = pd.read_csv('out.csv', names=HEADERS)
  result = predictFromHeaderTable(dataFrame)
  toReturn =  {
      HEADER_TABLE: dataFrame,
      SENTENCES: result[SENTENCES],
      OUT: result[OUT]
  }
  return toReturn

def makeIndexColumn(allInputs):
  def _makeIndexColumnFor(column):
    theseHeaders = allInputs[column].columns
    newHeaders = ['#', *theseHeaders]
    allInputs[column]['#'] = allInputs[column].index
    allInputs[column] = allInputs[column][newHeaders]

  if(SENTENCES in allInputs):
    _makeIndexColumnFor(SENTENCES)
  if(HEADER_TABLE in allInputs):
    _makeIndexColumnFor(HEADER_TABLE)
  if(OUT in allInputs):
    _makeIndexColumnFor(OUT)
  return allInputs

def predict(inputType, fileUpload, headerTable, sentenceTable, out):
  output = {};
  if(inputType == LOG):
    # Process File Upload
    output = makeIndexColumn(predictFromFileUpload(fileUpload))
    return [output[HEADER_TABLE], output[SENTENCES], output[OUT]]
  elif(inputType == HEADER_TABLE):
    # Process Header Table
    output = makeIndexColumn(predictFromHeaderTable(headerTable))
    return [headerTable, output[SENTENCES], output[OUT]]
  elif(inputType == SENTENCES):
    # Process From Sentences
    output = makeIndexColumn(predictFromSentences(sentenceTable))
    return [headerTable, sentenceTable, output[OUT]]

# Update UI
def updateInputOutputBlocks(inputType, steps):
  # Update visibility and Interactivity of Gradio Blocks based on Settings
  fileUpload = gr.File(
      visible=(True if inputType == LOG else False),
      interactive=(1 if inputType == LOG else 0)
  )
  headerTable = gr.Dataframe(
      visible=(True if (inputType == HEADER_TABLE or HEADER_TABLE in steps) else False),
      interactive=(1 if inputType == HEADER_TABLE else 0)
    )
  sentenceTable = gr.Dataframe(
      interactive=(1 if inputType == SENTENCES else 0),
      visible=(True if (inputType == SENTENCES or SENTENCES in steps) else False)
    )
  return fileUpload, headerTable, sentenceTable

# Create Gradio UI
with gr.Blocks() as app:
    gr.Markdown("""
    # Network Log Predictions
    Input log information below and click 'Run' to get predictions from our model!
    Access the settings at the bottom for different types of input and to see inbetween steps.
    """)
    # Inputs / Outputs
    fileUpload = gr.File(file_types=[".log", ".log.labeled", ".csv"], label="Zeek Log File", visible=False, file_count='single')
    headerTable = gr.Dataframe(row_count = (2, "dynamic"), col_count=(7,"fixed"), headers=['#', *HEADERS], label="Header Inputs", interactive=1)
    sentenceTable = gr.Dataframe(row_count = (2, "dynamic"), col_count=(2, "fixed"), headers=["#", "Sentence"], label="Sentences", interactive=0, visible=False)
    out = gr.Dataframe(row_count = (2, "dynamic"), col_count=(2, "fixed"), headers=['#', "Output"], label="Predictions", column_widths=["60px", "100%"])
    btn = gr.Button("Run")

    # Settings
    with gr.Accordion("Settings", open=False):
      inputType = gr.Radio(INPUT_TYPES, value="Headers Table", label="Input")
      steps = gr.CheckboxGroup(STEPS, label="Display Intermediary Steps")
      inputType.change(
        fn=updateInputOutputBlocks,
        inputs=[inputType, steps],
        outputs=[fileUpload, headerTable, sentenceTable]
      )
      steps.change(
        fn=updateInputOutputBlocks,
        inputs=[inputType, steps],
        outputs=[fileUpload, headerTable, sentenceTable]
      )
    # Assign Callback
    btn.click(
        fn=predict,
        inputs=[inputType, fileUpload, headerTable, sentenceTable, out],
        outputs=[headerTable, sentenceTable, out]
        )

app.launch()