Autor: Marco Minas | Publicado: 2015-10-04T13:08:00-07:00

This code analyse Elastic Load Balancing Access log files.

#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Sources:
  # http://codereview.stackexchange.com/questions/44413/diving-into-python-sequences-analyze-an-access-log
  # http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/access-log-collection.html

import os.path
from collections import defaultdict, Counter

class LogAnalyzer():
    """ Parses and summarizes 'Elastic Load Balancing Access' logfiles """

    def __init__(self):
        """ Initializing """
        self.summary = {
            "client": {},
            "backend": {},
            "requ_process_time": {},
            "back_process_time": {},
            "resp_process_time": {},
            "status_code": {},
            "backend_status": {},
            "date": {}
        }

        self.loglist = []
        self.linecount = 0
        self.counters = defaultdict(Counter)

    def analyze(self, logfile):
        """ Reads and splits the access-log into our dictionary """
        #is file?
        if not os.path.isfile(logfile):
            print(logfile, "does not exist! exiting")
            exit(1)
        
        with open(logfile) as f:
            for line in f:
                timestamp, balancer, client, backend, request_processing_time, backend_processing_time, response_processing_time, elb_status_code, backend_status_code, received_bytes, sent_bytes, a, b, c, user_agent, ssl_cipher, ssl_protocol = line.split(' ')
                request = a + ' ' + b + ' ' + c
                #writing a dictionary per line into a list...huh...dunno
                #print(request+'\n')
                self.loglist.append({
                    "client": client.split(':')[0],
                    "backend": backend,
                    "requ_process_time": request_processing_time,
                    "back_process_time": backend_processing_time,
                    "resp_process_time": response_processing_time,
                    "status_code": elb_status_code,
                    "backend_status": backend_status_code,
                    "date": timestamp.split(':')[0] + ":00"
                })

    def summarize(self, writefile, topcount=10):
        """ count occurences """
        for col in self.loglist:
            if not col['client'] in self.summary['client']:
                self.summary['client'][col['client']] = 0
            self.summary['client'][col['client']] += 1

            if not col['backend'] in self.summary['backend']:
                self.summary['backend'][col['backend']] = 0
            self.summary['backend'][col['backend']] += 1

            if not col['requ_process_time'] in self.summary['requ_process_time']:
                self.summary['requ_process_time'][col['requ_process_time']] = 0
            self.summary['requ_process_time'][col['requ_process_time']] += 1

            if not col['back_process_time'] in self.summary['back_process_time']:
                self.summary['back_process_time'][col['back_process_time']] = 0
            self.summary['back_process_time'][col['back_process_time']] += 1

            if not col['resp_process_time'] in self.summary['resp_process_time']:
                self.summary['resp_process_time'][col['resp_process_time']] = 0
            self.summary['resp_process_time'][col['resp_process_time']] += 1

            if not col['status_code'] in self.summary['status_code']:
                self.summary['status_code'][col['status_code']] = 0
            self.summary['status_code'][col['status_code']] += 1

            if not col['backend_status'] in self.summary['backend_status']:
                self.summary['backend_status'][col['backend_status']] = 0
            self.summary['backend_status'][col['backend_status']] += 1

            if not col['date'] in self.summary['date']:
                self.summary['date'][col['date']] = 0
            self.summary['date'][col['date']] += 1

        #sorts and writes occurences into file
        report = open(writefile, 'w')
        report.write("Log summary\n")
        for key in self.summary:
            list = sorted(self.summary[key].items(), key=lambda x: x[1], reverse=True)
            if (topcount != -1):
                list = list[:topcount]
            report.write("\nTop "+key+":\n")
            report.write("\nposition\tkey\ttotal\n")
            index = 0
            for l in list:
                index += 1
                sindex = str(index)
                if (topcount != -1):
                    sindex = str(index).zfill(len(str(topcount)))
                report.write( sindex + '\t' + l[0] + "\t" + str(l[1]) + "\n")
        report.close()

    @staticmethod
    def _parse(line):
        timestamp, balancer, client, backend, request_processing_time, backend_processing_time, response_processing_time, elb_status_code, backend_status_code, received_bytes, sent_bytes, a, b, c, user_agent, ssl_cipher, ssl_protocol = line.split(' ')
        request = a + ' ' + b + ' ' + c
        return {'timestamp': timestamp, 'balancer': balancer, 'client': client }

    def _update(self, **kwargs):
        self.linecount += 1
        for key, value in kwargs.items():
            self.counters[key][value] += 1

def analyseLogsOnDirectory(dir = "./logs-loadbalance"):
    print("\nAnalysing logs o directory %s\n" % dir)

    analysis = LogAnalyzer()
    for file in os.listdir(dir):
        if file.endswith(".log"):
            print('Analysing file \'' + os.path.join(dir, file) + '\'')
            analysis.analyze(os.path.join(dir, file))

    print("\nCreating to 10 symmary")
    analysis.summarize('solr_summary_top_10.txt')
    print("Creating to 50 symmary")
    analysis.summarize('solr_summary_top_50.txt', 50)
    print("Creating symmary of all items")
    analysis.summarize('solr_summary_all.txt', -1)
    print("\nCreating symmary finished")

if __name__ == '__main__':
    analyseLogsOnDirectory()

Enjoy