Autor: Marco Minas | Publicado: 2015-10-04T13:37:00-07:00

This code analyse Elastic Load Balancing Access log files. 
Homework done - this version apply codereview.stackexchange.com suggestions

#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
Research sources:
 * http://codereview.stackexchange.com/questions/44413/diving-into-python-sequences-analyze-an-access-log
 * http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/access-log-collection.html
 * http://stackoverflow.com/questions/1823058/how-to-print-number-with-commas-as-thousands-separators
'''

import os.path, locale
from collections import defaultdict, Counter

class LogAnalyzer():
    """ Parses and summarizes 'Elastic Load Balancing Access' logfiles """

    def __init__(self):
        """ Initializing """

        self.loglist = []
        self.linecount = 0
        self.counters = defaultdict(Counter)
        locale.setlocale(locale.LC_ALL, 'pt_BR')

    def analyze(self, logfile):
        """ Reads and splits the access-log into our dictionary """

        if not os.path.isfile(logfile):
            print(logfile, "does not exist! exiting")
            exit(1)

        with open(logfile) as f:
            for line in f:
                self._update(**self._parse(line))

    def summarize(self, writefile, topcount=10):
        """ count occurences """

        report = open(writefile, 'w')
        report.write("Log summary - Total lines read: %s\n" % locale.format("%d", self.linecount, grouping=True))

        for key in self.counters:
            list = sorted(self.counters[key].items(), key=lambda x: x[1], reverse=True)
            if (topcount != -1):
                list = list[:topcount]
            report.write("\nTop "+key+":\n")
            report.write("\nposition\tkey\ttotal\n")
            index = 0
            for l in list:
                index += 1
                sindex = str(index).zfill(len(str(self.linecount)))
                if (topcount != -1):
                    sindex = str(index).zfill(len(str(topcount)))
                total = locale.format("%d", l[1], grouping=True)
                report.write( sindex + '\t' + l[0] + "\t" + total + "\n")

        report.close()

    @staticmethod
    def _parse(line):
        timestamp, balancer, client, backend, request_processing_time, backend_processing_time, response_processing_time, elb_status_code, backend_status_code, received_bytes, sent_bytes, a, b, c, user_agent, ssl_cipher, ssl_protocol = line.split(' ')
        request = a + ' ' + b + ' ' + c
        date = timestamp.split(':')[0].replace('T', ' ') + ':00' # gets until hour only.
        return {
                'client': client, 
                'backend': backend, 
                'requ_process_time': request_processing_time, 
                'back_process_time': backend_processing_time, 
                'resp_process_time': response_processing_time, 
                'ebs_status': elb_status_code, 
                'backend_status': backend_status_code, 
                'date': date 
                }

    def _update(self, **kwargs):
        self.linecount += 1
        for key, value in kwargs.items():
            self.counters[key][value] += 1

def analyseLogsOnDirectory(dir = "./logs-loadbalance"):
    print("\nAnalysing logs o directory %s\n" % dir)

    analysis = LogAnalyzer()
    for file in os.listdir(dir):
        if file.endswith(".log"):
            print('Analysing file \'' + os.path.join(dir, file) + '\'')
            analysis.analyze(os.path.join(dir, file))

    print("\nCreating to 10 symmary")
    analysis.summarize('solr_summary_top_10.txt')
    print("Creating to 50 symmary")
    analysis.summarize('solr_summary_top_50.txt', 50)
    print("Creating symmary of all items")
    analysis.summarize('solr_summary_all.txt', -1)
    print("\nCreating symmary finished")

if __name__ == '__main__':
    analyseLogsOnDirectory('./logs-loadbalance/../02')