This code analyse Elastic Load Balancing Access log files.#!/usr/bin/env python# -*- coding: utf-8 -*-# Sources: # http://codereview.stackexchange.com/questions/44413/diving-into-python-sequences-analyze-an-access-log # http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/access-log-collection.htmlimport os.pathfrom collections import defaultdict, Counterclass LogAnalyzer(): """ Parses and summarizes 'Elastic Load Balancing Access' logfiles """ def __init__(self): """ Initializing """ self.summary = { "client": {}, "backend": {}, "requ_process_time": {}, "back_process_time": {}, "resp_process_time": {}, "status_code": {}, "backend_status": {}, "date": {} } self.loglist = [] self.linecount = 0 self.counters = defaultdict(Counter) def analyze(self, logfile): """ Reads and splits the access-log into our dictionary """ #is file? if not os.path.isfile(logfile): print(logfile, "does not exist! exiting") exit(1) with open(logfile) as f: for line in f: timestamp, balancer, client, backend, request_processing_time, backend_processing_time, response_processing_time, elb_status_code, backend_status_code, received_bytes, sent_bytes, a, b, c, user_agent, ssl_cipher, ssl_protocol = line.split(' ') request = a + ' ' + b + ' ' + c #writing a dictionary per line into a list...huh...dunno #print(request+'\n') self.loglist.append({ "client": client.split(':')[0], "backend": backend, "requ_process_time": request_processing_time, "back_process_time": backend_processing_time, "resp_process_time": response_processing_time, "status_code": elb_status_code, "backend_status": backend_status_code, "date": timestamp.split(':')[0] + ":00" }) def summarize(self, writefile, topcount=10): """ count occurences """ for col in self.loglist: if not col['client'] in self.summary['client']: self.summary['client'][col['client']] = 0 self.summary['client'][col['client']] += 1 if not col['backend'] in self.summary['backend']: self.summary['backend'][col['backend']] = 0 self.summary['backend'][col['backend']] += 1 if not col['requ_process_time'] in self.summary['requ_process_time']: self.summary['requ_process_time'][col['requ_process_time']] = 0 self.summary['requ_process_time'][col['requ_process_time']] += 1 if not col['back_process_time'] in self.summary['back_process_time']: self.summary['back_process_time'][col['back_process_time']] = 0 self.summary['back_process_time'][col['back_process_time']] += 1 if not col['resp_process_time'] in self.summary['resp_process_time']: self.summary['resp_process_time'][col['resp_process_time']] = 0 self.summary['resp_process_time'][col['resp_process_time']] += 1 if not col['status_code'] in self.summary['status_code']: self.summary['status_code'][col['status_code']] = 0 self.summary['status_code'][col['status_code']] += 1 if not col['backend_status'] in self.summary['backend_status']: self.summary['backend_status'][col['backend_status']] = 0 self.summary['backend_status'][col['backend_status']] += 1 if not col['date'] in self.summary['date']: self.summary['date'][col['date']] = 0 self.summary['date'][col['date']] += 1 #sorts and writes occurences into file report = open(writefile, 'w') report.write("Log summary\n") for key in self.summary: list = sorted(self.summary[key].items(), key=lambda x: x[1], reverse=True) if (topcount != -1): list = list[:topcount] report.write("\nTop "+key+":\n") report.write("\nposition\tkey\ttotal\n") index = 0 for l in list: index += 1 sindex = str(index) if (topcount != -1): sindex = str(index).zfill(len(str(topcount))) report.write( sindex + '\t' + l[0] + "\t" + str(l[1]) + "\n") report.close() @staticmethod def _parse(line): timestamp, balancer, client, backend, request_processing_time, backend_processing_time, response_processing_time, elb_status_code, backend_status_code, received_bytes, sent_bytes, a, b, c, user_agent, ssl_cipher, ssl_protocol = line.split(' ') request = a + ' ' + b + ' ' + c return {'timestamp': timestamp, 'balancer': balancer, 'client': client } def _update(self, **kwargs): self.linecount += 1 for key, value in kwargs.items(): self.counters[key][value] += 1def analyseLogsOnDirectory(dir = "./logs-loadbalance"): print("\nAnalysing logs o directory %s\n" % dir) analysis = LogAnalyzer() for file in os.listdir(dir): if file.endswith(".log"): print('Analysing file \'' + os.path.join(dir, file) + '\'') analysis.analyze(os.path.join(dir, file)) print("\nCreating to 10 symmary") analysis.summarize('solr_summary_top_10.txt') print("Creating to 50 symmary") analysis.summarize('solr_summary_top_50.txt', 50) print("Creating symmary of all items") analysis.summarize('solr_summary_all.txt', -1) print("\nCreating symmary finished")if __name__ == '__main__': analyseLogsOnDirectory()Enjoy Real-time log files watcher supporting log rotation. Source: http://code.activestate.com/recipes/577968-log-watcher-tail-f-log/#!/usr/bin/env python"""Real-time log files watcher supporting log rotation.Works with Python >= 2.6 and >= 3.2, on both POSIX and Windows.Author: Giampaolo Rodola' < g.rodola [AT] gmail [DOT] com >License: MIThttp://code.activestate.com/recipes/577968-log-watcher-tail-f-log/"""import os, time, errno, stat, sysclass LogWatcher(object): """Looks for changes in all files of a directory. This is useful for watching log file changes in real-time. It also supports files rotation. Example: >>> def