callback(filename, lines): ... print(filename, lines) ... >>> lw = LogWatcher("/var/log/", callback) >>> lw.loop() """ def __init__(self, folder, callback, extensions=["log"], tail_lines=0, sizehint=1048576): """Arguments: (str) @folder: the folder to watch (callable) @callback: a function which is called every time one of the file being watched is updated; this is called with "filename" and "lines" arguments. (list) @extensions: only watch files with these extensions (int) @tail_lines: read last N lines from files being watched before starting (int) @sizehint: passed to file.readlines(), represents an approximation of the maximum number of bytes to read from a file on every ieration (as opposed to load the entire file in memory until EOF is reached). Defaults to 1MB. """ self.folder = os.path.realpath(folder) self.extensions = extensions self._files_map = {} self._callback = callback self._sizehint = sizehint assert os.path.isdir(self.folder), self.folder assert callable(callback), repr(callback) self.update_files() for id, file in self._files_map.items(): file.seek(os.path.getsize(file.name)) # EOF if tail_lines: try: lines = self.tail(file.name, tail_lines) except IOError as err: if err.errno != errno.ENOENT: raise else: if lines: self._callback(file.name, lines) def __enter__(self): return self def __exit__(self, *args): self.close() def __del__(self): self.close() def loop(self, interval=0.1, blocking=True): """Start a busy loop checking for file changes every *interval* seconds. If *blocking* is False make one loop then return. """ # May be overridden in order to use pyinotify lib and block # until the directory being watched is updated. # Note that directly calling readlines() as we do is faster # than first checking file's last modification times. while True: self.update_files() for fid, file in list(self._files_map.items()): self.readlines(file) if not blocking: return time.sleep(interval) def log(self, line): """Log when a file is un/watched""" print(line) def listdir(self): """List directory and filter files by extension. You may want to override this to add extra logic or globbing support. """ ls = os.listdir(self.folder) if self.extensions: return [x for x in ls if os.path.splitext(x)[1][1:] \ in self.extensions] else: return ls @classmethod def open(cls, file): """Wrapper around open(). By default files are opened in binary mode and readlines() will return bytes on both Python 2 and 3. This means callback() will deal with a list of bytes. Can be overridden in order to deal with unicode strings instead, like this: import codecs, locale return codecs.open(file, 'r', encoding=locale.getpreferredencoding(), errors='ignore') """ return open(file, 'rb') @classmethod def tail(cls, fname, window): """Read last N lines from file fname.""" if window <= 0: raise ValueError('invalid window value %r' % window) with cls.open(fname) as f: BUFSIZ = 1024 # True if open() was overridden and file was opened in text # mode. In that case readlines() will return unicode strings # instead of bytes. encoded = getattr(f, 'encoding', False) CR = '\n' if encoded else b'\n' data = '' if encoded else b'' f.seek(0, os.SEEK_END) fsize = f.tell() block = -1 exit = False while not exit: step = (block * BUFSIZ) if abs(step) >= fsize: f.seek(0) newdata = f.read(BUFSIZ - (abs(step) - fsize)) exit = True else: f.seek(step, os.SEEK_END) newdata = f.read(BUFSIZ) data = newdata + data if data.count(CR) >= window: break else: block -= 1 return data.splitlines()[-window:] def update_files(self): ls = [] for name in self.listdir(): absname = os.path.realpath(os.path.join(self.folder, name)) try: st =