coopy/restore.py
def restore(system, basedir):
    #save current clock
    current_clock = system._clock

    #restore from snapshot
    system = SnapshotManager(basedir).recover_snapshot()

    files = fileutils.last_log_files(basedir)
    logger.debug(LOG_PREFIX + "Files found: " + str(files))
    if not files:
        return system

    actions = []
    for file in files:
        logger.debug(LOG_PREFIX + "Opening  " + str(file))
        unpickler = Unpickler(open(file,'rb'))
        try:
            while True:
                action = unpickler.load()
                logger.debug(LOG_PREFIX + action.action)
                actions.append(action)
        except BadPickleGet:
            logger.error(LOG_PREFIX + "Error unpickling %s" % (str(file)))
        except EOFError:
            pass

    if not actions:
        return system

    logger.debug(LOG_PREFIX + "Actions re-execution")
    for action in actions:
        try:
            if hasattr(action, 'results'):
                system._clock = RestoreClock(action.results)
            action.execute_action(system)
        except Exception as e:
            logger.debug(LOG_PREFIX + 'Error executing: %s' % (str(action)))
            logger.debug(LOG_PREFIX + 'Exception: %s' % (str(e)))

    system._clock = current_clock
    return system
coopy/network/default_select.py
class CopyNet(threading.Thread):
    def __init__(self,
                 obj,
                 host=None,
                 port=5466,
                 max_clients=5,
                 password=b'copynet',
                 ipc=False):

        threading.Thread.__init__(self)
        self.obj = obj
        self.host = host
        self.port = port
        self.max_clients = max_clients
        self.password = password
        self.ipc = ipc

        self.clientmap = {}
        self.queues = {}

        if not self.ipc:
            self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        else:
            self.server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)

        if not host:
            try:
                host = socket.gethostbyname(socket.gethostname())
            except Exception as e:
                _mwarn("Error on gethostname %s" % (str(e)))
                host = "127.0.0.1"

        self.server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

        if not self.ipc:
            self.server.bind((host, port))
        else:
            self.server.bind(COPYNET_SOCK)

        self.server.listen(max_clients)
        _minfo("Listening to %d" % max_clients)

    def close(self):
        self.running = False
        try:
            self.server.shutdown(socket.SHUT_RDWR)
        except socket.error:
            pass
        self.server.close()
        if self.ipc:
            os.remove(COPYNET_SOCK)

    def receive(self, message):
        _mdebug('Receive')

        if len(self.clientmap) == 0:
            return

        (header, data) = prepare_data(message)
        self.broadcast(header, data)

    def broadcast(self, header, data):
       _mdebug('Broadcast')

       for copynetclient in self.clientmap.values():
            if copynetclient.state == 'r':
                copynetclient.client.sendall(header)
                copynetclient.client.sendall(data)
            elif copynetclient.state == 'b':
                self.queues[copynetclient.client].put_nowait(
                                                   CopyNetPacket(header,data))
            else:
                _mdebug('Unknow client state')

    def send_direct(self, client, message):
        (header, data) = prepare_data(message)
        client.sendall(header)
        client.sendall(data)

    def check_if_authorized_client(self, client):
        password = client.recv(20)

        if self.password != password.rstrip():
            unauthdata = struct.pack(COPYNET_HEADER, 0, b'n')
            client.sendall(unauthdata)
            client.close()
            _minfo('Client rejected')
            return False

        return True

    def initialize_client(self, client, address):
        self.clientmap[client] = CopyNetClient(client, address, 'r')
        self.queues[client] = Queue(999999)

    def disconnect(self, sock):
        _mdebug('Master received data. Close client')
        sock.close()
        del self.clientmap[sock]

    def run(self):
        self.running = True

        while self.running:

            try:
                to_read, to_write, exception_mode = \
                    select([self.server] + list(self.clientmap.keys()), [], [], 5)
            except socket.error as e:
                _mdebug("Select error")
                self.running = False
                break

            for sock in to_read:
                if sock == self.server:
                    try:
                        client, address = self.server.accept()
                    except Exception as e:
                        _mdebug('Cannot accept client: %s'  % e.message)
                        continue

                    _minfo('Server: got connection %d from %s' %
                                                   (client.fileno(), address))

                    if not self.check_if_authorized_client(client):
                       continue

                    _minfo('Client connected')
                    self.initialize_client(client, address)
                    #CopyNetSnapshotThread(self.clientmap[client], self.obj).start()
                else:
                    self.disconnect(sock)

            for sock in to_write:
                while not self.queues[sock].empty():
                    self.send_direct(sock, self.queues[sock].get_nowait())

        self.server.close()
coopy/foundation.py
class Action(object):
    def __init__(self, caller_id, action, timestamp, args, kwargs):
        self.caller_id = caller_id
        self.action = action
        self.args = args
        self.kwargs = kwargs
        self.results = None

        # TODO do we need it?
        self.timestamp = timestamp

    def __str__(self):
        return "action %s \ncaller_id %s \nargs %s" % \
                (self.action, str(self.caller_id), self.args)

    def execute_action(self, system):
        #TODO not reliable
        method = getattr(system, self.action)
        return method(*self.args, **self.kwargs)
coopy/foundation.py
class RecordClock(object):
    def __init__(self):
        self.results = []

    def __getattr__(self, name):
        if name in DATETIME_FUNCS:
            return getattr(datetime, name)
        elif name in DATE_FUNCS:
            return getattr(date, name)
        return object.__getattribute__(self, name)

    def now(self):
        dt = datetime.now()
        self.results.append(dt)
        return dt

    def utcnow(self):
        dt = datetime.utcnow()
        self.results.append(dt)
        return dt

    def today(self):
        dt = date.today()
        self.results.append(dt)
        return dt
coopy/fileutils.py
DIGITS = 15
MAX_LOGFILE_SIZE = 1024 * 100 * 10 # 1 MB
LOG_PREFIX = "transaction_"
LOG_SUFIX = ".log"
SNAPSHOT_PREFIX = "snapshot_"
SNAPSHOT_SUFIX = ".dat"
SNAPSHOT_REGEX = re.compile("snapshot_(\d{15})\.dat", re.IGNORECASE)
LOG_REGEX = re.compile("transaction_(\d{15})\.log", re.IGNORECASE)
FILE_REGEX = re.compile("(transaction|snapshot)_(\d{15})\..{3}", re.IGNORECASE)
class RotateFileWrapper():
    def __init__(self, file, basedir, system_data_path, max_size=MAX_LOGFILE_SIZE):
    def write(self, data):
    def __getattr__(self, name):
    def name(self):
    def close(self):
    def set_pickler(self, pickler):
def number_as_string(number):
def list_coopy_files(basedir, regex):
def list_snapshot_files(basedir):
def list_log_files(basedir):
def check_files(files):
def last_snapshot_file(basedir):
def last_log_file(basedir):
def lastest_snapshot_number(basedir):
def lastest_log_number(basedir):
def to_log_file(basedir, number):
def to_snapshot_file(basedir, number):
def get_number(filename):
def next_number(basedir):
def next_snapshot_file(basedir):
def next_log_file(basedir):
def last_log_files(basedir):
def name_to_dir(basedir):
def obj_to_dir_name(obj):
coopy/network/default_select.py
class CopyNetSlave(threading.Thread):
    def __init__(self,
                 parent,
                 host='localhost',
                 port=5466,
                 password=b'copynet',
                 ipc=False):

        threading.Thread.__init__(self)
        self.parent = parent
        self.host = host
        self.port = port
        self.running = False
        self.lock = threading.RLock()

        try:
            if not ipc:
                self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                self.sock.connect((host, self.port))
            else:
                self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
                self.sock.connect(COPYNET_SOCK)

            self.sock.sendall(password)
            _sdebug('Slave connected to server@%d' % self.port)
        except socket.error as e:
            _sdebug("Error connecting to server %s" % (e))
            raise Exception("Network Error: Could not connect to: %s:%d" %
                    (host, port))

    def acquire(self):
        self.lock.acquire(1)

    def release(self):
        self.lock.release()

    def close(self):
        self.running = False
        self.sock.close()

    def run(self):
        self.running = True

        while self.running:
            try:
                to_read, to_write, exception_mode = \
                    select([self.sock], [],[], 1)
            except Exception as e:
                _mdebug('Error on select %s..\nShutting down' % (e))
                self.running = False
                break

            _mdebug("waitin for data..")

            for sock in to_read:
                if sock == self.sock:
                    try:
                        data = self.sock.recv(_HEADER_SIZE)
                    except Exception as e:
                        _mdebug('Exception %s'  % e)
                        data = None

                    if not data:
                        _mdebug('Shutting down')
                        self.running = False
                        break
                    else:
                        try:
                            (psize, stype) = struct.unpack(COPYNET_HEADER, data)
                        except struct.error as e:
                            self.running = False
                            self.sock.close()
                            raise Exception("Unexpected error")
                        if stype == 'n':
                            self.sock.close()
                            raise Exception("Not authorized")

                        _sdebug('Reading %d bytes' % (psize))
                        buf = ''

                        while len(buf) < psize:
                            buf += self.sock.recv(4096)

                        un_data = zlib.decompress(buf)

                        self.lock.acquire(1)
                        if stype == 'a':
                            _sdebug('Receiving action')
                            action = picke.loads(str(un_data))
                            action.execute_action(self.parent.obj)
                            _sdebug(str(action))
                        else:
                            _sdebug('Receiving system')
                            self.parent.obj = pickle.loads(str(un_data))
                        self.lock.release()

                else:
                    _sdebug("Unexcpected")

        _mdebug("Slave close")
        self.sock.close()
coopy/journal.py
class DiskJournal():
    def __init__(self, basedir, system_data_path):
        '''
            set basedir and declare file attribute
        '''
        self.basedir = basedir
        self.file = None
        self.system_data_path = system_data_path

    def setup(self):
        '''
            configure file attribute, create pickler
        '''
        self.file = self.current_journal_file(self.basedir)
        self.pickler = Pickler(self.file, pickle.HIGHEST_PROTOCOL)
        self.file.set_pickler(self.pickler)

    def current_journal_file(self, basedir):
        '''
            get current file on basedir(last created one)
            and return a Rotate wrapper.

            if current file size > 1Mb (configured on fileutils) crate
            another file with the next name
        '''

        last_file_name = fileutils.last_log_file(self.basedir)

        if last_file_name and os.path.getsize(self.basedir + last_file_name) \
            < fileutils.MAX_LOGFILE_SIZE:

            file = fileutils.RotateFileWrapper(
                       open(self.basedir + last_file_name, 'ab'),
                       self.basedir,
                       self.system_data_path)
        else:
            file = fileutils.RotateFileWrapper(
                       open(fileutils.next_log_file(self.basedir), 'wb'),
                       self.basedir,
                       self.system_data_path)

        return file

    def receive_before(self, message):
        pass

    def receive(self, message):
        '''
            receive a message and pickle
        '''
        self.pickler.dump(message)

    def receive_exception(self, message):
        pass

    def close(self):
        '''
            close file instance
        '''
        if not self.file.closed:
            self.file.close()
coopy/validation.py
def validate_system(system_instance):
    for (method_name, method) in inspect.getmembers(system_instance,
                                                    predicate=inspect.ismethod):
        if method_name.startswith("__") and method_name.endswith("__"):
            continue
        fixed_source = unindent_source(inspect.getsourcelines(method)[0])
        valid = validate_date_datetime_calls(fixed_source)
        if not valid:
            raise PrevalentError("%s contains methods with invalid calls "\
                                 "on method %s:\n%s" % (system_instance,
                                                        method_name,
                                                        fixed_source))
    return True
coopy/foundation.py
class Publisher:
    def __init__(self, subscribers):
        self.subscribers = subscribers

    def register(self, subscriber):
        self.subscribers.append(subscriber)

    def publish(self, message):
        for subscriber in self.subscribers:
            subscriber.receive(message)

    def publish_before(self, message):
        for subscriber in self.subscribers:
            subscriber.receive_before(message)

    def publish_exception(self, message):
        for subscriber in self.subscribers:
            subscriber.receive_exception(message)

    def close(self):
        for subscriber in self.subscribers:
            subscriber.close()
coopy/snapshot.py
class SnapshotTimer(threading.Thread):
    def __init__(self, snapshot_time, proxy):
        threading.Thread.__init__ (self)
        self.snapshot_time = snapshot_time
        self.proxy = proxy
        self.finished = threading.Event()

    def run(self):
        while not self.finished.isSet():
            self.proxy.take_snapshot()
            self.finished.wait(self.snapshot_time)

    def stop(self):
        self.finished.set()
        self.join()
coopy/snapshot.py
class SnapshotManager(object):
    def __init__(self, basedir):
        if len(basedir) > 0:
            basedir = basedir.replace('\\','/')
            if not basedir.endswith('/'):
                basedir += '/'
        else:
            basedir =  './'

        self.basedir = basedir

    def take_snapshot(self, object):
        file = open(fu.next_snapshot_file(self.basedir), "wb")
        logger.debug("Taking snapshot on: " + file.name)
        pickler = Pickler(file, pickle.HIGHEST_PROTOCOL)
        pickler.dump(object)
        file.flush()
        file.close()

    def recover_snapshot(self):
        file = open(path.join(self.basedir,fu.last_snapshot_file(self.basedir)),"rb")
        if not file:
            return None
        logger.debug("Recovering snapshot from: " + file.name)
        unpickler = Unpickler(file)
        return unpickler.load()
coopy/utils.py
def inject(obj, name, dependency):
    obj.__dict__[name] = dependency
coopy/utils.py
def action_check(obj):
    return (hasattr(obj, '__readonly'),
            hasattr(obj, '__unlocked'),
            hasattr(obj, '__abort_exception'))
coopy/utils.py
def method_or_none(instance, name):
    method = getattr(instance, name)
    if (name[0:2] == '__' and name[-2:] == '__') or \
                                    not callable(method) :
        return None
    return method


Question: import six
import os
import logging
import threading
import types
    import _thread as thread
    import thread
        import time
from logging.handlers import RotatingFileHandler
from datetime import datetime
from coopy import fileutils
from coopy.foundation import Action, RecordClock, Publisher
from coopy.journal import DiskJournal
from coopy.restore import restore
from coopy.snapshot import SnapshotManager, SnapshotTimer
from coopy.utils import method_or_none, action_check, inject
from coopy.validation import validate_system
from coopy.network.default_select import CopyNet, CopyNetSlave
'''
Created on Dec 6, 2011

@author: felipe j. cruz
'''
__author__ = "Felipe Cruz <felipecruz@loogica.net>"
__license__ = "BSD"
__version__ = "0.4.6b"


if six.PY3:
else:




CORE_LOG_PREFIX = '[CORE] '

def logging_config(basedir="./"):
    log_file_path = os.path.join(basedir, "coopy.log")
    result = logging.getLogger("coopy")
    result.setLevel(logging.DEBUG)

    handler = RotatingFileHandler(log_file_path, "a", 1000000, 10)
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s ' \
                                 '- %(message)s', '%a, %d %b %Y %H:%M:%S')
    handler.setFormatter(formatter)
    result.addHandler(handler)

logging_config()
logger = logging.getLogger("coopy")


def init_persistent_system(obj, basedir=None):
    # a system object is needed in order to coopy work
    if not obj:
        raise Exception(CORE_LOG_PREFIX +
                "Must input a valid object if there's no snapshot files")

    # if obj is a class, change obj to an insance
    if isinstance(obj, type):
        obj = obj()

    validate_system(obj)

    # first step is to check basedir argument. if isn't defined
    # coopy will create a directory name based on system class
    if not basedir:
        basedir = fileutils.obj_to_dir_name(obj)

    # convert some string to a valid directory name
    basedir = fileutils.name_to_dir(basedir)

    system_data_path = os.getcwd()

    # check if basedir exists, if not, create it
    try:
        os.listdir(basedir)
    except os.error:
        os.mkdir(basedir)

    # if no snapshot files, create first one with a 'empty' system
    if not fileutils.last_snapshot_file(basedir):
        logger.info(CORE_LOG_PREFIX + "No snapshot files..")
        SnapshotManager(basedir).take_snapshot(obj)

    # measure restore time
    start = datetime.utcnow()
    logger.info(CORE_LOG_PREFIX + "coopy init....")

    # inject clock on system object
    inject(obj, '_clock', RecordClock())
    inject(obj, '_system_data_path', system_data_path)

    # RestoreHelper will recover a system state based on snapshots and
    # transations files extracting actions executed previously and
    # re-executing them again
    obj = restore(obj, basedir)

    end = datetime.utcnow()
    delta = end - start
    logger.info(CORE_LOG_PREFIX + "spent " + str(delta) + "microseconds")


Answer:     journal = DiskJournal(basedir, system_data_path)