scales/message.py
class Deadline(object):
  KEY = "__Deadline"
  EVENT_KEY = "__Deadline_Event"

  def __init__(self, timeout):
    """
    Args:
      timeout - The timeout in seconds
    """
    import  time
    self._ts = Long(time.time()) * 1000000000 # Nanoseconds
    self._timeout = Long(timeout * 1000000000)
scales/sink.py
class ClientMessageSink(MessageSink):
  """ClientMessageSinks take a message, stream, and headers and perform
  processing on them.
  """
  __slots__ = '_on_faulted',
  Role = None
  Builder = None

  def __init__(self):
    self._on_faulted = Observable()
    super(ClientMessageSink, self).__init__()

  @property
  def state(self):
    return self.next_sink.state

  @property
  def is_open(self):
    """Returns True if the sink is Idle, Open, or Busy"""
    return self.state <= ChannelState.Busy

  @property
  def is_closed(self):
    """Returns True if the sink is Closed."""
    return self.state == ChannelState.Closed

  @property
  def is_ready(self):
    """Returns True if the channel is open, eg ready to process messages."""
    return self.state == ChannelState.Open

  @property
  def on_faulted(self):
    return self._on_faulted

  def Open(self):
    if self.next_sink:
      return self.next_sink.Open()
    else:
      return AsyncResult.Complete()

  def Close(self):
    if self.next_sink:
      self.next_sink.Close()

  @abstractmethod
  def AsyncProcessRequest(self, sink_stack, msg, stream, headers):
    """Process a request message, stream, and headers.

    Args:
      sink_stack - The SinkStack representing the processing state of the message.
                   Implementors should push their sink onto this stack before
                   forwarding the message in order to participate in processing
                   the response.
      msg - The message being processed.
      stream - A serialized version of the message.
      headers - Any additional headers to be sent.
    """
    raise NotImplementedError()

  @abstractmethod
  def AsyncProcessResponse(self, sink_stack, context, stream, msg):
    """Process a response stream.

    Args:
      sink_stack - The SinkStack representing the processing state of the message.
                   Implementors should call sink_stack.AsyncProcessMessage(...)
                   to forward the message to the next sink.
      context - The context that was pushed onto the stack in AsyncProcessRequest.
      stream - The stream representing the serialized response.
    """
    raise NotImplementedError()
scales/thriftmux/serializer.py
class MessageSerializer(object):
  """A serializer that can serialize/deserialize method calls into the ThriftMux
  wire format."""
  def __init__(self, service_cls):
    self._marshal_map = {
      MethodCallMessage: self._Marshal_Tdispatch,
      MethodDiscardMessage: self._Marshal_Tdiscarded,
    }
    self._unmarshal_map = {
      MessageType.Rdispatch: self._Unmarshal_Rdispatch,
      MessageType.Rerr: self._Unmarshal_Rerror,
      MessageType.BAD_Rerr: self._Unmarshal_Rerror,
    }
    if service_cls:
      self._thrift_serializer = ThriftMessageSerializer(service_cls)

  def _Marshal_Tdispatch(self, msg, buf, headers):
    ctx = {}
    ctx.update(msg.public_properties)
    ctx.update(headers)
    MessageSerializer._WriteContext(ctx, buf)

    headers[TransportHeaders.MessageType] = MessageType.Tdispatch
    buf.write(pack('!hh', 0, 0))  # len(dst), len(dtab), both unsupported
    self._thrift_serializer.SerializeThriftCall(msg, buf)

  @staticmethod
  def _Marshal_Tdiscarded(msg, buf, headers):
    headers[TransportHeaders.MessageType] = MessageType.Tdiscarded
    buf.write(pack('!BBB', *Tag(msg.which).Encode()))
    buf.write(msg.reason.encode('utf-8'))

  @staticmethod
  def _WriteContext(ctx, buf):
    buf.write(pack('!h', len(ctx)))
    for k, v in ctx.items():
      if not isinstance(k, string_types):
        raise NotImplementedError("Unsupported key type in context")
      k_len = len(k)
      buf.write(pack('!h%ds' % k_len, k_len, k.encode('utf-8')))
      if isinstance(v, Deadline):
        buf.write(pack('!h', 16))
        buf.write(pack('!qq', v._ts, v._timeout))
      elif isinstance(v, string_types):
        v_len = len(v)
        buf.write(pack('!h%ds' % v_len, v_len, v.encode('utf-8')))
      else:
        raise NotImplementedError("Unsupported value type in context.")

  @staticmethod
  def _ReadContext(buf):
    for _ in range(2):
      sz, = unpack('!h', buf.read(2))
      buf.read(sz)

  def _Unmarshal_Rdispatch(self, buf):
    status, nctx = unpack('!bh', buf.read(3))
    for n in range(0, nctx):
      self._ReadContext(buf)

    if status == Rstatus.OK:
      return self._thrift_serializer.DeserializeThriftCall(buf)
    elif status == Rstatus.NACK:
      return MethodReturnMessage(error=ServerError('The server returned a NACK'))
    else:
      return MethodReturnMessage(error=ServerError(buf.read().decode('utf-8')))

  @staticmethod
  def _Unmarshal_Rerror(buf):
    why = buf.read()
    return MethodReturnMessage(error=ServerError(why.decode('utf-8')))

  def Unmarshal(self, tag, msg_type, buf):
    """Deserialize a message from a stream.

    Args:
      tag - The tag of the message.
      msg_type - The message type intended to be deserialized.
      buf - The stream to deserialize from.
      ctx - The context from serialization.
    Returns:
      A MethodReturnMessage.
    """
    unmarshaller = self._unmarshal_map[msg_type]
    return unmarshaller(buf)

  def Marshal(self, msg, buf, headers):
    """Serialize a message into a stream.

    Args:
      msg - The message to serialize.
      buf - The stream to serialize into.
      headers - (out) Optional headers associated with the message.
    Returns:
      A context to be supplied during deserialization.
    """
    marshaller = self._marshal_map[msg.__class__]
    marshaller(msg, buf, headers)
scales/constants.py
class SinkProperties(object):
  Endpoint = 'endpoint'
  ServiceInterface = 'service_iface'
  Label = 'label'
scales/compat.py

scales/asynchronous.py
class AsyncResult(g_AsyncResult):
  @staticmethod
  def WhenAll(ars):
    """Returns an AsyncResult representing the state of all AsyncResults passed.

    Args:
      ars - An enumerable of AsyncResults.
    Returns:
      An AsyncResult representing the completion of all ars passed in.  When all
      complete, the AsyncResult will be set to an array of the results of each
      AsyncResult, in the order they were enumerated in.
      If any AsyncResult fails, the return result will fail.
    """

    ret = AsyncResult()
    num_ars = len(ars)
    total = [num_ars]
    results = [None] * num_ars
    def complete(_n, _ar):
      if _ar.exception:
        ret.set_exception(_ar.exception)
      elif not ret.ready():
        total[0] -= 1
        results[_n] = _ar.value
        if total[0] == 0:
          ret.set(results)

    for n, ar in enumerate(ars):
      ar.rawlink(functools.partial(complete, n))
    return ret

  @staticmethod
  def WhenAny(ars):
    """Returns an AsyncResult representing the state of any AsyncResult passed in.
    The return value represents the state of the first AsyncResult to complete, or,
    if all fail, the last to fail.

    Args:
      ars - An enumerable of AsyncResults.
    Returns:
      An AsyncResult representing the state of the first AsyncResult to complete.
      The AsyncResult's value will be set to the value of the first result to
      complete, or, if all fail, the exception thrown by the last to fail.
    """
    ready_ars = [ar for ar in ars if ar.ready()]
    if ready_ars:
      return ready_ars[0]

    ret = AsyncResult()
    total = [len(ars)]
    def complete(_ar):
      total[0] -= 1
      if total[0] == 0 and _ar.exception:
        ret.set_exception(_ar.exception)
      elif not ret.ready() and _ar.successful():
        ret.set(_ar.value)

    for ar in ars:
      ar.rawlink(complete)
    return ret

  @staticmethod
  def FromValue(val):
    if val is None:
      return AsyncResult.Complete()
    else:
      ar = AsyncResult()
      ar.set(val)
      return ar

  @staticmethod
  def Complete():
    """Return an AsyncResult that has completed."""
    return _COMPLETE

  @staticmethod
  def CompleteIn(n):
    """Returns an AsyncResult that completes in <n> seconds

    Args:
      n - The number of seconds to wait before completing.
    """
    ar = AsyncResult()
    def helper():
      ar.set()
    g = Greenlet(helper)
    g.start_later(float(n))
    return ar

  def _SafeLinkHelper(self, fn):
    try:
      self.set(fn())
    except:
      self.set_exception(sys.exc_info()[1])

  def SafeLink(self, fn):
    """Propagate the result of calling fn() on a new greenlet to ar

    Args:
      ar - An AsyncResult.
      fn - The function to execute.
    """
    gevent.spawn(self._SafeLinkHelper, fn)

  def ContinueWith(self, fn, on_hub=True):
    cw_ar = AsyncResult()
    def continue_with_callback(_ar):
      def run():
        try:
          val = fn(_ar)
          cw_ar.set(val)
        except:
          cw_ar.set_exception(sys.exc_info()[1])
      if on_hub:
        run()
      else:
        gevent.spawn(run)
    self.rawlink(continue_with_callback)
    return cw_ar

  def Map(self, fn):
    def mapper(_):
      if self.exception:
        return self
      else:
        return fn(self.value)
    return self.ContinueWith(mapper).Unwrap()

  def _UnwrapHelper(self, target):
    if self.ready():
      # We're ready, propagate the result
      if self.exception:
        target.set_exception(self.exception)
      else:
        if isinstance(self.value, AsyncResult):
          self.value._UnwrapHelper(target)
        else:
          target.set(self.value)
    else:
      self.rawlink(
        functools.partial(AsyncResult._UnwrapHelper, target=target))

  def Unwrap(self):
    unwrapped_ar = AsyncResult()
    self._UnwrapHelper(unwrapped_ar)
    return unwrapped_ar

  @staticmethod
  def TryGet(val):
    if isinstance(val, AsyncResult):
      return val.get()
    else:
      return val

  @staticmethod
  def Run(fn):
    ar = AsyncResult()
    ar.SafeLink(fn)
    return ar

  @staticmethod
  def RunInline(fn):
    ar = AsyncResult()
    ar._SafeLinkHelper(fn)
    return ar
scales/message.py
class MethodReturnMessage(Message):
  """A message representing the return value from a service call."""
  __slots__ = ('return_value', 'error', 'stack')

  def __init__(self, return_value=None, error=None):
    """
    Args:
      return_value - The return value of the call, or None
      error - The error that occurred during processing, or None.
              If not None, the current stack will be captured and included.
    """
    super(MethodReturnMessage, self).__init__()
    self.return_value = return_value
    self.error = error
    if error:
      exc_info = sys.exc_info()
      if len(exc_info) != 3 or exc_info[2] is None:
        try:
          raise ZeroDivisionError
        except ZeroDivisionError:
          tb = sys.exc_info()[2]
          frame = tb.tb_frame.f_back
      else:
        tb = exc_info[2]
        while tb.tb_next is not None:
          tb = tb.tb_next
        frame = tb.tb_frame

      stack = traceback.format_list(traceback.extract_stack(frame))
      stack = stack + traceback.format_exception_only(error.__class__, error)
      self.stack = stack
      # Prevent circular references
      del frame
      del tb
    else:
      self.stack = None
scales/sink.py
def SocketTransportSinkProvider(sink_cls):
  class _SocketTransportSinkProvider(SinkProviderBase):
    SINK_CLS = sink_cls
    Role = SinkRole.Transport

    def CreateSink(self, properties):
      server = properties[SinkProperties.Endpoint]
      service = properties[SinkProperties.Label]
      sock = ScalesSocket(server.host, server.port)
      healthy_sock = VarzSocketWrapper(sock, service)
      sink = self.SINK_CLS(healthy_sock, service)
      return sink

    @property
    def sink_class(self):
      return self.SINK_CLS

  return _SocketTransportSinkProvider
scales/varz.py
class VarzType(object):
class Source(object):
class VarzMetric(object):
class Gauge(VarzMetric): VARZ_TYPE = VarzType.Gauge
class Rate(VarzMetric): VARZ_TYPE = VarzType.Rate
class AverageRate(VarzMetric): VARZ_TYPE = VarzType.AverageRate
class Counter(Rate): VARZ_TYPE = VarzType.Counter
class VarzTimerBase(VarzMetric):
class AverageTimer(VarzTimerBase): VARZ_TYPE = VarzType.AverageTimer
class AggregateTimer(VarzTimerBase): VARZ_TYPE = VarzType.AggregateTimer
class VarzMeta(type):
class _VarzBase(object):
class _SampleSet(object):
class VarzReceiver(object):
class VarzAggregator(object):
  class _Agg(object):
class VarzSocketWrapper(object):
  class Varz(VarzBase):
class MonoClock(object):
class Ema(object):
  def __init__(self, method=None, service=None, endpoint=None, client_id=None):
  def to_tuple(self):
  def to_dict(self):
  def __cmp__(self, other):
  def __hash__(self):
  def _Adapt(fn):
    def __Adapt(metric, source, amount=1):
  def __init__(self, metric, source):
  def __call__(self, *args):
  def ForSource(self, source):
  def Measure(self, source=None):
  def __new__(mcs, name, bases, dct):
def VerifySource(source):
  def __init__(self, source):
  def __getattr__(self, item):
  def __init__(self, max_size, data=None, p=.1):
  def Sample(self, value):
  def RegisterMetric(metric, varz_type):
  def IncrementVarz(source, metric, amount=1):
  def SetVarz(source, metric, value):
  def RecordPercentileSample(cls, source, metric, value):
def DefaultKeySelector(k):
    def __init__(self):
  def CalculatePercentile(values, pct):
  def _Downsample(lst, target_size):
  def Aggregate(varz, metrics, key_selector=None):
  def __init__(self, socket, varz_tag):
  def host(self):
  def port(self):
  def isOpen(self):
  def read(self, sz):
  def recv_into(self, buf, sz):
  def flush(self):
  def write(self, buff):
  def open(self):
  def close(self):
  def readAll(self, sz):
  def __init__(self):
  def Sample(self):
  def __init__(self, window):
  def Update(self, ts, sample):
  VARZ_TYPE = None
class Gauge(VarzMetric): VARZ_TYPE = VarzType.Gauge
class Rate(VarzMetric): VARZ_TYPE = VarzType.Rate
class AverageRate(VarzMetric): VARZ_TYPE = VarzType.AverageRate
class Counter(Rate): VARZ_TYPE = VarzType.Counter
class AverageTimer(VarzTimerBase): VARZ_TYPE = VarzType.AverageTimer
class AggregateTimer(VarzTimerBase): VARZ_TYPE = VarzType.AggregateTimer
  _VARZ = {}
  _VARZ_BASE_NAME = None
  VARZ_METRICS = {}
  VARZ_DATA = defaultdict(lambda: defaultdict(int))
  VARZ_PERCENTILES = [.5, .90, .99, .999, .9999]
  _MAX_PERCENTILE_SIZE = 1000
  MAX_AGG_AGE = 5 * 60
    _VARZ_BASE_NAME = 'scales.socket'
    _VARZ = {
      'bytes_recv': Rate,
      'bytes_sent': Rate,
      'num_connections': Counter,
      'tests_failed': Counter,
      'connects': Rate,
      'open_latency': AverageTimer
    }
scales/message.py
class MethodDiscardMessage(Message):
  """A MethodDiscardMessage represents the intent of the client to discard the
  response to a message."""
  __slots__ = ('which', 'reason')

  def __init__(self, which, reason):
    """
    Args:
      which - The MethodCallMessage that was discarded.
      reason - The reason the message is being discarded.
    """
    super(MethodDiscardMessage, self).__init__()
    self.which = which
    self.reason = reason

  @property
  def is_one_way(self):
    return True
scales/mux/sink.py
class MuxSocketTransportSink(ClientMessageSink):
  """A transport sink for tmux servers.
  This sink supports concurrent requests over its transport.
  """

  SINK_LOG = ROOT_LOG.getChild('SocketTransportSink')
  _EMPTY_DCT = {}
  _CLOSE_INVOKED = "Close invoked"

  class Varz(VarzBase):
    """
    messages_sent - The number of messages sent over this sink.
    messages_recv - The number of messages received over this sink.
    active - 1 if the sink is open, else 0.
    send_queue_size - The length of the send queue.
    send_time - The aggregate amount of time spent sending data.
    recv_time - The aggregate amount of time spend receiving data.
    send_latency - The average amount of time taken to send a message.
    recv_latency - The average amount of time taken to receive a message
                   (once a response has reached the client).
    transport_latency - The average amount of time taken to perform a full
                        method call transaction (send data, wait for response,
                        read response).
    """
    _VARZ_BASE_NAME = 'scales.thriftmux.SocketTransportSink'
    _VARZ = {
      'messages_sent': Rate,
      'messages_recv': Rate,
      'active': Gauge,
      'send_queue_size': Gauge,
      'send_time': AggregateTimer,
      'recv_time': AggregateTimer,
      'send_latency': AverageTimer,
      'recv_latency': AverageTimer,
      'transport_latency': AverageTimer
    }

  def __init__(self, socket, service):
    super(MuxSocketTransportSink, self).__init__()
    self._socket = socket
    self._state = ChannelState.Idle
    self._log = self.SINK_LOG.getChild('[%s.%s:%d]' % (
      service, self._socket.host, self._socket.port))
    self._socket_source = '%s:%d' % (self._socket.host, self._socket.port)
    self._service = service
    self._open_result = None
    self._varz = self.Varz(Source(service=self._service,
      endpoint=self._socket_source))

  def _Init(self):
    self._tag_map = {}
    self._open_result = None
    self._tag_pool = TagPool((2 ** 24) - 1, self._service, self._socket_source)
    self._greenlets = []
    self._send_queue = Queue()

  @property
  def isActive(self):
    return self._state != ChannelState.Closed

  @property
  def state(self):
    return self._state

  def Open(self):
    """Initializes the dispatcher, opening a connection to the remote host.
    This method may only be called once.
    """
    if not self._open_result:
      self._Init()
      self._open_result = AsyncResult()
      self._open_result.SafeLink(self._OpenImpl)
    return self._open_result

  def _SpawnNamedGreenlet(self, name, *args, **kwargs):
    return NamedGreenlet.spawn(
      'Scales %s for %s [%s]' % (name, self._service, self._socket_source),
      *args,
      **kwargs)

  def _OpenImpl(self):
    try:
      self._log.debug('Opening transport.')
      self._socket.open()
      self._greenlets.append(self._SpawnNamedGreenlet('Recv Loop', self._RecvLoop))
      self._greenlets.append(self._SpawnNamedGreenlet('Send Loop', self._SendLoop))

      self._CheckInitialConnection()
      self._log.debug('Open successful')
      self._state = ChannelState.Open
      self._varz.active(1)
    except Exception as e:
      self._log.error('Exception opening socket')
      self._open_result.set_exception(e)
      self._Shutdown('Open failed')
      raise

  @abstractmethod
  def _CheckInitialConnection(self):
    raise NotImplementedError()

  def Close(self):
    self._Shutdown(self._CLOSE_INVOKED, False)

  def _Shutdown(self, reason, fault=True):
    if not self.isActive:
      return

    self._state = ChannelState.Closed

    if reason == self._CLOSE_INVOKED:
      log_fn = self._log.debug
    else:
      log_fn = self._log.warning
    log_fn('Shutting down transport [%s].' % str(reason))
    self._varz.active(0)
    self._socket.close()
    [g.kill(block=False) for g in self._greenlets]
    self._greenlets = []

    if not isinstance(reason, Exception):
      reason = Exception(str(reason))
    if fault:
      self.on_faulted.Set(reason)
    msg = MethodReturnMessage(error=ClientError(reason))

    for sink_stack, _, _ in self._tag_map.values():
      sink_stack.AsyncProcessResponseMessage(msg)

    if (self._open_result
        and not self._open_result.ready()
        and isinstance(reason, Exception)):
      self._open_result.set_exception(reason)

    self._tag_map = {}
    self._open_result = None
    self._send_queue = Queue()

  @abstractmethod
  def _OnTimeout(self, tag):
    raise NotImplementedError()

  def _HandleTimeout(self, msg_properties):
    """Determine if a message has timed out yet (because it waited in the queue
    for too long).  If it hasn't, initialize the timeout handler to fire if the
    message times out in transit.

    Args:
      msg_properties - The properties of the message.
    """
    timeout_event = msg_properties.get(Deadline.EVENT_KEY, None)
    if timeout_event and timeout_event.Get():
      # The message has timed out before it got out of the send queue
      # In this case, we can discard it immediately without even sending it.
      tag = msg_properties.pop(Tag.KEY, 0)
      if tag != 0:
        self._ReleaseTag(tag)
      return True
    elif timeout_event:
      # The event exists but hasn't been signaled yet, hook up a
      # callback so we can be notified on a timeout.
      def timeout_proc():
        timeout_tag = msg_properties.pop(Tag.KEY, 0)
        if timeout_tag:
          self._OnTimeout(timeout_tag)

      timeout_event.Subscribe(lambda evt: timeout_proc(), True)
      return False
    else:
      # No event was created, so this will never timeout.
      return False

  def _SendLoop(self):
    """Dispatch messages from the send queue to the remote server.

    Note: Messages in the queue have already been serialized into wire format.
    """
    while self.isActive:
      try:
        payload, dct = self._send_queue.get()
        queue_len = self._send_queue.qsize()
        self._varz.send_queue_size(queue_len)
        # HandleTimeout sets up the transport level timeout handling
        # for this message.  If the message times out in transit, this
        # transport will handle sending a Tdiscarded to the server.
        if self._HandleTimeout(dct): continue

        with self._varz.send_time.Measure():
          with self._varz.send_latency.Measure():
            self._socket.write(payload)
        self._varz.messages_sent()
      except Exception as e:
        self._Shutdown(e)
        break

  def _RecvLoop(self):
    """Dispatch messages from the remote server to their recipient.

    Note: Deserialization and dispatch occurs on a seperate greenlet, this only
    reads the message off the wire.
    """
    while self.isActive:
      try:
        sz, = unpack('!i', self._socket.readAll(4))
        with self._varz.recv_time.Measure():
          with self._varz.recv_latency.Measure():
            buf = BytesIO(self._socket.readAll(sz))
        self._varz.messages_recv()
        gevent.spawn(self._ProcessReply, buf)
      except Exception as e:
        self._Shutdown(e)
        break

  @abstractmethod
  def _ProcessReply(self, stream):
    raise NotImplementedError()

  def _ProcessTaggedReply(self, tag, stream):
    tup = self._ReleaseTag(tag)
    if tup:
      reply_stack, start_time, props = tup
      props[Tag.KEY] = None
      self._varz.transport_latency(time.time() - start_time)
      stream.seek(0)
      reply_stack.AsyncProcessResponseStream(stream)

  def _ReleaseTag(self, tag):
    """Return a tag to the tag pool.

    Note: Tags are only returned when the server has ACK'd them (or NACK'd) with
    and Rdispatch message (or similar).  Client initiated timeouts do NOT return
    tags to the pool.

    Args:
      tag - The tag to return.

    Returns:
      The ClientChannelSinkStack associated with the tag's response.
    """
    tup = self._tag_map.pop(tag, None)
    self._tag_pool.release(tag)
    return tup

  @abstractmethod
  def _BuildHeader(self, tag, msg_type, data_len):
    raise NotImplementedError()

  def AsyncProcessRequest(self, sink_stack, msg, stream, headers):
    if self._state == ChannelState.Idle and self._open_result:
      self._log.debug('Waiting for channel to be open')
      self._open_result.wait()

    if ((self._state == ChannelState.Idle and not self._open_result)
        or self._state == ChannelState.Closed) and sink_stack is not None:
      err_msg = MethodReturnMessage(error=Exception('Sink not open.'))
      sink_stack.AsyncProcessResponseMessage(err_msg)
      return

    if not msg.is_one_way:
      tag = self._tag_pool.get()
      msg.properties[Tag.KEY] = tag
      self._tag_map[tag] = (sink_stack, time.time(), msg.properties)
    else:
      tag = 0

    data_len = stream.tell()
    header = self._BuildHeader(tag, headers[TransportHeaders.MessageType], data_len)
    payload = header + stream.getvalue()
    self._send_queue.put((payload, msg.properties))

  def AsyncProcessResponse(self, sink_stack, context, stream, msg):
    pass
scales/thriftmux/protocol.py
class MessageType(Enum):
  Tdispatch = 2
  Rdispatch = -2
  Rerr = -128
  BAD_Rerr = 127

  Tping = 65
  Rping = -65

  Tdiscarded = 66
  BAD_Tdiscarded = -62
scales/sink.py
def SinkProvider(sink_cls, role=None, **defaults):
  """Factory for creating simple sink providers.

  Args:
    sink_cls - The type of sink to provide.
  Returns:
    A SinkProvider that provides sinks of type 'sink_cls'.
  """
  field_names = ' '.join(defaults.keys())
  params_cls = namedtuple('Params', field_names)

  def CreateSink(self, properties):
    return self.SINK_CLASS(self.next_provider, self.sink_properties, properties)

  def sink_class(self):
    return self.SINK_CLASS

  provider = type(
    sink_cls.__name__ + 'Provider',
    (SinkProviderBase, ),
    {
      'SINK_CLASS': sink_cls,
      'PARAMS_CLASS': params_cls,
      'Role': role,
      'CreateSink': CreateSink,
      'sink_class': property(sink_class),
      '_defaults': defaults
    }
  )
  return provider


Question: import logging
import random
import time
import gevent
from struct import (pack, unpack)
from ..asynchronous import AsyncResult
from ..compat import BytesIO
from ..constants import SinkProperties
from ..message import (
  Deadline,
  MethodDiscardMessage,
  MethodReturnMessage
)
from ..sink import (
  ClientMessageSink,
  SinkProvider,
  SocketTransportSinkProvider
)
from ..mux.sink import MuxSocketTransportSink
from ..varz import (
  AverageRate,
  Counter,
  Source,
  VarzBase
)
from .serializer import (
  MessageSerializer,
)
from .protocol import (
  MessageType,
)
    self._ping_ar = AsyncResult()
    self._last_ping_start = time.time()
    self._send_queue.put((self._ping_msg, self._EMPTY_DCT))
    gevent.spawn(self._PingTimeoutHelper)
    return self._ping_ar

  def _OnPingResponse(self, msg_type, stream):
    """Handles the response to a ping.  On failure, shuts down the dispatcher.
    """
    ar, self._ping_ar = self._ping_ar, None
    if msg_type == MessageType.Rping:
      ar.set()
      ping_duration = time.time() - self._last_ping_start
      self._log.debug('Got ping response in %d ms' % int(ping_duration * 1000))
    else:
      self._log.error('Unexpected response for tag 1 (msg_type was %d)' % msg_type)
      ar.set_exception(Exception("Invalid ping response"))

  def _PingTimeoutHelper(self):
    ar = self._ping_ar
    ar.wait(self._ping_timeout)
    if not ar.successful():
      ar.set_exception(Exception('Ping timed out'))
      self._Shutdown('Ping Timeout')

  def _CheckInitialConnection(self):
    ar = self._SendPingMessage()
    ar.get()
    self._log.debug('Ping successful')
    self._greenlets.append(self._SpawnNamedGreenlet('Ping Loop', self._PingLoop))

  @staticmethod
  def _CreateDiscardMessage(tag):
    """Create a Tdiscarded message for 'tag'

    Args:
      tag - The message tag to discard.
    Returns
      A (message, buffer, headers) tuple suitable for passing to AsyncProcessRequest.
    """
    discard_message = MethodDiscardMessage(tag, 'Client timeout')
    discard_message.which = tag
    buf = BytesIO()
    headers = {}
    MessageSerializer(None).Marshal(discard_message, buf, headers)
    return discard_message, buf, headers

  def _OnTimeout(self, tag):
    if tag:
      msg, buf, headers = self._CreateDiscardMessage(tag)
      self.AsyncProcessRequest(None, msg, buf, headers)

  def _ProcessReply(self, stream):
    try:
      msg_type, tag = ThriftMuxMessageSerializerSink.ReadHeader(stream)
      if tag == 1 and msg_type == MessageType.Rping: #Ping
        self._OnPingResponse(msg_type, stream)
      elif tag != 0:
        self._ProcessTaggedReply(tag, stream)
      else:
        self._log.error('Unexpected message, msg_type = %d, tag = %d' % (msg_type, tag))
    except Exception:
      self._log.exception('Exception processing reply message.')

  def _Shutdown(self, reason, fault=True):
    super(SocketTransportSink, self)._Shutdown(reason, fault)
    if self._ping_ar:
      self._ping_ar.set_exception(reason)


SocketTransportSink.Builder = SocketTransportSinkProvider(SocketTransportSink)


class ThriftMuxMessageSerializerSink(ClientMessageSink):
  """A serializer sink that serializes thrift messages to the finagle mux
  wire format"""

  class Varz(VarzBase):
    _VARZ_BASE_NAME = 'scales.thriftmux.ThriftMuxMessageSerializerSink'
    _VARZ = {
      'deserialization_failures': Counter,
      'serialization_failures': Counter,
      'message_bytes_sent': AverageRate,
      'message_bytes_recv': AverageRate
    }

  def __init__(self, next_provider, sink_properties, global_properties):
    super(ThriftMuxMessageSerializerSink, self).__init__()
    self.next_sink = next_provider.CreateSink(global_properties)
    self._serializer = MessageSerializer(global_properties[SinkProperties.ServiceInterface])
    self._varz = self.Varz(Source(
      service=global_properties[SinkProperties.Label]))

  @staticmethod
  def ReadHeader(stream):
    """Read a mux header off a message.

    Args:
      msg - a byte buffer of raw data.

    Returns:
      A tuple of (message_type, tag)
    """
    header, = unpack('!i', stream.read(4))
    msg_type = (256 - (header >> 24 & 0xff)) * -1
    tag = ((header << 8) & 0xFFFFFFFF) >> 8
    return msg_type, tag

  def AsyncProcessRequest(self, sink_stack, msg, stream, headers):
    buf = BytesIO()
    headers = {}

    deadline = msg.properties.get(Deadline.KEY)
    if deadline:
      headers['com.twitter.finagle.Deadline'] = Deadline(deadline)

    try:
      self._serializer.Marshal(msg, buf, headers)
    except Exception as ex:
      self._varz.serialization_failures()

Answer:       msg = MethodReturnMessage(error=ex)