func (state *ServicesState) Format(list *memberlist.Memberlist) string {
	var outStr string
	refTime := time.Now().UTC()
	var servers []*Server
	for _, svr := range state.Servers {
		servers = append(servers, svr)
	}
	sort.Sort(ServerByName(servers))
	outStr += "Services ------------------------------\n"
	for _, server := range servers {
		outStr += fmt.Sprintf("  %s: (%s)\n", server.Name, output.TimeAgo(server.LastUpdated, refTime))
		var services []*service.Service
		for _, svc := range server.Services {
			services = append(services, svc)
		}
		sort.Sort(ServicesByName(services))
		for _, svc := range ServicesByName(services) {
			outStr += svc.Format()
		}
		outStr += "\n"
	}
	if list == nil {
		return outStr
	}
	outStr += "\nCluster Hosts -------------------------\n"
	for _, host := range list.Members() {
		outStr += fmt.Sprintf("    %s\n", host.Name)
	}
	outStr += "---------------------------------------"
	return outStr
}
Pretty-print(ish) a services state struct so a human can read
it on the terminal. Makes for awesome web apps.
package catalog
import (
	"encoding/json"
	"fmt"
	"io"
	"os"
	"sort"
	"sync"
	"time"
	"github.com/NinesStack/memberlist"
	"github.com/NinesStack/sidecar/output"
	"github.com/NinesStack/sidecar/service"
	"github.com/armon/go-metrics"
	"github.com/relistan/go-director"
	log "github.com/sirupsen/logrus"
)
const (
	TOMBSTONE_LIFESPAN         = 3 * time.Hour                   
	TOMBSTONE_COUNT            = 10                              
	ALIVE_COUNT                = 5                               
	TOMBSTONE_SLEEP_INTERVAL   = 2 * time.Second                 
	TOMBSTONE_RETRANSMIT       = 1 * time.Second                 
	ALIVE_LIFESPAN             = 1*time.Minute + 20*time.Second  
	DRAINING_LIFESPAN          = 10 * time.Minute                
	ALIVE_SLEEP_INTERVAL       = 1 * time.Second                 
	ALIVE_BROADCAST_INTERVAL   = 1 * time.Minute                 
	LISTENER_EVENT_BUFFER_SIZE = 20                              
)
type ChangeEvent struct {
	Service        service.Service
	PreviousStatus int
	Time           time.Time
}
type Server struct {
	Name        string
	Services    map[string]*service.Service
	LastUpdated time.Time
	LastChanged time.Time
}
func NewServer(name string) *Server {
	server := &Server{
		Name: name,
		Services:    make(map[string]*service.Service, 5),
		LastUpdated: time.Unix(0, 0),
		LastChanged: time.Unix(0, 0),
	}
	return server
}
type ServicesState struct {
	Servers             map[string]*Server
	LastChanged         time.Time
	ClusterName         string
	Hostname            string
	Broadcasts          chan [][]byte        `json:"-"`
	ServiceMsgs         chan service.Service `json:"-"`
	listeners           map[string]Listener
	tombstoneRetransmit time.Duration
	sync.RWMutex
}
type Listener interface {
	Chan() chan ChangeEvent  
	Name() string            
	Managed() bool           
}
func NewServicesState() *ServicesState {
	var err error
	state := &ServicesState{
		Servers:             make(map[string]*Server, 5),
		Broadcasts:          make(chan [][]byte),
		LastChanged:         time.Unix(0, 0),
		tombstoneRetransmit: TOMBSTONE_RETRANSMIT,
		ServiceMsgs:         make(chan service.Service, 25),
		listeners:           make(map[string]Listener),
	}
	state.Hostname, err = os.Hostname()
	if err != nil {
		log.Errorf("Error getting hostname! %s", err.Error())
	}
	return state
}
func (server *Server) HasService(id string) bool {
	_, ok := server.Services[id]
	return ok
}
func (state *ServicesState) Encode() []byte {
	jsonData, err := state.MarshalJSON()
	if err != nil {
		log.Error("ERROR: Failed to Marshal state")
		return []byte{}
	}
	return jsonData
}
func (state *ServicesState) ProcessServiceMsgs(looper director.Looper) {
	looper.Loop(func() error {
		service := <-state.ServiceMsgs
		state.AddServiceEntry(service)
		return nil
	})
}
func (state *ServicesState) UpdateService(svc service.Service) {
	state.ServiceMsgs <- svc
}
func (state *ServicesState) HasServer(hostname string) bool {
	_, ok := state.Servers[hostname]
	return ok
}
func (state *ServicesState) ExpireServer(hostname string) {
	state.Lock()
	defer state.Unlock()
	if !state.HasServer(hostname) || len(state.Servers[hostname].Services) == 0 {
		log.Infof("No records to expire for %s", hostname)
		return
	}
	hasLiveServices := false
	for _, svc := range state.Servers[hostname].Services {
		if !svc.IsTombstone() {
			hasLiveServices = true
			break
		}
	}
	if !hasLiveServices {
		log.Infof("No records to expire for %s (no live services)", hostname)
		return
	}
	log.Infof("Expiring %s", hostname)
	var tombstones []service.Service
	for _, svc := range state.Servers[hostname].Services {
		previousStatus := svc.Status
		svc.Tombstone()
		state.ServiceChanged(svc, previousStatus, svc.Updated)
		tombstones = append(tombstones, *svc)
	}
	if len(tombstones) < 1 {
		log.Warn("Tried to announce a zero length list of tombstones")
		return
	}
	state.SendServices(
		tombstones,
		director.NewTimedLooper(TOMBSTONE_COUNT, state.tombstoneRetransmit, nil),
	)
}
func (state *ServicesState) ServiceChanged(svc *service.Service, previousStatus int, updated time.Time) {
	state.serverChanged(svc.Hostname, updated)
	state.NotifyListeners(svc, previousStatus, state.LastChanged)
}
func (state *ServicesState) serverChanged(hostname string, updated time.Time) {
	if !state.HasServer(hostname) {
		log.Errorf("Attempt to change a server we don't have! (%s)", hostname)
		return
	}
	state.Servers[hostname].LastUpdated = updated
	state.Servers[hostname].LastChanged = updated
	state.LastChanged = updated
}
func (state *ServicesState) NotifyListeners(svc *service.Service, previousStatus int, changedTime time.Time) {
	listeners := state.listeners
	if len(listeners) < 1 {
		log.Debugf("Skipping listeners, there are none")
		return
	}
	log.Debugf("Notifying listeners of change at %s", changedTime.String())
	event := ChangeEvent{Service: *svc, PreviousStatus: previousStatus, Time: changedTime}
	for _, listener := range listeners {
		if listener == nil {
			continue
		}
		select {
		case listener.Chan() <- event:
			continue
		default:
			log.Warnf("Can't notify listener (%s). May not be ready yet.", listener.Name())
		}
	}
}
func (state *ServicesState) AddListener(listener Listener) {
	if listener.Chan() == nil {
		log.Errorf("Refusing to add listener %s with nil channel!", listener.Name())
		return
	}
	if cap(listener.Chan()) < 1 {
		log.Errorf("Refusing to add blocking channel as listener: %s", listener.Name())
		return
	}
	state.Lock()
	defer state.Unlock()
	state.listeners[listener.Name()] = listener
	log.Debugf("AddListener(): added %s, new count %d", listener.Name(), len(state.listeners))
}
func (state *ServicesState) RemoveListener(name string) error {
	state.Lock()
	defer state.Unlock()
	if _, ok := state.listeners[name]; !ok {
		return fmt.Errorf("no listener found with the name %q", name)
	}
	delete(state.listeners, name)
	log.Debugf("RemoveListener(): removed %s, new count %d", name, len(state.listeners))
	return nil
}
func (state *ServicesState) GetListeners() []Listener {
	state.RLock()
	var listeners []Listener
	for _, listener := range state.listeners {
		listeners = append(listeners, listener)
	}
	state.RUnlock()
	return listeners
}
func (state *ServicesState) AddServiceEntry(newSvc service.Service) {
	defer metrics.MeasureSince([]string{"services_state", "AddServiceEntry"}, time.Now())
	state.Lock()
	defer state.Unlock()
	if !state.HasServer(newSvc.Hostname) {
		state.Servers[newSvc.Hostname] = NewServer(newSvc.Hostname)
	}
	server := state.Servers[newSvc.Hostname]
	if !server.HasService(newSvc.ID) {
		server.Services[newSvc.ID] = &newSvc
		state.ServiceChanged(&newSvc, service.UNKNOWN, newSvc.Updated)
		state.retransmit(newSvc)
	} else if newSvc.Invalidates(server.Services[newSvc.ID]) {
		server.LastUpdated = newSvc.Updated
		oldEntry := server.Services[newSvc.ID]
		if oldEntry.Status == service.DRAINING && newSvc.Status == service.ALIVE {
			newSvc.Status = oldEntry.Status
		}
		server.Services[newSvc.ID] = &newSvc
		if oldEntry.Status != newSvc.Status {
			state.ServiceChanged(&newSvc, oldEntry.Status, newSvc.Updated)
		}
		state.retransmit(newSvc)
	}
}
func (state *ServicesState) GetLocalServiceByID(id string) (service.Service, error) {
	state.RLock()
	defer state.RUnlock()
	if server, ok := state.Servers[state.Hostname]; ok {
		if svc, ok := server.Services[id]; ok {
			return *svc, nil
		}
	}
	return service.Service{},
		fmt.Errorf("service with ID %q not found on host %q", id, state.Hostname)
}
func (state *ServicesState) Merge(otherState *ServicesState) {
	for _, server := range otherState.Servers {
		for _, svc := range server.Services {
			state.UpdateService(*svc)
		}
	}
}
func (state *ServicesState) retransmit(svc service.Service) {
	if svc.Hostname == state.Hostname {
		return
	}
	go func() {
		encoded, err := svc.Encode()
		if err != nil {
			log.Errorf("ERROR encoding message to forward: (%s)", err.Error())
			return
		}
		state.Broadcasts <- [][]byte{encoded}
	}()
}