switzerland/common/PcapLogger.py
class PcapLogger:
  """
  This device organises the elaborate switzerland logging structure, which
  is a standard log containing references to a directory of incident-specific
  pcap log files.
  """
  
  # XXXXXX fix this default before release
  def __init__(self, log_dir):
    self.lock = threading.RLock()
    self.make_or_check_directory(log_dir, "log dir")
    #ts = self.timestamp()
    self.log_dir = log_dir     
    # XXX do not expect this to be secure if log_dir is world writeable
    # on a multi-user system
    self.make_or_check_directory(self.log_dir, "pcap log dir")
 
  def make_or_check_directory(self, path, alias="directory"):
    alias += " "
    try:
      os.mkdir(path, 0755)
    except OSError, e:
      assert not os.path.islink(path), alias + path + " should not be a symlink!"
      assert os.path.isdir(path), "cannot create " + alias + path + ":\n"+`e`

  def new_flow(self, flow, id):
    log.info("New bidirectional flow %d : %s\n" % (id,`flow`))

  def timestamp(self, t=None):
    if t == None:
      t = time.time()
    frac = `t - int(t)`[2:5]
    return time.strftime("%Y%m%d%H%M%S", time.localtime(t)) + frac

  def log_forged_in(self, context, id):
    self.lock.acquire()
    try:
      in_path, out_path = self.__gen_filenames(context)
      pcap = PcapWriter(in_path)
      self.log_to_pcap(context, pcap)
      pcap.file.close() 
    finally:
      self.lock.release()
    return out_path

  def log_forged_out(self, context, out_path):
    self.lock.acquire()
    try:
      pcap = PcapWriter(out_path)
      self.log_to_pcap(context, pcap)
      pcap.file.close() 
    finally:
      self.lock.release()

  def __gen_filenames(self, context):
    "Determine the filenames for the inbound and outbound logs"

    ts = self.timestamp(context[0][0])
    filename1 = ts + "-in.pcap"
    filename2 = ts + "-out.pcap"
    path1 = self.log_dir + os.path.sep + filename1
    path2 = self.log_dir + os.path.sep + filename2
    n = 0
    while os.path.exists(path1):
      # This inbound filename has already been used; find another
      filename1 = ts + "-"  + `n` + "-in.pcap"
      filename2 = ts + "-"  + `n` + "-out.pcap"
      path1 = self.log_dir + os.path.sep + filename1 
      path2 = self.log_dir + os.path.sep + filename2 
      n +=1
    log.info("Recording inbound modified packets & context in %s\n" % path1)

    return (path1,path2)
    
  def log_to_pcap(self, packets, pcap_log):
    ## consider adding linktype= to this:

    for timestamp, hash, packet in packets:
      # be careful what we feed to scapy!
      if type(packet) != types.StringType:
        raise LoggerError, "Packet data %s is not a string!\n" % packet

      try:
        pcap_log.write(packet,timestamp)
      except:
        log.error("Tripped on %s", `pcap_log`)
        raise
      pcap_log.file.flush()
switzerland/client/FlowManager.py
class FlowManager:
    "FlowManager tracks active Flows (and also the Peers who are in our circle"

    def __init__(self, config, parent=None):
        self.flows = { }
        self.flow_id_to_address = { }
        # jered used these to implement the Reporter:
        self.new_flows = [ ]
        self.deleted_flows = [ ]
        # I'mma use these to implement the API:
        self.api_new_flows = []

        self.unique_id = 0
        self.config = config
        self.private_ip = s.inet_aton(config.private_ip)
        self.lock = threading.RLock()
        self.batch_to_process = threading.Condition(self.lock)
        self.peers = {}
        self.parent = parent

        self.ip_ids = {}        # for debugging

    def farewell(self, old_peer):
        "A peer has left. Remove any outstanding flows with it."
        self.lock.acquire()
        try:
            ip = s.inet_aton(old_peer)
            try:
                del self.peers[ip] 
            except KeyError:
                msg = "Farewell message for non-existent peer %s" % old_peer
                log.error(msg)
                self.parent.link.send_message("error-cont", [msg])

            for f in self.flows.values():
                if f.src_ip == ip or f.dest_ip == ip:
                    f.marked_for_deletion = True
            log.info("Peer %s has disconnected" % old_peer)
        finally:
            self.lock.release()

    def listen_for(self, new_peers):
        "Start sniffing packets from these peers."
        self.lock.acquire()
        try:
            note = "Listening for traffic with peers: "
            for p,firewalled in new_peers:
                # XXX add a test for private IPs here
                # s.inet_ntoa(p) asserts that p is a valid peer address
                new = s.inet_ntoa(p)
                assert s.inet_ntoa(p) != self.parent.link.private_ip
                assert s.inet_ntoa(p) != self.parent.link.public_ip

                self.peers[p] = Peer(firewalled)
                note += new
                if firewalled: note += '*'
                note += ", "
            note = note[:-2] # chop off the last comma & space
            log.info(note)
        finally:
            self.lock.release()


    def is_peer_firewalled(self, peer):
        "Here to let a packet know if its peer is firewalled"
        self.lock.acquire()
        try:
          return self.peers[peer].firewalled
        finally:
          self.lock.release()

    def is_local_flow(self, packet):
        """
        determine if this flow is to or from the local machine (Alice)
        so we can filter out broadcast traffic.
        """
        if self.config.ignore_nonlocal_packets:
          if packet.dest_ip == self.private_ip and packet.source_ip == self.private_ip: 
            log.warning("sniffing packets from self!")
          if packet.source_ip == self.private_ip or packet.dest_ip == self.private_ip:
            return True
          else:
            return False
        else:
          raise NotImplementedError, "Promiscuity not implemented"

        return True

    def mark_if_firewalled(self, packet):
        """
        flag whether peer in this packet is firewalled
        """
        if packet.dest_ip == self.private_ip:
          peer_ip = packet.source_ip
        elif packet.source_ip == self.private_ip:
          peer_ip = packet.dest_ip
        else:
          raise NotImplementedError, "Promiscuity not implemented"

        try:
          packet.peer_firewalled = self.is_peer_firewalled(peer_ip)
        except KeyError:
          # we don't know if they're firewalled, since they're
          # not a true peer
          packet.peer_firewalled = True

    def is_in_circle(self, packet):
        """
        determine if a packet is to or from a host in Alice's circle.
        packets to hosts outside Alice's circle will not be logged (unless
        config.filter_packets is False), however statistics will be kept
        and sent to switzerland (anonymously)
        """
        if not self.config.filter_packets:
          return True

        self.lock.acquire()
        try:
            if (packet.dest_ip not in self.peers) and \
               (packet.source_ip not in self.peers):
                return False
        finally:
            self.lock.release()
          
        return True

    def archive_packet(self, packet):
      global a_lock
      global archives
      if not self.config.keep_archives:
        return
      a_lock.acquire()
      try:
          thread = self.parent.link.thread_no()
          archive = archives.setdefault(thread,{})
          hash = packet.get_hash()
          archive.setdefault(hash,[]).append(packet)

      finally:
          a_lock.release()

    def create_flow(self, packet, flow_addr):
        """ create a new flow with given initial packet and flow id string """
        opening_hash = packet.get_hash()
        flow = AliceFlow(packet.dest_ip == self.private_ip, # inbound flow?
            packet.source_ip, packet.source_port, 
            packet.dest_ip, packet.dest_port, packet.proto, 
            packet.timestamp, opening_hash, self.is_in_circle(packet))
        flow.id = self.unique_id # internal serial# for flow, != flow_addr
        self.unique_id += 1
        if flow.in_circle: # only report flows in circle to switzerland
            self.new_flows.append(flow)
            self.api_new_flows.append(flow)
            log.info("Attempting to test flow #%d %s" % (flow.id, flow.__str__()))
            log.debug("(flow opens with hash: %s\npacket data: %s" % \
                (hexlify(opening_hash), hexlify(packet.data.tostring())))
            log.debug("masked from: %s)" % hexlify(packet.original_data))
        self.flows[flow_addr] = flow
        # map IP datagram flow info to our internal flow id
        self.flow_id_to_address[flow.id] = flow_addr


    def handle_packet(self, packet):
        """
        called for each incoming packet. if packet destined for alice from
        host in circle, find existing flow or detect new flow, then queue
        for that flow.  return True if the packet completes a PacketBatch,
        False otherwise.
        """
        self.lock.acquire()
        try:
            assert isinstance(packet, Packet.Packet), 'expecting Packet'
            if not self.is_local_flow(packet): # ignore packets not from/to alice
                return False
            self.mark_if_firewalled(packet)
            self.archive_packet(packet)

            flow_addr = packet.flow_addr()
            if not self.flows.has_key(flow_addr): # create flow if it doesn't already exist
                self.create_flow(packet, flow_addr)

            flow = self.flows[flow_addr]
            assert flow != None, 'expecting to find flow'

            # track flow data
            flow.activity = True
            flow.time_last_active = packet.timestamp
            flow.bytes_transferred += len(packet)
            flow.packets_transferred += 1

            if flow.in_circle:
                return flow.queue.append(packet)
            return False
        finally:
            self.lock.release()

    def clean(self, now):
        """ remove inactive flows or stale packets in active flows """
        self.lock.acquire()
        try:
          for f in self.flows.keys():
              flow = self.flows[f]
              inactive = now - flow.time_last_active
              if inactive > Flow.timeout or flow.marked_for_deletion:
                  log.debug("Flow %s timed out, deleting", `flow`)
                  flow.active = False
                  if flow.in_circle: # switzerland only knows about this flow if it's in alice's circle
                      self.deleted_flows.append(flow)
                  del self.flow_id_to_address[flow.id]
                  del self.flows[f]
              else:
                  flow.clean(now)
        finally:
          self.lock.release()
switzerland/client/TimeManager.py
class UncertainTimeError(Exception):
  pass
switzerland/client/AliceLink.py
class AliceLink(Protocol.Protocol):
  "Extend the protocol base class for Alice's end."

  def __init__(self, quit_event, parent, config):
    self.in_messages = Messages.alice_in_messages 
    self.out_messages = Messages.alice_out_messages
    try:
      self.flow_manager = parent.fm
    except AttributeError:
      self.flow_manager = None
    self.parent = parent
    self.config = config
    self.quit_event = quit_event
    self.dangling_note = False
    self.forged_notification = True
    try:
      self.server_ip = s.gethostbyname(config.host)
    except:
      log.error("DNS error looking up %s" % config.host)
      sys.exit(1)

    self.socket = s.socket(s.AF_INET, s.SOCK_STREAM)
    log.info("Connecting to %s:%s", config.host, `config.port`)
    self.socket.settimeout(10.0)
    try:
      self.socket.connect((self.server_ip, config.port))
    except:
      log.error("couldn't connect to %s:%s", config.host, `config.port`)
      sys.exit(1)
    self.socket.settimeout(40)

    self.peer = (config.host, config.port)
    self.time_connected = time.time()
    self.member_queue = []  # members to be added once we public_ip is known

    Protocol.Protocol.__init__(self, log, config.seriousness, private_ip=config.private_ip)
    # up until now it has been safe to send messages on the socket because
    # we are still blocking the caller thread

  def run(self):
    excepted = False
    try:
      Protocol.Protocol.run(self)
    except:
      print traceback.format_exc()
      excepted=True
    if not excepted:
      log.info("Adieu...")
    self.quit_event.set()
    self.ready.set() # to hurry Alice up

  def handshake(self):
    """Confirm that we are talking to a Switzerland server"""
    
    self.socket.send(Protocol.handshake1)

    self.socket.settimeout(30)
    try:
      msg = self.socket.recv(len(Protocol.handshake2))
    except s.timeout:
      self.debug_note("Timeout during handshake", seriousness=11)
      return False
    except s.error, e:
      log.error("error during handshake: %s" % e)
      return False
    except:
      log.error("error during handshake")
      return False

    if msg[:-2] == Protocol.no_common_version[:-2]:
      log.error("Server refuses to speak protocols before version %d; we need version %d" % (Protocol.parse_version(msg[-2:]), Protocol.protocol_version))
      return False
    
    elif msg[:-2] != Protocol.handshake2[:-2]:
      log.error("Handshake failed, wasn't expecting:\n" + msg)
      return False

    log.debug("Started session with version %d" % Protocol.parse_version(msg[-2:]))
    self.socket.setblocking(1)
    return True

  def setup(self):
    "Handshake successful; now do initial housekeeping"
    self.send_myip()

  def lookup_flow_by_id(self, flow_id, caller):
    """
    Called in handle_forged_{in,out}
    Return the flow, or None if we fail.
    "caller" is modified-{in,out} & is only used for error reporting.
    """
    if flow_id not in self.flow_manager.flow_id_to_address:
      msg = "stale %s flow id %s" % (caller, `flow_id`)
      log.info(msg)
      self.send_message("error-cont", [msg])
      return None

    flow_addr = self.flow_manager.flow_id_to_address[flow_id]
    if flow_addr not in self.flow_manager.flows:
      msg= "bad %s flow addr %s in id to addr dict" % (caller, `flow_addr`)
      log.error(msg)
      self.send_message("error-cont", [msg])
      return None

    flow = self.flow_manager.flows[flow_addr]
    return flow

  def handle_forged_out(self, args, seq_no):
    """ Send context surrounding a forged hash """
    if self.flow_manager == None:
      # we're in a test case
      return
    self.flow_manager.lock.acquire()
    try:
      flow_id, forgeries = args
      log.warn("Switzerland has informed us of %d modified/spoofed outbound packets in flow #%d" % (len(forgeries), flow_id))
      self.forged_note()
      flow = self.lookup_flow_by_id(flow_id, "modified-out")
      self.nat_firewall_warnings(flow)
      contexts = {}
      reports = {}
      for wanted in forgeries:
        hash = wanted[0]             # hash of forged packet 
        receipt_context = wanted[1]
        filename = self.parent.pcap_logger.log_forged_in(receipt_context, flow_id)
        ctxt, report = flow.get_fo_context_and_diff(receipt_context, self.parent)
        if report:
          log.error("Here is what we know about these modified packets:\n%s\n" %
                    fmt_forged_report(report))
          reports[hash] = report
        else:
          log.error("This packet appears to have been injected")
          log.error("(at least, we couldn't find a packet we're sure was modified)")
          log.error("(AND THIS CODE SHOULDN'T RUN)")
        if ctxt:
          contexts[hash] = ctxt
          self.parent.pcap_logger.log_forged_out(ctxt, filename)
        else:
          log.error("No modified-out context for %s" % hexlify(hash))
          contexts[hash] = None
      self.send_message("fo-context", [contexts, reports], reply_seq_no=seq_no)
      
    finally:
      self.flow_manager.lock.release()


  def handle_forged_in(self, args, seq_no):
    """ Send context about forged packets whose batches have exactly
        the newest_timestamps specified in timestamps. """
    self.flow_manager.lock.acquire()
    try:
      try:
        flow_id, packets_wanted = args
        log.warn("Switzerland has informed us of %d modified inbound packets in flow #%d" % (len(args[1]), flow_id))
        self.forged_note()
        flow = self.lookup_flow_by_id(flow_id, "modified-in")
        if not flow or self.nat_firewall_warnings(flow) == -1:
          # errors have already been reported
          return

        contexts = {}
        out_filenames = []
        for ts,hash in packets_wanted:
          context = contexts[hash] = flow.get_fi_context(ts, hash)
          if not context:
            log.warn("No context was available for hash %s" % hexlify(hash))
            out_filenames.append("")
          else:
            fn = self.parent.pcap_logger.log_forged_in(context, flow_id)
            out_filenames.append(fn)

        self.send_message("fi-context", [contexts], reply_seq_no=seq_no,\
                          data_for_reply=out_filenames)
      except:
        log.error("exception in handle_forged_in:")
        log.error(traceback.format_exc())
    finally:
      self.flow_manager.lock.release()

  def forged_note(self):
    """
    Point the user towards some (hopefully) helpful documentation about the
    forged -in and -out PCAPs.  But only do this once!
    """
    if not self.forged_notification:
      log.warn("For information about Switzerland's modified/forged PCAP "+\
               "records, please see") 
      log.warn("http://switzerland.wiki.sourceforge.net/forged")
      self.forged_notification = True

  def nat_firewall_warnings(self, flow):
    """
    Warn the user if they are seeing reports of modified packets when there
    are NAT firewalls along the path, since those NATs are mischievous.
    Return -1 if we encounter & report a fatal error along the way.
    """

    if flow.inbound:
      them = flow.src_ip
    else:
      them = flow.dest_ip
    them_p = s.inet_ntoa(them)
    try:
      peer_firewalled = self.flow_manager.is_peer_firewalled(them)
    except:
      msg = "Can no longer find " + them_p + " in peer records ("
      msg += `self.flow_manager.peers` + ")"
      log.error(msg)
      self.send_message("error-cont", [msg])
      return -1
    if not (peer_firewalled or self.firewalled):
      return
    msg = "(Note that "
    if self.firewalled: 
      msg += "you are behind a NAT firewall"
      if peer_firewalled: 
        msg += " and "
    if peer_firewalled:
      msg += "this other machine (at %s) is behind a NAT firewall" % them_p
    msg += ", so it is likely that these modifications are being made "
    msg += "by firewalls, not ISPs)"
    log.warn(msg)

  def handle_forged_details(self, args, reply_seq_no):
    """
    A "forged-details" message is the followup to for forged-in; it shows 
    us Alice's side of the story.
    """
    meta = args[0]
    in_reply_to, remembered = meta
    id = args[1]
    msgs = args[2]

    if in_reply_to != "fi-context":
      self.protocol_error("reply %d should not be a forged-details message\n" % reply_seq_no)
      sys.exit(0)

    out_filenames = remembered

    for filename, (timestamp, context, report) in zip(out_filenames, msgs):
      if filename:
        # This nonsense ensures we're prettier for the users
        log.error("Report on forged/modified packet:\n%s\n" % \
                  fmt_forged_report(report))
        if context:
          self.parent.pcap_logger.log_forged_out(context, filename)
        else:
          log.error("no outbound context to write to " + filename) 

  def dummy_handle_forged_in(self, args, seq_no):
    "This variant of handle_forged_in is only used for testing"
    self.debug_note("Heard about %d dummy forged packets: %s" % \
                    (len(args[1]), `args[0]`), 2)
    forgeries = args[1]
    contexts = {}
    for timestamp,hash in forgeries:
      contexts[hash] = [(1,"packet1"), (2,"packet2"),(3,"x" + `timestamp`)]
    self.send_message("fi-context", [contexts], reply_seq_no=seq_no)

  def handle_new_members(self, args):
    new_peers = args[0]
    try:
      self.flow_manager.listen_for(new_peers)
    except:
      # it's most likely this is due to an invalid new_peers list
      self.protocol_error("Problem with new-members: %s\n" % new_peers)
      raise

  def handle_farewell(self, args):
    old_peer = args[0]
    try:
      if self.flow_manager:
        self.flow_manager.farewell(old_peer)
    except:
      self.protocol_error("Problem with farwell: %s\n" % old_peer)
      raise

  def send_myip(self):
    "Figure out the local IP address, and send it."
    if self.config.force_public_ip: # if force_public_ip is set, assume this public ip
      self.send_message("my-ip", [[self.private_ip, self.config.force_public_ip]])
    else:
      self.send_message("my-ip", [[self.private_ip]])

  def handle_public_ip(self, args):
    "Switzerland tells alice her publicly visible ip"
    stored_data = args[0]
    log.debug("Switzerland says our public ip is %s", `args[1]`)
    self.public_ip = args[1]
    self.firewalled = self.public_ip != self.private_ip
    log.info("Private IP: %s ; Public IP: %s", `self.private_ip`, `self.public_ip`)
    self.now_ready()

  def handle_dangling_flow(self, args):
    "Switzerland has failed to matchmake our view of this flow with out peers"
    msg = "Switzerland is unable to test flow #%d.  " % args[0]
    if not self.dangling_note:
      msg += "Reasons for this may include:\n"
      msg += " - Modifications to the first packet in the flow (hash 0x%s)\n" % \
             hexlify(args[1])
      # XXX we could make the inclusion of some of these lines conditional on
      # whether the peer is actually firewalled
      msg += " - The flow is not actually with the other Switzerland client\n" 
      msg += "   (it could be with their firewall, another machine on their LAN,\n"
      msg += "   or an impostor)\n"
      msg+=" - Alice and Bob seeing a different packet as the first packet in the\n"
      msg+="   flow (most likely if the flow was active before switzerland started)\n"
    else:
      msg += "(opening hash 0x%s)" % hexlify(args[1])
    log.warn(msg)
    self.dangling_note = True
    # XXX we should also take steps to prevent the flow manager from send sent
    # and recd messages for this flow

  def debug_ip_id(self, args):
    ip_id = args[0]
    print "\n\n\n\n"
    print "Debugging ip_id", hexlify(ip_id)
    print "ip_id table:", self.flow_manager.ip_ids
    #if ip_id in self.flow_manager.scapy_ip_ids:
    #  print "ip_id is in the scapy table:",self.flow_manager.scapy_ip_ids[ip_id]
    
    #else:
    #  print "ip_id is not in scapy table"
    try:
      print "Found:", hexlify(self.flow_manager.ip_ids[ip_id])
    except:
      print "Unable to find ip_id in flow_manager.ip_ids!!!"
    print "\n\n\n\n"

  def now_ready(self):
    if self.member_queue:
      map (self.handle_new_members, self.member_queue)
    Protocol.Protocol.now_ready(self)

  def determine_response(self, msg_type, args, seq_no, reply_seq_no):

    # First look for messages common to Alice & Switzerland
    if Protocol.Protocol.determine_response(self, msg_type, args, seq_no, reply_seq_no):
      return True

    # Okay, that didn't work...
    if msg_type == "forged-in":
      self.handle_forged_in(args, seq_no)
      return True
    elif msg_type == "new-members":
      if self.flow_manager:  # only false in unit tests
        if self.ready.isSet():
          self.handle_new_members(args)
        else:
          self.member_queue.append(args)
      return True
    elif msg_type == "forged-out":
      self.handle_forged_out(args, seq_no)
      return True
    elif msg_type == "forged-details":
      self.handle_forged_details(args, reply_seq_no)
      return True
    elif msg_type == "farewell":
      self.handle_farewell(args)
      return True
    elif msg_type == "debug-ipid":
      self.debug_ip_id(args)
      return True
    elif msg_type == "public-ip":
      self.handle_public_ip(args)
      return True
    elif msg_type == "flow-status":
      log.info(args[0])
      return True
    elif msg_type == "dangling-flow":
      self.handle_dangling_flow(args)
      return True
    else:
      return False

    return True   # Tricksy
   
  def tst(self):
    self.send_message("ping")
    self.send_message("sent", ["flow","time",[1,2,4]])
    time.sleep(1)
    self.send_message("sent", ["flow","time",[1,2,5]])
    time.sleep(2)
    self.send_message("sent", ["flow","time",[1,2,6]])
    self.send_message("ping")
    self.send_message("ping")
    self.send_message("ping")
    self.send_message("signoff")
switzerland/client/AliceConfig.py
class AliceConfig:
    def __init__(self, getopt=False, **keywords):

        # all of the variables above become attributes of AliceConfig;
        # we pick the value passed to this function (if there was one)
        # or the default value otherwise

        self.private_ip = None

        for variable in alice_options.keys():
            if variable in keywords:
                self.__dict__[variable] = keywords[variable]
            else:
                self.__dict__[variable] = alice_options[variable][0]

        if getopt:
            self.get_options()

        # Defaults remaining if getopt hasn't set things:
        if self.interface == None and getopt:
            self.interface = local_ip.get_interface()
            log.info("interface is now %s", self.interface)

        if self.force_private_ip and self.private_ip == None: # could have been set by getopt?
            self.private_ip = force_private_ip
        if self.private_ip == None:
            self.private_ip = local_ip.get_local_ip(self.interface)

    def get_options(self):
        if len(sys.argv) > 1 and sys.argv[1] == "help":
            self.usage()
        try:
            (opts, args) = \
                getopt.gnu_getopt(sys.argv[1:], 's:p:i:l:u:L:P:f:b:hqv',
                    ['server=', 'port=', 'interface=', 'ip=', 'help',
                    'private-ip=', 'public-ip=', 'logfile=', 'pcap-logs=',
                    'quiet', 'uncertain-time', 'verbose', 'buffer='])
        except getopt.GetoptError:
            self.usage()

        for opt, arg in opts:
            if opt in ('-s', '--server'):
                try:
                    if ":" in arg:
                        self.host, self.port = arg.split(":")
                        self.port = int(self.port)
                    else:
                        self.host = arg
                except ValueError:
                    self.usage(False)
                    print "Invalid argument for the", opt, "flag (specify a",
                    print "server to connect to)"
                    sys.exit(1)
            elif opt in ('-f', '--public-ip'):
                self.force_public_ip = arg
            elif opt in ('-p', '--port'):
                try:
                    self.port = int(arg)
                except ValueError:
                    self.usage(False)
                    print "Invalid argument for the", opt, "flag (need a",
                    print "port number"
                    sys.exit(1)
            elif opt in ('-i', '--interface'):
                self.interface = arg
            elif opt in ('-b', '--buffer'):
                try:
                    self.packet_buffer_size = int(arg)
                    assert(self.packet_buffer_size > 1)
                except:
                    print "Invalid packet buffer size (suggestion: 25000 or more)"
                    sys.exit(1)
            elif opt in ('-l', '--ip', '--private-ip'):
                self.private_ip = arg
            elif opt in ('-L', '--logfile'):
                self.logfile = arg
            elif opt in ('-P', '--pcap-logs'):
                self.pcap_logdir = arg
            elif opt in ('-h', '--help'):
                self.usage()
            elif opt in ('-q', '--quiet'):
                self.quiet = True
            elif opt in ('-u', '--uncertain-time'):
                self.allow_uncertain_time = True
                try:
                    self.manual_clock_error = float(arg)
                except ValueError:
                    self.usage(False)
                    print "Invalid argument for the", opt, "flag (please",
                    print "specify the maximum error of your clock in seconds)"
                    sys.exit(1)
            elif opt in ('-v', '--verbose'):
                self.log_level -= (logging.INFO - logging.DEBUG)

    def usage(self, exit=True):
        print
        print "%s [OPTIONS]" % sys.argv[0]
        print
        print "This is the client for EFF's Switzerland network traffic auditing system"
        print
        print "  -h, --help                 Print usage info"
        print "  -s, --server <host[:port]> Switzerland server [and port]"
        print "  -p, --port <port number>   Switzerland server port"
        print "  -i, --interface <iface>    Interface on which to monitor traffic"
        print "  -l, --private-ip <ip>      (LAN) ip address of monitored interface"
        print "  -f, --public-ip <ip>       Force a public IP address.  Useful if you want to"
        print "                             tunnel to a server using ssh, tor,  a proxy, etc. "
        print "                             NB:this is disallowed by the default server config"
        print "  -u, --uncertain-time       Work without NTP.  This is dangerous;"
        print "       <time in seconds>     acurately specify the error in your system clock"
        print '  -L, --logfile <file>       Write a copy of the output to <file>. "-" for none'
        print "                             (defaults to " + default_logfile + ")"
        print "  -P, --pcap-logs <dir>      Sets the directory to which PCAPs of modified"
        print '                             packets will be written. "-" for none.'
        print "                             (defaults to " + default_pcap_logdir + ")"
        print "  -q, --quiet                Do not print output"
        if sys.argv[0].endswith('WebGUI.py'):
            print "  -a, --webaddr              Sets the ip address to run the Switzerland client"
            print "                             web server (defaults to 0.0.0.0)"
            print "  -w, --webport              Sets the port to run the Switzerland client"
            print "                             web server (defaults to 8080)"
            print "  -F, --fake                 Runs WebGUI in an off-network demonstration mode"
            print "                             with fake data"   
        print
        if exit: sys.exit(0)

    def check(self):
        if not self.host:
            self.host = "switzerland.eff.org"
            log.info("no switzerland server specified, defaulting to %s" % self.host)

        # check for a valid ip address
        if self.private_ip == None:
            log.error("Switzerland wasn't able to determine your local IP address.")
            log.error("Make sure you're online; if you are, use the -l flag to specify you IP")
            sys.exit(1)
        try:
            s.inet_aton(self.private_ip)
        except s.error:
            log.error("invalid local address format %s", `self.private_ip`)
            sys.exit(1)

        if self.pcap_logdir and self.pcap_logdir != "-":
            try:
                st = os.stat(self.pcap_logdir)
                if not stat.S_ISDIR(st[stat.ST_MODE]): # not a directory
                    log.error("%s isn't a directory", self.pcap_logdir)
                    log.error('Please make this a directory, or use "-P -" for no logging')
                    sys.exit(1)
            except OSError, e:
                if e.errno == errno.ENOENT: # no such file or directory
                    log.warn("PCAP log directory %s doesn't exist", self.pcap_logdir)
                    log.warn("trying to create it")
                    dir = os.path.split(self.pcap_logdir)
                    create = [dir[1]]
                    while ((not os.path.exists(dir[0])) and (not dir[0] == os.path.dirname(dir[0]))):
                        dir = os.path.split(dir[0])
                        create.insert(0, dir[1])
                    try:
                        dir=dir[0]
                        for newdir in create:
                            dir=os.path.join(dir, newdir)
                            os.mkdir(dir)
                    except OSError:
                        log.error("can't create PCAP log directory")
                        log.error('Please create it, or use "-P -" for no logging')
                        sys.exit(1)
                else: # can't stat, that's probably bad
                    log.error("can't stat PCAP log directory %s: %s", self.pcap_logdir, str(e))
                    log.error('Use "-P -" for no logging')
                    sys.exit(1)
            if not util.writable(self.pcap_logdir):
                log.error("Cannot write to PCAP log directory %s", self.pcap_logdir)
                log.error("Change its permissions or specify another directory with -P")
                log.error('Use "-P -" for no logging')
                sys.exit(1)
switzerland/client/Cleaner.py
class Cleaner(threading.Thread):
    """Periodically clean up stale packets and expired flows."""

    interval = 10              # seconds between waking up and cleaning

    def __init__(self, parent):
        threading.Thread.__init__(self)
        self.parent = parent

    def run(self):
        try:
            while True:
                time.sleep(Cleaner.interval)
                self.parent.fm.clean(int(time.time()))
        except:
            # Exit-time crash avoidance.
            if self.parent.quit_event.set != None:
                log.info("Cleaner exception:\n" + traceback.format_exc())
                self.parent.quit_event.set()
switzerland/client/PacketListener.py
class PacketListener(threading.Thread):
    """ sniff incoming IP datagrams on a network interface or from a save file.
        pass packets off to FlowManager. """

    def __init__(self, parent):
        """self.parent.fm: FlowManager to receive packets"""
        threading.Thread.__init__(self)
        self.parent = parent
        self.packets_in_buf = parent.config.packet_buffer_size
        self.entry_size = packet_size + packlen_size + ts_size + valid_size + \
                          padding2_size
        self.buffer_size = self.packets_in_buf * self.entry_size
        self._zcat_pipe = None
        self._zcat = None
        self.tmpfile = None
        self.lock = threading.RLock()
        self.latest = 0
        self.max_timewarp = 0
        self._skew = parent.config.skew
        self.kernel_tweaked = False
        self.frag_warn = False # have we warned about fragmented packets?
        self.done = threading.Event() # are we done replaying an 
                                      # offline capture?
        self.half_done = False  # True when FastCollector has finished
                                # healthily but we haven't read all the
                                # packets from the buffer yet
        
        if platform.system() == 'Windows':
          self.sniffer_thread = os.path.sep.join(['FastCollector.exe'])
          self.sniffer_thread2 = os.path.sep.join(['bin','FastCollector.exe'])
        else:
          self.sniffer_thread = os.path.sep.join(['FastCollector'])
          self.sniffer_thread2 = os.path.sep.join(['bin','FastCollector'])
        if parent.config.pcap_playback:
          self.live = False
          self.target = self.filename_magic(parent.config.pcap_playback)
        else:
          assert parent.config.interface, "Can't create PacketListner with neither a pcap file nor an interface"
          self.live = True
          self.target = parent.config.interface

    def minimize_packet_loss(self):
        """
        Platform-specific code to maximise buffer/queue space in the kernel
        captured packets, to minimize the risk that the kernel drops packets
        before they get to the sniffer.
        """
        p = platform.system()
        # Implementing the recommendations from
        # http://www.net.t-labs.tu-berlin.de/research/hppc/
        if p[-3:] == "BSD" or p == "Darwin":
          cmd = ["sysctl","-w","net.bpf.bufsize=10485760"]
          try:     # Recent FreeBSDs
            proc = Popen(cmd, stdin=PIPE, stdout=PIPE)
            assert proc.wait() == 0
            cmd[2] = "net.bpf.maxbufsize=10485760"
            proc = Popen(cmd, stdin=PIPE, stdout=PIPE)
            assert proc.wait() == 0
          except:  # Older FreeBSDs
            cmd[2] = "debug.bpf_bufsize=10485760"
            proc = Popen(cmd, stdin=PIPE, stdout=PIPE)
            assert proc.wait() == 0
            cmd[2] = "debug.maxbpf_bufsize=10485760"
            proc = Popen(cmd, stdin=PIPE, stdout=PIPE)
            assert proc.wait() == 0

        elif p == "Linux":
          vars = [("/proc/sys/net/core/rmem_default", "33554432"),
                  ("/proc/sys/net/core/rmem_max", "33554432"),
                  ("/proc/sys/net/core/netdev_max_backlog", "10000")]
          self.old_kernel_state = []
          for file, value in vars:
            # save kernel state before overwriting it
            f = open(file, 'r')
            self.old_kernel_state.append((file, f.read()))
            f.close()
            f = open(file, 'w')
            f.write(value)
            f.close()
          self.kernel_tweaked = True

        # on Windows, we should call pcap_setbuff from inside FastCollector.

    def restore_kernel_state(self):
        p = platform.system()
        if p == "Linux":
            for file, value in self.old_kernel_state:
                # save kernel state before overwriting it
                f = open(file, 'w')
                f.write(value)
                f.close()
            self.kernel_tweaked = False

    def launch_collector(self):
        """
        Launch the PacketCollector, which lives in another process.
        For live captures this approach is necessary to avoid packet loss;
        for offline playback it is helpful for testing.
        Because of the weirdness of pypcap, target can be *either* an
        interface name or a pcap filename! :/
        """
        
        try:
          if self.live: self.minimize_packet_loss()
        except:
          log.info(traceback.format_exc())
          print "Failed to set kernel buffers to minimize packet loss!!!"
          print "You may encounter problems as a result of this..."

        try:
          if platform.system() == 'Windows':
            use_shell = True
          else:
            use_shell = False
          try: 
            mode = '-i' # interactive
            if not self.live:
              mode = '-f'
            sz = "-b%ld" % self.packets_in_buf
            log.debug("Launching packet collector: "+ `[self.sniffer_thread, mode, self.target]`)
            try:
              self.sniff = Popen(
                [self.sniffer_thread, mode, sz, self.target], shell=use_shell,
                stdout=PIPE, stdin=PIPE, stderr=PIPE
              )         
            except:
              # try bin/FastCollector
              self.sniff = Popen(
                [self.sniffer_thread2, mode, sz, self.target], shell=use_shell,
                stdout=PIPE, stdin=PIPE, stderr=PIPE
              )         
          except:
            log.error("failed to start %s", self.sniffer_thread)
            raise

          if self.live: self.prioritise_sniffer()

        except OSError:
          value = sys.exc_info()[1]
          if "No such file or directory" in value:
            print self.sniffer_thread, "does not appear to be in the PATH..."
            #print "(which is", os.environ["PATH"] + ")"
          else:
            raise
          self.parent.quit_event.set()
          print "exiting"
          print threading.enumerate()
          sys.exit(1)

        self.tmpfile = self.get_tempfile_name()
        log.debug("Opening tempfile %s" % self.tmpfile)
        self.file = open(self.tmpfile, "a+b")
        fd = self.file.fileno()
        self.mem = mmap.mmap(fd, self.buffer_size, access=mmap.ACCESS_WRITE)

        self.parent.config.pcap_datalink = self.get_pcap_datalink()
        log.debug("Got pcap_datalink: %s" % `self.parent.config.pcap_datalink`)

    def prioritise_sniffer(self):
        """
        OS-dependent code to ensure the sniffer runs with the highest
        available priority level.
        """
        if platform.system() == 'Windows':
          util.set_win32_priority(self.sniff.pid, 5)
        else:  # UNIX is nice
          try:
            n = Popen(["renice", "-19", `self.sniff.pid`], stdout=PIPE)
            log.debug(n.stdout.read())
          except OSError:
            log.error("Attempt to prioritise sniffer failed with" + `sys.exc_info[0:2]`)

    def get_tempfile_name(self):
      """
      Communicate with the sniffer process to obtain the name of the
      mmap()ed tempfile through which it will pass us packets.
      """
      log.debug("looking for temp file..")
      self.sniff.stdin.close()
      line = self.sniff.stdout.readline()
      while line == "":
        line = self.sniff.stdout.readline()
        rval = self.sniff.poll()
        if rval != None:              # The sniffer died
          msg = "packet collector exited with return code %d\n" % rval
          msg += self.sniff.stderr.read()
          raise SnifferError, msg

      words = line.split()
      if len(words) == 0 or words[0] != "Tempfile:":
        raise SnifferError("packet collector didn't print a Tempfile: line")
      return words[1]

    def get_pcap_datalink(self):
        """
        Communicate with sniffer process to obtain the output of
        pcap_datalink on the capture (this will later determine header
        length of link layer portion of the packet).
        """
        line = self.sniff.stdout.readline() # XXX This is for "Initialized..." line
        line = self.sniff.stdout.readline()
        while line == "":
            line = self.sniff.stdout.readline()
            rval = self.sniff.poll()
            if rval != None:              # The sniffer died
                msg = "packet collector exited with return code %d\n" % (rval)
                msg += self.sniff.stderr.read()
                raise SnifferError, msg
        words = line.split()
        if len(words) == 0 or words[0] != "pcap_datalink:":
            raise SnifferError,"packet collector didn't print a pcap_datalink: line"
        return int(words[1])


    def run(self):
        """ sit in a tight loop and collect packets
            in interactive mode, run until the parent thread exits.
            in offline mode, break when the dump file ends """

        # This only returns on an error condition...
        try:
          self.launch_collector()
          self.read_packets()

          # The obvious thing to do with exceptions here would be to set the
          # quit event and then "raise".  But if there's a context switch in
          # between those two lines, we might never see the exception.  So
          # print it before sending quit_event.
        except SnifferError,e:    # sniffer error; clean up on the way out
          self.cleanup()
          log.error(e)
          if "handle a packet of size" in repr(e):
            self.large_segment_message()
          self.parent.quit_event.set()
          sys.exit(1)
        except:                   # What kind of error is this?
          self.cleanup()
          log.error(traceback.format_exc())
          self.parent.quit_event.set()
          sys.exit(1)

        # notify parent thread that there's no more input
        self.done.set()
        self.cleanup()
        # close pipe from zcat for gzipped traces
        if self._zcat != None:
            self._zcat.wait()
            if self._zcat_pipe:
                os.unlink(self._zcat_pipe)
        # organise shutdown for the entire client process
        log.info("Goodbye...")
        self.parent.quit_event.set()


    def filename_magic(self, filename):
        """
        Do all our housekeeping for playbacks of stored pcaps.  Sometimes
        these are gzipped, so we try to use named pipes to decompress them
        for the pcap library code.  Returns a filename for a pcap library.
        """
        # pcapy can't deal with gzipped input
        if filename.endswith('.gz'):
            trim_gz = filename[:-3]

            # if there's an ungzipped copy in the same place, use it
            if os.path.exists(trim_gz):
                filename = trim_gz

            # XXX this isn't going to work on windows
            elif platform.system() == 'Windows':
                print "sorry, I don't know how to open gzipped traces on windows"
                print "please gunzip %s and try again\n" % (filename)
                raise NotImplementedError

            # use a named pipe if we got em
            else:
                tmp = tempfile.mktemp()
                posix.mkfifo(tmp)

                self._zcat = Popen('zcat "%s" > "%s"' % (filename, tmp), shell=True)
                self._zcat_pipe = tmp
                filename = tmp
            
        return filename

    def read_packets(self):
      """
      Read packets out of the mmap()ed buffer that PacketCollector.py
      is busy writing them into.
      """

      self.pos = 0
      count = 0

      if platform.system() == 'Windows':
        self.check_for_sniffer_error = self.check_for_sniffer_error_win32
        self.sniff.fd_stderr = self.sniff.stderr.fileno()
        self.sniff.os_stderr = msvcrt.get_osfhandle(self.sniff.fd_stderr)
      else:
        self.check_for_sniffer_error = self.check_for_sniffer_error_posix

      while True:
        packet = self.read_one_packet()
        if packet == None:
          return
        else:
          timestamp, data = packet
          
        count +=1
        if count % lag_check_frequency == 0:
          delta = time.time() - timestamp
          log.info("Capturing a packet currently takes us %lf seconds" % delta)
        self.process_packet(timestamp, data)
        if double_check:
          print "Pos is now 0x%x" % self.pos
          print Ether(data).summary()

        #if count % 10 == 0:
        #  # at this point we aren't interested in normal playback termination
        #  # from the sniffer (ie, half_done==True).  We just need to know
        #  # about urgent conditions like dropped packets in live deployment
        #  self.check_for_sniffer_error()

    def read_one_packet(self):
      "Read a single packet out of the mmaped buffer; return (timestamp, data)."
      self.lock.acquire()
      try:
        if not self.tmpfile: return
        valid = self.mem[self.pos]
        while valid == '\x00':
          if self.check_for_sniffer_error(): return     # playback complete
          time.sleep(poll_interval)
          valid = self.mem[self.pos]
          if double_check: sys.stdout.write(".")

        base_pos = self.pos
        self.pos += vs

        len = struct.unpack("H", self.mem[self.pos:self.pos+pls])[0]
        if double_check: print "Packet of length", len
        if len > 1514:
          print "Wild packet length %d at pos 0x%x" % (len, self.pos)
          self.parent.quit_event.set()
          sys.exit(1)
        self.pos +=pls + padding2_size

        timestamp = struct.unpack("d", self.mem[self.pos:self.pos+tss])[0]
        if timestamp > self.latest:
          self.latest = timestamp
        else:
          t = self.latest - timestamp

          # Tell the user and the server every time we see packets
          # more out-of-order than anything we've seen yet
          if t > self.max_timewarp:
            msg = "Timestamp monotonicity violated by up to %g seconds" % t
            self.max_timewarp = t
            if t > 0.0002:  # 200 microseconds
              log.error(msg)
              # XXX if this gets up into the millisecond range we should
              # probably switch to a hard error.
              self.parent.link.send_message("error-cont", [msg])
            else:
              log.debug(msg)


        if double_check: print "Timestamp", timestamp
        self.pos += tss

        data = self.mem[self.pos:self.pos+len]
        # mark this packet as read
        self.mem[base_pos] = "\x00"
        self.pos = (self.pos + packet_size) % self.buffer_size
        return (timestamp, data)
      finally:
        self.lock.release()

    def process_packet(self, timestamp, data):
      "Integrate this packet appropriately into Alice's data structures."
      try:
        packet = Packet.Packet(
          timestamp + self._skew,
          data, 
          self.parent
        )
      except:
        msg = "Packet parser failed on this packet (raw hex):\n"
        msg += hexlify(data) + "\n"
        if have_scapy:
          msg += "Scapy analysis\n" + `Ether(data).summary()` + "\n"
        msg += "The error was:\n" + traceback.format_exc()
        log.warn(msg)
        self.parent.link.send_message("error-cont", [msg])
        # HELP I'M STUCK IN AN EXCEPTION HANDLER!
        return
        
      if dummy:
        return
      if packet.is_fragment(): # skip fragmented packets
          if not self.frag_warn:
              log.warn("We saw fragments! This may result in false drop reports.")
              self.frag_warn = True
          return
      self.parent.fm.batch_to_process.acquire()
      # notify reporter thread if this packet completes a batch
      if self.parent.fm.handle_packet(packet):
          self.parent.fm.batch_to_process.notify()
      self.parent.fm.batch_to_process.release()

    def check_for_sniffer_error_win32(self):
      """
      Windows doesn't have "select" for pipes, so we have to resort to dirty tricks
      adapted from http://mail.python.org/pipermail/python-list/2006-August/396531.html
      """
      if self.half_done:
        return True
      try:
        data, avail, _ = PeekNamedPipe(self.sniff.os_stderr, 0)
      except: # pipe closed, we must be done
        return self.report_sniffer_error()
      if avail != 0:
        return self.report_sniffer_error()
      return False

    def check_for_sniffer_error_posix(self):
      """
      Return False for no condition; -1 for an error and +1 for a healthy
      shutdown
      """
      if self.half_done:
        return True
      # XXX perhaps we should use some combination of os.open() flags
      # and read() here instead of select...
      s = select.select([self.sniff.stderr], [], [], 0)
      if s[0]:
        self.report_sniffer_error()
        return True
      return False

    def report_sniffer_error(self):
      """
      go ahead and read from the sniffer's stderr, there's something there
      """
      print "We have news from PacketCollector:"
      ret = self.sniff.wait()
      output= self.sniff.stderr.read()
      if output:
        msg = "Packet sniffer error\n" + output
        try:
          self.parent.link.send_message("error-bye", [msg])
        except:
          pass
        raise SnifferError, output
      else:
        print "(Normal termination)..."
        print self.sniff.stdout.read()
        self.half_done = True
        return True

    def cleanup(self):
      # Alice may call this code too, so avoid deadlocks
      self.lock.acquire()
      try:
        if self.kernel_tweaked:
          self.restore_kernel_state()
      except:
        log.error(traceback.format_exc())
      try:
        if not self.tmpfile:
          return
        self.mem.close()
        self.file.close()
        try:            # to kill FastCollector
          if platform.system() == "Windows":
            # According to http://code.activestate.com/recipes/347462/, 
            # this is how we killa process on Windows
            win32api.TerminateProcess(int(self.sniff._handle), -1)
          else:
            os.kill(self.sniff.pid, signal.SIGTERM)
        except:
          pass          # it's probably already dead
        time.sleep(0.3) # give the FastCollector time to exit
        if self.packets_in_buf > 25000:   # the old/default value
          log.info("Closing packet capture thread...")
          # this takes longer with large buffers!
          time.sleep (0.5 * self.packets_in_buf / 25000.0)
        try:
          cmd = ["shred", "-n", "1", self.tmpfile]
          shred = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
          assert shred.wait() == 0
        except:
          # XXX some platforms won't have shred.  Ideally we should use their
          # native RNGs here... also, is this a good mode for open()?
          if not self.tmpfile:
            log.error("tmpfile already gone")
            return
          f = open(self.tmpfile, 'w')
          blank_entry = "\x00" * self.entry_size
          for n in xrange(self.packets_in_buf):
            f.write(blank_entry)
          f.close()
        try:
          os.unlink(self.tmpfile)
        except OSError, e:
          if "No such file or directory" in e:
            pass
          else:
            raise
      finally:
        self.lock.release()
        self.tmpfile = None
          

    segment_message = \
"""This error is usually a results of having a feature called "TCP Segmentation
Offloading" or "Large Segment Offloading" enabled in your operating
system.  See http://en.wikipedia.org/wiki/Large_segment_offload for more info.
Try running Switzerland with Segmentation Offloading disabled.
"""

    def large_segment_message(self):
      "Tell the user about large segement offloading and how to disable it"
      msg = self.segment_message
      plat = platform.system()
      if plat == "Linux":
        msg += "On linux, you can disable it with the following command:\n"
        msg += "ethtool -K %s tso off" % self.parent.config.interface
      elif plat == "Darwin" or plat[-3:] == "BSD":
        msg += "On %s, you can disable it with the following command:\n" % plat
        msg += "ifconfig %s -tso" % self.parent.config.interface
      elif plat == "Windows":
        msg += "On Windows, you should be able to disable it using the advanced "
        msg += "settings for your network driver." 
      else:
        msg += "Sorry, we can't yet tell you how to disable it on %s" % plat
      log.warn(msg)
switzerland/client/Reporter.py
class Reporter(threading.Thread):
    """ send packet batches and report flow activity """
    flow_activity_interval = 10 # seconds between sending flow updates
    batch_wait_timeout = 1 # maximum time to wait for a full batch

    def __init__(self, parent):
        threading.Thread.__init__(self)
        self.parent = parent
        self.link = parent.link
        self.flow_manager = parent.fm
        self.batch_to_process = parent.fm.batch_to_process 
        self.last_flow_activity_update = None
        if self.parent.config.debug_monotonicity:
            self.latest = 0

    def send_flow_stats(self):
        """ send stats about flows """
        pass

    def send_flow_announcements(self):
        """ send information about which flows have been opened/closed """
        if len(self.flow_manager.new_flows) == 0 and \
           len(self.flow_manager.deleted_flows) == 0:
           return # don't bother reporting flow activity if nothing new to report
        new     = [f.summary() for f in self.flow_manager.new_flows]
        deleted = [f.id        for f in self.flow_manager.deleted_flows]

        self.link.send_message("active_flows", [new, deleted])

        for flow in self.flow_manager.new_flows:
            flow.reported = True
        self.flow_manager.new_flows = [ ]
        self.flow_manager.deleted_flows = [ ]

    def send_flow_activity(self):
        self.send_flow_announcements()
        self.send_flow_stats()

    def send_batch(self, batch, flow, now):
        """ send packet batch """
        # have to report flow before sending batches from it
        if not flow.reported: 
            # XXX this opening_hash juggling could be cleaned up...
            self.send_flow_announcements()
        hashes = batch.get_hashes()
        timestamp = batch.newest_timestamp
        if self.parent.config.debug_monotonicity:
            if timestamp > self.latest:
                self.latest = timestamp
            else:
                log.debug("send-batch out of order by %gs" % (self.latest - timestamp))
        if flow.inbound:
            self.link.send_message("recd", [flow.id, timestamp, hashes])
        else:
            self.link.send_message("sent", [flow.id, timestamp, hashes])
	#for p in batch.packets:
	#    log.debug("%s %s", p.hash, p.data)
        #log.debug("%d [%d,%d] %d packets for %s", \
        #    now, batch.oldest_timestamp, batch.newest_timestamp, batch.size, flow)

    def run(self):
      try:
        while True:
            self.batch_to_process.acquire()
            # wait for a full batch
            self.batch_to_process.wait(Reporter.batch_wait_timeout) 
            # now either there's a full batch or we timed out waiting

            # This looks crazy, but it prevents lots of exceptions during
            # multi-threaded shutdown (the previous line is a long wait)
            try:
                now = int(time.time())
            except:
                return

            # scan batches
            for flow in self.flow_manager.flows.values():
                while flow.queue.has_unsent_batches():
                    batch = flow.queue.get_oldest_unsent_batch()
                    time_batch_open = now - batch.oldest_timestamp
                    # if batch completed or timed out, send it and look at the next
                    if batch.full or time_batch_open > PacketBatch.timeout:
                        self.send_batch(batch, flow, now)
                        flow.queue.mark_oldest_batch_sent()
                        batch.sent = True
                        batch.full = True
                    # otherwise nothing more to do for this flow
                    else:
                        break

            # send flow updates
            if self.last_flow_activity_update == None or \
               now - self.last_flow_activity_update > Reporter.flow_activity_interval:
                self.send_flow_activity()
                self.last_flow_activity_update = now

            self.batch_to_process.release()
      except:
        if traceback:    # only false during interpreter shutdown
          traceback.print_exc()
          self.parent.quit_event.set()
        return
switzerland/client/TimeManager.py
class TimeManager:

  def __init__(self):
    
    self.ntp_sysinfo = {}
    self.method = "none"  # what we tell the user about how ntp is working

    # get_time_error is the function Alice will call to get an estimate
    # of the inaccuracy of her clock.  Ideally we use ntpd's measure of
    # root dispersion, but that can be impossible for many reasons, in which
    # case we fall back to something else by switching out this function 
    self.get_time_error = self.ntpd_root_dispersion

    log.info("Looking for ntpd...")
    try:
      ntpdc = Popen("ntpdc", stdin=PIPE, stderr=PIPE, stdout=PIPE)
    except:
      log.info("Couldn't run ntpdc.  Will try something else...")
      log.debug(traceback.format_exc())
      self.get_time_error = self.ntpdate_poll
      return
      
    ntpdc.stdin.write("sysinfo\n")
    ntpdc.stdin.close()
    self.lines = ntpdc.stdout.readlines()
    if not self.lines or "onnection refused" in self.lines[0]:
      log.debug('ntpdc is present but ntpd does not appear to be alive: ("%s").  Will try something else...' % self.lines)
      self.get_time_error = self.ntpdate_poll
      return

    err = ntpdc.stderr.readlines()
    if err:
      log.debug('error running ntpdc: ("%s")' % err)
      self.get_time_error = self.ntpdate_poll
      return
      
    for line in self.lines:
      pos = line.find(":")
      key = line[:pos].lower()
      val = line[pos+1:].strip()
      self.ntp_sysinfo[key]=val

    mode = self.ntp_sysinfo["system peer mode"]
    if mode == "unspec":
      log.debug("""
      NTP is in "unspec" mode.  This is probably because your system clock
      is wrong by seconds or more.  Switzerland won't work if your clock is
      that wrong.  Unspec mode could also mean that you're offline or that 
      you need to give NTP another minute or two to find time servers.  If 
      this doesn't work, try stopping NTP, use the ntpdate program to adjust 
      the clock and then start NTP again.""")
      log.info("ntpdc is in unspec mode.  Will try something else...")
      self.get_time_error = self.ntpdate_poll
    elif mode != "client":
      log.info("Note that NTP is in mode %s", mode)


  def query_timeserver(self, timeserver):
    '''Query a timeserver using ntpdate and return the clock delta'''
    try:
      regex = re.compile("offset (-?[0-9]+\.[0-9]+) sec")
      executable = "ntpdate"
      # try bin/ntpdate if ntpdate doesn't work
      try:
        Popen(executable, stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=False)
      except:
        executable = os.path.join("bin", "ntpdate")
        try:
          Popen(executable, stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=False)
        except:
          log.error('''Can't even find a copy of "ntpdate" :(''')
          raise UncertainTimeError, "Cannot find an ntpdate executable"

      if platform.system() == "Windows":
        cmd = [executable,"-b","-q",timeserver]
        cmd = " ".join(cmd)
        ntpdate = Popen(cmd, stdin=PIPE, stderr=PIPE, stdout=PIPE, shell=True)
      else:
        cmd = [executable,"-q",timeserver]        
        ntpdate = Popen(cmd, stdin=PIPE, stderr=PIPE, stdout=PIPE)
      ntpdate_lines = ntpdate.stdout.read()
      match = regex.search(ntpdate_lines)
      if not match:
        log.error("Failed to parse the output from ntpdate.")
        log.debug(ntpdate_lines)
        return None
      delta = float(match.group(1))
      return abs(delta)

    except UncertainTimeError:
      # let this pass upwards
      raise
    except:
      log.warn("Weird but possibly non-fatal error:\n"+traceback.format_exc())
      return None

  def ntpdate_poll(self):
    '''An alternative method for finding the root dispersion'''
    servers = ["0.pool.ntp.org", "1.pool.ntp.org", "2.pool.ntp.org"]
    try:
      results = [self.query_timeserver(s) for s in servers]
      results = filter(lambda r : r != None, results)
      if results:
        log.debug("ntpdate poll results are: %s" % results)
        log.info("Believing the maximum clock error reported by %d timeservers"\
                  % len(results))
        log.debug("Not quite as good as a working ntpd, but this should be ok")
        self.method = "ntpdate -q"
        return max(results)
      else:
        log.info("We haven't succeeded in polling any NTP servers, either.")
        raise UncertainTimeError
    except UncertainTimeError:
      # let this pass upwards
      raise
    except:
      log.debug("ntpdate_poll error:\n%s" % traceback.format_exc())
      raise UncertainTimeError

  def ntpd_root_dispersion(self):
    try:
        
      mode = self.ntp_sysinfo["system peer mode"]
      self.method = "ntpd, root dispersion measured with ntpdc (%s mode)" % mode
      return float(self.ntp_sysinfo["root dispersion"].split()[0])
    except KeyError:		 
      # try to use ntpdate to calculate the root dispersion by finding the max 
      # deviation as reported by 3 time servers
      log.warning("Couldn't find root dispersion in ntpdc output.  Trying ntpdate...")
      return self.ntpdate_poll()
switzerland/client/Tracerouter.py
class Tracerouter(threading.Thread):
    def __init__(self):
        self.ready = False
        self.tr_cmd = CALCULATING
        self.routes = {}
        self.q_work = threading.Condition()
        self.queue = []
        threading.Thread.__init__(self)
        self.setDaemon(True)

    def run(self):
        self.setup()
        self.do_traceroutes()

    def setup(self):
        if platform.system() == "Windows":
            tr_cmd = ["tracert"]
            use_shell = True
        else:
            tr_cmd = ["traceroute"]
            use_shell = False

        try:
            run = Popen(["mtr", "--report", "-n", "localhost"],
                        shell=use_shell, stdout=PIPE)
            test = run.stdout.read()
            if "1" in test:
                tr_cmd = ["mtr", "--report", "--report-cycles", "2"]
                # print "mtr works..."
        except:
            try:
                run = Popen(tr_cmd + [ "localhost"], 
                            shell=use_shell, stdout=PIPE)
                test = run.stdout.read()
            except:
                print traceback.format_exc()
                tr_cmd = None
        self.tr_cmd = tr_cmd
        self.use_shell = use_shell
        self.ready = True
        print "ready"

    def do_traceroutes(self):
        while True:

            try:
                target_ip = self.get_from_queue()
                run = Popen(self.tr_cmd + ["-n", target_ip], stdout=PIPE)
                self.routes[target_ip] = run.stdout.read()
            except:
                traceback.print_exc() 

    def get_from_queue(self):
        # Wait til there's something in the queue
        self.q_work.acquire()
        if len(self.queue) == 0:
            self.q_work.wait()
        else:
            self.q_work.release()

        # Grab the next traceroute target off the queue
        target_ip = self.queue[0]
        del self.queue[0]
        print "got", target_ip
        return target_ip

    def enqueue(self, target):
        self.q_work.acquire()
        try:
            self.queue.append(target)
            self.routes[target] = "Waiting"
            self.q_work.notify()
        finally:
            self.q_work.release()
       

    def route_to(self, target_ip):
        if not self.tr_cmd:
            return False
        elif self.tr_cmd == CALCULATING:
            self.enqueue(target_ip)
            return "Waiting"
        elif target_ip in self.routes:
            return self.routes[target_ip]
        else:
            self.enqueue(target_ip)
            # This *probably* means we return "waiting", unless things happen
            # very quickly
            return self.routes[target_ip]


Question: import sys
import time
import os
import platform
import traceback
import threading
import socket as s
import logging
            import psyco
from switzerland.client import PacketListener
from switzerland.client import Cleaner
from switzerland.client import Reporter
from switzerland.client.AliceLink import AliceLink
from switzerland.client.FlowManager import FlowManager
from switzerland.client.AliceConfig import AliceConfig
from switzerland.client.TimeManager import TimeManager, UncertainTimeError
from switzerland.client.Tracerouter import Tracerouter
from switzerland.common.PcapLogger import PcapLogger
#                    format="[%(name)s]: %(message)s")

log = logging.getLogger('')
log.setLevel(logging.DEBUG)
startup_handler = logging.StreamHandler()
log.addHandler(startup_handler)

welcome_msg = \
"""--------------------------------------------------------------------------------
Welcome to Switzerland.  
READ THE PRIVACY AND SECURITY SECTIONS OF THE README.txt!!

Once you've done that... this is a Version Zero alpha release.  It's sure to
break at some point.  If/when that happens, please let us know by email 
(switzerland-devel@eff.org), IRC (#switzerland on irc.oftc.net) or bug report
( https://sourceforge.net/tracker/?func=browse&group_id=233013&atid=1088569 )
--------------------------------------------------------------------------------"""

class Alice:
    """
    Alice is the Switzerland client.
    config is an AliceConfig object saying how she should operate;
    linkobj is only changed in test cases
    """
    def __init__(self, config, linkobj=AliceLink):

        self.config = config

        #logging.basicConfig(level=config.log_level,
        #                    format="[%(name)s]: %(message)s")
        config.check()

        self.init_logging()
         
        self.params = {}
        if config.use_ntp:
            self.ntp_setup()

        # local address may be fake for testing
        self.fm = FlowManager(config, parent=self)
        self.quit_event = threading.Event()
        self.cleaner = Cleaner.Cleaner(self)
        
        self.listener = PacketListener.PacketListener(self)
        self.link = linkobj(self.quit_event, self, config)
        self.reporter = Reporter.Reporter(self)
        self.tr = Tracerouter()
    
        self.cleaner.setDaemon(True)
        self.listener.setDaemon(True)
        self.reporter.setDaemon(True)
        self.link.setDaemon(True)
        self.tr.setDaemon(True)

        # XXX we need a privacy setting to disable this, since it might in some
        # cases create link between separate client sessions at separate IPs.
        # But this is about the minimum amount of information we need to spot
        # weird platform specific crashes in early releases
        version = (platform.system(), platform.release(), \
                    platform.machine(), sys.version_info)
        self.params["version"] = version
        
        log.debug("Starting AliceLink...")
        self.link.start()
        self.tr.start() # do something while we wait
        self.link.ready.wait(20.0)
        if self.link.quit_event.isSet():
          # error during handshake
          self.listener.cleanup()
          sys.exit(0)
        if not self.link.ready.isSet():
            log.error("failed to start link with switzerland")
            sys.exit(1)
        
        if config.use_ntp:
            self.link.send_message("parameters", [self.params])

    def init_logging(self):
        "Setup all logging for the client"
        global log

        rootlogger = logging.getLogger()
        if not self.config.quiet:
            output = logging.StreamHandler(sys.stdout)
            rootlogger.addHandler(output)

        # now remove the temporary stderr output we were using during startup
        log.removeHandler(startup_handler)
        log = rootlogger

        if self.config.logfile not in ("-", None)  :
            fileout = logging.FileHandler(self.config.logfile)
            rootlogger.addHandler(fileout)
            log.error("Logging events to " + self.config.logfile)

        if self.config.pcap_logdir not in ("-", None):
            self.pcap_logger = PcapLogger(self.config.pcap_logdir)
        else:
            self.pcap_logger = None


    def start(self):
        "Launch various sub-threads."
        #self.listener.start()
        self.reporter.start()
        if self.config.do_cleaning:
            self.cleaner.start()
        try:
            psyco.profile()
        except ImportError:
            log.info("The psyco package is unavailable (that's okay, but the client is more\nefficient if psyco is installed).")

    def ntp_setup(self):
        try:
            self.time_manager = TimeManager()
            self.root_dispersion = self.time_manager.get_time_error()
            log.info("We believe this system's clock is accurate to within %f seconds" % self.root_dispersion)
            if self.root_dispersion > 5.0:
              log.error("Sorry, your clock is out by more than 5 secs. Switzerland isn't going to work!")
              sys.exit(1)

Answer:         except UncertainTimeError: