sirikata/sirikata — Issue #484: SST causes crash when out of ports
Author: ewencp
Created: 2012-01-26 02:44:00+00:00
State: open

Generated this crash on the space server when around 30k objects had connected:

```
Program received signal SIGSEGV, Segmentation fault.
0x0000000000575321 in Sirikata::XDP::Port::receive(const std::tr1::function &) (this=0x0, cb=...) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/xdp/Port.hpp:76
76	receiveFrom(Endpoint::any(), cb);
(gdb)
(gdb) bt
#0 0x0000000000575321 in Sirikata::XDP::Port::receive(const std::tr1::function &) (this=0x0, cb=...) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/xdp/Port.hpp:76
#1 0x00000000005716df in Sirikata::SST::BaseDatagramLayer::listenOn(Sirikata::SST::EndPoint &, std::tr1::function) (this=0x7fffe8012980, listeningEndPoint=..., cb=...) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/odp/SST.hpp:78
#2 0x000000000057c4cc in Sirikata::SST::Connection::Connection (this=0x7fffc836c260, sstConnVars=0x7fffe8001148, localEndPoint=..., remoteEndPoint=...) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/network/SSTImpl.hpp:411
#3 0x00000000005764a8 in Sirikata::SST::Connection::handleReceive (sstConnVars=0x7fffe8001148, remoteEndPoint=..., localEndPoint=..., recv_buffer=0x7fffdafe08a8, len=12) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/network/SSTImpl.hpp:1199
#4 0x0000000000571e5b in Sirikata::SST::BaseDatagramLayer::receiveMessage (this=0x7fffe8012980, src=..., dst=..., payload=...) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/odp/SST.hpp:162
#5 0x00000000005945c0 in std::tr1::_Mem_fn::*)(Sirikata::ODP::Endpoint const&, Sirikata::ODP::Endpoint const&, Sirikata::MemoryReference)>::operator() (this=0x7fffe8013070, __object=0x7fffe8012980, __args#0=..., __args#1=..., __args#2=...) at /usr/include/c++/4.4/tr1_impl/functional:552
#6 0x0000000000589d38 in std::tr1::_Bind::*)(const Sirikata::ODP::Endpoint&, const Sirikata::ODP::Endpoint&, Sirikata::MemoryReference)>(Sirikata::SST::BaseDatagramLayer*, std::tr1::_Placeholder<1>, std::tr1::_Placeholder<2>, std::tr1::_Placeholder<3>)>::__call&, 0, 1, 2, 3>(const std::tr1::tuple&> &, std::tr1::_Index_tuple<0, 1, 2, 3>) (this=0x7fffe8013070, __args=...)
at /usr/include/c++/4.4/tr1_impl/functional:1137
#7 0x0000000000581b5d in std::tr1::_Bind::*)(const Sirikata::ODP::Endpoint&, const Sirikata::ODP::Endpoint&, Sirikata::MemoryReference)>(Sirikata::SST::BaseDatagramLayer*, std::tr1::_Placeholder<1>, std::tr1::_Placeholder<2>, std::tr1::_Placeholder<3>)>::operator() >(const Sirikata::ODP::Endpoint &, const Sirikata::ODP::Endpoint &, Sirikata::DataReference &) (this=0x7fffe8013070, __args#0=..., __args#1=..., __args#2=...) at /usr/include/c++/4.4/tr1_impl/functional:1191
#8 0x000000000057b3fd in std::tr1::_Function_handler), std::tr1::_Bind::*)(const Sirikata::ODP::Endpoint&, const Sirikata::ODP::Endpoint&, Sirikata::MemoryReference)>(Sirikata::SST::BaseDatagramLayer*, std::tr1::_Placeholder<1>, std::tr1::_Placeholder<2>, std::tr1::_Placeholder<3>)> >::_M_invoke(const std::tr1::_Any_data &, const Sirikata::ODP::Endpoint &, const Sirikata::ODP::Endpoint &, Sirikata::DataReference) (__functor=..., __args#0=..., __args#1=..., __args#2=...)
at /usr/include/c++/4.4/tr1_impl/functional:1668
#9 0x00007ffff6871330 in std::tr1::function::operator()(const Sirikata::ODP::Endpoint &, const Sirikata::ODP::Endpoint &, Sirikata::MemoryReference) const (this=0x7fffe8013038, __args#0=..., __args#1=..., __args#2=...)
at /usr/include/c++/4.4/tr1_impl/functional:2024
#10 0x00007ffff687227c in Sirikata::XDP::DelegatePort >::tryDeliver (this=0x7fffe8012e20, src_match_ep=..., src_real_ep=..., dst=..., data=...) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/xdp/DelegatePort.hpp:105
#11 0x00007ffff6871239 in Sirikata::XDP::DelegatePort >::deliver (this=0x7fffe8012e20, src=..., dst=..., data=...) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/xdp/DelegatePort.hpp:76
#12 0x00007ffff686fafd in Sirikata::ODP::DelegateService::deliver (this=0x7fffe8011cd8, src=..., dst=..., data=...) at /home/ewencp/sirikata.sirikata/libcore/src/odp/DelegateService.cpp:134
#13 0x00000000005288d3 in Sirikata::Forwarder::dispatchMessage (this=0x7fffe8007710, obj_msg=0x7fffc8327270) at /home/ewencp/sirikata.sirikata/space/src/Forwarder.cpp:182
#14 0x000000000052aba2 in Sirikata::Forwarder::routeObjectHostMessage (this=0x7fffe8007710, obj_msg=0x7fffc8327270) at /home/ewencp/sirikata.sirikata/space/src/Forwarder.cpp:386
#15 0x0000000000565873 in Sirikata::Server::handleSingleObjectHostMessageRouting (this=0x7fffe8011cc0) at /home/ewencp/sirikata.sirikata/space/src/Server.cpp:446
#16 0x00000000005645ca in Sirikata::Server::handleObjectHostMessageRouting (this=0x7fffe8011cc0) at /home/ewencp/sirikata.sirikata/space/src/Server.cpp:369
#17 0x000000000059c351 in std::tr1::_Mem_fn::operator() (this=0x7fffda09fea0, __object=0x7fffe8011cc0) at /usr/include/c++/4.4/tr1_impl/functional:552
#18 0x00000000005932f4 in std::tr1::_Bind(Sirikata::Server*)>::__call<, 0>(const std::tr1::tuple<> &, std::tr1::_Index_tuple<0>) (this=0x7fffda09fea0, __args=...) at /usr/include/c++/4.4/tr1_impl/functional:1137
#19 0x00000000005881a1 in std::tr1::_Bind(Sirikata::Server*)>::operator()<>(void) (this=0x7fffda09fea0) at /usr/include/c++/4.4/tr1_impl/functional:1191
#20 0x00000000005804e5 in std::tr1::_Function_handler(Sirikata::Server*)> >::_M_invoke(const std::tr1::_Any_data &) (__functor=...) at /usr/include/c++/4.4/tr1_impl/functional:1668
#21 0x00007ffff679f3af in std::tr1::function::operator()(void) const (this=0x7fffffffd440) at /usr/include/c++/4.4/tr1_impl/functional:2024
#22 0x00007ffff68955c6 in boost::asio::asio_handler_invoke >(std::tr1::function, ...) (function=...) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/handler_invoke_hook.hpp:62
#23 0x00007ffff689502b in boost_asio_handler_invoke_helpers::invoke, std::tr1::function >(const std::tr1::function &, std::tr1::function &) (function=..., context=...) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/handler_invoke_helpers.hpp:41
#24 0x00007ffff689c7eb in boost::asio::detail::strand_service::handler_wrapper >::do_invoke(boost::asio::detail::strand_service::handler_base *, boost::asio::detail::strand_service &, boost::asio::detail::strand_service::strand_impl *&) (base=0x7fffdb81cbd0, service_impl=..., impl=@0x7fffffffd5d8)
at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/strand_service.hpp:261
#25 0x00007ffff689af71 in boost::asio::detail::strand_service::handler_base::invoke (this=0x7fffdb81cbd0, service_impl=..., impl=@0x7fffffffd5d8) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/strand_service.hpp:109
#26 0x00007ffff689afe0 in boost::asio::detail::strand_service::invoke_current_handler::operator() (this=0x7fffffffd5d0) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/strand_service.hpp:144
#27 0x00007ffff689c6f3 in boost::asio::asio_handler_invoke (function=...) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/handler_invoke_hook.hpp:62
#28 0x00007ffff689c4e1 in boost_asio_handler_invoke_helpers::invoke (function=..., context=...) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/handler_invoke_helpers.hpp:41
#29 0x00007ffff689c653 in boost::asio::detail::task_io_service >::dispatch (this=0x7fffe8000920, handler=...) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/task_io_service.hpp:180
#30 0x00007ffff689c41b in boost::asio::io_service::dispatch (this=0x7fffe80008b0, handler=...) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/impl/io_service.ipp:130
#31 0x00007ffff689bebe in boost::asio::detail::strand_service::dispatch >(boost::asio::detail::strand_service::strand_impl *&, std::tr1::function) (this=0x7fffe8000a00, impl=@0x7fffebcc2438, handler=...) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/strand_service.hpp:374
#32 0x00007ffff689bc65 in boost::asio::io_service::strand::dispatch >(std::tr1::function) (this=0x7fffebcc2430, handler=...) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/strand.hpp:158
#33 0x00007ffff689b894 in boost::asio::detail::wrapped_handler >::operator()(void) (this=0x7fffebcc2430) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/wrapped_handler.hpp:48
#34 0x00007ffff689b6d7 in std::tr1::_Function_handler > >::_M_invoke(const std::tr1::_Any_data &) (__functor=...) at /usr/include/c++/4.4/tr1_impl/functional:1668
#35 0x00007ffff679f3af in std::tr1::function::operator()(void) const (this=0x7fffffffd990) at /usr/include/c++/4.4/tr1_impl/functional:2024
#36 0x00007ffff68955c6 in boost::asio::asio_handler_invoke >(std::tr1::function, ...) (function=...) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/handler_invoke_hook.hpp:62
#37 0x00007ffff689502b in boost_asio_handler_invoke_helpers::invoke, std::tr1::function >(const std::tr1::function &, std::tr1::function &) (function=..., context=...) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/handler_invoke_helpers.hpp:41
#38 0x00007ffff689607d in handler_wrapper >::do_call(boost::asio::detail::handler_queue::handler *) (base=0x7fffebcc2430) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/handler_queue.hpp:192
#39 0x00007ffff6816ac1 in boost::asio::detail::handler_queue::handler::invoke (this=0x7fffebcc2430) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/handler_queue.hpp:39
#40 0x00007ffff681ceb7 in boost::asio::detail::task_io_service >::do_one (this=0x7fffe8000920, lock=..., this_idle_thread=0x7fffffffdb70, ec=...) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/task_io_service.hpp:268
#41 0x00007ffff68199ff in boost::asio::detail::task_io_service >::run (this=0x7fffe8000920, ec=...) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/detail/task_io_service.hpp:103
#42 0x00007ffff6816fe1 in boost::asio::io_service::run (this=0x7fffe80008b0) at /home/ewencp/sirikata.sirikata/dependencies/installed-boost/include/boost/asio/impl/io_service.ipp:68
#43 0x00007ffff6893a13 in Sirikata::Network::IOService::run (this=0x94cf60) at /home/ewencp/sirikata.sirikata/libcore/src/network/IOService.cpp:102
#44 0x00007ffff68ae79f in Sirikata::Context::run (this=0x7fffe80013d0, nthreads=3, exthreads=Sirikata::Context::IncludeOriginal) at /home/ewencp/sirikata.sirikata/libcore/src/service/Context.cpp:96
#45 0x00000000005d8962 in main (argc=1, argv=0x7fffffffe328) at /home/ewencp/sirikata.sirikata/space/src/main.cpp:344
```

It looks like it might be due to trying to use port 0:

```
(gdb) up
#1 0x00000000005716df in Sirikata::SST::BaseDatagramLayer::listenOn(Sirikata::SST::EndPoint &, std::tr1::function) (this=0x7fffe8012980, listeningEndPoint=..., cb=...) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/odp/SST.hpp:78
78	);
(gdb) list
69	void listenOn(EndPoint& listeningEndPoint, DataCallback cb) {
70	ODP::Port* port = allocatePort(listeningEndPoint);
71	port->receive(
72	std::tr1::bind(&BaseDatagramLayer::receiveMessageToCallback, this,
73	std::tr1::placeholders::_1,
74	std::tr1::placeholders::_2,
75	std::tr1::placeholders::_3,
76	cb
77	)
78	);
79	}
(gdb) p listeningEndPoint.port
$5 = 0
(gdb)
```

which in turn is failing because I guess it can't find an available port:

```
(gdb) up
#2 0x000000000057c4cc in Sirikata::SST::Connection::Connection (this=0x7fffc836c260, sstConnVars=0x7fffe8001148, localEndPoint=..., remoteEndPoint=...) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/network/SSTImpl.hpp:411
411	mDatagramLayer->listenOn(
(gdb)
#3 0x00000000005764a8 in Sirikata::SST::Connection::handleReceive (sstConnVars=0x7fffe8001148, remoteEndPoint=..., localEndPoint=..., recv_buffer=0x7fffdafe08a8, len=12) at /home/ewencp/sirikata.sirikata/libcore/include/sirikata/core/network/SSTImpl.hpp:1199
1199	new Connection(sstConnVars, newLocalEndPoint, remoteEndPoint));
(gdb) list
1179	else if (channelID == 0) {
1180	/* it's a new channel request negotiation protocol
1181	packet ; allocate a new channel.*/
1182
1183	StreamReturnCallbackMap& listeningConnectionsCallbackMap = sstConnVars->sListeningConnectionsCallbackMap;
1184	if (listeningConnectionsCallbackMap.find(localEndPoint) != listeningConnectionsCallbackMap.end()) {
1185	uint16* received_payload = (uint16*) received_msg->payload().data();
1186
1187	uint16 payload[2];
1188
1189	uint16 availableChannel = sstConnVars->getAvailableChannel(localEndPoint.endPoint);
1190	payload[0] = htons(availableChannel);
1191	uint16 availablePort = availableChannel; //availableChannel is picked from the same 16-bit
1192	//address space and has to be unique. So why not use
1193	//use it to identify the port as well...
1194	payload[1] = htons(availablePort);
1195
1196	EndPoint newLocalEndPoint(localEndPoint.endPoint, availablePort);
1197	std::tr1::shared_ptr conn =
1198	std::tr1::shared_ptr(
1199	new Connection(sstConnVars, newLocalEndPoint, remoteEndPoint));
1200
1201	conn->listenStream(newLocalEndPoint.port, listeningConnectionsCallbackMap[localEndPoint]);
1202	conn->setWeakThis(conn);
1203	connectionMap[newLocalEndPoint] = conn;
1204
1205	conn->setLocalChannelID(availableChannel);
1206	conn->setRemoteChannelID(ntohs(received_payload[0]));
1207	conn->setState(CONNECTION_PENDING_RECEIVE_CONNECT);
1208
1209	conn->sendData(payload, sizeof(payload), false);
1210	}
1211	else {
1212	SST_LOG(warn, "No one listening on this connection\n");
1213	}
1214	}
1215
1216	delete received_msg;
1217	}
1218
```

At a minimum, we shouldn't crash when this occurs. Instead we should catch the condition and fail gracefully, probably emit an error since this should be an exceptional condition, and maybe send the appropriate response to indicate failure to the sender.