Repository: AntelopeIO/spring
Pull request #1308: Eliminate duplicate P2P connections
Author: login: heifner
Labels: name: OCI
description: Work exclusive to OCI team
Created: 2025-04-01T19:37:59Z | Merged: 2025-04-03T13:42:25Z

Fix for an issue where `net_plugin` would allow some duplicate connections to remain open.

Simplified to just checking both incoming and outgoing connections for duplicate.

Resolves #1255 

Linked issue #1255: P2P: Fix duplicate connection logic
See #1243 for additional info.

The current scheme still leaves duplicates in some race conditions. A simpler approach was recently attempted. See https://github.com/AntelopeIO/spring/pull/1108. However, this approach had a race condition so that it would sometimes disconnect both connections; reverted in https://github.com/AntelopeIO/spring/pull/1186. The issue is that the connection and handshake messages can be sent from each node at the same time. Both nodes can consider the other a duplicate at the same time and both disconnect.

We believe the race condition can be solved using the following technique.

1. On disconnect because of duplicate connection, do not attempt an immediate retry of the connections.
2. When connecting to a peer, verify there is not already a open connection to that peer node_id.
  a. Note the node_id of the connection must be saved on close to make this determination.
3. Maintain the logic of closing a duplicate if node's `node_id` less than peer `node_id`.

Linked issue comments: user: login: heifner
created_at: 2025-03-31T17:05:16Z
body: 1. Already was not attempting to immediately retry on duplicate disconnect.
2. In order to know the node_id a handshake has to be received from the connection. This can be accomplished by checking both incoming and outgoing initial handshakes for duplicate.
3. Nothing to do.
author_association: CONTRIBUTOR

Diff:
diff --git a/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp b/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp
index 1ddb241086..25dd24f65d 100644
--- a/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp
+++ b/plugins/net_plugin/include/eosio/net_plugin/protocol.hpp
@@ -42,7 +42,7 @@ namespace eosio {
    };
 
 
-  enum go_away_reason {
+  enum class go_away_reason {
     no_reason, ///< no reason to go away
     self, ///< the connection is to itself
     duplicate, ///< the connection is redundant
@@ -59,25 +59,24 @@ namespace eosio {
 
   constexpr auto reason_str( go_away_reason rsn ) {
     switch (rsn ) {
-    case no_reason : return "no reason";
-    case self : return "self connect";
-    case duplicate : return "duplicate";
-    case wrong_chain : return "wrong chain";
-    case wrong_version : return "wrong version";
-    case forked : return "chain is forked";
-    case unlinkable : return "unlinkable block received";
-    case bad_transaction : return "bad transaction";
-    case validation : return "invalid block";
-    case authentication : return "authentication failure";
-    case fatal_other : return "some other failure";
-    case benign_other : return "some other non-fatal condition, possibly unknown block";
+    case go_away_reason::no_reason : return "no reason";
+    case go_away_reason::self : return "self connect";
+    case go_away_reason::duplicate : return "duplicate";
+    case go_away_reason::wrong_chain : return "wrong chain";
+    case go_away_reason::wrong_version : return "wrong version";
+    case go_away_reason::forked : return "chain is forked";
+    case go_away_reason::unlinkable : return "unlinkable block received";
+    case go_away_reason::bad_transaction : return "bad transaction";
+    case go_away_reason::validation : return "invalid block";
+    case go_away_reason::authentication : return "authentication failure";
+    case go_away_reason::fatal_other : return "some other failure";
+    case go_away_reason::benign_other : return "some other non-fatal condition, possibly unknown block";
     default : return "some crazy reason";
     }
   }
 
   struct go_away_message {
-    go_away_message(go_away_reason r = no_reason) : reason(r), node_id() {}
-    go_away_reason reason{no_reason};
+    go_away_reason reason{go_away_reason::no_reason};
     fc::sha256 node_id; ///< for duplicate notification
   };
 
diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp
index bd104e07e5..06721991f2 100644
--- a/plugins/net_plugin/net_plugin.cpp
+++ b/plugins/net_plugin/net_plugin.cpp
@@ -924,7 +924,7 @@ namespace eosio {
       boost::asio::steady_timer        sync_response_expected_timer GUARDED_BY(sync_response_expected_timer_mtx);
 
       alignas(hardware_destructive_interference_sz)
-      std::atomic<go_away_reason>      no_retry{no_reason};
+      std::atomic<go_away_reason>      no_retry{go_away_reason::no_reason};
 
       alignas(hardware_destructive_interference_sz)
       mutable fc::mutex                conn_mtx; //< mtx for last_handshake_recv .. remote_endpoint_ip
@@ -1545,8 +1545,8 @@ namespace eosio {
       auto [on_fork, unknown_block] = block_on_fork(msg_head_id);
       if( unknown_block ) {
          peer_ilog( this, "Peer asked for unknown block ${mn}, sending: benign_other go away", ("mn", msg_head_num) );
-         no_retry = benign_other;
-         enqueue( go_away_message( benign_other ) );
+         no_retry = go_away_reason::benign_other;
+         enqueue( go_away_message{ go_away_reason::benign_other } );
       } else {
          // if peer on fork, start at their last fork_db_root_num, otherwise we can start at msg_head+1
          if (on_fork)
@@ -1619,7 +1619,7 @@ namespace eosio {
    void connection::check_heartbeat( std::chrono::steady_clock::time_point current_time ) {
       if( latest_msg_time > std::chrono::steady_clock::time_point::min() ) {
          if( current_time > latest_msg_time + hb_timeout ) {
-            no_retry = benign_other;
+            no_retry = go_away_reason::benign_other;
             if( !peer_address().empty() ) {
                peer_wlog(this, "heartbeat timed out for peer address");
                close(true);
@@ -1821,8 +1821,8 @@ namespace eosio {
          peer_requested.reset(); // unable to provide requested blocks
          block_sync_send_start = 0ns;
          block_sync_frame_bytes_sent = 0;
-         no_retry = benign_other;
-         enqueue( go_away_message( benign_other ) );
+         no_retry = go_away_reason::benign_other;
+         enqueue( go_away_message{ go_away_reason::benign_other } );
       }
       return true;
    }
@@ -1957,7 +1957,7 @@ namespace eosio {
    // called from connection strand
    void connection::enqueue( const net_message& m ) {
       verify_strand_in_this_thread( strand, __func__, __LINE__ );
-      go_away_reason close_after_send = no_reason;
+      go_away_reason close_after_send = go_away_reason::no_reason;
       if (std::holds_alternative<go_away_message>(m)) {
          close_after_send = std::get<go_away_message>(m).reason;
       }
@@ -1975,7 +1975,7 @@ namespace eosio {
       block_buffer_factory buff_factory;
       const auto& sb = buff_factory.get_send_buffer( b );
       latest_blk_time = std::chrono::steady_clock::now();
-      enqueue_buffer( msg_type_t::signed_block, block_num, queue, sb, no_reason);
+      enqueue_buffer( msg_type_t::signed_block, block_num, queue, sb, go_away_reason::no_reason);
       return sb->size();
    }
 
@@ -1997,7 +1997,7 @@ namespace eosio {
                         }
                         if (net_msg == msg_type_t::signed_block && block_num)
                            fc_dlog(logger, "Connection - ${cid} - done sending block ${bn}", ("cid", conn->connection_id)("bn", *block_num));
-                        if (close_after_send != no_reason) {
+                        if (close_after_send != go_away_reason::no_reason) {
                            fc_ilog( logger, "sent a go away message: ${r}, closing connection ${cid}",
                                     ("r", reason_str(close_after_send))("cid", conn->connection_id) );
                            conn->close();
@@ -2839,7 +2839,7 @@ namespace eosio {
                   boost::asio::post(cp->strand, [cp, send_buffer{std::move(send_buffer)}, bnum]() {
                      cp->latest_blk_time = std::chrono::steady_clock::now();
                      peer_dlog( cp, "bcast block_notice ${b}", ("b", bnum) );
-                     cp->enqueue_buffer( msg_type_t::block_notice_message, std::nullopt, queued_buffer::queue_t::general, send_buffer, no_reason );
+                     cp->enqueue_buffer( msg_type_t::block_notice_message, std::nullopt, queued_buffer::queue_t::general, send_buffer, go_away_reason::no_reason );
                   });
                   return;
                }
@@ -2853,7 +2853,7 @@ namespace eosio {
             bool has_block = cp->peer_fork_db_root_num >= bnum;
             if( !has_block ) {
                peer_dlog( cp, "bcast block ${b}", ("b", bnum) );
-               cp->enqueue_buffer( msg_type_t::signed_block, bnum, queued_buffer::queue_t::general, sb, no_reason );
+               cp->enqueue_buffer( msg_type_t::signed_block, bnum, queued_buffer::queue_t::general, sb, go_away_reason::no_reason );
             }
          });
       } );
@@ -2867,7 +2867,7 @@ namespace eosio {
          boost::asio::post(cp->strand, [cp, msg]() {
             if (vote_logger.is_enabled(fc::log_level::debug))
                peer_dlog(cp, "sending vote msg");
-            cp->enqueue_buffer( msg_type_t::vote_message, std::nullopt, queued_buffer::queue_t::general, msg, no_reason );
+            cp->enqueue_buffer( msg_type_t::vote_message, std::nullopt, queued_buffer::queue_t::general, msg, go_away_reason::no_reason );
          });
          return true;
       } );
@@ -2888,7 +2888,7 @@ namespace eosio {
          send_buffer_type sb = buff_factory.get_send_buffer( trx );
          fc_dlog( logger, "sending trx: ${id}, to connection - ${cid}", ("id", trx->id())("cid", cp->connection_id) );
          boost::asio::post(cp->strand, [cp, sb{std::move(sb)}]() {
-            cp->enqueue_buffer( msg_type_t::packed_transaction, std::nullopt, queued_buffer::queue_t::general, sb, no_reason );
+            cp->enqueue_buffer( msg_type_t::packed_transaction, std::nullopt, queued_buffer::queue_t::general, sb, go_away_reason::no_reason );
          } );
       } );
    }
@@ -3338,7 +3338,7 @@ namespace eosio {
       buffer_factory buff_factory;
       auto send_buffer = buff_factory.get_send_buffer( block_nack_message{block_id} );
 
-      enqueue_buffer( msg_type_t::block_nack_message, std::nullopt, queued_buffer::queue_t::general, send_buffer, no_reason );
+      enqueue_buffer( msg_type_t::block_nack_message, std::nullopt, queued_buffer::queue_t::general, send_buffer, go_away_reason::no_reason );
    }
 
    void net_plugin_impl::plugin_shutdown() {
@@ -3446,7 +3446,7 @@ namespace eosio {
       if( !is_valid( msg ) ) {
          peer_wlog( this, "bad handshake message");
          no_retry = go_away_reason::fatal_other;
-         enqueue( go_away_message( fatal_other ) );
+         enqueue( go_away_message{ go_away_reason::fatal_other } );
          return;
       }
       peer_dlog( this, "received handshake gen ${g}, froot ${fr}, fhead ${fh}",
@@ -3456,7 +3456,6 @@ namespace eosio {
       peer_fork_db_head_block_num = msg.fork_db_head_num;
       fc::unique_lock g_conn( conn_mtx );
       last_handshake_recv = msg;
-      auto c_time = last_handshake_sent.time;
       g_conn.unlock();
 
       set_state(connection_state::connected);
@@ -3492,57 +3491,35 @@ namespace eosio {
                set_connection_type( msg.p2p_address);
             else
                peer_dlog(this, "Invalid handshake p2p_address ${p}", ("p", msg.p2p_address));
+         } else {
+            // peer p2p_address may contain trx or blk only request, honor requested connection type
+            set_peer_connection_type(msg.p2p_address);
+         }
 
-            peer_dlog( this, "checking for duplicate" );
-            auto is_duplicate = [&](const connection_ptr& check) {
-               if(check.get() == this)
+         peer_dlog( this, "checking for duplicate" );
+         auto is_duplicate = [&](const connection_ptr& check) {
+            if(check.get() == this)
+               return false;
+            fc::unique_lock g_check_conn( check->conn_mtx );
+            fc_dlog( logger, "dup check: connected ${c}, ${l} =? ${r}",
+                     ("c", check->connected())("l", check->last_handshake_recv.node_id)("r", msg.node_id) );
+            if(check->connected() && check->last_handshake_recv.node_id == msg.node_id) {
+               if (my_impl->node_id < msg.node_id) {
+                  fc_dlog( logger, "not duplicate, my_impl->node_id '${lhs}' < msg.node_id '${rhs}'",
+                           ("lhs", my_impl->node_id)("rhs", msg.node_id) );
+                  // only the connection from lower node_id to higher node_id will be considered as a duplicate,
+                  // so there is no chance for both connections to be closed
                   return false;
-               fc::unique_lock g_check_conn( check->conn_mtx );
-               fc_dlog( logger, "dup check: connected ${c}, ${l} =? ${r}",
-                        ("c", check->connected())("l", check->last_handshake_recv.node_id)("r", msg.node_id) );
-               if(check->connected() && check->last_handshake_recv.node_id == msg.node_id) {
-                  if (net_version < proto_dup_goaway_resolution || msg.network_version < proto_dup_goaway_resolution) {
-                     // It's possible that both peers could arrive here at relatively the same time, so
-                     // we need to avoid the case where they would both tell a different connection to go away.
-                     // Using the sum of the initial handshake times of the two connections, we will
-                     // arbitrarily (but consistently between the two peers) keep one of them.
-
-                     auto check_time = check->last_handshake_sent.time + check->last_handshake_recv.time;
-                     g_check_conn.unlock();
-                     if (msg.time + c_time <= check_time)
-                        return false;
-                  } else if (net_version < proto_dup_node_id_goaway || msg.network_version < proto_dup_node_id_goaway) {
-                     if (listen_address < msg.p2p_address) {
-                        fc_dlog( logger, "listen_address '${lhs}' < msg.p2p_address '${rhs}'",
-                                 ("lhs", listen_address)( "rhs", msg.p2p_address ) );
-                        // only the connection from lower p2p_address to higher p2p_address will be considered as a duplicate,
-                        // so there is no chance for both connections to be closed
-                        return false;
-                     }
-                  } else if (my_impl->node_id < msg.node_id) {
-                     fc_dlog( logger, "not duplicate, my_impl->node_id '${lhs}' < msg.node_id '${rhs}'",
-                              ("lhs", my_impl->node_id)("rhs", msg.node_id) );
-                     // only the connection from lower node_id to higher node_id will be considered as a duplicate,
-                     // so there is no chance for both connections to be closed
-                     return false;
-                  }
-                  return true;
                }
-               return false;
-            };
-            if (my_impl->connections.any_of_connections(std::move(is_duplicate))) {
-               peer_dlog( this, "sending go_away duplicate, msg.p2p_address: ${add}", ("add", msg.p2p_address) );
-               go_away_message gam(duplicate);
-               gam.node_id = conn_node_id;
-               enqueue(gam);
-               no_retry = duplicate;
-               return;
+               return true;
             }
-         } else {
-            peer_dlog(this, "skipping duplicate check, addr == ${pa}, id = ${ni}", ("pa", peer_address())("ni", msg.node_id));
-
-            // check if peer requests no trx or no blocks
-            set_peer_connection_type(msg.p2p_address);
+            return false;
+         };
+         if (my_impl->connections.any_of_connections(is_duplicate)) {
+            peer_dlog( this, "sending go_away duplicate, msg.p2p_address: ${add}", ("add", msg.p2p_address) );
+            enqueue(go_away_message{go_away_reason::duplicate, conn_node_id});
+            no_retry = go_away_reason::duplicate;
+            return;
          }
 
          if( msg.chain_id != my_impl->chain_id ) {
@@ -3625,15 +3602,15 @@ namespace eosio {
    void connection::handle_message( const go_away_message& msg ) {
       peer_wlog( this, "received go_away_message, reason = ${r}", ("r", reason_str( msg.reason )) );
 
-      bool retry = no_retry == no_reason; // if no previous go away message
+      bool retry = no_retry == go_away_reason::no_reason; // if no previous go away message
       no_retry = msg.reason;
-      if( msg.reason == duplicate ) {
+      if( msg.reason == go_away_reason::duplicate ) {
          conn_node_id = msg.node_id;
       }
-      if( msg.reason == wrong_version ) {
-         if( !retry ) no_retry = fatal_other; // only retry once on wrong version
+      if( msg.reason == go_away_reason::wrong_version ) {
+         if( !retry ) no_retry = go_away_reason::fatal_other; // only retry once on wrong version
       }
-      else if ( msg.reason == benign_other ) {
+      else if ( msg.reason == go_away_reason::benign_other ) {
          if ( retry ) peer_dlog( this, "received benign_other reason, retrying to connect");
       }
       else {
@@ -4839,10 +4816,10 @@ namespace eosio {
    // called from any thread
    bool connection::resolve_and_connect() {
       switch ( no_retry ) {
-         case no_reason:
-         case wrong_version:
-         case benign_other:
-         case duplicate: // attempt reconnect in case connection has been dropped, should quickly disconnect if duplicate
+         case go_away_reason::no_reason:
+         case go_away_reason::wrong_version:
+         case go_away_reason::benign_other:
+         case go_away_reason::duplicate: // attempt reconnect in case connection has been dropped, should quickly disconnect if duplicate
             break;
          default:
             fc_dlog( logger, "Skipping connect due to go_away reason ${r}",("r", reason_str( no_retry )));
@@ -4855,7 +4832,7 @@ namespace eosio {
 
       connection_ptr c = shared_from_this();
 
-      if( consecutive_immediate_connection_close > def_max_consecutive_immediate_connection_close || no_retry == benign_other ) {
+      if( consecutive_immediate_connection_close > def_max_consecutive_immediate_connection_close || no_retry == go_away_reason::benign_other ) {
          fc::microseconds connector_period = my_impl->connections.get_connector_period();
          fc::lock_guard g( conn_mtx );
          if( last_close == fc::time_point() || last_close > fc::time_point::now() - connector_period ) {
diff --git a/tests/auto_bp_peering_test.py b/tests/auto_bp_peering_test.py
index 4821df4a01..36fb71fda4 100755
--- a/tests/auto_bp_peering_test.py
+++ b/tests/auto_bp_peering_test.py
@@ -96,6 +96,7 @@
     for nodeId in range(0, producerNodes):
         # retrieve the connections in each node and check if each connects to the other bps in the schedule
         connections = cluster.nodes[nodeId].processUrllibRequest("net", "connections")
+        if Utils.Debug: Utils.Print(f"Node {nodeId} connections {connections}")
         peers = []
         for conn in connections["payload"]:
             peer_addr = conn["peer"]


Reviews:
user: login: linh2931
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T13:43:09Z
user: login: heifner
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T14:30:41Z
user: login: linh2931
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T14:31:40Z
user: login: greg7mdp
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T15:12:10Z
user: login: heifner
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T15:25:43Z
user: login: heifner
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T15:25:44Z
user: login: greg7mdp
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T19:10:41Z
user: login: greg7mdp
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T19:14:09Z
user: login: heifner
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T19:39:28Z
user: login: greg7mdp
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T20:02:18Z

Review comments:
diff_hunk: @@ -3492,57 +3491,37 @@ namespace eosio {
                set_connection_type( msg.p2p_address);
             else
                peer_dlog(this, "Invalid handshake p2p_address ${p}", ("p", msg.p2p_address));
+         } else {
+            // check if peer requests no trx or no blocks
path: plugins/net_plugin/net_plugin.cpp
user: login: linh2931
body: This comment is a little confusing.
created_at: 2025-04-02T13:40:10Z
author_association: CONTRIBUTOR
diff_hunk: @@ -3492,57 +3491,37 @@ namespace eosio {
                set_connection_type( msg.p2p_address);
             else
                peer_dlog(this, "Invalid handshake p2p_address ${p}", ("p", msg.p2p_address));
+         } else {
+            // check if peer requests no trx or no blocks
+            set_peer_connection_type(msg.p2p_address);
+         }
 
-            peer_dlog( this, "checking for duplicate" );
-            auto is_duplicate = [&](const connection_ptr& check) {
-               if(check.get() == this)
+         peer_dlog( this, "checking for duplicate" );
+         auto is_duplicate = [&](const connection_ptr& check) {
+            if(check.get() == this)
+               return false;
+            fc::unique_lock g_check_conn( check->conn_mtx );
+            fc_dlog( logger, "dup check: connected ${c}, ${l} =? ${r}",
+                     ("c", check->connected())("l", check->last_handshake_recv.node_id)("r", msg.node_id) );
+            if(check->connected() && check->last_handshake_recv.node_id == msg.node_id) {
+               if (my_impl->node_id < msg.node_id) {
+                  fc_dlog( logger, "not duplicate, my_impl->node_id '${lhs}' < msg.node_id '${rhs}'",
+                           ("lhs", my_impl->node_id)("rhs", msg.node_id) );
+                  // only the connection from lower node_id to higher node_id will be considered as a duplicate,
+                  // so there is no chance for both connections to be closed
path: plugins/net_plugin/net_plugin.cpp
user: login: linh2931
body: The is simpler than the complicated prior version.
created_at: 2025-04-02T13:43:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -3492,57 +3491,37 @@ namespace eosio {
                set_connection_type( msg.p2p_address);
             else
                peer_dlog(this, "Invalid handshake p2p_address ${p}", ("p", msg.p2p_address));
+         } else {
+            // check if peer requests no trx or no blocks
path: plugins/net_plugin/net_plugin.cpp
user: login: heifner
body: Updated.
created_at: 2025-04-02T14:30:40Z
author_association: CONTRIBUTOR
diff_hunk: @@ -3492,57 +3491,37 @@ namespace eosio {
                set_connection_type( msg.p2p_address);
             else
                peer_dlog(this, "Invalid handshake p2p_address ${p}", ("p", msg.p2p_address));
+         } else {
+            // peer p2p_address may contain trx or blk only request, honor requested connection type
+            set_peer_connection_type(msg.p2p_address);
+         }
 
-            peer_dlog( this, "checking for duplicate" );
-            auto is_duplicate = [&](const connection_ptr& check) {
-               if(check.get() == this)
+         peer_dlog( this, "checking for duplicate" );
+         auto is_duplicate = [&](const connection_ptr& check) {
+            if(check.get() == this)
+               return false;
+            fc::unique_lock g_check_conn( check->conn_mtx );
+            fc_dlog( logger, "dup check: connected ${c}, ${l} =? ${r}",
+                     ("c", check->connected())("l", check->last_handshake_recv.node_id)("r", msg.node_id) );
+            if(check->connected() && check->last_handshake_recv.node_id == msg.node_id) {
+               if (my_impl->node_id < msg.node_id) {
path: plugins/net_plugin/net_plugin.cpp
user: login: greg7mdp
body: Does this work? Suppose you are `node_id` 2, and you already have a connection to `node_id` 5.

Now you receive an incoming connection from `node_id` 5. 

`check->last_handshake_recv.node_id == msg.node_id` will be true, but `my_impl->node_id < msg.node_id` will be true, so the lambda will return false. So I think you'll keep both connections.
created_at: 2025-04-02T15:11:42Z
author_association: CONTRIBUTOR
diff_hunk: @@ -3492,57 +3491,37 @@ namespace eosio {
                set_connection_type( msg.p2p_address);
             else
                peer_dlog(this, "Invalid handshake p2p_address ${p}", ("p", msg.p2p_address));
+         } else {
+            // peer p2p_address may contain trx or blk only request, honor requested connection type
+            set_peer_connection_type(msg.p2p_address);
+         }
 
-            peer_dlog( this, "checking for duplicate" );
-            auto is_duplicate = [&](const connection_ptr& check) {
-               if(check.get() == this)
+         peer_dlog( this, "checking for duplicate" );
+         auto is_duplicate = [&](const connection_ptr& check) {
+            if(check.get() == this)
+               return false;
+            fc::unique_lock g_check_conn( check->conn_mtx );
+            fc_dlog( logger, "dup check: connected ${c}, ${l} =? ${r}",
+                     ("c", check->connected())("l", check->last_handshake_recv.node_id)("r", msg.node_id) );
+            if(check->connected() && check->last_handshake_recv.node_id == msg.node_id) {
+               if (my_impl->node_id < msg.node_id) {
+                  fc_dlog( logger, "not duplicate, my_impl->node_id '${lhs}' < msg.node_id '${rhs}'",
+                           ("lhs", my_impl->node_id)("rhs", msg.node_id) );
+                  // only the connection from lower node_id to higher node_id will be considered as a duplicate,
+                  // so there is no chance for both connections to be closed
                   return false;
-               fc::unique_lock g_check_conn( check->conn_mtx );
-               fc_dlog( logger, "dup check: connected ${c}, ${l} =? ${r}",
-                        ("c", check->connected())("l", check->last_handshake_recv.node_id)("r", msg.node_id) );
-               if(check->connected() && check->last_handshake_recv.node_id == msg.node_id) {
-                  if (net_version < proto_dup_goaway_resolution || msg.network_version < proto_dup_goaway_resolution) {
-                     // It's possible that both peers could arrive here at relatively the same time, so
-                     // we need to avoid the case where they would both tell a different connection to go away.
-                     // Using the sum of the initial handshake times of the two connections, we will
-                     // arbitrarily (but consistently between the two peers) keep one of them.
-
-                     auto check_time = check->last_handshake_sent.time + check->last_handshake_recv.time;
-                     g_check_conn.unlock();
-                     if (msg.time + c_time <= check_time)
-                        return false;
-                  } else if (net_version < proto_dup_node_id_goaway || msg.network_version < proto_dup_node_id_goaway) {
-                     if (listen_address < msg.p2p_address) {
-                        fc_dlog( logger, "listen_address '${lhs}' < msg.p2p_address '${rhs}'",
-                                 ("lhs", listen_address)( "rhs", msg.p2p_address ) );
-                        // only the connection from lower p2p_address to higher p2p_address will be considered as a duplicate,
-                        // so there is no chance for both connections to be closed
-                        return false;
-                     }
-                  } else if (my_impl->node_id < msg.node_id) {
-                     fc_dlog( logger, "not duplicate, my_impl->node_id '${lhs}' < msg.node_id '${rhs}'",
-                              ("lhs", my_impl->node_id)("rhs", msg.node_id) );
-                     // only the connection from lower node_id to higher node_id will be considered as a duplicate,
-                     // so there is no chance for both connections to be closed
-                     return false;
-                  }
-                  return true;
                }
-               return false;
-            };
-            if (my_impl->connections.any_of_connections(std::move(is_duplicate))) {
-               peer_dlog( this, "sending go_away duplicate, msg.p2p_address: ${add}", ("add", msg.p2p_address) );
-               go_away_message gam(duplicate);
-               gam.node_id = conn_node_id;
-               enqueue(gam);
-               no_retry = duplicate;
-               return;
+               return true;
             }
-         } else {
-            peer_dlog(this, "skipping duplicate check, addr == ${pa}, id = ${ni}", ("pa", peer_address())("ni", msg.node_id));
-
-            // check if peer requests no trx or no blocks
-            set_peer_connection_type(msg.p2p_address);
+            return false;
+         };
+         if (my_impl->connections.any_of_connections(std::move(is_duplicate))) {
path: plugins/net_plugin/net_plugin.cpp
user: login: greg7mdp
body: why `std::move`?
created_at: 2025-04-02T15:12:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -3492,57 +3491,37 @@ namespace eosio {
                set_connection_type( msg.p2p_address);
             else
                peer_dlog(this, "Invalid handshake p2p_address ${p}", ("p", msg.p2p_address));
+         } else {
+            // peer p2p_address may contain trx or blk only request, honor requested connection type
+            set_peer_connection_type(msg.p2p_address);
+         }
 
-            peer_dlog( this, "checking for duplicate" );
-            auto is_duplicate = [&](const connection_ptr& check) {
-               if(check.get() == this)
+         peer_dlog( this, "checking for duplicate" );
+         auto is_duplicate = [&](const connection_ptr& check) {
+            if(check.get() == this)
+               return false;
+            fc::unique_lock g_check_conn( check->conn_mtx );
+            fc_dlog( logger, "dup check: connected ${c}, ${l} =? ${r}",
+                     ("c", check->connected())("l", check->last_handshake_recv.node_id)("r", msg.node_id) );
+            if(check->connected() && check->last_handshake_recv.node_id == msg.node_id) {
+               if (my_impl->node_id < msg.node_id) {
+                  fc_dlog( logger, "not duplicate, my_impl->node_id '${lhs}' < msg.node_id '${rhs}'",
+                           ("lhs", my_impl->node_id)("rhs", msg.node_id) );
+                  // only the connection from lower node_id to higher node_id will be considered as a duplicate,
+                  // so there is no chance for both connections to be closed
                   return false;
-               fc::unique_lock g_check_conn( check->conn_mtx );
-               fc_dlog( logger, "dup check: connected ${c}, ${l} =? ${r}",
-                        ("c", check->connected())("l", check->last_handshake_recv.node_id)("r", msg.node_id) );
-               if(check->connected() && check->last_handshake_recv.node_id == msg.node_id) {
-                  if (net_version < proto_dup_goaway_resolution || msg.network_version < proto_dup_goaway_resolution) {
-                     // It's possible that both peers could arrive here at relatively the same time, so
-                     // we need to avoid the case where they would both tell a different connection to go away.
-                     // Using the sum of the initial handshake times of the two connections, we will
-                     // arbitrarily (but consistently between the two peers) keep one of them.
-
-                     auto check_time = check->last_handshake_sent.time + check->last_handshake_recv.time;
-                     g_check_conn.unlock();
-                     if (msg.time + c_time <= check_time)
-                        return false;
-                  } else if (net_version < proto_dup_node_id_goaway || msg.network_version < proto_dup_node_id_goaway) {
-                     if (listen_address < msg.p2p_address) {
-                        fc_dlog( logger, "listen_address '${lhs}' < msg.p2p_address '${rhs}'",
-                                 ("lhs", listen_address)( "rhs", msg.p2p_address ) );
-                        // only the connection from lower p2p_address to higher p2p_address will be considered as a duplicate,
-                        // so there is no chance for both connections to be closed
-                        return false;
-                     }
-                  } else if (my_impl->node_id < msg.node_id) {
-                     fc_dlog( logger, "not duplicate, my_impl->node_id '${lhs}' < msg.node_id '${rhs}'",
-                              ("lhs", my_impl->node_id)("rhs", msg.node_id) );
-                     // only the connection from lower node_id to higher node_id will be considered as a duplicate,
-                     // so there is no chance for both connections to be closed
-                     return false;
-                  }
-                  return true;
                }
-               return false;
-            };
-            if (my_impl->connections.any_of_connections(std::move(is_duplicate))) {
-               peer_dlog( this, "sending go_away duplicate, msg.p2p_address: ${add}", ("add", msg.p2p_address) );
-               go_away_message gam(duplicate);
-               gam.node_id = conn_node_id;
-               enqueue(gam);
-               no_retry = duplicate;
-               return;
+               return true;
             }
-         } else {
-            peer_dlog(this, "skipping duplicate check, addr == ${pa}, id = ${ni}", ("pa", peer_address())("ni", msg.node_id));
-
-            // check if peer requests no trx or no blocks
-            set_peer_connection_type(msg.p2p_address);
+            return false;
+         };
+         if (my_impl->connections.any_of_connections(std::move(is_duplicate))) {
path: plugins/net_plugin/net_plugin.cpp
user: login: heifner
body: No reason, I removed it.
created_at: 2025-04-02T15:25:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -3492,57 +3491,37 @@ namespace eosio {
                set_connection_type( msg.p2p_address);
             else
                peer_dlog(this, "Invalid handshake p2p_address ${p}", ("p", msg.p2p_address));
+         } else {
+            // peer p2p_address may contain trx or blk only request, honor requested connection type
+            set_peer_connection_type(msg.p2p_address);
+         }
 
-            peer_dlog( this, "checking for duplicate" );
-            auto is_duplicate = [&](const connection_ptr& check) {
-               if(check.get() == this)
+         peer_dlog( this, "checking for duplicate" );
+         auto is_duplicate = [&](const connection_ptr& check) {
+            if(check.get() == this)
+               return false;
+            fc::unique_lock g_check_conn( check->conn_mtx );
+            fc_dlog( logger, "dup check: connected ${c}, ${l} =? ${r}",
+                     ("c", check->connected())("l", check->last_handshake_recv.node_id)("r", msg.node_id) );
+            if(check->connected() && check->last_handshake_recv.node_id == msg.node_id) {
+               if (my_impl->node_id < msg.node_id) {
path: plugins/net_plugin/net_plugin.cpp
user: login: heifner
body: In that scenario the node does keep both connections. The peer will then drop one of the two connections leaving you with one connection.
created_at: 2025-04-02T15:25:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -3492,57 +3491,37 @@ namespace eosio {
                set_connection_type( msg.p2p_address);
             else
                peer_dlog(this, "Invalid handshake p2p_address ${p}", ("p", msg.p2p_address));
+         } else {
+            // peer p2p_address may contain trx or blk only request, honor requested connection type
+            set_peer_connection_type(msg.p2p_address);
+         }
 
-            peer_dlog( this, "checking for duplicate" );
-            auto is_duplicate = [&](const connection_ptr& check) {
-               if(check.get() == this)
+         peer_dlog( this, "checking for duplicate" );
+         auto is_duplicate = [&](const connection_ptr& check) {
+            if(check.get() == this)
+               return false;
+            fc::unique_lock g_check_conn( check->conn_mtx );
+            fc_dlog( logger, "dup check: connected ${c}, ${l} =? ${r}",
+                     ("c", check->connected())("l", check->last_handshake_recv.node_id)("r", msg.node_id) );
+            if(check->connected() && check->last_handshake_recv.node_id == msg.node_id) {
+               if (my_impl->node_id < msg.node_id) {
+                  fc_dlog( logger, "not duplicate, my_impl->node_id '${lhs}' < msg.node_id '${rhs}'",
+                           ("lhs", my_impl->node_id)("rhs", msg.node_id) );
+                  // only the connection from lower node_id to higher node_id will be considered as a duplicate,
+                  // so there is no chance for both connections to be closed
                   return false;
-               fc::unique_lock g_check_conn( check->conn_mtx );
-               fc_dlog( logger, "dup check: connected ${c}, ${l} =? ${r}",
-                        ("c", check->connected())("l", check->last_handshake_recv.node_id)("r", msg.node_id) );
-               if(check->connected() && check->last_handshake_recv.node_id == msg.node_id) {
-                  if (net_version < proto_dup_goaway_resolution || msg.network_version < proto_dup_goaway_resolution) {
-                     // It's possible that both peers could arrive here at relatively the same time, so
-                     // we need to avoid the case where they would both tell a different connection to go away.
-                     // Using the sum of the initial handshake times of the two connections, we will
-                     // arbitrarily (but consistently between the two peers) keep one of them.
-
-                     auto check_time = check->last_handshake_sent.time + check->last_handshake_recv.time;
-                     g_check_conn.unlock();
-                     if (msg.time + c_time <= check_time)
-                        return false;
-                  } else if (net_version < proto_dup_node_id_goaway || msg.network_version < proto_dup_node_id_goaway) {
-                     if (listen_address < msg.p2p_address) {
-                        fc_dlog( logger, "listen_address '${lhs}' < msg.p2p_address '${rhs}'",
-                                 ("lhs", listen_address)( "rhs", msg.p2p_address ) );
-                        // only the connection from lower p2p_address to higher p2p_address will be considered as a duplicate,
-                        // so there is no chance for both connections to be closed
-                        return false;
-                     }
-                  } else if (my_impl->node_id < msg.node_id) {
-                     fc_dlog( logger, "not duplicate, my_impl->node_id '${lhs}' < msg.node_id '${rhs}'",
-                              ("lhs", my_impl->node_id)("rhs", msg.node_id) );
-                     // only the connection from lower node_id to higher node_id will be considered as a duplicate,
-                     // so there is no chance for both connections to be closed
-                     return false;
-                  }
-                  return true;
                }
-               return false;
-            };
-            if (my_impl->connections.any_of_connections(std::move(is_duplicate))) {
-               peer_dlog( this, "sending go_away duplicate, msg.p2p_address: ${add}", ("add", msg.p2p_address) );
-               go_away_message gam(duplicate);
-               gam.node_id = conn_node_id;
-               enqueue(gam);
-               no_retry = duplicate;
-               return;
+               return true;
             }
-         } else {
-            peer_dlog(this, "skipping duplicate check, addr == ${pa}, id = ${ni}", ("pa", peer_address())("ni", msg.node_id));
-
-            // check if peer requests no trx or no blocks
-            set_peer_connection_type(msg.p2p_address);
+            return false;
+         };
+         if (my_impl->connections.any_of_connections(is_duplicate)) {
+            peer_dlog( this, "sending go_away duplicate, msg.p2p_address: ${add}", ("add", msg.p2p_address) );
+            go_away_message gam(duplicate);
path: plugins/net_plugin/net_plugin.cpp
user: login: greg7mdp
body: I don't think the constructor of `go_away_message` serves any useful purpose, and makes the code (here at least) more awkward. Without it we could do:
`enqueue(go_away_message{duplicate, conn_node_id});`
which I think is clearer.
created_at: 2025-04-02T19:10:40Z
author_association: CONTRIBUTOR
diff_hunk: @@ -3492,57 +3491,37 @@ namespace eosio {
                set_connection_type( msg.p2p_address);
             else
                peer_dlog(this, "Invalid handshake p2p_address ${p}", ("p", msg.p2p_address));
+         } else {
+            // peer p2p_address may contain trx or blk only request, honor requested connection type
+            set_peer_connection_type(msg.p2p_address);
+         }
 
-            peer_dlog( this, "checking for duplicate" );
-            auto is_duplicate = [&](const connection_ptr& check) {
-               if(check.get() == this)
+         peer_dlog( this, "checking for duplicate" );
+         auto is_duplicate = [&](const connection_ptr& check) {
+            if(check.get() == this)
+               return false;
+            fc::unique_lock g_check_conn( check->conn_mtx );
+            fc_dlog( logger, "dup check: connected ${c}, ${l} =? ${r}",
+                     ("c", check->connected())("l", check->last_handshake_recv.node_id)("r", msg.node_id) );
+            if(check->connected() && check->last_handshake_recv.node_id == msg.node_id) {
+               if (my_impl->node_id < msg.node_id) {
+                  fc_dlog( logger, "not duplicate, my_impl->node_id '${lhs}' < msg.node_id '${rhs}'",
+                           ("lhs", my_impl->node_id)("rhs", msg.node_id) );
+                  // only the connection from lower node_id to higher node_id will be considered as a duplicate,
+                  // so there is no chance for both connections to be closed
                   return false;
-               fc::unique_lock g_check_conn( check->conn_mtx );
-               fc_dlog( logger, "dup check: connected ${c}, ${l} =? ${r}",
-                        ("c", check->connected())("l", check->last_handshake_recv.node_id)("r", msg.node_id) );
-               if(check->connected() && check->last_handshake_recv.node_id == msg.node_id) {
-                  if (net_version < proto_dup_goaway_resolution || msg.network_version < proto_dup_goaway_resolution) {
-                     // It's possible that both peers could arrive here at relatively the same time, so
-                     // we need to avoid the case where they would both tell a different connection to go away.
-                     // Using the sum of the initial handshake times of the two connections, we will
-                     // arbitrarily (but consistently between the two peers) keep one of them.
-
-                     auto check_time = check->last_handshake_sent.time + check->last_handshake_recv.time;
-                     g_check_conn.unlock();
-                     if (msg.time + c_time <= check_time)
-                        return false;
-                  } else if (net_version < proto_dup_node_id_goaway || msg.network_version < proto_dup_node_id_goaway) {
-                     if (listen_address < msg.p2p_address) {
-                        fc_dlog( logger, "listen_address '${lhs}' < msg.p2p_address '${rhs}'",
-                                 ("lhs", listen_address)( "rhs", msg.p2p_address ) );
-                        // only the connection from lower p2p_address to higher p2p_address will be considered as a duplicate,
-                        // so there is no chance for both connections to be closed
-                        return false;
-                     }
-                  } else if (my_impl->node_id < msg.node_id) {
-                     fc_dlog( logger, "not duplicate, my_impl->node_id '${lhs}' < msg.node_id '${rhs}'",
-                              ("lhs", my_impl->node_id)("rhs", msg.node_id) );
-                     // only the connection from lower node_id to higher node_id will be considered as a duplicate,
-                     // so there is no chance for both connections to be closed
-                     return false;
-                  }
-                  return true;
                }
-               return false;
-            };
-            if (my_impl->connections.any_of_connections(std::move(is_duplicate))) {
-               peer_dlog( this, "sending go_away duplicate, msg.p2p_address: ${add}", ("add", msg.p2p_address) );
-               go_away_message gam(duplicate);
-               gam.node_id = conn_node_id;
-               enqueue(gam);
-               no_retry = duplicate;
-               return;
+               return true;
             }
-         } else {
-            peer_dlog(this, "skipping duplicate check, addr == ${pa}, id = ${ni}", ("pa", peer_address())("ni", msg.node_id));
-
-            // check if peer requests no trx or no blocks
-            set_peer_connection_type(msg.p2p_address);
+            return false;
+         };
+         if (my_impl->connections.any_of_connections(is_duplicate)) {
+            peer_dlog( this, "sending go_away duplicate, msg.p2p_address: ${add}", ("add", msg.p2p_address) );
+            go_away_message gam(duplicate);
path: plugins/net_plugin/net_plugin.cpp
user: login: greg7mdp
body: bonus point if making `go_away_reason` an `enum class` :-)
created_at: 2025-04-02T19:13:17Z
author_association: CONTRIBUTOR
diff_hunk: @@ -3492,57 +3491,37 @@ namespace eosio {
                set_connection_type( msg.p2p_address);
             else
                peer_dlog(this, "Invalid handshake p2p_address ${p}", ("p", msg.p2p_address));
+         } else {
+            // peer p2p_address may contain trx or blk only request, honor requested connection type
+            set_peer_connection_type(msg.p2p_address);
+         }
 
-            peer_dlog( this, "checking for duplicate" );
-            auto is_duplicate = [&](const connection_ptr& check) {
-               if(check.get() == this)
+         peer_dlog( this, "checking for duplicate" );
+         auto is_duplicate = [&](const connection_ptr& check) {
+            if(check.get() == this)
+               return false;
+            fc::unique_lock g_check_conn( check->conn_mtx );
+            fc_dlog( logger, "dup check: connected ${c}, ${l} =? ${r}",
+                     ("c", check->connected())("l", check->last_handshake_recv.node_id)("r", msg.node_id) );
+            if(check->connected() && check->last_handshake_recv.node_id == msg.node_id) {
+               if (my_impl->node_id < msg.node_id) {
+                  fc_dlog( logger, "not duplicate, my_impl->node_id '${lhs}' < msg.node_id '${rhs}'",
+                           ("lhs", my_impl->node_id)("rhs", msg.node_id) );
+                  // only the connection from lower node_id to higher node_id will be considered as a duplicate,
+                  // so there is no chance for both connections to be closed
                   return false;
-               fc::unique_lock g_check_conn( check->conn_mtx );
-               fc_dlog( logger, "dup check: connected ${c}, ${l} =? ${r}",
-                        ("c", check->connected())("l", check->last_handshake_recv.node_id)("r", msg.node_id) );
-               if(check->connected() && check->last_handshake_recv.node_id == msg.node_id) {
-                  if (net_version < proto_dup_goaway_resolution || msg.network_version < proto_dup_goaway_resolution) {
-                     // It's possible that both peers could arrive here at relatively the same time, so
-                     // we need to avoid the case where they would both tell a different connection to go away.
-                     // Using the sum of the initial handshake times of the two connections, we will
-                     // arbitrarily (but consistently between the two peers) keep one of them.
-
-                     auto check_time = check->last_handshake_sent.time + check->last_handshake_recv.time;
-                     g_check_conn.unlock();
-                     if (msg.time + c_time <= check_time)
-                        return false;
-                  } else if (net_version < proto_dup_node_id_goaway || msg.network_version < proto_dup_node_id_goaway) {
-                     if (listen_address < msg.p2p_address) {
-                        fc_dlog( logger, "listen_address '${lhs}' < msg.p2p_address '${rhs}'",
-                                 ("lhs", listen_address)( "rhs", msg.p2p_address ) );
-                        // only the connection from lower p2p_address to higher p2p_address will be considered as a duplicate,
-                        // so there is no chance for both connections to be closed
-                        return false;
-                     }
-                  } else if (my_impl->node_id < msg.node_id) {
-                     fc_dlog( logger, "not duplicate, my_impl->node_id '${lhs}' < msg.node_id '${rhs}'",
-                              ("lhs", my_impl->node_id)("rhs", msg.node_id) );
-                     // only the connection from lower node_id to higher node_id will be considered as a duplicate,
-                     // so there is no chance for both connections to be closed
-                     return false;
-                  }
-                  return true;
                }
-               return false;
-            };
-            if (my_impl->connections.any_of_connections(std::move(is_duplicate))) {
-               peer_dlog( this, "sending go_away duplicate, msg.p2p_address: ${add}", ("add", msg.p2p_address) );
-               go_away_message gam(duplicate);
-               gam.node_id = conn_node_id;
-               enqueue(gam);
-               no_retry = duplicate;
-               return;
+               return true;
             }
-         } else {
-            peer_dlog(this, "skipping duplicate check, addr == ${pa}, id = ${ni}", ("pa", peer_address())("ni", msg.node_id));
-
-            // check if peer requests no trx or no blocks
-            set_peer_connection_type(msg.p2p_address);
+            return false;
+         };
+         if (my_impl->connections.any_of_connections(is_duplicate)) {
+            peer_dlog( this, "sending go_away duplicate, msg.p2p_address: ${add}", ("add", msg.p2p_address) );
+            go_away_message gam(duplicate);
path: plugins/net_plugin/net_plugin.cpp
user: login: heifner
body: Done.
created_at: 2025-04-02T19:39:28Z
author_association: CONTRIBUTOR

PR comments:
user: login: greg7mdp
created_at: 2025-04-02T19:05:10Z
body: I wonder if it would not be preferable to do the duplicate connection check when `(msg.generation >= 1) ` (instead of `==`)?
author_association: CONTRIBUTOR
user: login: heifner
created_at: 2025-04-02T19:40:11Z
body: > Seems excessive to check on every handshake message.

I think the cost is probaly negligeable, but OK.
author_association: CONTRIBUTOR
user: login: ericpassmore
created_at: 2025-04-07T00:15:41Z
body: Note:start
category: Other
component: P2P
summary: Fix for an issue where `net_plugin` would allow some duplicate connections to remain open. 
Note:end
author_association: CONTRIBUTOR