Apply PR feedback:

Only use pop method if graceful failure handling is desired

libp2p/network/swarm.py

@@ -270,9 +270,11 @@ class Swarm(INetwork):
         """Simply remove the connection from Swarm's records, without closing
         the connection."""
         peer_id = swarm_conn.muxed_conn.peer_id
+        if peer_id not in self.connections:
+            return
         # TODO: Should be changed to remove the exact connection,
         #   if we have several connections per peer in the future.
-        self.connections.pop(peer_id, None)
+        del self.connections[peer_id]
 
     # Notifee