diff --git a/libp2p/network/connection/swarm_connection.py b/libp2p/network/connection/swarm_connection.py
index d21da482..f6e60784 100644
--- a/libp2p/network/connection/swarm_connection.py
+++ b/libp2p/network/connection/swarm_connection.py
@@ -61,12 +61,11 @@ class SwarmConn(INetConn):
                 try:
                     stream = await self.muxed_conn.accept_stream()
                 except MuxedConnUnavailable:
+                    await self.close()
                     break
                 # Asynchronously handle the accepted stream, to avoid blocking the next stream.
                 nursery.start_soon(self._handle_muxed_stream, stream)
 
-        await self.close()
-
     async def _handle_muxed_stream(self, muxed_stream: IMuxedStream) -> None:
         net_stream = self._add_stream(muxed_stream)
         try: