Merge branch 'main' into dependency-chore

docs/release_notes.rst

@@ -3,6 +3,65 @@ Release Notes
 
 .. towncrier release notes start
 
+py-libp2p v0.2.9 (2025-07-09)
+-----------------------------
+
+Breaking Changes
+~~~~~~~~~~~~~~~~
+
+- Reordered the arguments to ``upgrade_security`` to place ``is_initiator`` before ``peer_id``, and made ``peer_id`` optional.
+  This allows the method to reflect the fact that peer identity is not required for inbound connections. (`#681 <https://github.com/libp2p/py-libp2p/issues/681>`__)
+
+
+Bugfixes
+~~~~~~~~
+
+- Add timeout wrappers in:
+  1. ``multiselect.py``: ``negotiate`` function
+  2. ``multiselect_client.py``: ``select_one_of`` , ``query_multistream_command`` functions
+  to prevent indefinite hangs when a remote peer does not respond. (`#696 <https://github.com/libp2p/py-libp2p/issues/696>`__)
+- Align stream creation logic with yamux specification (`#701 <https://github.com/libp2p/py-libp2p/issues/701>`__)
+- Fixed an issue in ``Pubsub`` where async validators were not handled reliably under concurrency. Now uses a safe aggregator list for consistent behavior. (`#702 <https://github.com/libp2p/py-libp2p/issues/702>`__)
+
+
+Features
+~~~~~~~~
+
+- Added support for ``Kademlia DHT`` in py-libp2p. (`#579 <https://github.com/libp2p/py-libp2p/issues/579>`__)
+- Limit concurrency in ``push_identify_to_peers`` to prevent resource congestion under high peer counts. (`#621 <https://github.com/libp2p/py-libp2p/issues/621>`__)
+- Store public key and peer ID in peerstore during handshake
+
+  Modified the InsecureTransport class to accept an optional peerstore parameter and updated the handshake process to store the received public key and peer ID in the peerstore when available.
+
+  Added test cases to verify:
+  1. The peerstore remains unchanged when handshake fails due to peer ID mismatch
+  2. The handshake correctly adds a public key to a peer ID that already exists in the peerstore but doesn't have a public key yet (`#631 <https://github.com/libp2p/py-libp2p/issues/631>`__)
+- Fixed several flow-control and concurrency issues in the ``YamuxStream`` class. Previously, stress-testing revealed that transferring data over ``DEFAULT_WINDOW_SIZE`` would break the stream due to inconsistent window update handling and lock management. The fixes include:
+
+  - Removed sending of window updates during writes to maintain correct flow-control.
+  - Added proper timeout handling when releasing and acquiring locks to prevent concurrency errors.
+  - Corrected the ``read`` function to properly handle window updates for both ``read_until_EOF`` and ``read_n_bytes``.
+  - Added event logging at ``send_window_updates`` and ``waiting_for_window_updates`` for better observability. (`#639 <https://github.com/libp2p/py-libp2p/issues/639>`__)
+- Added support for ``Multicast DNS`` in py-libp2p (`#649 <https://github.com/libp2p/py-libp2p/issues/649>`__)
+- Optimized pubsub publishing to send multiple topics in a single message instead of separate messages per topic. (`#685 <https://github.com/libp2p/py-libp2p/issues/685>`__)
+- Optimized pubsub message writing by implementing a write_msg() method that uses pre-allocated buffers and single write operations, improving performance by eliminating separate varint prefix encoding and write operations in FloodSub and GossipSub. (`#687 <https://github.com/libp2p/py-libp2p/issues/687>`__)
+- Added peer exchange and backoff logic as part of Gossipsub v1.1 upgrade (`#690 <https://github.com/libp2p/py-libp2p/issues/690>`__)
+
+
+Internal Changes - for py-libp2p Contributors
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- Added sparse connect utility function to pubsub test utilities for creating test networks with configurable connectivity. (`#679 <https://github.com/libp2p/py-libp2p/issues/679>`__)
+- Added comprehensive tests for pubsub connection utility functions to verify degree limits are enforced, excess peers are handled correctly, and edge cases (degree=0, negative values, empty lists) are managed gracefully. (`#707 <https://github.com/libp2p/py-libp2p/issues/707>`__)
+- Added extra tests for identify push concurrency cap under high peer load (`#708 <https://github.com/libp2p/py-libp2p/issues/708>`__)
+
+
+Miscellaneous Changes
+~~~~~~~~~~~~~~~~~~~~~
+
+- `#678 <https://github.com/libp2p/py-libp2p/issues/678>`__, `#684 <https://github.com/libp2p/py-libp2p/issues/684>`__
+
+
 py-libp2p v0.2.8 (2025-06-10)
 -----------------------------
 

libp2p/abc.py

@@ -50,6 +50,11 @@ if TYPE_CHECKING:
         Pubsub,
     )
 
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from libp2p.protocol_muxer.multiselect import Multiselect
+
 from libp2p.pubsub.pb import (
     rpc_pb2,
 )
@@ -1545,9 +1550,8 @@ class IHost(ABC):
 
         """
 
-    # FIXME: Replace with correct return type
     @abstractmethod
-    def get_mux(self) -> Any:
+    def get_mux(self) -> "Multiselect":
         """
         Retrieve the muxer instance for the host.
 
@@ -2158,6 +2162,7 @@ class IMultiselectMuxer(ABC):
 
         """
 
+    @abstractmethod
     def get_protocols(self) -> tuple[TProtocol | None, ...]:
         """
         Retrieve the protocols for which handlers have been registered.
@@ -2168,7 +2173,6 @@ class IMultiselectMuxer(ABC):
             A tuple of registered protocol names.
 
         """
-        return tuple(self.handlers.keys())
 
     @abstractmethod
     async def negotiate(

libp2p/identity/identify/identify.py

@@ -59,7 +59,7 @@ def _mk_identify_protobuf(
 ) -> Identify:
     public_key = host.get_public_key()
     laddrs = host.get_addrs()
-    protocols = host.get_mux().get_protocols()
+    protocols = tuple(str(p) for p in host.get_mux().get_protocols() if p is not None)
 
     observed_addr = observed_multiaddr.to_bytes() if observed_multiaddr else b""
     return Identify(

libp2p/peer/peerstore.py

@@ -64,7 +64,11 @@ class PeerStore(IPeerStore):
         return list(self.peer_data_map.keys())
 
     def clear_peerdata(self, peer_id: ID) -> None:
-        """Clears the peer data of the peer"""
+        """Clears all data associated with the given peer_id."""
+        if peer_id in self.peer_data_map:
+            del self.peer_data_map[peer_id]
+        else:
+            raise PeerStoreError("peer ID not found")
 
     def valid_peer_ids(self) -> list[ID]:
         """

libp2p/protocol_muxer/multiselect.py

@@ -101,6 +101,18 @@ class Multiselect(IMultiselectMuxer):
         except trio.TooSlowError:
             raise MultiselectError("handshake read timeout")
 
+    def get_protocols(self) -> tuple[TProtocol | None, ...]:
+        """
+        Retrieve the protocols for which handlers have been registered.
+
+        Returns
+        -------
+        tuple[TProtocol, ...]
+            A tuple of registered protocol names.
+
+        """
+        return tuple(self.handlers.keys())
+
     async def handshake(self, communicator: IMultiselectCommunicator) -> None:
         """
         Perform handshake to agree on multiselect protocol.

libp2p/relay/circuit_v2/discovery.py

@@ -234,7 +234,8 @@ class RelayDiscovery(Service):
 
             if not callable(proto_getter):
                 return None
-
+            if peer_id not in peerstore.peer_ids():
+                return None
             try:
                 # Try to get protocols
                 proto_result = proto_getter(peer_id)
@@ -283,8 +284,6 @@ class RelayDiscovery(Service):
                 return None
 
             mux = self.host.get_mux()
-            if not hasattr(mux, "protocols"):
-                return None
 
             peer_protocols = set()
             # Get protocols from mux with proper type safety
@@ -293,7 +292,9 @@ class RelayDiscovery(Service):
                 # Get protocols with proper typing
                 mux_protocols = mux.get_protocols()
                 if isinstance(mux_protocols, (list, tuple)):
-                    available_protocols = list(mux_protocols)
+                    available_protocols = [
+                        p for p in mux.get_protocols() if p is not None
+                    ]
 
             for protocol in available_protocols:
                 try:
@@ -313,7 +314,7 @@ class RelayDiscovery(Service):
 
             self._protocol_cache[peer_id] = peer_protocols
             protocol_str = str(PROTOCOL_ID)
-            for protocol in peer_protocols:
+            for protocol in map(TProtocol, peer_protocols):
                 if protocol == protocol_str:
                     return True
             return False

newsfragments/579.feature.rst

@@ -1 +0,0 @@
-Added support for ``Kademlia DHT`` in py-libp2p.

newsfragments/621.feature.rst

@@ -1 +0,0 @@
-Limit concurrency in `push_identify_to_peers` to prevent resource congestion under high peer counts.

newsfragments/631.feature.rst

@@ -1,7 +0,0 @@
-Store public key and peer ID in peerstore during handshake
-
-Modified the InsecureTransport class to accept an optional peerstore parameter and updated the handshake process to store the received public key and peer ID in the peerstore when available.
-
-Added test cases to verify:
-1. The peerstore remains unchanged when handshake fails due to peer ID mismatch
-2. The handshake correctly adds a public key to a peer ID that already exists in the peerstore but doesn't have a public key yet

newsfragments/639.feature.rst

@@ -1,6 +0,0 @@
-Fixed several flow-control and concurrency issues in the `YamuxStream` class. Previously, stress-testing revealed that transferring data over `DEFAULT_WINDOW_SIZE` would break the stream due to inconsistent window update handling and lock management. The fixes include:
-
-- Removed sending of window updates during writes to maintain correct flow-control.
-- Added proper timeout handling when releasing and acquiring locks to prevent concurrency errors.
-- Corrected the `read` function to properly handle window updates for both `read_until_EOF` and `read_n_bytes`.
-- Added event logging at `send_window_updates` and `waiting_for_window_updates` for better observability.

newsfragments/649.feature.rst

@@ -1 +0,0 @@
-Added support for ``Multicast DNS`` in py-libp2p

newsfragments/678.misc.rst

@@ -1 +0,0 @@
-Refactored gossipsub heartbeat logic to use a single helper method `_handle_topic_heartbeat` that handles both fanout and gossip heartbeats.

newsfragments/679.feature.rst

@@ -1 +0,0 @@
-Added sparse connect utility function to pubsub test utilities for creating test networks with configurable connectivity.

newsfragments/681.breaking.rst

@@ -1,2 +0,0 @@
-Reordered the arguments to `upgrade_security` to place `is_initiator` before `peer_id`, and made `peer_id` optional.
-This allows the method to reflect the fact that peer identity is not required for inbound connections.

newsfragments/684.misc.rst

@@ -1 +0,0 @@
-Uses the `decapsulate` method of the `Multiaddr` class to clean up the observed address.

newsfragments/685.feature.rst

@@ -1 +0,0 @@
-Optimized pubsub publishing to send multiple topics in a single message instead of separate messages per topic.

newsfragments/687.feature.rst

@@ -1 +0,0 @@
-Optimized pubsub message writing by implementing a write_msg() method that uses pre-allocated buffers and single write operations, improving performance by eliminating separate varint prefix encoding and write operations in FloodSub and GossipSub.

newsfragments/690.feature.rst

@@ -1 +0,0 @@
-added peer exchange and backoff logic as part of Gossipsub v1.1 upgrade

newsfragments/696.bugfix.rst

@@ -1,4 +0,0 @@
-Add timeout wrappers in:
-1. multiselect.py: `negotiate` function
-2. multiselect_client.py: `select_one_of` , `query_multistream_command` functions
-to prevent indefinite hangs when a remote peer does not respond.

newsfragments/701.bugfix.rst

@@ -1 +0,0 @@
-align stream creation logic with yamux specification

newsfragments/702.bugfix.rst

@@ -1 +0,0 @@
-Fixed an issue in `Pubsub` where async validators were not handled reliably under concurrency. Now uses a safe aggregator list for consistent behavior.

newsfragments/707.feature.rst

@@ -1 +0,0 @@
-Added comprehensive tests for pubsub connection utility functions to verify degree limits are enforced, excess peers are handled correctly, and edge cases (degree=0, negative values, empty lists) are managed gracefully.

newsfragments/708.performance.rst

@@ -1 +0,0 @@
-Added extra tests for identify push concurrency cap under high peer load

newsfragments/746.bugfix.rst

@@ -0,0 +1,3 @@
+Improved type safety in `get_mux()` and `get_protocols()` by returning properly typed values instead
+of `Any`. Also updated `identify.py` and `discovery.py` to handle `None` values safely and
+compare protocols correctly.

newsfragments/749.internal.rst

@@ -0,0 +1 @@
+Add comprehensive tests for relay_discovery method in circuit_relay_v2

newsfragments/750.feature.rst

@@ -0,0 +1 @@
+Add logic to clear_peerdata method in peerstore

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "libp2p"
-version = "0.2.8"
+version = "0.2.9"
 description = "libp2p: The Python implementation of the libp2p networking stack"
 readme = "README.md"
 requires-python = ">=3.10, <4.0"
@@ -188,7 +188,7 @@ name = "Removals"
 showcontent = true
 
 [tool.bumpversion]
-current_version = "0.2.8"
+current_version = "0.2.9"
 parse = """
     (?P<major>\\d+)
     \\.(?P<minor>\\d+)

tests/core/protocol_muxer/test_protocol_muxer.py

@@ -3,6 +3,7 @@ import pytest
 from libp2p.custom_types import (
     TProtocol,
 )
+from libp2p.protocol_muxer.multiselect import Multiselect
 from libp2p.tools.utils import (
     create_echo_stream_handler,
 )
@@ -138,3 +139,23 @@ async def test_multistream_command(security_protocol):
         # Dialer asks for unspoorted command
         with pytest.raises(ValueError, match="Command not supported"):
             await dialer.send_command(listener.get_id(), "random")
+
+
+@pytest.mark.trio
+async def test_get_protocols_returns_all_registered_protocols():
+    ms = Multiselect()
+
+    async def dummy_handler(stream):
+        pass
+
+    p1 = TProtocol("/echo/1.0.0")
+    p2 = TProtocol("/foo/1.0.0")
+    p3 = TProtocol("/bar/1.0.0")
+
+    ms.add_handler(p1, dummy_handler)
+    ms.add_handler(p2, dummy_handler)
+    ms.add_handler(p3, dummy_handler)
+
+    protocols = ms.get_protocols()
+
+    assert set(protocols) == {p1, p2, p3}

tests/core/relay/test_circuit_v2_discovery.py

@@ -105,11 +105,11 @@ async def test_relay_discovery_initialization():
 
 
 @pytest.mark.trio
-async def test_relay_discovery_find_relay():
-    """Test finding a relay node via discovery."""
+async def test_relay_discovery_find_relay_peerstore_method():
+    """Test finding a relay node via discovery using the peerstore method."""
     async with HostFactory.create_batch_and_listen(2) as hosts:
         relay_host, client_host = hosts
-        logger.info("Created hosts for test_relay_discovery_find_relay")
+        logger.info("Created host for test_relay_discovery_find_relay_peerstore_method")
         logger.info("Relay host ID: %s", relay_host.get_id())
         logger.info("Client host ID: %s", client_host.get_id())
 
@@ -144,19 +144,19 @@ async def test_relay_discovery_find_relay():
         # Start discovery service
         async with background_trio_service(client_discovery):
             await client_discovery.event_started.wait()
-            logger.info("Client discovery service started")
+            logger.info("Client discovery service started (peerstore method)")
 
-            # Wait for discovery to find the relay
-            logger.info("Waiting for relay discovery...")
+            # Wait for discovery to find the relay using the peerstore method
+            logger.info("Waiting for relay discovery using peerstore...")
 
-            # Manually trigger discovery instead of waiting
+            # Manually trigger discovery which uses peerstore as default
             await client_discovery.discover_relays()
 
             # Check if relay was found
             with trio.fail_after(DISCOVERY_TIMEOUT):
                 for _ in range(20):  # Try multiple times
                     if relay_host.get_id() in client_discovery._discovered_relays:
-                        logger.info("Relay discovered successfully")
+                        logger.info("Relay discovered successfully (peerstore method)")
                         break
 
                     # Wait and try again
@@ -164,14 +164,194 @@ async def test_relay_discovery_find_relay():
                     # Manually trigger discovery again
                     await client_discovery.discover_relays()
                 else:
-                    pytest.fail("Failed to discover relay node within timeout")
+                    pytest.fail(
+                        "Failed to discover relay node within timeout(peerstore method)"
+                    )
 
             # Verify that relay was found and is valid
             assert relay_host.get_id() in client_discovery._discovered_relays, (
-                "Relay should be discovered"
+                "Relay should be discovered (peerstore method)"
             )
             relay_info = client_discovery._discovered_relays[relay_host.get_id()]
-            assert relay_info.peer_id == relay_host.get_id(), "Peer ID should match"
+            assert relay_info.peer_id == relay_host.get_id(), (
+                "Peer ID should match (peerstore method)"
+            )
+
+
+@pytest.mark.trio
+async def test_relay_discovery_find_relay_direct_connection_method():
+    """Test finding a relay node via discovery using the direct connection method."""
+    async with HostFactory.create_batch_and_listen(2) as hosts:
+        relay_host, client_host = hosts
+        logger.info("Created hosts for test_relay_discovery_find_relay_direct_method")
+        logger.info("Relay host ID: %s", relay_host.get_id())
+        logger.info("Client host ID: %s", client_host.get_id())
+
+        # Explicitly register the protocol handlers on relay_host
+        relay_host.set_stream_handler(PROTOCOL_ID, simple_stream_handler)
+        relay_host.set_stream_handler(STOP_PROTOCOL_ID, simple_stream_handler)
+
+        # Manually add protocol to peerstore for testing, then remove to force fallback
+        client_host.get_peerstore().add_protocols(
+            relay_host.get_id(), [str(PROTOCOL_ID)]
+        )
+
+        # Set up discovery on the client host
+        client_discovery = RelayDiscovery(
+            client_host, discovery_interval=5
+        )  # Use shorter interval for testing
+
+        try:
+            # Connect peers so they can discover each other
+            with trio.fail_after(CONNECT_TIMEOUT):
+                logger.info("Connecting client host to relay host")
+                await connect(client_host, relay_host)
+                assert relay_host.get_network().connections[client_host.get_id()], (
+                    "Peers not connected"
+                )
+                logger.info("Connection established between peers")
+        except Exception as e:
+            logger.error("Failed to connect peers: %s", str(e))
+            raise
+
+        # Remove the relay from the peerstore to test fallback to direct connection
+        client_host.get_peerstore().clear_peerdata(relay_host.get_id())
+        # Make sure that peer_id is not present in peerstore
+        assert relay_host.get_id() not in client_host.get_peerstore().peer_ids()
+
+        # Start discovery service
+        async with background_trio_service(client_discovery):
+            await client_discovery.event_started.wait()
+            logger.info("Client discovery service started (direct connection method)")
+
+            # Wait for discovery to find the relay using the direct connection method
+            logger.info(
+                "Waiting for relay discovery using direct connection fallback..."
+            )
+
+            # Manually trigger discovery which should fallback to direct connection
+            await client_discovery.discover_relays()
+
+            # Check if relay was found
+            with trio.fail_after(DISCOVERY_TIMEOUT):
+                for _ in range(20):  # Try multiple times
+                    if relay_host.get_id() in client_discovery._discovered_relays:
+                        logger.info("Relay discovered successfully (direct method)")
+                        break
+
+                    # Wait and try again
+                    await trio.sleep(1)
+                    # Manually trigger discovery again
+                    await client_discovery.discover_relays()
+                else:
+                    pytest.fail(
+                        "Failed to discover relay node within timeout (direct method)"
+                    )
+
+            # Verify that relay was found and is valid
+            assert relay_host.get_id() in client_discovery._discovered_relays, (
+                "Relay should be discovered (direct method)"
+            )
+            relay_info = client_discovery._discovered_relays[relay_host.get_id()]
+            assert relay_info.peer_id == relay_host.get_id(), (
+                "Peer ID should match (direct method)"
+            )
+
+
+@pytest.mark.trio
+async def test_relay_discovery_find_relay_mux_method():
+    """
+    Test finding a relay node via discovery using the mux method
+    (fallback after direct connection fails).
+    """
+    async with HostFactory.create_batch_and_listen(2) as hosts:
+        relay_host, client_host = hosts
+        logger.info("Created hosts for test_relay_discovery_find_relay_mux_method")
+        logger.info("Relay host ID: %s", relay_host.get_id())
+        logger.info("Client host ID: %s", client_host.get_id())
+
+        # Explicitly register the protocol handlers on relay_host
+        relay_host.set_stream_handler(PROTOCOL_ID, simple_stream_handler)
+        relay_host.set_stream_handler(STOP_PROTOCOL_ID, simple_stream_handler)
+
+        client_host.set_stream_handler(PROTOCOL_ID, simple_stream_handler)
+        client_host.set_stream_handler(STOP_PROTOCOL_ID, simple_stream_handler)
+
+        # Set up discovery on the client host
+        client_discovery = RelayDiscovery(
+            client_host, discovery_interval=5
+        )  # Use shorter interval for testing
+
+        try:
+            # Connect peers so they can discover each other
+            with trio.fail_after(CONNECT_TIMEOUT):
+                logger.info("Connecting client host to relay host")
+                await connect(client_host, relay_host)
+                assert relay_host.get_network().connections[client_host.get_id()], (
+                    "Peers not connected"
+                )
+                logger.info("Connection established between peers")
+        except Exception as e:
+            logger.error("Failed to connect peers: %s", str(e))
+            raise
+
+        # Remove the relay from the peerstore to test fallback
+        client_host.get_peerstore().clear_peerdata(relay_host.get_id())
+        # Make sure that peer_id is not present in peerstore
+        assert relay_host.get_id() not in client_host.get_peerstore().peer_ids()
+
+        # Mock the _check_via_direct_connection method to return None
+        # This forces the discovery to fall back to the mux method
+        async def mock_direct_check_fails(peer_id):
+            """Mock that always returns None to force mux fallback."""
+            return None
+
+        client_discovery._check_via_direct_connection = mock_direct_check_fails
+
+        # Start discovery service
+        async with background_trio_service(client_discovery):
+            await client_discovery.event_started.wait()
+            logger.info("Client discovery service started (mux method)")
+
+            # Wait for discovery to find the relay using the mux method
+            logger.info("Waiting for relay discovery using mux fallback...")
+
+            # Manually trigger discovery which should fallback to mux method
+            await client_discovery.discover_relays()
+
+            # Check if relay was found
+            with trio.fail_after(DISCOVERY_TIMEOUT):
+                for _ in range(20):  # Try multiple times
+                    if relay_host.get_id() in client_discovery._discovered_relays:
+                        logger.info("Relay discovered successfully (mux method)")
+                        break
+
+                    # Wait and try again
+                    await trio.sleep(1)
+                    # Manually trigger discovery again
+                    await client_discovery.discover_relays()
+                else:
+                    pytest.fail(
+                        "Failed to discover relay node within timeout (mux method)"
+                    )
+
+            # Verify that relay was found and is valid
+            assert relay_host.get_id() in client_discovery._discovered_relays, (
+                "Relay should be discovered (mux method)"
+            )
+            relay_info = client_discovery._discovered_relays[relay_host.get_id()]
+            assert relay_info.peer_id == relay_host.get_id(), (
+                "Peer ID should match (mux method)"
+            )
+
+            # Verify that the protocol was cached via mux method
+            assert relay_host.get_id() in client_discovery._protocol_cache, (
+                "Protocol should be cached (mux method)"
+            )
+            assert (
+                str(PROTOCOL_ID)
+                in client_discovery._protocol_cache[relay_host.get_id()]
+            ), "Relay protocol should be in cache (mux method)"
 
 
 @pytest.mark.trio