ft. modernise py-libp2p (#618)

* fix pyproject.toml , add ruff

* rm lock

* make progress

* add poetry lock ignore

* fix type issues

* fix tcp type errors

* fix text example - type error - wrong args

* add setuptools to dev

* test ci

* fix docs build

* fix type issues for new_swarm & new_host

* fix types in gossipsub

* fix type issues in noise

* wip: factories

* revert factories

* fix more type issues

* more type fixes

* fix: add null checks for noise protocol initialization and key handling

* corrected argument-errors in peerId and Multiaddr in peer tests

* fix: Noice - remove redundant type casts in BaseNoiseMsgReadWriter

* fix: update test_notify.py to use SwarmFactory.create_batch_and_listen, fix type hints, and comment out ClosedStream assertions

* Fix type checks for pubsub module

Signed-off-by: sukhman <sukhmansinghsaluja@gmail.com>

* Fix type checks for pubsub module-tests

Signed-off-by: sukhman <sukhmansinghsaluja@gmail.com>

* noise: add checks for uninitialized protocol and key states in PatternXX

Signed-off-by: varun-r-mallya <varunrmallya@gmail.com>

* pubsub: add None checks for optional fields in FloodSub and Pubsub

Signed-off-by: varun-r-mallya <varunrmallya@gmail.com>

* Fix type hints and improve testing

Signed-off-by: varun-r-mallya <varunrmallya@gmail.com>

* remove redundant checks

Signed-off-by: varun-r-mallya <varunrmallya@gmail.com>

* fix build issues

* add optional to trio service

* fix types

* fix type errors

* Fix type errors

Signed-off-by: varun-r-mallya <varunrmallya@gmail.com>

* fixed more-type checks in crypto and peer_data files

* wip: factories

* replaced union with optional

* fix: type-error in interp-utils and peerinfo

* replace pyright with pyrefly

* add pyrefly.toml

* wip: fix multiselect issues

* try typecheck

* base check

* mcache test fixes , typecheck ci update

* fix ci

* will this work

* minor fix

* use poetry

* fix wokflow

* use cache,fix err

* fix pyrefly.toml

* fix pyrefly.toml

* fix cache in ci

* deploy commit

* add main baseline

* update to v5

* improve typecheck ci (#14)

* fix typo

* remove holepunching code (#16)

* fix gossipsub typeerrors (#17)

* fix: ensure initiator user includes remote peer id in handshake (#15)

* fix ci (#19)

* typefix: custom_types | core/peerinfo/test_peer_info | io/abc | pubsub/floodsub | protocol_muxer/multiselect (#18)

* fix: Typefixes in PeerInfo  (#21)

* fix minor type issue (#22)

* fix type errors in pubsub (#24)

* fix: Minor typefixes in tests (#23)

* Fix failing tests for type-fixed test/pubsub (#8)

* move pyrefly & ruff to pyproject.toml & rm .project-template (#28)

* move the async_context file to tests/core

* move crypto test to crypto folder

* fix: some typefixes (#25)

* fix type errors

* fix type issues

* fix: update gRPC API usage in autonat_pb2_grpc.py (#31)

* md: typecheck ci

* rm comments

* clean up : from review suggestions

* use | None over Optional as per new python standards

* drop supporto for py3.9

* newsfragments

---------

Signed-off-by: sukhman <sukhmansinghsaluja@gmail.com>
Signed-off-by: varun-r-mallya <varunrmallya@gmail.com>
Co-authored-by: acul71 <luca.pisani@birdo.net>
Co-authored-by: kaneki003 <sakshamchauhan707@gmail.com>
Co-authored-by: sukhman <sukhmansinghsaluja@gmail.com>
Co-authored-by: varun-r-mallya <varunrmallya@gmail.com>
Co-authored-by: varunrmallya <100590632+varun-r-mallya@users.noreply.github.com>
Co-authored-by: lla-dane <abhinavagarwalla6@gmail.com>
Co-authored-by: Collins <ArtemisfowlX@protonmail.com>
Co-authored-by: Abhinav Agarwalla <120122716+lla-dane@users.noreply.github.com>
Co-authored-by: guha-rahul <52607971+guha-rahul@users.noreply.github.com>
Co-authored-by: Sukhman Singh <63765293+sukhman-sukh@users.noreply.github.com>
Co-authored-by: acul71 <34693171+acul71@users.noreply.github.com>
Co-authored-by: pacrob <5199899+pacrob@users.noreply.github.com>
This commit is contained in:
Arush Kurundodi
2025-06-09 23:09:59 +05:30
committed by GitHub
parent d020bbc066
commit bdadec7519
111 changed files with 1537 additions and 1401 deletions

View File

@ -0,0 +1,189 @@
import pytest
import trio
from libp2p.abc import ISecureConn
from libp2p.crypto.keys import PrivateKey, PublicKey
from libp2p.peer.id import ID
from libp2p.stream_muxer.exceptions import (
MuxedStreamClosed,
MuxedStreamError,
)
from libp2p.stream_muxer.mplex.datastructures import (
StreamID,
)
from libp2p.stream_muxer.mplex.mplex import Mplex
from libp2p.stream_muxer.mplex.mplex_stream import (
MplexStream,
)
from libp2p.stream_muxer.yamux.yamux import (
Yamux,
YamuxStream,
)
DUMMY_PEER_ID = ID(b"dummy_peer_id")
class DummySecuredConn(ISecureConn):
def __init__(self, is_initiator: bool = False):
self.is_initiator = is_initiator
async def write(self, data: bytes) -> None:
pass
async def read(self, n: int | None = -1) -> bytes:
return b""
async def close(self) -> None:
pass
def get_remote_address(self):
return None
def get_local_address(self):
return None
def get_local_peer(self) -> ID:
return ID(b"local")
def get_local_private_key(self) -> PrivateKey:
return PrivateKey() # Dummy key
def get_remote_peer(self) -> ID:
return ID(b"remote")
def get_remote_public_key(self) -> PublicKey:
return PublicKey() # Dummy key
class MockMuxedConn:
def __init__(self):
self.streams = {}
self.streams_lock = trio.Lock()
self.event_shutting_down = trio.Event()
self.event_closed = trio.Event()
self.event_started = trio.Event()
self.secured_conn = DummySecuredConn() # For YamuxStream
async def send_message(self, flag, data, stream_id):
pass
def get_remote_address(self):
return None
class MockMplexMuxedConn:
def __init__(self):
self.streams_lock = trio.Lock()
self.event_shutting_down = trio.Event()
self.event_closed = trio.Event()
self.event_started = trio.Event()
async def send_message(self, flag, data, stream_id):
pass
def get_remote_address(self):
return None
class MockYamuxMuxedConn:
def __init__(self):
self.secured_conn = DummySecuredConn()
self.event_shutting_down = trio.Event()
self.event_closed = trio.Event()
self.event_started = trio.Event()
async def send_message(self, flag, data, stream_id):
pass
def get_remote_address(self):
return None
@pytest.mark.trio
async def test_mplex_stream_async_context_manager():
muxed_conn = Mplex(DummySecuredConn(), DUMMY_PEER_ID)
stream_id = StreamID(1, True) # Use real StreamID
stream = MplexStream(
name="test_stream",
stream_id=stream_id,
muxed_conn=muxed_conn,
incoming_data_channel=trio.open_memory_channel(8)[1],
)
async with stream as s:
assert s is stream
assert not stream.event_local_closed.is_set()
assert not stream.event_remote_closed.is_set()
assert not stream.event_reset.is_set()
assert stream.event_local_closed.is_set()
@pytest.mark.trio
async def test_yamux_stream_async_context_manager():
muxed_conn = Yamux(DummySecuredConn(), DUMMY_PEER_ID)
stream = YamuxStream(stream_id=1, conn=muxed_conn, is_initiator=True)
async with stream as s:
assert s is stream
assert not stream.closed
assert not stream.send_closed
assert not stream.recv_closed
assert stream.send_closed
@pytest.mark.trio
async def test_mplex_stream_async_context_manager_with_error():
muxed_conn = Mplex(DummySecuredConn(), DUMMY_PEER_ID)
stream_id = StreamID(1, True)
stream = MplexStream(
name="test_stream",
stream_id=stream_id,
muxed_conn=muxed_conn,
incoming_data_channel=trio.open_memory_channel(8)[1],
)
with pytest.raises(ValueError):
async with stream as s:
assert s is stream
assert not stream.event_local_closed.is_set()
assert not stream.event_remote_closed.is_set()
assert not stream.event_reset.is_set()
raise ValueError("Test error")
assert stream.event_local_closed.is_set()
@pytest.mark.trio
async def test_yamux_stream_async_context_manager_with_error():
muxed_conn = Yamux(DummySecuredConn(), DUMMY_PEER_ID)
stream = YamuxStream(stream_id=1, conn=muxed_conn, is_initiator=True)
with pytest.raises(ValueError):
async with stream as s:
assert s is stream
assert not stream.closed
assert not stream.send_closed
assert not stream.recv_closed
raise ValueError("Test error")
assert stream.send_closed
@pytest.mark.trio
async def test_mplex_stream_async_context_manager_write_after_close():
muxed_conn = Mplex(DummySecuredConn(), DUMMY_PEER_ID)
stream_id = StreamID(1, True)
stream = MplexStream(
name="test_stream",
stream_id=stream_id,
muxed_conn=muxed_conn,
incoming_data_channel=trio.open_memory_channel(8)[1],
)
async with stream as s:
assert s is stream
with pytest.raises(MuxedStreamClosed):
await stream.write(b"test data")
@pytest.mark.trio
async def test_yamux_stream_async_context_manager_write_after_close():
muxed_conn = Yamux(DummySecuredConn(), DUMMY_PEER_ID)
stream = YamuxStream(stream_id=1, conn=muxed_conn, is_initiator=True)
async with stream as s:
assert s is stream
with pytest.raises(MuxedStreamError):
await stream.write(b"test data")

View File

@ -1,6 +1,7 @@
import logging
import pytest
from multiaddr.multiaddr import Multiaddr
import trio
from libp2p import (
@ -11,6 +12,8 @@ from libp2p import (
new_host,
set_default_muxer,
)
from libp2p.custom_types import TProtocol
from libp2p.peer.peerinfo import PeerInfo
# Enable logging for debugging
logging.basicConfig(level=logging.DEBUG)
@ -24,13 +27,14 @@ async def host_pair(muxer_preference=None, muxer_opt=None):
host_b = new_host(muxer_preference=muxer_preference, muxer_opt=muxer_opt)
# Start both hosts
await host_a.get_network().listen("/ip4/127.0.0.1/tcp/0")
await host_b.get_network().listen("/ip4/127.0.0.1/tcp/0")
await host_a.get_network().listen(Multiaddr("/ip4/127.0.0.1/tcp/0"))
await host_b.get_network().listen(Multiaddr("/ip4/127.0.0.1/tcp/0"))
# Connect hosts with a timeout
listen_addrs_a = host_a.get_addrs()
with trio.move_on_after(5): # 5 second timeout
await host_b.connect(host_a.get_id(), listen_addrs_a)
peer_info_a = PeerInfo(host_a.get_id(), listen_addrs_a)
await host_b.connect(peer_info_a)
yield host_a, host_b
@ -57,14 +61,14 @@ async def test_multiplexer_preference_parameter(muxer_preference):
try:
# Start both hosts
await host_a.get_network().listen("/ip4/127.0.0.1/tcp/0")
await host_b.get_network().listen("/ip4/127.0.0.1/tcp/0")
await host_a.get_network().listen(Multiaddr("/ip4/127.0.0.1/tcp/0"))
await host_b.get_network().listen(Multiaddr("/ip4/127.0.0.1/tcp/0"))
# Connect hosts with timeout
listen_addrs_a = host_a.get_addrs()
with trio.move_on_after(5): # 5 second timeout
await host_b.connect(host_a.get_id(), listen_addrs_a)
peer_info_a = PeerInfo(host_a.get_id(), listen_addrs_a)
await host_b.connect(peer_info_a)
# Check if connection was established
connections = host_b.get_network().connections
assert len(connections) > 0, "Connection not established"
@ -74,7 +78,7 @@ async def test_multiplexer_preference_parameter(muxer_preference):
muxed_conn = conn.muxed_conn
# Define a simple echo protocol
ECHO_PROTOCOL = "/echo/1.0.0"
ECHO_PROTOCOL = TProtocol("/echo/1.0.0")
# Setup echo handler on host_a
async def echo_handler(stream):
@ -89,7 +93,7 @@ async def test_multiplexer_preference_parameter(muxer_preference):
# Open a stream with timeout
with trio.move_on_after(5):
stream = await muxed_conn.open_stream(ECHO_PROTOCOL)
stream = await muxed_conn.open_stream()
# Check stream type
if muxer_preference == MUXER_YAMUX:
@ -132,13 +136,14 @@ async def test_explicit_muxer_options(muxer_option_func, expected_stream_class):
try:
# Start both hosts
await host_a.get_network().listen("/ip4/127.0.0.1/tcp/0")
await host_b.get_network().listen("/ip4/127.0.0.1/tcp/0")
await host_a.get_network().listen(Multiaddr("/ip4/127.0.0.1/tcp/0"))
await host_b.get_network().listen(Multiaddr("/ip4/127.0.0.1/tcp/0"))
# Connect hosts with timeout
listen_addrs_a = host_a.get_addrs()
with trio.move_on_after(5): # 5 second timeout
await host_b.connect(host_a.get_id(), listen_addrs_a)
peer_info_a = PeerInfo(host_a.get_id(), listen_addrs_a)
await host_b.connect(peer_info_a)
# Check if connection was established
connections = host_b.get_network().connections
@ -149,7 +154,7 @@ async def test_explicit_muxer_options(muxer_option_func, expected_stream_class):
muxed_conn = conn.muxed_conn
# Define a simple echo protocol
ECHO_PROTOCOL = "/echo/1.0.0"
ECHO_PROTOCOL = TProtocol("/echo/1.0.0")
# Setup echo handler on host_a
async def echo_handler(stream):
@ -164,7 +169,7 @@ async def test_explicit_muxer_options(muxer_option_func, expected_stream_class):
# Open a stream with timeout
with trio.move_on_after(5):
stream = await muxed_conn.open_stream(ECHO_PROTOCOL)
stream = await muxed_conn.open_stream()
# Check stream type
assert expected_stream_class in stream.__class__.__name__
@ -200,13 +205,14 @@ async def test_global_default_muxer(global_default):
try:
# Start both hosts
await host_a.get_network().listen("/ip4/127.0.0.1/tcp/0")
await host_b.get_network().listen("/ip4/127.0.0.1/tcp/0")
await host_a.get_network().listen(Multiaddr("/ip4/127.0.0.1/tcp/0"))
await host_b.get_network().listen(Multiaddr("/ip4/127.0.0.1/tcp/0"))
# Connect hosts with timeout
listen_addrs_a = host_a.get_addrs()
with trio.move_on_after(5): # 5 second timeout
await host_b.connect(host_a.get_id(), listen_addrs_a)
peer_info_a = PeerInfo(host_a.get_id(), listen_addrs_a)
await host_b.connect(peer_info_a)
# Check if connection was established
connections = host_b.get_network().connections
@ -217,7 +223,7 @@ async def test_global_default_muxer(global_default):
muxed_conn = conn.muxed_conn
# Define a simple echo protocol
ECHO_PROTOCOL = "/echo/1.0.0"
ECHO_PROTOCOL = TProtocol("/echo/1.0.0")
# Setup echo handler on host_a
async def echo_handler(stream):
@ -232,7 +238,7 @@ async def test_global_default_muxer(global_default):
# Open a stream with timeout
with trio.move_on_after(5):
stream = await muxed_conn.open_stream(ECHO_PROTOCOL)
stream = await muxed_conn.open_stream()
# Check stream type based on global default
if global_default == MUXER_YAMUX:

View File

@ -7,6 +7,9 @@ from trio.testing import (
memory_stream_pair,
)
from libp2p.abc import (
IRawConnection,
)
from libp2p.crypto.ed25519 import (
create_new_key_pair,
)
@ -29,18 +32,19 @@ from libp2p.stream_muxer.yamux.yamux import (
)
class TrioStreamAdapter:
def __init__(self, send_stream, receive_stream):
class TrioStreamAdapter(IRawConnection):
def __init__(self, send_stream, receive_stream, is_initiator: bool = False):
self.send_stream = send_stream
self.receive_stream = receive_stream
self.is_initiator = is_initiator
async def write(self, data):
async def write(self, data: bytes) -> None:
logging.debug(f"Writing {len(data)} bytes")
with trio.move_on_after(2):
await self.send_stream.send_all(data)
async def read(self, n=-1):
if n == -1:
async def read(self, n: int | None = None) -> bytes:
if n is None or n == -1:
raise ValueError("Reading unbounded not supported")
logging.debug(f"Attempting to read {n} bytes")
with trio.move_on_after(2):
@ -48,9 +52,13 @@ class TrioStreamAdapter:
logging.debug(f"Read {len(data)} bytes")
return data
async def close(self):
async def close(self) -> None:
logging.debug("Closing stream")
def get_remote_address(self) -> tuple[str, int] | None:
# Return None since this is a test adapter without real network info
return None
@pytest.fixture
def key_pair():
@ -68,8 +76,8 @@ async def secure_conn_pair(key_pair, peer_id):
client_send, server_receive = memory_stream_pair()
server_send, client_receive = memory_stream_pair()
client_rw = TrioStreamAdapter(client_send, client_receive)
server_rw = TrioStreamAdapter(server_send, server_receive)
client_rw = TrioStreamAdapter(client_send, client_receive, is_initiator=True)
server_rw = TrioStreamAdapter(server_send, server_receive, is_initiator=False)
insecure_transport = InsecureTransport(key_pair)
@ -196,9 +204,9 @@ async def test_yamux_stream_close(yamux_pair):
await trio.sleep(0.1)
# Now both directions are closed, so stream should be fully closed
assert (
client_stream.closed
), "Client stream should be fully closed after bidirectional close"
assert client_stream.closed, (
"Client stream should be fully closed after bidirectional close"
)
# Writing should still fail
with pytest.raises(MuxedStreamError):
@ -215,8 +223,12 @@ async def test_yamux_stream_reset(yamux_pair):
server_stream = await server_yamux.accept_stream()
await client_stream.reset()
# After reset, reading should raise MuxedStreamReset or MuxedStreamEOF
with pytest.raises((MuxedStreamEOF, MuxedStreamError)):
try:
await server_stream.read()
except (MuxedStreamEOF, MuxedStreamError):
pass
else:
pytest.fail("Expected MuxedStreamEOF or MuxedStreamError")
# Verify subsequent operations fail with StreamReset or EOF
with pytest.raises(MuxedStreamError):
await server_stream.read()
@ -269,9 +281,9 @@ async def test_yamux_flow_control(yamux_pair):
await client_stream.write(large_data)
# Check that window was reduced
assert (
client_stream.send_window < initial_window
), "Window should be reduced after sending"
assert client_stream.send_window < initial_window, (
"Window should be reduced after sending"
)
# Read the data on the server side
received = b""
@ -307,9 +319,9 @@ async def test_yamux_flow_control(yamux_pair):
f" {client_stream.send_window},"
f"initial half: {initial_window // 2}"
)
assert (
client_stream.send_window > initial_window // 2
), "Window should be increased after update"
assert client_stream.send_window > initial_window // 2, (
"Window should be increased after update"
)
await client_stream.close()
await server_stream.close()
@ -349,17 +361,17 @@ async def test_yamux_half_close(yamux_pair):
test_data = b"server response after client close"
# The server shouldn't be marked as send_closed yet
assert (
not server_stream.send_closed
), "Server stream shouldn't be marked as send_closed"
assert not server_stream.send_closed, (
"Server stream shouldn't be marked as send_closed"
)
await server_stream.write(test_data)
# Client can still read
received = await client_stream.read(len(test_data))
assert (
received == test_data
), "Client should still be able to read after sending FIN"
assert received == test_data, (
"Client should still be able to read after sending FIN"
)
# Now server closes its sending side
await server_stream.close()
@ -406,9 +418,9 @@ async def test_yamux_go_away_with_error(yamux_pair):
await trio.sleep(0.2)
# Verify server recognized shutdown
assert (
server_yamux.event_shutting_down.is_set()
), "Server should be shutting down after GO_AWAY"
assert server_yamux.event_shutting_down.is_set(), (
"Server should be shutting down after GO_AWAY"
)
logging.debug("test_yamux_go_away_with_error complete")