Merge upstream/main into add-ws-transport

Resolved conflicts in:
- .gitignore: Combined JavaScript interop and Sphinx build ignores
- libp2p/__init__.py: Integrated QUIC transport support with WebSocket transport
- libp2p/network/swarm.py: Used upstream's improved listener handling
- pyproject.toml: Kept both WebSocket and QUIC dependencies

This merge brings in:
- QUIC transport implementation
- Enhanced swarm functionality
- Improved peer discovery
- Better error handling
- Updated dependencies and documentation

WebSocket transport implementation remains intact and functional.

tests/interop/nim_libp2p/.gitignore

@@ -0,0 +1,8 @@
+nimble.develop
+nimble.paths
+
+*.nimble
+nim-libp2p/
+
+nim_echo_server
+config.nims

tests/interop/nim_libp2p/conftest.py

@@ -0,0 +1,119 @@
+import fcntl
+import logging
+from pathlib import Path
+import shutil
+import subprocess
+import time
+
+import pytest
+
+logger = logging.getLogger(__name__)
+
+
+def check_nim_available():
+    """Check if nim compiler is available."""
+    return shutil.which("nim") is not None and shutil.which("nimble") is not None
+
+
+def check_nim_binary_built():
+    """Check if nim echo server binary is built."""
+    current_dir = Path(__file__).parent
+    binary_path = current_dir / "nim_echo_server"
+    return binary_path.exists() and binary_path.stat().st_size > 0
+
+
+def run_nim_setup_with_lock():
+    """Run nim setup with file locking to prevent parallel execution."""
+    current_dir = Path(__file__).parent
+    lock_file = current_dir / ".setup_lock"
+    setup_script = current_dir / "scripts" / "setup_nim_echo.sh"
+
+    if not setup_script.exists():
+        raise RuntimeError(f"Setup script not found: {setup_script}")
+
+    # Try to acquire lock
+    try:
+        with open(lock_file, "w") as f:
+            # Non-blocking lock attempt
+            fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+
+            # Double-check binary doesn't exist (another worker might have built it)
+            if check_nim_binary_built():
+                logger.info("Binary already exists, skipping setup")
+                return
+
+            logger.info("Acquired setup lock, running nim-libp2p setup...")
+
+            # Make setup script executable and run it
+            setup_script.chmod(0o755)
+            result = subprocess.run(
+                [str(setup_script)],
+                cwd=current_dir,
+                capture_output=True,
+                text=True,
+                timeout=300,  # 5 minute timeout
+            )
+
+            if result.returncode != 0:
+                raise RuntimeError(
+                    f"Setup failed (exit {result.returncode}):\n"
+                    f"stdout: {result.stdout}\n"
+                    f"stderr: {result.stderr}"
+                )
+
+            # Verify binary was built
+            if not check_nim_binary_built():
+                raise RuntimeError("nim_echo_server binary not found after setup")
+
+            logger.info("nim-libp2p setup completed successfully")
+
+    except BlockingIOError:
+        # Another worker is running setup, wait for it to complete
+        logger.info("Another worker is running setup, waiting...")
+
+        # Wait for setup to complete (check every 2 seconds, max 5 minutes)
+        for _ in range(150):  # 150 * 2 = 300 seconds = 5 minutes
+            if check_nim_binary_built():
+                logger.info("Setup completed by another worker")
+                return
+            time.sleep(2)
+
+        raise TimeoutError("Timed out waiting for setup to complete")
+
+    finally:
+        # Clean up lock file
+        try:
+            lock_file.unlink(missing_ok=True)
+        except Exception:
+            pass
+
+
+@pytest.fixture(scope="function")  # Changed to function scope
+def nim_echo_binary():
+    """Get nim echo server binary path."""
+    current_dir = Path(__file__).parent
+    binary_path = current_dir / "nim_echo_server"
+
+    if not binary_path.exists():
+        pytest.skip(
+            "nim_echo_server binary not found. "
+            "Run setup script: ./scripts/setup_nim_echo.sh"
+        )
+
+    return binary_path
+
+
+@pytest.fixture
+async def nim_server(nim_echo_binary):
+    """Start and stop nim echo server for tests."""
+    # Import here to avoid circular imports
+    # pyrefly: ignore
+    from test_echo_interop import NimEchoServer
+
+    server = NimEchoServer(nim_echo_binary)
+
+    try:
+        peer_id, listen_addr = await server.start()
+        yield server, peer_id, listen_addr
+    finally:
+        await server.stop()

tests/interop/nim_libp2p/nim_echo_server.nim

@@ -0,0 +1,108 @@
+{.used.}
+
+import chronos
+import stew/byteutils
+import libp2p
+
+##
+# Simple Echo Protocol Implementation for py-libp2p Interop Testing
+##
+const EchoCodec = "/echo/1.0.0"
+
+type EchoProto = ref object of LPProtocol
+
+proc new(T: typedesc[EchoProto]): T =
+  proc handle(conn: Connection, proto: string) {.async: (raises: [CancelledError]).} =
+    try:
+      echo "Echo server: Received connection from ", conn.peerId
+
+      # Read and echo messages in a loop
+      while not conn.atEof:
+        try:
+          # Read length-prefixed message using nim-libp2p's readLp
+          let message = await conn.readLp(1024 * 1024) # Max 1MB
+          if message.len == 0:
+            echo "Echo server: Empty message, closing connection"
+            break
+
+          let messageStr = string.fromBytes(message)
+          echo "Echo server: Received (", message.len, " bytes): ", messageStr
+
+          # Echo back using writeLp
+          await conn.writeLp(message)
+          echo "Echo server: Echoed message back"
+
+        except CatchableError as e:
+          echo "Echo server: Error processing message: ", e.msg
+          break
+
+    except CancelledError as e:
+      echo "Echo server: Connection cancelled"
+      raise e
+    except CatchableError as e:
+      echo "Echo server: Exception in handler: ", e.msg
+    finally:
+      echo "Echo server: Connection closed"
+      await conn.close()
+
+  return T.new(codecs = @[EchoCodec], handler = handle)
+
+##
+# Create QUIC-enabled switch
+##
+proc createSwitch(ma: MultiAddress, rng: ref HmacDrbgContext): Switch =
+  var switch = SwitchBuilder
+    .new()
+    .withRng(rng)
+    .withAddress(ma)
+    .withQuicTransport()
+    .build()
+  result = switch
+
+##
+# Main server
+##
+proc main() {.async.} =
+  let
+    rng = newRng()
+    localAddr = MultiAddress.init("/ip4/0.0.0.0/udp/0/quic-v1").tryGet()
+    echoProto = EchoProto.new()
+
+  echo "=== Nim Echo Server for py-libp2p Interop ==="
+
+  # Create switch
+  let switch = createSwitch(localAddr, rng)
+  switch.mount(echoProto)
+
+  # Start server
+  await switch.start()
+
+  # Print connection info
+  echo "Peer ID: ", $switch.peerInfo.peerId
+  echo "Listening on:"
+  for addr in switch.peerInfo.addrs:
+    echo "  ", $addr, "/p2p/", $switch.peerInfo.peerId
+  echo "Protocol: ", EchoCodec
+  echo "Ready for py-libp2p connections!"
+  echo ""
+
+  # Keep running
+  try:
+    await sleepAsync(100.hours)
+  except CancelledError:
+    echo "Shutting down..."
+  finally:
+    await switch.stop()
+
+# Graceful shutdown handler
+proc signalHandler() {.noconv.} =
+  echo "\nShutdown signal received"
+  quit(0)
+
+when isMainModule:
+  setControlCHook(signalHandler)
+  try:
+    waitFor(main())
+  except CatchableError as e:
+    echo "Error: ", e.msg
+    quit(1)

tests/interop/nim_libp2p/scripts/setup_nim_echo.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# tests/interop/nim_libp2p/scripts/setup_nim_echo.sh
+# Cache-aware setup that skips installation if packages exist
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_DIR="${SCRIPT_DIR}/.."
+
+# Colors
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+YELLOW='\033[1;33m'
+NC='\033[0m'
+
+log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
+log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
+log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
+
+main() {
+    log_info "Setting up nim echo server for interop testing..."
+
+    # Check if nim is available
+    if ! command -v nim &> /dev/null || ! command -v nimble &> /dev/null; then
+        log_error "Nim not found. Please install nim first."
+        exit 1
+    fi
+
+    cd "${PROJECT_DIR}"
+
+    # Create logs directory
+    mkdir -p logs
+
+    # Check if binary already exists
+    if [[ -f "nim_echo_server" ]]; then
+        log_info "nim_echo_server already exists, skipping build"
+        return 0
+    fi
+
+    # Check if libp2p is already installed (cache-aware)
+    if nimble list -i | grep -q "libp2p"; then
+        log_info "libp2p already installed, skipping installation"
+    else
+        log_info "Installing nim-libp2p globally..."
+        nimble install -y libp2p
+    fi
+
+    log_info "Building nim echo server..."
+    # Compile the echo server
+    nim c \
+        -d:release \
+        -d:chronicles_log_level=INFO \
+        -d:libp2p_quic_support \
+        -d:chronos_event_loop=iocp \
+        -d:ssl \
+        --opt:speed \
+        --mm:orc \
+        --verbosity:1 \
+        -o:nim_echo_server \
+        nim_echo_server.nim
+
+    # Verify binary was created
+    if [[ -f "nim_echo_server" ]]; then
+        log_info "✅ nim_echo_server built successfully"
+        log_info "Binary size: $(ls -lh nim_echo_server | awk '{print $5}')"
+    else
+        log_error "❌ Failed to build nim_echo_server"
+        exit 1
+    fi
+
+    log_info "🎉 Setup complete!"
+}
+
+main "$@"

tests/interop/nim_libp2p/test_echo_interop.py

@@ -0,0 +1,195 @@
+import logging
+from pathlib import Path
+import subprocess
+import time
+
+import pytest
+import multiaddr
+import trio
+
+from libp2p import new_host
+from libp2p.crypto.secp256k1 import create_new_key_pair
+from libp2p.custom_types import TProtocol
+from libp2p.peer.peerinfo import info_from_p2p_addr
+from libp2p.utils.varint import encode_varint_prefixed, read_varint_prefixed_bytes
+
+# Configuration
+PROTOCOL_ID = TProtocol("/echo/1.0.0")
+TEST_TIMEOUT = 30
+SERVER_START_TIMEOUT = 10.0
+
+# Setup logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class NimEchoServer:
+    """Simple nim echo server manager."""
+
+    def __init__(self, binary_path: Path):
+        self.binary_path = binary_path
+        self.process: None | subprocess.Popen = None
+        self.peer_id = None
+        self.listen_addr = None
+
+    async def start(self):
+        """Start nim echo server and get connection info."""
+        logger.info(f"Starting nim echo server: {self.binary_path}")
+
+        self.process = subprocess.Popen(
+            [str(self.binary_path)],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            universal_newlines=True,
+            bufsize=1,
+        )
+
+        # Parse output for connection info
+        start_time = time.time()
+        while time.time() - start_time < SERVER_START_TIMEOUT:
+            if self.process and self.process.poll() and self.process.stdout:
+                output = self.process.stdout.read()
+                raise RuntimeError(f"Server exited early: {output}")
+
+            reader = self.process.stdout if self.process else None
+            if reader:
+                line = reader.readline().strip()
+                if not line:
+                    continue
+
+            logger.info(f"Server: {line}")
+
+            if line.startswith("Peer ID:"):
+                self.peer_id = line.split(":", 1)[1].strip()
+
+            elif "/quic-v1/p2p/" in line and self.peer_id:
+                if line.strip().startswith("/"):
+                    self.listen_addr = line.strip()
+                    logger.info(f"Server ready: {self.listen_addr}")
+                    return self.peer_id, self.listen_addr
+
+        await self.stop()
+        raise TimeoutError(f"Server failed to start within {SERVER_START_TIMEOUT}s")
+
+    async def stop(self):
+        """Stop the server."""
+        if self.process:
+            logger.info("Stopping nim echo server...")
+            try:
+                self.process.terminate()
+                self.process.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                self.process.kill()
+                self.process.wait()
+            self.process = None
+
+
+async def run_echo_test(server_addr: str, messages: list[str]):
+    """Test echo protocol against nim server with proper timeout handling."""
+    # Create py-libp2p QUIC client with shorter timeouts
+
+    host = new_host(
+        enable_quic=True,
+        key_pair=create_new_key_pair(),
+    )
+
+    listen_addr = multiaddr.Multiaddr("/ip4/0.0.0.0/udp/0/quic-v1")
+    responses = []
+
+    try:
+        async with host.run(listen_addrs=[listen_addr]):
+            logger.info(f"Connecting to nim server: {server_addr}")
+
+            # Connect to nim server
+            maddr = multiaddr.Multiaddr(server_addr)
+            info = info_from_p2p_addr(maddr)
+            await host.connect(info)
+
+            # Create stream
+            stream = await host.new_stream(info.peer_id, [PROTOCOL_ID])
+            logger.info("Stream created")
+
+            # Test each message
+            for i, message in enumerate(messages, 1):
+                logger.info(f"Testing message {i}: {message}")
+
+                # Send with varint length prefix
+                data = message.encode("utf-8")
+                prefixed_data = encode_varint_prefixed(data)
+                await stream.write(prefixed_data)
+
+                # Read response
+                response_data = await read_varint_prefixed_bytes(stream)
+                response = response_data.decode("utf-8")
+
+                logger.info(f"Got echo: {response}")
+                responses.append(response)
+
+                # Verify echo
+                assert message == response, (
+                    f"Echo failed: sent {message!r}, got {response!r}"
+                )
+
+            await stream.close()
+            logger.info("✅ All messages echoed correctly")
+
+    finally:
+        await host.close()
+
+    return responses
+
+
+@pytest.mark.trio
+@pytest.mark.timeout(TEST_TIMEOUT)
+async def test_basic_echo_interop(nim_server):
+    """Test basic echo functionality between py-libp2p and nim-libp2p."""
+    server, peer_id, listen_addr = nim_server
+
+    test_messages = [
+        "Hello from py-libp2p!",
+        "QUIC transport working",
+        "Echo test successful!",
+        "Unicode: Ñoël, 测试, Ψυχή",
+    ]
+
+    logger.info(f"Testing against nim server: {peer_id}")
+
+    # Run test with timeout
+    with trio.move_on_after(TEST_TIMEOUT - 2):  # Leave 2s buffer for cleanup
+        responses = await run_echo_test(listen_addr, test_messages)
+
+        # Verify all messages echoed correctly
+        assert len(responses) == len(test_messages)
+        for sent, received in zip(test_messages, responses):
+            assert sent == received
+
+        logger.info("✅ Basic echo interop test passed!")
+
+
+@pytest.mark.trio
+@pytest.mark.timeout(TEST_TIMEOUT)
+async def test_large_message_echo(nim_server):
+    """Test echo with larger messages."""
+    server, peer_id, listen_addr = nim_server
+
+    large_messages = [
+        "x" * 1024,
+        "y" * 5000,
+    ]
+
+    logger.info("Testing large message echo...")
+
+    # Run test with timeout
+    with trio.move_on_after(TEST_TIMEOUT - 2):  # Leave 2s buffer for cleanup
+        responses = await run_echo_test(listen_addr, large_messages)
+
+        assert len(responses) == len(large_messages)
+        for sent, received in zip(large_messages, responses):
+            assert sent == received
+
+        logger.info("✅ Large message echo test passed!")
+
+
+if __name__ == "__main__":
+    # Run tests directly
+    pytest.main([__file__, "-v", "--tb=short"])