fix: fix nim interop env setup file

.github/workflows/tox.yml

@@ -36,34 +36,48 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
-      - run: |
-          python -m pip install --upgrade pip
-          python -m pip install tox
-      - run: |
-          python -m tox run -r
 
-  windows:
-    runs-on: windows-latest
-    strategy:
-      matrix:
-        python-version: ["3.11", "3.12", "3.13"]
-        toxenv: [core, wheel]
-      fail-fast: false
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
+      # Add Nim installation for interop tests
+      - name: Install Nim for interop testing
+        if: matrix.toxenv == 'interop'
         run: |
+          echo "Installing Nim for nim-libp2p interop testing..."
+          curl -sSf https://nim-lang.org/choosenim/init.sh | sh -s -- -y --firstInstall
+          echo "$HOME/.nimble/bin" >> $GITHUB_PATH
+          echo "$HOME/.choosenim/toolchains/nim-stable/bin" >> $GITHUB_PATH
+
+      # Cache nimble packages - ADD THIS
+      - name: Cache nimble packages
+        if: matrix.toxenv == 'interop'
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.nimble
+            ~/.choosenim/toolchains/*/lib
+          key: ${{ runner.os }}-nimble-${{ hashFiles('**/nim_echo_server.nim') }}
+          restore-keys: |
+            ${{ runner.os }}-nimble-
+
+      - name: Build nim interop binaries
+        if: matrix.toxenv == 'interop'
+        run: |
+          export PATH="$HOME/.nimble/bin:$HOME/.choosenim/toolchains/nim-stable/bin:$PATH"
+          cd tests/interop/nim_libp2p
+          ./scripts/setup_nim_echo.sh
+
+      - run: |
           python -m pip install --upgrade pip
           python -m pip install tox
-      - name: Test with tox
-        shell: bash
+
+      - name: Run Tests or Generate Docs
         run: |
-          if [[ "${{ matrix.toxenv }}" == "wheel" ]]; then
-            python -m tox run -e windows-wheel
+          if [[ "${{ matrix.toxenv }}" == 'docs' ]]; then
+            export TOXENV=docs
           else
-            python -m tox run -e py311-${{ matrix.toxenv }}
+            export TOXENV=py${{ matrix.python }}-${{ matrix.toxenv }}
           fi
+          # Set PATH for nim commands during tox
+          if [[ "${{ matrix.toxenv }}" == 'interop' ]]; then
+            export PATH="$HOME/.nimble/bin:$HOME/.choosenim/toolchains/nim-stable/bin:$PATH"
+          fi
+          python -m tox run -r

pyproject.toml

@@ -78,6 +78,7 @@ dev = [
     "pytest>=7.0.0",
     "pytest-xdist>=2.4.0",
     "pytest-trio>=0.5.2",
+    "pytest-timeout>=2.4.0",
     "factory-boy>=2.12.0,<3.0.0",
     "ruff>=0.11.10",
     "pyrefly (>=0.17.1,<0.18.0)",
@@ -89,11 +90,12 @@ docs = [
     "tomli; python_version < '3.11'",
 ]
 test = [
+    "factory-boy>=2.12.0,<3.0.0",
     "p2pclient==0.2.0",
     "pytest>=7.0.0",
-    "pytest-xdist>=2.4.0",
+    "pytest-timeout>=2.4.0",
     "pytest-trio>=0.5.2",
-    "factory-boy>=2.12.0,<3.0.0",
+    "pytest-xdist>=2.4.0",
 ]
 
 [tool.setuptools]

tests/interop/nim_libp2p/conftest.py

@@ -0,0 +1,119 @@
+import fcntl
+import logging
+from pathlib import Path
+import shutil
+import subprocess
+import time
+
+import pytest
+
+logger = logging.getLogger(__name__)
+
+
+def check_nim_available():
+    """Check if nim compiler is available."""
+    return shutil.which("nim") is not None and shutil.which("nimble") is not None
+
+
+def check_nim_binary_built():
+    """Check if nim echo server binary is built."""
+    current_dir = Path(__file__).parent
+    binary_path = current_dir / "nim_echo_server"
+    return binary_path.exists() and binary_path.stat().st_size > 0
+
+
+def run_nim_setup_with_lock():
+    """Run nim setup with file locking to prevent parallel execution."""
+    current_dir = Path(__file__).parent
+    lock_file = current_dir / ".setup_lock"
+    setup_script = current_dir / "scripts" / "setup_nim_echo.sh"
+
+    if not setup_script.exists():
+        raise RuntimeError(f"Setup script not found: {setup_script}")
+
+    # Try to acquire lock
+    try:
+        with open(lock_file, "w") as f:
+            # Non-blocking lock attempt
+            fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+
+            # Double-check binary doesn't exist (another worker might have built it)
+            if check_nim_binary_built():
+                logger.info("Binary already exists, skipping setup")
+                return
+
+            logger.info("Acquired setup lock, running nim-libp2p setup...")
+
+            # Make setup script executable and run it
+            setup_script.chmod(0o755)
+            result = subprocess.run(
+                [str(setup_script)],
+                cwd=current_dir,
+                capture_output=True,
+                text=True,
+                timeout=300,  # 5 minute timeout
+            )
+
+            if result.returncode != 0:
+                raise RuntimeError(
+                    f"Setup failed (exit {result.returncode}):\n"
+                    f"stdout: {result.stdout}\n"
+                    f"stderr: {result.stderr}"
+                )
+
+            # Verify binary was built
+            if not check_nim_binary_built():
+                raise RuntimeError("nim_echo_server binary not found after setup")
+
+            logger.info("nim-libp2p setup completed successfully")
+
+    except BlockingIOError:
+        # Another worker is running setup, wait for it to complete
+        logger.info("Another worker is running setup, waiting...")
+
+        # Wait for setup to complete (check every 2 seconds, max 5 minutes)
+        for _ in range(150):  # 150 * 2 = 300 seconds = 5 minutes
+            if check_nim_binary_built():
+                logger.info("Setup completed by another worker")
+                return
+            time.sleep(2)
+
+        raise TimeoutError("Timed out waiting for setup to complete")
+
+    finally:
+        # Clean up lock file
+        try:
+            lock_file.unlink(missing_ok=True)
+        except Exception:
+            pass
+
+
+@pytest.fixture(scope="function")  # Changed to function scope
+def nim_echo_binary():
+    """Get nim echo server binary path."""
+    current_dir = Path(__file__).parent
+    binary_path = current_dir / "nim_echo_server"
+
+    if not binary_path.exists():
+        pytest.skip(
+            "nim_echo_server binary not found. "
+            "Run setup script: ./scripts/setup_nim_echo.sh"
+        )
+
+    return binary_path
+
+
+@pytest.fixture
+async def nim_server(nim_echo_binary):
+    """Start and stop nim echo server for tests."""
+    # Import here to avoid circular imports
+    # pyrefly: ignore
+    from test_echo_interop import NimEchoServer
+
+    server = NimEchoServer(nim_echo_binary)
+
+    try:
+        peer_id, listen_addr = await server.start()
+        yield server, peer_id, listen_addr
+    finally:
+        await server.stop()

tests/interop/nim_libp2p/scripts/setup_nim_echo.sh

@@ -1,8 +1,12 @@
 #!/usr/bin/env bash
-# Simple setup script for nim echo server interop testing
+# tests/interop/nim_libp2p/scripts/setup_nim_echo.sh
+# Cache-aware setup that skips installation if packages exist
 
 set -euo pipefail
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_DIR="${SCRIPT_DIR}/.."
+
 # Colors
 GREEN='\033[0;32m'
 RED='\033[0;31m'
@@ -13,86 +17,58 @@ log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
 log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
 log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
 
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-PROJECT_ROOT="${SCRIPT_DIR}/.."
-NIM_LIBP2P_DIR="${PROJECT_ROOT}/nim-libp2p"
+main() {
+    log_info "Setting up nim echo server for interop testing..."
 
-# Check prerequisites
-check_nim() {
-    if ! command -v nim &> /dev/null; then
-        log_error "Nim not found. Install with: curl -sSf https://nim-lang.org/choosenim/init.sh | sh"
+    # Check if nim is available
+    if ! command -v nim &> /dev/null || ! command -v nimble &> /dev/null; then
+        log_error "Nim not found. Please install nim first."
         exit 1
     fi
-    if ! command -v nimble &> /dev/null; then
-        log_error "Nimble not found. Please install Nim properly."
-        exit 1
-    fi
-}
 
-# Setup nim-libp2p dependency
-setup_nim_libp2p() {
-    log_info "Setting up nim-libp2p dependency..."
+    cd "${PROJECT_DIR}"
 
-    if [ ! -d "${NIM_LIBP2P_DIR}" ]; then
-        log_info "Cloning nim-libp2p..."
-        git clone https://github.com/status-im/nim-libp2p.git "${NIM_LIBP2P_DIR}"
+    # Create logs directory
+    mkdir -p logs
+
+    # Check if binary already exists
+    if [[ -f "nim_echo_server" ]]; then
+        log_info "nim_echo_server already exists, skipping build"
+        return 0
     fi
 
-    cd "${NIM_LIBP2P_DIR}"
-    log_info "Installing nim-libp2p dependencies..."
-    nimble install -y --depsOnly
-}
+    # Check if libp2p is already installed (cache-aware)
+    if nimble list -i | grep -q "libp2p"; then
+        log_info "libp2p already installed, skipping installation"
+    else
+        log_info "Installing nim-libp2p globally..."
+        nimble install -y libp2p
+    fi
 
-# Build nim echo server
-build_echo_server() {
     log_info "Building nim echo server..."
+    # Compile the echo server
+    nim c \
+        -d:release \
+        -d:chronicles_log_level=INFO \
+        -d:libp2p_quic_support \
+        -d:chronos_event_loop=iocp \
+        -d:ssl \
+        --opt:speed \
+        --mm:orc \
+        --verbosity:1 \
+        -o:nim_echo_server \
+        nim_echo_server.nim
 
-    cd "${PROJECT_ROOT}"
-
-    # Create nimble file if it doesn't exist
-    cat > nim_echo_test.nimble << 'EOF'
-# Package
-version = "0.1.0"
-author = "py-libp2p interop"
-description = "nim echo server for interop testing"
-license = "MIT"
-
-# Dependencies
-requires "nim >= 1.6.0"
-requires "libp2p"
-requires "chronos"
-requires "stew"
-
-# Binary
-bin = @["nim_echo_server"]
-EOF
-
-    # Build the server
-    log_info "Compiling nim echo server..."
-    nim c -d:release -d:chronicles_log_level=INFO -d:libp2p_quic_support --opt:speed --gc:orc -o:nim_echo_server nim_echo_server.nim
-
-    if [ -f "nim_echo_server" ]; then
+    # Verify binary was created
+    if [[ -f "nim_echo_server" ]]; then
         log_info "✅ nim_echo_server built successfully"
+        log_info "Binary size: $(ls -lh nim_echo_server | awk '{print $5}')"
     else
         log_error "❌ Failed to build nim_echo_server"
         exit 1
     fi
-}
 
-main() {
-    log_info "Setting up nim echo server for interop testing..."
-
-    # Create logs directory
-    mkdir -p "${PROJECT_ROOT}/logs"
-
-    # Clean up any existing processes
-    pkill -f "nim_echo_server" || true
-
-    check_nim
-    setup_nim_libp2p
-    build_echo_server
-
-    log_info "🎉 Setup complete! You can now run: python -m pytest test_echo_interop.py -v"
+    log_info "🎉 Setup complete!"
 }
 
 main "$@"

tests/interop/nim_libp2p/test_echo_interop.py

@@ -1,14 +1,6 @@
-#!/usr/bin/env python3
-"""
-Simple echo protocol interop test between py-libp2p and nim-libp2p.
-
-Tests that py-libp2p QUIC clients can communicate with nim-libp2p echo servers.
-"""
-
 import logging
 from pathlib import Path
 import subprocess
-from subprocess import Popen
 import time
 
 import pytest
@@ -24,7 +16,7 @@ from libp2p.utils.varint import encode_varint_prefixed, read_varint_prefixed_byt
 
 # Configuration
 PROTOCOL_ID = TProtocol("/echo/1.0.0")
-TEST_TIMEOUT = 15.0  # Reduced timeout
+TEST_TIMEOUT = 30
 SERVER_START_TIMEOUT = 10.0
 
 # Setup logging
@@ -37,7 +29,7 @@ class NimEchoServer:
 
     def __init__(self, binary_path: Path):
         self.binary_path = binary_path
-        self.process: None | Popen = None
+        self.process: None | subprocess.Popen = None
         self.peer_id = None
         self.listen_addr = None
 
@@ -45,31 +37,24 @@ class NimEchoServer:
         """Start nim echo server and get connection info."""
         logger.info(f"Starting nim echo server: {self.binary_path}")
 
-        self.process: Popen[str] = subprocess.Popen(
+        self.process = subprocess.Popen(
             [str(self.binary_path)],
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
-            text=True,
+            universal_newlines=True,
             bufsize=1,
         )
 
-        if self.process is None:
-            return None, None
-
         # Parse output for connection info
         start_time = time.time()
-        while (
-            self.process is not None and time.time() - start_time < SERVER_START_TIMEOUT
-        ):
-            if self.process.poll() is not None:
-                IOout = self.process.stdout
-                if IOout:
-                    output = IOout.read()
-                    raise RuntimeError(f"Server exited early: {output}")
+        while time.time() - start_time < SERVER_START_TIMEOUT:
+            if self.process and self.process.poll() and self.process.stdout:
+                output = self.process.stdout.read()
+                raise RuntimeError(f"Server exited early: {output}")
 
-            IOin = self.process.stdout
-            if IOin:
-                line = IOin.readline().strip()
+            reader = self.process.stdout if self.process else None
+            if reader:
+                line = reader.readline().strip()
                 if not line:
                     continue
 
@@ -147,8 +132,6 @@ async def run_echo_test(server_addr: str, messages: list[str]):
                 logger.info(f"Got echo: {response}")
                 responses.append(response)
 
-                assert False, "FORCED FAILURE"
-
                 # Verify echo
                 assert message == response, (
                     f"Echo failed: sent {message!r}, got {response!r}"
@@ -163,33 +146,8 @@ async def run_echo_test(server_addr: str, messages: list[str]):
     return responses
 
 
-@pytest.fixture
-def nim_echo_binary():
-    """Path to nim echo server binary."""
-    current_dir = Path(__file__).parent
-    binary_path = current_dir / "nim_echo_server"
-
-    if not binary_path.exists():
-        pytest.skip(
-            f"Nim echo server not found at {binary_path}. Run setup script first."
-        )
-
-    return binary_path
-
-
-@pytest.fixture
-async def nim_server(nim_echo_binary):
-    """Start and stop nim echo server for tests."""
-    server = NimEchoServer(nim_echo_binary)
-
-    try:
-        peer_id, listen_addr = await server.start()
-        yield server, peer_id, listen_addr
-    finally:
-        await server.stop()
-
-
 @pytest.mark.trio
+@pytest.mark.timeout(TEST_TIMEOUT)
 async def test_basic_echo_interop(nim_server):
     """Test basic echo functionality between py-libp2p and nim-libp2p."""
     server, peer_id, listen_addr = nim_server
@@ -216,13 +174,14 @@ async def test_basic_echo_interop(nim_server):
 
 
 @pytest.mark.trio
+@pytest.mark.timeout(TEST_TIMEOUT)
 async def test_large_message_echo(nim_server):
     """Test echo with larger messages."""
     server, peer_id, listen_addr = nim_server
 
     large_messages = [
-        "x" * 1024,  # 1KB
-        "y" * 10000,
+        "x" * 1024,
+        "y" * 5000,
     ]
 
     logger.info("Testing large message echo...")