From e1141ee376647c7f63685ebd89e281937a06b0e8 Mon Sep 17 00:00:00 2001 From: Akash Mondal Date: Sun, 31 Aug 2025 06:47:15 +0000 Subject: [PATCH] fix: fix nim interop env setup file --- .github/workflows/tox.yml | 62 +++++---- pyproject.toml | 6 +- tests/interop/nim_libp2p/conftest.py | 119 ++++++++++++++++++ .../nim_libp2p/scripts/setup_nim_echo.sh | 106 ++++++---------- tests/interop/nim_libp2p/test_echo_interop.py | 71 +++-------- 5 files changed, 217 insertions(+), 147 deletions(-) create mode 100644 tests/interop/nim_libp2p/conftest.py diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index ef963f80..e90c3688 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -36,34 +36,48 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - - run: | - python -m pip install --upgrade pip - python -m pip install tox - - run: | - python -m tox run -r - windows: - runs-on: windows-latest - strategy: - matrix: - python-version: ["3.11", "3.12", "3.13"] - toxenv: [core, wheel] - fail-fast: false - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies + # Add Nim installation for interop tests + - name: Install Nim for interop testing + if: matrix.toxenv == 'interop' run: | + echo "Installing Nim for nim-libp2p interop testing..." + curl -sSf https://nim-lang.org/choosenim/init.sh | sh -s -- -y --firstInstall + echo "$HOME/.nimble/bin" >> $GITHUB_PATH + echo "$HOME/.choosenim/toolchains/nim-stable/bin" >> $GITHUB_PATH + + # Cache nimble packages - ADD THIS + - name: Cache nimble packages + if: matrix.toxenv == 'interop' + uses: actions/cache@v4 + with: + path: | + ~/.nimble + ~/.choosenim/toolchains/*/lib + key: ${{ runner.os }}-nimble-${{ hashFiles('**/nim_echo_server.nim') }} + restore-keys: | + ${{ runner.os }}-nimble- + + - name: Build nim interop binaries + if: matrix.toxenv == 'interop' + run: | + export PATH="$HOME/.nimble/bin:$HOME/.choosenim/toolchains/nim-stable/bin:$PATH" + cd tests/interop/nim_libp2p + ./scripts/setup_nim_echo.sh + + - run: | python -m pip install --upgrade pip python -m pip install tox - - name: Test with tox - shell: bash + + - name: Run Tests or Generate Docs run: | - if [[ "${{ matrix.toxenv }}" == "wheel" ]]; then - python -m tox run -e windows-wheel + if [[ "${{ matrix.toxenv }}" == 'docs' ]]; then + export TOXENV=docs else - python -m tox run -e py311-${{ matrix.toxenv }} + export TOXENV=py${{ matrix.python }}-${{ matrix.toxenv }} fi + # Set PATH for nim commands during tox + if [[ "${{ matrix.toxenv }}" == 'interop' ]]; then + export PATH="$HOME/.nimble/bin:$HOME/.choosenim/toolchains/nim-stable/bin:$PATH" + fi + python -m tox run -r diff --git a/pyproject.toml b/pyproject.toml index f97edbb1..8af0f5a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ dev = [ "pytest>=7.0.0", "pytest-xdist>=2.4.0", "pytest-trio>=0.5.2", + "pytest-timeout>=2.4.0", "factory-boy>=2.12.0,<3.0.0", "ruff>=0.11.10", "pyrefly (>=0.17.1,<0.18.0)", @@ -89,11 +90,12 @@ docs = [ "tomli; python_version < '3.11'", ] test = [ + "factory-boy>=2.12.0,<3.0.0", "p2pclient==0.2.0", "pytest>=7.0.0", - "pytest-xdist>=2.4.0", + "pytest-timeout>=2.4.0", "pytest-trio>=0.5.2", - "factory-boy>=2.12.0,<3.0.0", + "pytest-xdist>=2.4.0", ] [tool.setuptools] diff --git a/tests/interop/nim_libp2p/conftest.py b/tests/interop/nim_libp2p/conftest.py new file mode 100644 index 00000000..5765a09d --- /dev/null +++ b/tests/interop/nim_libp2p/conftest.py @@ -0,0 +1,119 @@ +import fcntl +import logging +from pathlib import Path +import shutil +import subprocess +import time + +import pytest + +logger = logging.getLogger(__name__) + + +def check_nim_available(): + """Check if nim compiler is available.""" + return shutil.which("nim") is not None and shutil.which("nimble") is not None + + +def check_nim_binary_built(): + """Check if nim echo server binary is built.""" + current_dir = Path(__file__).parent + binary_path = current_dir / "nim_echo_server" + return binary_path.exists() and binary_path.stat().st_size > 0 + + +def run_nim_setup_with_lock(): + """Run nim setup with file locking to prevent parallel execution.""" + current_dir = Path(__file__).parent + lock_file = current_dir / ".setup_lock" + setup_script = current_dir / "scripts" / "setup_nim_echo.sh" + + if not setup_script.exists(): + raise RuntimeError(f"Setup script not found: {setup_script}") + + # Try to acquire lock + try: + with open(lock_file, "w") as f: + # Non-blocking lock attempt + fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + + # Double-check binary doesn't exist (another worker might have built it) + if check_nim_binary_built(): + logger.info("Binary already exists, skipping setup") + return + + logger.info("Acquired setup lock, running nim-libp2p setup...") + + # Make setup script executable and run it + setup_script.chmod(0o755) + result = subprocess.run( + [str(setup_script)], + cwd=current_dir, + capture_output=True, + text=True, + timeout=300, # 5 minute timeout + ) + + if result.returncode != 0: + raise RuntimeError( + f"Setup failed (exit {result.returncode}):\n" + f"stdout: {result.stdout}\n" + f"stderr: {result.stderr}" + ) + + # Verify binary was built + if not check_nim_binary_built(): + raise RuntimeError("nim_echo_server binary not found after setup") + + logger.info("nim-libp2p setup completed successfully") + + except BlockingIOError: + # Another worker is running setup, wait for it to complete + logger.info("Another worker is running setup, waiting...") + + # Wait for setup to complete (check every 2 seconds, max 5 minutes) + for _ in range(150): # 150 * 2 = 300 seconds = 5 minutes + if check_nim_binary_built(): + logger.info("Setup completed by another worker") + return + time.sleep(2) + + raise TimeoutError("Timed out waiting for setup to complete") + + finally: + # Clean up lock file + try: + lock_file.unlink(missing_ok=True) + except Exception: + pass + + +@pytest.fixture(scope="function") # Changed to function scope +def nim_echo_binary(): + """Get nim echo server binary path.""" + current_dir = Path(__file__).parent + binary_path = current_dir / "nim_echo_server" + + if not binary_path.exists(): + pytest.skip( + "nim_echo_server binary not found. " + "Run setup script: ./scripts/setup_nim_echo.sh" + ) + + return binary_path + + +@pytest.fixture +async def nim_server(nim_echo_binary): + """Start and stop nim echo server for tests.""" + # Import here to avoid circular imports + # pyrefly: ignore + from test_echo_interop import NimEchoServer + + server = NimEchoServer(nim_echo_binary) + + try: + peer_id, listen_addr = await server.start() + yield server, peer_id, listen_addr + finally: + await server.stop() diff --git a/tests/interop/nim_libp2p/scripts/setup_nim_echo.sh b/tests/interop/nim_libp2p/scripts/setup_nim_echo.sh index bf8aa307..f80b2d27 100755 --- a/tests/interop/nim_libp2p/scripts/setup_nim_echo.sh +++ b/tests/interop/nim_libp2p/scripts/setup_nim_echo.sh @@ -1,8 +1,12 @@ #!/usr/bin/env bash -# Simple setup script for nim echo server interop testing +# tests/interop/nim_libp2p/scripts/setup_nim_echo.sh +# Cache-aware setup that skips installation if packages exist set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="${SCRIPT_DIR}/.." + # Colors GREEN='\033[0;32m' RED='\033[0;31m' @@ -13,86 +17,58 @@ log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } log_error() { echo -e "${RED}[ERROR]${NC} $1"; } -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="${SCRIPT_DIR}/.." -NIM_LIBP2P_DIR="${PROJECT_ROOT}/nim-libp2p" +main() { + log_info "Setting up nim echo server for interop testing..." -# Check prerequisites -check_nim() { - if ! command -v nim &> /dev/null; then - log_error "Nim not found. Install with: curl -sSf https://nim-lang.org/choosenim/init.sh | sh" + # Check if nim is available + if ! command -v nim &> /dev/null || ! command -v nimble &> /dev/null; then + log_error "Nim not found. Please install nim first." exit 1 fi - if ! command -v nimble &> /dev/null; then - log_error "Nimble not found. Please install Nim properly." - exit 1 - fi -} -# Setup nim-libp2p dependency -setup_nim_libp2p() { - log_info "Setting up nim-libp2p dependency..." + cd "${PROJECT_DIR}" - if [ ! -d "${NIM_LIBP2P_DIR}" ]; then - log_info "Cloning nim-libp2p..." - git clone https://github.com/status-im/nim-libp2p.git "${NIM_LIBP2P_DIR}" + # Create logs directory + mkdir -p logs + + # Check if binary already exists + if [[ -f "nim_echo_server" ]]; then + log_info "nim_echo_server already exists, skipping build" + return 0 fi - cd "${NIM_LIBP2P_DIR}" - log_info "Installing nim-libp2p dependencies..." - nimble install -y --depsOnly -} + # Check if libp2p is already installed (cache-aware) + if nimble list -i | grep -q "libp2p"; then + log_info "libp2p already installed, skipping installation" + else + log_info "Installing nim-libp2p globally..." + nimble install -y libp2p + fi -# Build nim echo server -build_echo_server() { log_info "Building nim echo server..." + # Compile the echo server + nim c \ + -d:release \ + -d:chronicles_log_level=INFO \ + -d:libp2p_quic_support \ + -d:chronos_event_loop=iocp \ + -d:ssl \ + --opt:speed \ + --mm:orc \ + --verbosity:1 \ + -o:nim_echo_server \ + nim_echo_server.nim - cd "${PROJECT_ROOT}" - - # Create nimble file if it doesn't exist - cat > nim_echo_test.nimble << 'EOF' -# Package -version = "0.1.0" -author = "py-libp2p interop" -description = "nim echo server for interop testing" -license = "MIT" - -# Dependencies -requires "nim >= 1.6.0" -requires "libp2p" -requires "chronos" -requires "stew" - -# Binary -bin = @["nim_echo_server"] -EOF - - # Build the server - log_info "Compiling nim echo server..." - nim c -d:release -d:chronicles_log_level=INFO -d:libp2p_quic_support --opt:speed --gc:orc -o:nim_echo_server nim_echo_server.nim - - if [ -f "nim_echo_server" ]; then + # Verify binary was created + if [[ -f "nim_echo_server" ]]; then log_info "✅ nim_echo_server built successfully" + log_info "Binary size: $(ls -lh nim_echo_server | awk '{print $5}')" else log_error "❌ Failed to build nim_echo_server" exit 1 fi -} -main() { - log_info "Setting up nim echo server for interop testing..." - - # Create logs directory - mkdir -p "${PROJECT_ROOT}/logs" - - # Clean up any existing processes - pkill -f "nim_echo_server" || true - - check_nim - setup_nim_libp2p - build_echo_server - - log_info "🎉 Setup complete! You can now run: python -m pytest test_echo_interop.py -v" + log_info "🎉 Setup complete!" } main "$@" diff --git a/tests/interop/nim_libp2p/test_echo_interop.py b/tests/interop/nim_libp2p/test_echo_interop.py index 45a87a18..ce03d939 100644 --- a/tests/interop/nim_libp2p/test_echo_interop.py +++ b/tests/interop/nim_libp2p/test_echo_interop.py @@ -1,14 +1,6 @@ -#!/usr/bin/env python3 -""" -Simple echo protocol interop test between py-libp2p and nim-libp2p. - -Tests that py-libp2p QUIC clients can communicate with nim-libp2p echo servers. -""" - import logging from pathlib import Path import subprocess -from subprocess import Popen import time import pytest @@ -24,7 +16,7 @@ from libp2p.utils.varint import encode_varint_prefixed, read_varint_prefixed_byt # Configuration PROTOCOL_ID = TProtocol("/echo/1.0.0") -TEST_TIMEOUT = 15.0 # Reduced timeout +TEST_TIMEOUT = 30 SERVER_START_TIMEOUT = 10.0 # Setup logging @@ -37,7 +29,7 @@ class NimEchoServer: def __init__(self, binary_path: Path): self.binary_path = binary_path - self.process: None | Popen = None + self.process: None | subprocess.Popen = None self.peer_id = None self.listen_addr = None @@ -45,31 +37,24 @@ class NimEchoServer: """Start nim echo server and get connection info.""" logger.info(f"Starting nim echo server: {self.binary_path}") - self.process: Popen[str] = subprocess.Popen( + self.process = subprocess.Popen( [str(self.binary_path)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - text=True, + universal_newlines=True, bufsize=1, ) - if self.process is None: - return None, None - # Parse output for connection info start_time = time.time() - while ( - self.process is not None and time.time() - start_time < SERVER_START_TIMEOUT - ): - if self.process.poll() is not None: - IOout = self.process.stdout - if IOout: - output = IOout.read() - raise RuntimeError(f"Server exited early: {output}") + while time.time() - start_time < SERVER_START_TIMEOUT: + if self.process and self.process.poll() and self.process.stdout: + output = self.process.stdout.read() + raise RuntimeError(f"Server exited early: {output}") - IOin = self.process.stdout - if IOin: - line = IOin.readline().strip() + reader = self.process.stdout if self.process else None + if reader: + line = reader.readline().strip() if not line: continue @@ -147,8 +132,6 @@ async def run_echo_test(server_addr: str, messages: list[str]): logger.info(f"Got echo: {response}") responses.append(response) - assert False, "FORCED FAILURE" - # Verify echo assert message == response, ( f"Echo failed: sent {message!r}, got {response!r}" @@ -163,33 +146,8 @@ async def run_echo_test(server_addr: str, messages: list[str]): return responses -@pytest.fixture -def nim_echo_binary(): - """Path to nim echo server binary.""" - current_dir = Path(__file__).parent - binary_path = current_dir / "nim_echo_server" - - if not binary_path.exists(): - pytest.skip( - f"Nim echo server not found at {binary_path}. Run setup script first." - ) - - return binary_path - - -@pytest.fixture -async def nim_server(nim_echo_binary): - """Start and stop nim echo server for tests.""" - server = NimEchoServer(nim_echo_binary) - - try: - peer_id, listen_addr = await server.start() - yield server, peer_id, listen_addr - finally: - await server.stop() - - @pytest.mark.trio +@pytest.mark.timeout(TEST_TIMEOUT) async def test_basic_echo_interop(nim_server): """Test basic echo functionality between py-libp2p and nim-libp2p.""" server, peer_id, listen_addr = nim_server @@ -216,13 +174,14 @@ async def test_basic_echo_interop(nim_server): @pytest.mark.trio +@pytest.mark.timeout(TEST_TIMEOUT) async def test_large_message_echo(nim_server): """Test echo with larger messages.""" server, peer_id, listen_addr = nim_server large_messages = [ - "x" * 1024, # 1KB - "y" * 10000, + "x" * 1024, + "y" * 5000, ] logger.info("Testing large message echo...")