Feat/issue 605 debug logging via env variable (#608)

* feat: Debug Logging via Environment Variable

* refactor: deleted libp2p/utils.py

* fix: double messages logging fix

* doc: add logging info to getting_started.rst

libp2p/utils/__init__.py

@@ -0,0 +1,23 @@
+"""Utility functions for libp2p."""
+
+from libp2p.utils.varint import (
+    decode_uvarint_from_stream,
+    encode_delim,
+    encode_uvarint,
+    encode_varint_prefixed,
+    read_delim,
+    read_varint_prefixed_bytes,
+)
+from libp2p.utils.version import (
+    get_agent_version,
+)
+
+__all__ = [
+    "decode_uvarint_from_stream",
+    "encode_delim",
+    "encode_uvarint",
+    "encode_varint_prefixed",
+    "get_agent_version",
+    "read_delim",
+    "read_varint_prefixed_bytes",
+]

libp2p/utils/logging.py

@@ -0,0 +1,217 @@
+import atexit
+from datetime import (
+    datetime,
+)
+import logging
+import logging.handlers
+import os
+from pathlib import (
+    Path,
+)
+import queue
+import sys
+import threading
+from typing import (
+    Any,
+    Optional,
+    Union,
+)
+
+# Create a log queue
+log_queue: "queue.Queue[Any]" = queue.Queue()
+
+# Store the current listener to stop it on exit
+_current_listener: Optional[logging.handlers.QueueListener] = None
+
+# Event to track when the listener is ready
+_listener_ready = threading.Event()
+
+# Default format for log messages
+DEFAULT_LOG_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+
+
+def _parse_debug_modules(debug_str: str) -> dict[str, int]:
+    """
+    Parse the LIBP2P_DEBUG environment variable to determine module-specific log levels.
+
+    Format examples:
+    - "DEBUG"  # All modules at DEBUG level
+    - "libp2p.identity.identify:DEBUG"  # Only identify module at DEBUG
+    - "identity.identify:DEBUG"  # Same as above, libp2p prefix is optional
+    - "libp2p.identity:DEBUG,libp2p.transport:INFO"  # Multiple modules
+    """
+    module_levels: dict[str, int] = {}
+
+    # Handle empty or whitespace-only string
+    if not debug_str or debug_str.isspace():
+        return module_levels
+
+    # If it's a plain log level without any colons, apply to all
+    if ":" not in debug_str and debug_str.upper() in logging._nameToLevel:
+        return {"": getattr(logging, debug_str.upper())}
+
+    # Handle module-specific levels
+    for part in debug_str.split(","):
+        if ":" not in part:
+            continue
+
+        module, level = part.split(":")
+        level = level.upper()
+
+        if level not in logging._nameToLevel:
+            continue
+
+        # Handle module name
+        module = module.strip()
+        # Remove libp2p prefix if present, it will be added back when creating logger
+        module = module.replace("libp2p.", "")
+        # Convert any remaining dots to ensure proper format
+        module = module.replace("/", ".").strip(".")
+
+        module_levels[module] = getattr(logging, level)
+
+    return module_levels
+
+
+def setup_logging() -> None:
+    """
+    Set up logging configuration based on environment variables.
+
+    Environment Variables:
+        LIBP2P_DEBUG
+            Controls logging levels. Examples:
+            - "DEBUG" (all modules at DEBUG level)
+            - "libp2p.identity.identify:DEBUG" (only identify module at DEBUG)
+            - "identity.identify:DEBUG" (same as above, libp2p prefix optional)
+            - "libp2p.identity:DEBUG,libp2p.transport:INFO" (multiple modules)
+
+        LIBP2P_DEBUG_FILE
+            If set, specifies the file path for log output. When this variable is set,
+            logs will only be written to the specified file. If not set, logs will be
+            written to both a default file (in the system's temp directory) and to
+            stderr (console output).
+
+    The logging system uses Python's native hierarchical logging:
+        - Loggers are organized in a hierarchy using dots
+          (e.g., libp2p.identity.identify)
+        - Child loggers inherit their parent's level unless explicitly set
+        - The root libp2p logger controls the default level
+    """
+    global _current_listener, _listener_ready
+
+    # Reset the event
+    _listener_ready.clear()
+
+    # Stop existing listener if any
+    if _current_listener is not None:
+        _current_listener.stop()
+        _current_listener = None
+
+    # Get the log level from environment variable
+    debug_str = os.environ.get("LIBP2P_DEBUG", "")
+
+    if not debug_str:
+        # If LIBP2P_DEBUG is not set, disable logging
+        root_logger = logging.getLogger("libp2p")
+        root_logger.handlers.clear()
+        root_logger.setLevel(logging.WARNING)
+        root_logger.propagate = False
+        _listener_ready.set()  # Signal that we're done
+        return
+
+    # Parse module-specific levels
+    module_levels = _parse_debug_modules(debug_str)
+
+    # If no valid levels specified, default to WARNING
+    if not module_levels:
+        root_logger = logging.getLogger("libp2p")
+        root_logger.handlers.clear()
+        root_logger.setLevel(logging.WARNING)
+        root_logger.propagate = False
+        _listener_ready.set()  # Signal that we're done
+        return
+
+    # Create formatter
+    formatter = logging.Formatter(DEFAULT_LOG_FORMAT)
+
+    # Configure handlers
+    handlers: list[Union[logging.StreamHandler[Any], logging.FileHandler]] = []
+
+    # Console handler
+    console_handler = logging.StreamHandler(sys.stderr)
+    console_handler.setFormatter(formatter)
+    handlers.append(console_handler)
+
+    # File handler (if configured)
+    log_file = os.environ.get("LIBP2P_DEBUG_FILE")
+
+    if log_file:
+        # Ensure the directory exists
+        log_path = Path(log_file)
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+    else:
+        # Default log file with timestamp and unique identifier
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
+        unique_id = os.urandom(4).hex()  # Add a unique identifier to prevent collisions
+        if os.name == "nt":  # Windows
+            log_file = f"C:\\Windows\\Temp\\py-libp2p_{timestamp}_{unique_id}.log"
+        else:  # Unix-like
+            log_file = f"/tmp/py-libp2p_{timestamp}_{unique_id}.log"
+
+        # Print the log file path so users know where to find it
+        print(f"Logging to: {log_file}", file=sys.stderr)
+
+    try:
+        file_handler = logging.FileHandler(
+            log_file, mode="w"
+        )  # Use 'w' mode to clear file
+        file_handler.setFormatter(formatter)
+        handlers.append(file_handler)
+    except Exception as e:
+        print(f"Error creating file handler: {e}", file=sys.stderr)
+        raise
+
+    # Create a QueueHandler and QueueListener
+    queue_handler = logging.handlers.QueueHandler(log_queue)
+
+    # Configure root logger for libp2p
+    root_logger = logging.getLogger("libp2p")
+    root_logger.handlers.clear()
+    root_logger.addHandler(queue_handler)
+    root_logger.propagate = False
+
+    # Set default level based on configuration
+    if "" in module_levels:
+        # Global level specified
+        root_logger.setLevel(module_levels[""])
+    else:
+        # Default to INFO for module-specific logging
+        root_logger.setLevel(logging.INFO)
+
+    # Configure module-specific levels
+    for module, level in module_levels.items():
+        if module:  # Skip the default "" key
+            logger = logging.getLogger(f"libp2p.{module}")
+            logger.handlers.clear()
+            logger.addHandler(queue_handler)
+            logger.setLevel(level)
+            logger.propagate = False  # Prevent message duplication
+
+    # Start the listener AFTER configuring all loggers
+    _current_listener = logging.handlers.QueueListener(
+        log_queue, *handlers, respect_handler_level=True
+    )
+    _current_listener.start()
+
+    # Signal that the listener is ready
+    _listener_ready.set()
+
+
+# Register cleanup function
+@atexit.register
+def cleanup_logging() -> None:
+    """Clean up logging resources on exit."""
+    global _current_listener
+    if _current_listener is not None:
+        _current_listener.stop()
+        _current_listener = None

libp2p/utils/varint.py

@@ -0,0 +1,87 @@
+import itertools
+import logging
+import math
+
+from libp2p.exceptions import (
+    ParseError,
+)
+from libp2p.io.abc import (
+    Reader,
+)
+from libp2p.io.utils import (
+    read_exactly,
+)
+
+logger = logging.getLogger("libp2p.utils.varint")
+
+# Unsigned LEB128(varint codec)
+# Reference: https://github.com/ethereum/py-wasm/blob/master/wasm/parsers/leb128.py
+
+LOW_MASK = 2**7 - 1
+HIGH_MASK = 2**7
+
+# The maximum shift width for a 64 bit integer.  We shouldn't have to decode
+# integers larger than this.
+SHIFT_64_BIT_MAX = int(math.ceil(64 / 7)) * 7
+
+
+def encode_uvarint(number: int) -> bytes:
+    """Pack `number` into varint bytes."""
+    buf = b""
+    while True:
+        towrite = number & 0x7F
+        number >>= 7
+        if number:
+            buf += bytes((towrite | 0x80,))
+        else:
+            buf += bytes((towrite,))
+            break
+    return buf
+
+
+async def decode_uvarint_from_stream(reader: Reader) -> int:
+    """https://en.wikipedia.org/wiki/LEB128."""
+    res = 0
+    for shift in itertools.count(0, 7):
+        if shift > SHIFT_64_BIT_MAX:
+            raise ParseError("TODO: better exception msg: Integer is too large...")
+
+        byte = await read_exactly(reader, 1)
+        value = byte[0]
+
+        res += (value & LOW_MASK) << shift
+
+        if not value & HIGH_MASK:
+            break
+    return res
+
+
+def encode_varint_prefixed(msg_bytes: bytes) -> bytes:
+    varint_len = encode_uvarint(len(msg_bytes))
+    return varint_len + msg_bytes
+
+
+async def read_varint_prefixed_bytes(reader: Reader) -> bytes:
+    len_msg = await decode_uvarint_from_stream(reader)
+    data = await read_exactly(reader, len_msg)
+    return data
+
+
+# Delimited read/write, used by multistream-select.
+# Reference: https://github.com/gogo/protobuf/blob/07eab6a8298cf32fac45cceaac59424f98421bbc/io/varint.go#L109-L126  # noqa: E501
+
+
+def encode_delim(msg: bytes) -> bytes:
+    delimited_msg = msg + b"\n"
+    return encode_varint_prefixed(delimited_msg)
+
+
+async def read_delim(reader: Reader) -> bytes:
+    msg_bytes = await read_varint_prefixed_bytes(reader)
+    if len(msg_bytes) == 0:
+        raise ParseError("`len(msg_bytes)` should not be 0")
+    if msg_bytes[-1:] != b"\n":
+        raise ParseError(
+            f'`msg_bytes` is not delimited by b"\\n": `msg_bytes`={msg_bytes!r}'
+        )
+    return msg_bytes[:-1]

libp2p/utils/version.py

@@ -0,0 +1,22 @@
+from importlib.metadata import (
+    version,
+)
+import logging
+
+logger = logging.getLogger("libp2p.utils.version")
+
+
+def get_agent_version() -> str:
+    """
+    Return the version of libp2p.
+
+    If the version cannot be determined due to an exception, return "py-libp2p/unknown".
+
+    :return: The version of libp2p.
+    :rtype: str
+    """
+    try:
+        return f"py-libp2p/{version('libp2p')}"
+    except Exception as e:
+        logger.warning("Could not fetch libp2p version: %s", e)
+        return "py-libp2p/unknown"