varun-r-mallya/py-libp2p
1
0
Fork 0
mirror of https://github.com/varun-r-mallya/py-libp2p.git synced 2026-02-09 22:50:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

drop async-service dep and copy relevant code into a local async_service

Browse Source 
tool, updated for modern handling of ExceptionGroup
This commit is contained in:
pacrob
2024-05-19 14:48:03 -06:00
committed by Paul Robinson
parent 7de6cbaab0
commit d9b92635c1
28 changed files with 2176 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
tests/core/network/test_notify.py
View File

@ -10,15 +10,15 @@ features are implemented in swarm
"""
import enum
from async_service import (
background_trio_service,
)
import pytest
import trio
from libp2p.network.notifee_interface import (
INotifee,
)
from libp2p.tools.async_service import (
background_trio_service,
)
from libp2p.tools.constants import (
LISTEN_MADDR,
)

20
tests/core/protocol_muxer/test_protocol_muxer.py
View File

@ -1,4 +1,7 @@
import pytest
from trio.testing import (
RaisesGroup,
)
from libp2p.host.exceptions import (
StreamFailure,
@ -58,7 +61,13 @@ async def test_single_protocol_succeeds(security_protocol):
@pytest.mark.trio
async def test_single_protocol_fails(security_protocol):
with pytest.raises(StreamFailure):
# using trio.testing.RaisesGroup b/c pytest.raises does not handle ExceptionGroups
# yet: https://github.com/pytest-dev/pytest/issues/11538
# but switch to that once they do
# the StreamFailure is within 2 nested ExceptionGroups, so we use strict=False
# to unwrap down to the core Exception
with RaisesGroup(StreamFailure, strict=False):
await perform_simple_test(
"", [PROTOCOL_ECHO], [PROTOCOL_POTATO], security_protocol
)
@ -96,7 +105,14 @@ async def test_multiple_protocol_second_is_valid_succeeds(security_protocol):
async def test_multiple_protocol_fails(security_protocol):
protocols_for_client = [PROTOCOL_ROCK, PROTOCOL_FOO, "/bar/1.0.0"]
protocols_for_listener = ["/aspyn/1.0.0", "/rob/1.0.0", "/zx/1.0.0", "/alex/1.0.0"]
with pytest.raises(StreamFailure):
# using trio.testing.RaisesGroup b/c pytest.raises does not handle ExceptionGroups
# yet: https://github.com/pytest-dev/pytest/issues/11538
# but switch to that once they do
# the StreamFailure is within 2 nested ExceptionGroups, so we use strict=False
# to unwrap down to the core Exception
with RaisesGroup(StreamFailure, strict=False):
await perform_simple_test(
"", protocols_for_client, protocols_for_listener, security_protocol
)

668
tests/core/tools/async_service/test_trio_based_service.py Normal file
View File

@ -0,0 +1,668 @@
import sys
if sys.version_info >= (3, 11):
from builtins import (
ExceptionGroup,
)
else:
from exceptiongroup import (
ExceptionGroup,
)
import pytest
import trio
from trio.testing import (
Matcher,
RaisesGroup,
)
from libp2p.tools.async_service import (
DaemonTaskExit,
LifecycleError,
Service,
TrioManager,
as_service,
background_trio_service,
)
class WaitCancelledService(Service):
async def run(self) -> None:
await self.manager.wait_finished()
async def do_service_lifecycle_check(
manager, manager_run_fn, trigger_exit_condition_fn, should_be_cancelled
):
async with trio.open_nursery() as nursery:
assert manager.is_started is False
assert manager.is_running is False
assert manager.is_cancelled is False
assert manager.is_finished is False
nursery.start_soon(manager_run_fn)
with trio.fail_after(0.1):
await manager.wait_started()
assert manager.is_started is True
assert manager.is_running is True
assert manager.is_cancelled is False
assert manager.is_finished is False
# trigger the service to exit
trigger_exit_condition_fn()
with trio.fail_after(0.1):
await manager.wait_finished()
if should_be_cancelled:
assert manager.is_started is True
# We cannot determine whether the service should be running at this
# stage because a service is considered running until it is
# finished. Since it may be cancelled but still not finished we
# can't know.
assert manager.is_cancelled is True
# We also cannot determine whether a service should be finished at this
# stage as it could have exited cleanly and is now finished or it
# might be doing some cleanup after which it will register as being
# finished.
assert manager.is_running is True or manager.is_finished is True
assert manager.is_started is True
assert manager.is_running is False
assert manager.is_cancelled is should_be_cancelled
assert manager.is_finished is True
def test_service_manager_initial_state():
service = WaitCancelledService()
manager = TrioManager(service)
assert manager.is_started is False
assert manager.is_running is False
assert manager.is_cancelled is False
assert manager.is_finished is False
@pytest.mark.trio
async def test_trio_service_lifecycle_run_and_clean_exit():
trigger_exit = trio.Event()
@as_service
async def ServiceTest(manager):
await trigger_exit.wait()
service = ServiceTest()
manager = TrioManager(service)
await do_service_lifecycle_check(
manager=manager,
manager_run_fn=manager.run,
trigger_exit_condition_fn=trigger_exit.set,
should_be_cancelled=False,
)
@pytest.mark.trio
async def test_trio_service_lifecycle_run_and_external_cancellation():
@as_service
async def ServiceTest(manager):
await trio.sleep_forever()
service = ServiceTest()
manager = TrioManager(service)
await do_service_lifecycle_check(
manager=manager,
manager_run_fn=manager.run,
trigger_exit_condition_fn=manager.cancel,
should_be_cancelled=True,
)
@pytest.mark.trio
async def test_trio_service_lifecycle_run_and_exception():
trigger_error = trio.Event()
@as_service
async def ServiceTest(manager):
await trigger_error.wait()
raise RuntimeError("Service throwing error")
service = ServiceTest()
manager = TrioManager(service)
async def do_service_run():
with RaisesGroup(
Matcher(RuntimeError, match="Service throwing error"), strict=False
):
await manager.run()
await do_service_lifecycle_check(
manager=manager,
manager_run_fn=do_service_run,
trigger_exit_condition_fn=trigger_error.set,
should_be_cancelled=True,
)
@pytest.mark.trio
async def test_trio_service_lifecycle_run_and_task_exception():
trigger_error = trio.Event()
@as_service
async def ServiceTest(manager):
async def task_fn():
await trigger_error.wait()
raise RuntimeError("Service throwing error")
manager.run_task(task_fn)
service = ServiceTest()
manager = TrioManager(service)
async def do_service_run():
with RaisesGroup(
Matcher(RuntimeError, match="Service throwing error"), strict=False
):
await manager.run()
await do_service_lifecycle_check(
manager=manager,
manager_run_fn=do_service_run,
trigger_exit_condition_fn=trigger_error.set,
should_be_cancelled=True,
)
@pytest.mark.trio
async def test_sub_service_cancelled_when_parent_stops():
ready_cancel = trio.Event()
# This test runs a service that runs a sub-service that sleeps forever. When the
# parent exits, the sub-service should be cancelled as well.
@as_service
async def WaitForeverService(manager):
ready_cancel.set()
await manager.wait_finished()
sub_manager = TrioManager(WaitForeverService())
@as_service
async def ServiceTest(manager):
async def run_sub():
await sub_manager.run()
manager.run_task(run_sub)
await manager.wait_finished()
s = ServiceTest()
async with background_trio_service(s) as manager:
await ready_cancel.wait()
assert not manager.is_running
assert manager.is_cancelled
assert manager.is_finished
assert not sub_manager.is_running
assert not sub_manager.is_cancelled
assert sub_manager.is_finished
@pytest.mark.trio
async def test_trio_service_lifecycle_run_and_daemon_task_exit():
trigger_error = trio.Event()
@as_service
async def ServiceTest(manager):
async def daemon_task_fn():
await trigger_error.wait()
manager.run_daemon_task(daemon_task_fn)
await manager.wait_finished()
service = ServiceTest()
manager = TrioManager(service)
async def do_service_run():
with RaisesGroup(Matcher(DaemonTaskExit, match="Daemon task"), strict=False):
await manager.run()
await do_service_lifecycle_check(
manager=manager,
manager_run_fn=do_service_run,
trigger_exit_condition_fn=trigger_error.set,
should_be_cancelled=True,
)
@pytest.mark.trio
async def test_exceptiongroup_in_run():
# This test should cause TrioManager.run() to explicitly raise an ExceptionGroup
# containing two exceptions -- one raised inside its run() method and another
# raised by the daemon task exiting early.
trigger_error = trio.Event()
class ServiceTest(Service):
async def run(self):
ready = trio.Event()
self.manager.run_task(self.error_fn, ready)
await ready.wait()
trigger_error.set()
raise RuntimeError("Exception inside Service.run()")
async def error_fn(self, ready):
ready.set()
await trigger_error.wait()
raise ValueError("Exception inside error_fn")
with pytest.raises(ExceptionGroup) as exc_info:
await TrioManager.run_service(ServiceTest())
exc = exc_info.value
assert len(exc.exceptions) == 2
assert any(isinstance(err, RuntimeError) for err in exc.exceptions)
assert any(isinstance(err, ValueError) for err in exc.exceptions)
@pytest.mark.trio
async def test_trio_service_background_service_context_manager():
service = WaitCancelledService()
async with background_trio_service(service) as manager:
# ensure the manager property is set.
assert hasattr(service, "manager")
assert service.get_manager() is manager
assert manager.is_started is True
assert manager.is_running is True
assert manager.is_cancelled is False
assert manager.is_finished is False
assert manager.is_started is True
assert manager.is_running is False
assert manager.is_cancelled is True
assert manager.is_finished is True
@pytest.mark.trio
async def test_trio_service_manager_stop():
service = WaitCancelledService()
async with background_trio_service(service) as manager:
assert manager.is_started is True
assert manager.is_running is True
assert manager.is_cancelled is False
assert manager.is_finished is False
await manager.stop()
assert manager.is_started is True
assert manager.is_running is False
assert manager.is_cancelled is True
assert manager.is_finished is True
@pytest.mark.trio
async def test_trio_service_manager_run_task():
task_event = trio.Event()
@as_service
async def RunTaskService(manager):
async def task_fn():
task_event.set()
manager.run_task(task_fn)
await manager.wait_finished()
async with background_trio_service(RunTaskService()):
with trio.fail_after(0.1):
await task_event.wait()
@pytest.mark.trio
async def test_trio_service_manager_run_task_waits_for_task_completion():
task_event = trio.Event()
@as_service
async def RunTaskService(manager):
async def task_fn():
await trio.sleep(0.01)
task_event.set()
manager.run_task(task_fn)
# the task is set to run in the background but then the service exits.
# We want to be sure that the task is allowed to continue till
# completion unless explicitely cancelled.
async with background_trio_service(RunTaskService()):
with trio.fail_after(0.1):
await task_event.wait()
@pytest.mark.trio
async def test_trio_service_manager_run_task_can_still_cancel_after_run_finishes():
task_event = trio.Event()
service_finished = trio.Event()
@as_service
async def RunTaskService(manager):
async def task_fn():
# this will never complete
await task_event.wait()
manager.run_task(task_fn)
# the task is set to run in the background but then the service exits.
# We want to be sure that the task is allowed to continue till
# completion unless explicitely cancelled.
service_finished.set()
async with background_trio_service(RunTaskService()) as manager:
with trio.fail_after(0.01):
await service_finished.wait()
# show that the service hangs waiting for the task to complete.
with trio.move_on_after(0.01) as cancel_scope:
await manager.wait_finished()
assert cancel_scope.cancelled_caught is True
# trigger cancellation and see that the service actually stops
manager.cancel()
with trio.fail_after(0.01):
await manager.wait_finished()
@pytest.mark.trio
async def test_trio_service_manager_run_task_reraises_exceptions():
task_event = trio.Event()
@as_service
async def RunTaskService(manager):
async def task_fn():
await task_event.wait()
raise Exception("task exception in run_task")
manager.run_task(task_fn)
with trio.fail_after(1):
await trio.sleep_forever()
with RaisesGroup(
Matcher(Exception, match="task exception in run_task"), strict=False
):
async with background_trio_service(RunTaskService()):
task_event.set()
with trio.fail_after(1):
await trio.sleep_forever()
@pytest.mark.trio
async def test_trio_service_manager_run_daemon_task_cancels_if_exits():
task_event = trio.Event()
@as_service
async def RunTaskService(manager):
async def daemon_task_fn():
await task_event.wait()
manager.run_daemon_task(daemon_task_fn, name="daemon_task_fn")
with trio.fail_after(1):
await trio.sleep_forever()
with RaisesGroup(
Matcher(
DaemonTaskExit, match=r"Daemon task daemon_task_fn\[daemon=True\] exited"
),
strict=False,
):
async with background_trio_service(RunTaskService()):
task_event.set()
with trio.fail_after(1):
await trio.sleep_forever()
@pytest.mark.trio
async def test_trio_service_manager_propogates_and_records_exceptions():
@as_service
async def ThrowErrorService(manager):
raise RuntimeError("this is the error")
service = ThrowErrorService()
manager = TrioManager(service)
assert manager.did_error is False
with RaisesGroup(Matcher(RuntimeError, match="this is the error"), strict=False):
await manager.run()
assert manager.did_error is True
@pytest.mark.trio
async def test_trio_service_lifecycle_run_and_clean_exit_with_child_service():
trigger_exit = trio.Event()
@as_service
async def ChildServiceTest(manager):
await trigger_exit.wait()
@as_service
async def ServiceTest(manager):
child_manager = manager.run_child_service(ChildServiceTest())
await child_manager.wait_started()
service = ServiceTest()
manager = TrioManager(service)
await do_service_lifecycle_check(
manager=manager,
manager_run_fn=manager.run,
trigger_exit_condition_fn=trigger_exit.set,
should_be_cancelled=False,
)
@pytest.mark.trio
async def test_trio_service_with_daemon_child_service():
ready = trio.Event()
@as_service
async def ChildServiceTest(manager):
await manager.wait_finished()
@as_service
async def ServiceTest(manager):
child_manager = manager.run_daemon_child_service(ChildServiceTest())
await child_manager.wait_started()
ready.set()
await manager.wait_finished()
service = ServiceTest()
async with background_trio_service(service):
await ready.wait()
@pytest.mark.trio
async def test_trio_service_with_daemon_child_task():
ready = trio.Event()
started = trio.Event()
async def _task():
started.set()
await trio.sleep(100)
@as_service
async def ServiceTest(manager):
manager.run_daemon_task(_task)
await started.wait()
ready.set()
await manager.wait_finished()
service = ServiceTest()
async with background_trio_service(service):
await ready.wait()
@pytest.mark.trio
async def test_trio_service_with_async_generator():
is_within_agen = trio.Event()
async def do_agen():
while True:
yield
@as_service
async def ServiceTest(manager):
async for _ in do_agen(): # noqa: F841
await trio.lowlevel.checkpoint()
is_within_agen.set()
async with background_trio_service(ServiceTest()) as manager:
await is_within_agen.wait()
manager.cancel()
@pytest.mark.trio
async def test_trio_service_disallows_task_scheduling_when_not_running():
class ServiceTest(Service):
async def run(self):
await self.manager.wait_finished()
def do_schedule(self):
self.manager.run_task(trio.sleep, 1)
service = ServiceTest()
async with background_trio_service(service):
service.do_schedule()
with pytest.raises(LifecycleError):
service.do_schedule()
@pytest.mark.trio
async def test_trio_service_disallows_task_scheduling_after_cancel():
@as_service
async def ServiceTest(manager):
manager.cancel()
manager.run_task(trio.sleep, 1)
await TrioManager.run_service(ServiceTest())
@pytest.mark.trio
async def test_trio_service_cancellation_with_running_daemon_task():
in_daemon = trio.Event()
class ServiceTest(Service):
async def run(self):
self.manager.run_daemon_task(self._do_daemon)
await self.manager.wait_finished()
async def _do_daemon(self):
in_daemon.set()
while self.manager.is_running:
await trio.lowlevel.checkpoint()
async with background_trio_service(ServiceTest()) as manager:
await in_daemon.wait()
manager.cancel()
@pytest.mark.trio
async def test_trio_service_with_try_finally_cleanup():
ready_cancel = trio.Event()
class TryFinallyService(Service):
cleanup_up = False
async def run(self) -> None:
try:
ready_cancel.set()
await self.manager.wait_finished()
finally:
self.cleanup_up = True
service = TryFinallyService()
async with background_trio_service(service) as manager:
await ready_cancel.wait()
assert not service.cleanup_up
manager.cancel()
assert service.cleanup_up
@pytest.mark.trio
async def test_trio_service_with_try_finally_cleanup_with_unshielded_await():
ready_cancel = trio.Event()
class TryFinallyService(Service):
cleanup_up = False
async def run(self) -> None:
try:
ready_cancel.set()
await self.manager.wait_finished()
finally:
await trio.lowlevel.checkpoint()
self.cleanup_up = True
service = TryFinallyService()
async with background_trio_service(service) as manager:
await ready_cancel.wait()
assert not service.cleanup_up
manager.cancel()
assert not service.cleanup_up
@pytest.mark.trio
async def test_trio_service_with_try_finally_cleanup_with_shielded_await():
ready_cancel = trio.Event()
class TryFinallyService(Service):
cleanup_up = False
async def run(self) -> None:
try:
ready_cancel.set()
await self.manager.wait_finished()
finally:
with trio.CancelScope(shield=True):
await trio.lowlevel.checkpoint()
self.cleanup_up = True
service = TryFinallyService()
async with background_trio_service(service) as manager:
await ready_cancel.wait()
assert not service.cleanup_up
manager.cancel()
assert service.cleanup_up
@pytest.mark.trio
async def test_error_in_service_run():
class ServiceTest(Service):
async def run(self):
self.manager.run_daemon_task(self.manager.wait_finished)
raise ValueError("Exception inside run()")
with RaisesGroup(ValueError, strict=False):
await TrioManager.run_service(ServiceTest())
@pytest.mark.trio
async def test_daemon_task_finishes_leaving_children():
class ServiceTest(Service):
async def sleep_and_fail(self):
await trio.sleep(1)
raise AssertionError(
"This should not happen as the task should be cancelled"
)
async def buggy_daemon(self):
self.manager.run_task(self.sleep_and_fail)
async def run(self):
self.manager.run_daemon_task(self.buggy_daemon)
with RaisesGroup(DaemonTaskExit, strict=False):
await TrioManager.run_service(ServiceTest())

109
tests/core/tools/async_service/test_trio_external_api.py Normal file
View File

@ -0,0 +1,109 @@
# Copied from https://github.com/ethereum/async-service
import pytest
import trio
from trio.testing import (
RaisesGroup,
)
from libp2p.tools.async_service import (
LifecycleError,
Service,
background_trio_service,
)
from libp2p.tools.async_service.trio_service import (
external_api,
)
class ExternalAPIService(Service):
async def run(self):
await self.manager.wait_finished()
@external_api
async def get_7(self, wait_return=None, signal_event=None):
if signal_event is not None:
signal_event.set()
if wait_return is not None:
await wait_return.wait()
return 7
@pytest.mark.trio
async def test_trio_service_external_api_fails_before_start():
service = ExternalAPIService()
# should raise if the service has not yet been started.
with pytest.raises(LifecycleError):
await service.get_7()
@pytest.mark.trio
async def test_trio_service_external_api_works_while_running():
service = ExternalAPIService()
async with background_trio_service(service):
assert await service.get_7() == 7
@pytest.mark.trio
async def test_trio_service_external_api_raises_when_cancelled():
service = ExternalAPIService()
async with background_trio_service(service) as manager:
with RaisesGroup(LifecycleError, strict=False):
async with trio.open_nursery() as nursery:
# an event to ensure that we are indeed within the body of the
is_within_fn = trio.Event()
trigger_return = trio.Event()
nursery.start_soon(service.get_7, trigger_return, is_within_fn)
# ensure we're within the body of the task.
await is_within_fn.wait()
# now cancel the service and trigger the return of the function.
manager.cancel()
# exiting the context block here will cause the background task
# to complete and shold raise the exception
# A direct call should also fail. This *should* be hitting the early
# return mechanism.
with pytest.raises(LifecycleError):
assert await service.get_7()
@pytest.mark.trio
async def test_trio_service_external_api_raises_when_finished():
service = ExternalAPIService()
async with background_trio_service(service) as manager:
pass
assert manager.is_finished
# A direct call should also fail. This *should* be hitting the early
# return mechanism.
with pytest.raises(LifecycleError):
assert await service.get_7()
@pytest.mark.trio
async def test_trio_external_api_call_that_schedules_task():
done = trio.Event()
class MyService(Service):
async def run(self):
await self.manager.wait_finished()
@external_api
async def do_scheduling(self):
self.manager.run_task(self.set_done)
async def set_done(self):
done.set()
service = MyService()
async with background_trio_service(service):
await service.do_scheduling()
with trio.fail_after(1):
await done.wait()

86
tests/core/tools/async_service/test_trio_manager_stats.py Normal file
View File

@ -0,0 +1,86 @@
import pytest
import trio
from libp2p.tools.async_service import (
Service,
background_trio_service,
)
@pytest.mark.trio
async def test_trio_manager_stats():
ready = trio.Event()
class StatsTest(Service):
async def run(self):
# 2 that run forever
self.manager.run_task(trio.sleep_forever)
self.manager.run_task(trio.sleep_forever)
# 2 that complete
self.manager.run_task(trio.lowlevel.checkpoint)
self.manager.run_task(trio.lowlevel.checkpoint)
# 1 that spawns some children
self.manager.run_task(self.run_with_children, 4)
async def run_with_children(self, num_children):
for _ in range(num_children):
self.manager.run_task(trio.sleep_forever)
ready.set()
def run_external_root(self):
self.manager.run_task(trio.lowlevel.checkpoint)
service = StatsTest()
async with background_trio_service(service) as manager:
service.run_external_root()
assert len(manager._root_tasks) == 2
with trio.fail_after(1):
await ready.wait()
# we need to yield to the event loop a few times to allow the various
# tasks to schedule themselves and get running.
for _ in range(50):
await trio.lowlevel.checkpoint()
assert manager.stats.tasks.total_count == 10
assert manager.stats.tasks.finished_count == 3
assert manager.stats.tasks.pending_count == 7
# This is a simple test to ensure that finished tasks are removed from
# tracking to prevent unbounded memory growth.
assert len(manager._root_tasks) == 1
# now check after exiting
assert manager.stats.tasks.total_count == 10
assert manager.stats.tasks.finished_count == 10
assert manager.stats.tasks.pending_count == 0
@pytest.mark.trio
async def test_trio_manager_stats_does_not_count_main_run_method():
ready = trio.Event()
class StatsTest(Service):
async def run(self):
self.manager.run_task(trio.sleep_forever)
ready.set()
async with background_trio_service(StatsTest()) as manager:
with trio.fail_after(1):
await ready.wait()
# we need to yield to the event loop a few times to allow the various
# tasks to schedule themselves and get running.
for _ in range(10):
await trio.lowlevel.checkpoint()
assert manager.stats.tasks.total_count == 1
assert manager.stats.tasks.finished_count == 0
assert manager.stats.tasks.pending_count == 1
# now check after exiting
assert manager.stats.tasks.total_count == 1
assert manager.stats.tasks.finished_count == 1
assert manager.stats.tasks.pending_count == 0