mirror of
https://github.com/varun-r-mallya/py-libp2p.git
synced 2025-12-31 20:36:24 +00:00
Fix all modules except for security
This commit is contained in:
mhchia
@ -1,12 +1,10 @@
|
||||
# type: ignore
|
||||
# To add typing to this module, it's better to do it after refactoring test cases into classes
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
import trio
|
||||
|
||||
from libp2p.tools.constants import FLOODSUB_PROTOCOL_ID, LISTEN_MADDR
|
||||
from libp2p.tools.factories import PubsubFactory
|
||||
from libp2p.tools.constants import FLOODSUB_PROTOCOL_ID
|
||||
from libp2p.tools.utils import connect
|
||||
|
||||
SUPPORTED_PROTOCOLS = [FLOODSUB_PROTOCOL_ID]
|
||||
@ -15,6 +13,7 @@ FLOODSUB_PROTOCOL_TEST_CASES = [
|
||||
{
|
||||
"name": "simple_two_nodes",
|
||||
"supported_protocols": SUPPORTED_PROTOCOLS,
|
||||
"nodes": ["A", "B"],
|
||||
"adj_list": {"A": ["B"]},
|
||||
"topic_map": {"topic1": ["B"]},
|
||||
"messages": [{"topics": ["topic1"], "data": b"foo", "node_id": "A"}],
|
||||
@ -22,6 +21,7 @@ FLOODSUB_PROTOCOL_TEST_CASES = [
|
||||
{
|
||||
"name": "three_nodes_two_topics",
|
||||
"supported_protocols": SUPPORTED_PROTOCOLS,
|
||||
"nodes": ["A", "B", "C"],
|
||||
"adj_list": {"A": ["B"], "B": ["C"]},
|
||||
"topic_map": {"topic1": ["B", "C"], "topic2": ["B", "C"]},
|
||||
"messages": [
|
||||
@ -32,6 +32,7 @@ FLOODSUB_PROTOCOL_TEST_CASES = [
|
||||
{
|
||||
"name": "two_nodes_one_topic_single_subscriber_is_sender",
|
||||
"supported_protocols": SUPPORTED_PROTOCOLS,
|
||||
"nodes": ["A", "B"],
|
||||
"adj_list": {"A": ["B"]},
|
||||
"topic_map": {"topic1": ["B"]},
|
||||
"messages": [{"topics": ["topic1"], "data": b"Alex is tall", "node_id": "B"}],
|
||||
@ -39,6 +40,7 @@ FLOODSUB_PROTOCOL_TEST_CASES = [
|
||||
{
|
||||
"name": "two_nodes_one_topic_two_msgs",
|
||||
"supported_protocols": SUPPORTED_PROTOCOLS,
|
||||
"nodes": ["A", "B"],
|
||||
"adj_list": {"A": ["B"]},
|
||||
"topic_map": {"topic1": ["B"]},
|
||||
"messages": [
|
||||
@ -49,6 +51,7 @@ FLOODSUB_PROTOCOL_TEST_CASES = [
|
||||
{
|
||||
"name": "seven_nodes_tree_one_topics",
|
||||
"supported_protocols": SUPPORTED_PROTOCOLS,
|
||||
"nodes": ["1", "2", "3", "4", "5", "6", "7"],
|
||||
"adj_list": {"1": ["2", "3"], "2": ["4", "5"], "3": ["6", "7"]},
|
||||
"topic_map": {"astrophysics": ["2", "3", "4", "5", "6", "7"]},
|
||||
"messages": [{"topics": ["astrophysics"], "data": b"e=mc^2", "node_id": "1"}],
|
||||
@ -56,6 +59,7 @@ FLOODSUB_PROTOCOL_TEST_CASES = [
|
||||
{
|
||||
"name": "seven_nodes_tree_three_topics",
|
||||
"supported_protocols": SUPPORTED_PROTOCOLS,
|
||||
"nodes": ["1", "2", "3", "4", "5", "6", "7"],
|
||||
"adj_list": {"1": ["2", "3"], "2": ["4", "5"], "3": ["6", "7"]},
|
||||
"topic_map": {
|
||||
"astrophysics": ["2", "3", "4", "5", "6", "7"],
|
||||
@ -71,6 +75,7 @@ FLOODSUB_PROTOCOL_TEST_CASES = [
|
||||
{
|
||||
"name": "seven_nodes_tree_three_topics_diff_origin",
|
||||
"supported_protocols": SUPPORTED_PROTOCOLS,
|
||||
"nodes": ["1", "2", "3", "4", "5", "6", "7"],
|
||||
"adj_list": {"1": ["2", "3"], "2": ["4", "5"], "3": ["6", "7"]},
|
||||
"topic_map": {
|
||||
"astrophysics": ["1", "2", "3", "4", "5", "6", "7"],
|
||||
@ -86,6 +91,7 @@ FLOODSUB_PROTOCOL_TEST_CASES = [
|
||||
{
|
||||
"name": "three_nodes_clique_two_topic_diff_origin",
|
||||
"supported_protocols": SUPPORTED_PROTOCOLS,
|
||||
"nodes": ["1", "2", "3"],
|
||||
"adj_list": {"1": ["2", "3"], "2": ["3"]},
|
||||
"topic_map": {"astrophysics": ["1", "2", "3"], "school": ["1", "2", "3"]},
|
||||
"messages": [
|
||||
@ -97,6 +103,7 @@ FLOODSUB_PROTOCOL_TEST_CASES = [
|
||||
{
|
||||
"name": "four_nodes_clique_two_topic_diff_origin_many_msgs",
|
||||
"supported_protocols": SUPPORTED_PROTOCOLS,
|
||||
"nodes": ["1", "2", "3", "4"],
|
||||
"adj_list": {
|
||||
"1": ["2", "3", "4"],
|
||||
"2": ["1", "3", "4"],
|
||||
@ -120,6 +127,7 @@ FLOODSUB_PROTOCOL_TEST_CASES = [
|
||||
{
|
||||
"name": "five_nodes_ring_two_topic_diff_origin_many_msgs",
|
||||
"supported_protocols": SUPPORTED_PROTOCOLS,
|
||||
"nodes": ["1", "2", "3", "4", "5"],
|
||||
"adj_list": {"1": ["2"], "2": ["3"], "3": ["4"], "4": ["5"], "5": ["1"]},
|
||||
"topic_map": {
|
||||
"astrophysics": ["1", "2", "3", "4", "5"],
|
||||
@ -143,7 +151,7 @@ floodsub_protocol_pytest_params = [
|
||||
]
|
||||
|
||||
|
||||
async def perform_test_from_obj(obj, router_factory) -> None:
|
||||
async def perform_test_from_obj(obj, pubsub_factory) -> None:
|
||||
"""
|
||||
Perform pubsub tests from a test obj.
|
||||
test obj are composed as follows:
|
||||
@ -174,88 +182,75 @@ async def perform_test_from_obj(obj, router_factory) -> None:
|
||||
|
||||
# Step 1) Create graph
|
||||
adj_list = obj["adj_list"]
|
||||
node_list = obj["nodes"]
|
||||
node_map = {}
|
||||
pubsub_map = {}
|
||||
|
||||
async def add_node(node_id_str: str) -> None:
|
||||
pubsub_router = router_factory(protocols=obj["supported_protocols"])
|
||||
pubsub = PubsubFactory(router=pubsub_router)
|
||||
await pubsub.host.get_network().listen(LISTEN_MADDR)
|
||||
node_map[node_id_str] = pubsub.host
|
||||
pubsub_map[node_id_str] = pubsub
|
||||
async with pubsub_factory(
|
||||
number=len(node_list), protocols=obj["supported_protocols"]
|
||||
) as pubsubs:
|
||||
for node_id_str, pubsub in zip(node_list, pubsubs):
|
||||
node_map[node_id_str] = pubsub.host
|
||||
pubsub_map[node_id_str] = pubsub
|
||||
|
||||
tasks_connect = []
|
||||
for start_node_id in adj_list:
|
||||
# Create node if node does not yet exist
|
||||
if start_node_id not in node_map:
|
||||
await add_node(start_node_id)
|
||||
# Connect nodes and wait at least for 2 seconds
|
||||
async with trio.open_nursery() as nursery:
|
||||
for start_node_id in adj_list:
|
||||
# For each neighbor of start_node, create if does not yet exist,
|
||||
# then connect start_node to neighbor
|
||||
for neighbor_id in adj_list[start_node_id]:
|
||||
nursery.start_soon(
|
||||
connect, node_map[start_node_id], node_map[neighbor_id]
|
||||
)
|
||||
nursery.start_soon(trio.sleep, 2)
|
||||
|
||||
# For each neighbor of start_node, create if does not yet exist,
|
||||
# then connect start_node to neighbor
|
||||
for neighbor_id in adj_list[start_node_id]:
|
||||
# Create neighbor if neighbor does not yet exist
|
||||
if neighbor_id not in node_map:
|
||||
await add_node(neighbor_id)
|
||||
tasks_connect.append(
|
||||
connect(node_map[start_node_id], node_map[neighbor_id])
|
||||
)
|
||||
# Connect nodes and wait at least for 2 seconds
|
||||
await asyncio.gather(*tasks_connect, asyncio.sleep(2))
|
||||
# Step 2) Subscribe to topics
|
||||
queues_map = {}
|
||||
topic_map = obj["topic_map"]
|
||||
|
||||
# Step 2) Subscribe to topics
|
||||
queues_map = {}
|
||||
topic_map = obj["topic_map"]
|
||||
async def subscribe_node(node_id, topic):
|
||||
if node_id not in queues_map:
|
||||
queues_map[node_id] = {}
|
||||
# Avoid repeated works
|
||||
if topic in queues_map[node_id]:
|
||||
# Checkpoint
|
||||
await trio.hazmat.checkpoint()
|
||||
return
|
||||
sub = await pubsub_map[node_id].subscribe(topic)
|
||||
queues_map[node_id][topic] = sub
|
||||
|
||||
tasks_topic = []
|
||||
tasks_topic_data = []
|
||||
for topic, node_ids in topic_map.items():
|
||||
for node_id in node_ids:
|
||||
tasks_topic.append(pubsub_map[node_id].subscribe(topic))
|
||||
tasks_topic_data.append((node_id, topic))
|
||||
tasks_topic.append(asyncio.sleep(2))
|
||||
async with trio.open_nursery() as nursery:
|
||||
for topic, node_ids in topic_map.items():
|
||||
for node_id in node_ids:
|
||||
nursery.start_soon(subscribe_node, node_id, topic)
|
||||
nursery.start_soon(trio.sleep, 2)
|
||||
|
||||
# Gather is like Promise.all
|
||||
responses = await asyncio.gather(*tasks_topic)
|
||||
for i in range(len(responses) - 1):
|
||||
node_id, topic = tasks_topic_data[i]
|
||||
if node_id not in queues_map:
|
||||
queues_map[node_id] = {}
|
||||
# Store queue in topic-queue map for node
|
||||
queues_map[node_id][topic] = responses[i]
|
||||
# Step 3) Publish messages
|
||||
topics_in_msgs_ordered = []
|
||||
messages = obj["messages"]
|
||||
|
||||
# Allow time for subscribing before continuing
|
||||
await asyncio.sleep(0.01)
|
||||
for msg in messages:
|
||||
topics = msg["topics"]
|
||||
data = msg["data"]
|
||||
node_id = msg["node_id"]
|
||||
|
||||
# Step 3) Publish messages
|
||||
topics_in_msgs_ordered = []
|
||||
messages = obj["messages"]
|
||||
tasks_publish = []
|
||||
# Publish message
|
||||
# TODO: Should be single RPC package with several topics
|
||||
for topic in topics:
|
||||
await pubsub_map[node_id].publish(topic, data)
|
||||
|
||||
for msg in messages:
|
||||
topics = msg["topics"]
|
||||
data = msg["data"]
|
||||
node_id = msg["node_id"]
|
||||
# For each topic in topics, add (topic, node_id, data) tuple to ordered test list
|
||||
for topic in topics:
|
||||
topics_in_msgs_ordered.append((topic, node_id, data))
|
||||
# Allow time for publishing before continuing
|
||||
await trio.sleep(1)
|
||||
|
||||
# Publish message
|
||||
# TODO: Should be single RPC package with several topics
|
||||
for topic in topics:
|
||||
tasks_publish.append(pubsub_map[node_id].publish(topic, data))
|
||||
|
||||
# For each topic in topics, add (topic, node_id, data) tuple to ordered test list
|
||||
for topic in topics:
|
||||
topics_in_msgs_ordered.append((topic, node_id, data))
|
||||
|
||||
# Allow time for publishing before continuing
|
||||
await asyncio.gather(*tasks_publish, asyncio.sleep(2))
|
||||
|
||||
# Step 4) Check that all messages were received correctly.
|
||||
for topic, origin_node_id, data in topics_in_msgs_ordered:
|
||||
# Look at each node in each topic
|
||||
for node_id in topic_map[topic]:
|
||||
# Get message from subscription queue
|
||||
msg = await queues_map[node_id][topic].get()
|
||||
assert data == msg.data
|
||||
# Check the message origin
|
||||
assert node_map[origin_node_id].get_id().to_bytes() == msg.from_id
|
||||
|
||||
# Success, terminate pending tasks.
|
||||
# Step 4) Check that all messages were received correctly.
|
||||
for topic, origin_node_id, data in topics_in_msgs_ordered:
|
||||
# Look at each node in each topic
|
||||
for node_id in topic_map[topic]:
|
||||
# Get message from subscription queue
|
||||
msg = await queues_map[node_id][topic].receive()
|
||||
assert data == msg.data
|
||||
# Check the message origin
|
||||
assert node_map[origin_node_id].get_id().to_bytes() == msg.from_id
|
||||
|
||||
Reference in New Issue
Block a user