varun-r-mallya/py-libp2p
1
0
Fork 0
mirror of https://github.com/varun-r-mallya/py-libp2p.git synced 2025-12-31 20:36:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Run black over repo

Browse Source
This commit is contained in:
Alex Stokes
2019-07-31 15:00:12 -07:00
parent a2133d8c7c
commit 0ae9840928
69 changed files with 791 additions and 1095 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
tests/pubsub/conftest.py
View File

@ -11,15 +11,12 @@ from libp2p.pubsub.floodsub import FloodSub
from libp2p.pubsub.gossipsub import GossipSub
from libp2p.pubsub.pubsub import Pubsub
from .configs import (
FLOODSUB_PROTOCOL_ID,
GOSSIPSUB_PROTOCOL_ID,
LISTEN_MADDR,
)
from .configs import FLOODSUB_PROTOCOL_ID, GOSSIPSUB_PROTOCOL_ID, LISTEN_MADDR
# pylint: disable=redefined-outer-name
@pytest.fixture
def num_hosts():
return 3
@ -27,14 +24,12 @@ def num_hosts():
@pytest.fixture
async def hosts(num_hosts):
_hosts = await asyncio.gather(*[
new_node(transport_opt=[str(LISTEN_MADDR)])
for _ in range(num_hosts)
])
await asyncio.gather(*[
_host.get_network().listen(LISTEN_MADDR)
for _host in _hosts
])
_hosts = await asyncio.gather(
*[new_node(transport_opt=[str(LISTEN_MADDR)]) for _ in range(num_hosts)]
)
await asyncio.gather(
*[_host.get_network().listen(LISTEN_MADDR) for _host in _hosts]
)
yield _hosts
# Clean up
listeners = []
@ -42,18 +37,12 @@ async def hosts(num_hosts):
for listener in _host.get_network().listeners.values():
listener.server.close()
listeners.append(listener)
await asyncio.gather(*[
listener.server.wait_closed()
for listener in listeners
])
await asyncio.gather(*[listener.server.wait_closed() for listener in listeners])
@pytest.fixture
def floodsubs(num_hosts):
return tuple(
FloodSub(protocols=[FLOODSUB_PROTOCOL_ID])
for _ in range(num_hosts)
)
return tuple(FloodSub(protocols=[FLOODSUB_PROTOCOL_ID]) for _ in range(num_hosts))
class GossipsubParams(NamedTuple):
@ -74,10 +63,7 @@ def gossipsub_params():
@pytest.fixture
def gossipsubs(num_hosts, gossipsub_params):
yield tuple(
GossipSub(
protocols=[GOSSIPSUB_PROTOCOL_ID],
**gossipsub_params._asdict(),
)
GossipSub(protocols=[GOSSIPSUB_PROTOCOL_ID], **gossipsub_params._asdict())
for _ in range(num_hosts)
)
# TODO: Clean up
@ -90,11 +76,7 @@ def _make_pubsubs(hosts, pubsub_routers):
f"length of hosts={len(hosts)}"
)
return tuple(
Pubsub(
host=host,
router=router,
my_id=host.get_id(),
)
Pubsub(host=host, router=router, my_id=host.get_id())
for host, router in zip(hosts, pubsub_routers)
)

25
tests/pubsub/dummy_account_node.py
View File

@ -29,15 +29,12 @@ class DummyAccountNode:
a dummy crypto blockchain. There is no actual blockchain, just a simple
map indicating how much crypto each user in the mappings holds
"""
libp2p_node: IHost
pubsub: Pubsub
floodsub: FloodSub
def __init__(
self,
libp2p_node: IHost,
pubsub: Pubsub,
floodsub: FloodSub):
def __init__(self, libp2p_node: IHost, pubsub: Pubsub, floodsub: FloodSub):
self.libp2p_node = libp2p_node
self.pubsub = pubsub
self.floodsub = floodsub
@ -56,19 +53,13 @@ class DummyAccountNode:
"""
libp2p_node = await new_node(transport_opt=["/ip4/127.0.0.1/tcp/0"])
await libp2p_node.get_network().listen(multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0"))
await libp2p_node.get_network().listen(
multiaddr.Multiaddr("/ip4/127.0.0.1/tcp/0")
)
floodsub = FloodSub(SUPPORTED_PUBSUB_PROTOCOLS)
pubsub = Pubsub(
libp2p_node,
floodsub,
"a",
)
return cls(
libp2p_node=libp2p_node,
pubsub=pubsub,
floodsub=floodsub,
)
pubsub = Pubsub(libp2p_node, floodsub, "a")
return cls(libp2p_node=libp2p_node, pubsub=pubsub, floodsub=floodsub)
async def handle_incoming_msgs(self):
"""
@ -76,7 +67,7 @@ class DummyAccountNode:
"""
while True:
incoming = await self.q.get()
msg_comps = incoming.data.decode('utf-8').split(",")
msg_comps = incoming.data.decode("utf-8").split(",")
if msg_comps[0] == "send":
self.handle_send_crypto(msg_comps[1], msg_comps[2], int(msg_comps[3]))

293
tests/pubsub/floodsub_integration_test_settings.py
View File

@ -6,15 +6,9 @@ from libp2p import new_node
from libp2p.peer.id import ID
from libp2p.pubsub.pubsub import Pubsub
from tests.utils import (
cleanup,
connect,
)
from tests.utils import cleanup, connect
from .configs import (
FLOODSUB_PROTOCOL_ID,
LISTEN_MADDR,
)
from .configs import FLOODSUB_PROTOCOL_ID, LISTEN_MADDR
SUPPORTED_PROTOCOLS = [FLOODSUB_PROTOCOL_ID]
@ -23,192 +17,84 @@ FLOODSUB_PROTOCOL_TEST_CASES = [
{
"name": "simple_two_nodes",
"supported_protocols": SUPPORTED_PROTOCOLS,
"adj_list": {
"A": ["B"]
},
"topic_map": {
"topic1": ["B"]
},
"messages": [
{
"topics": ["topic1"],
"data": b"foo",
"node_id": "A"
}
]
"adj_list": {"A": ["B"]},
"topic_map": {"topic1": ["B"]},
"messages": [{"topics": ["topic1"], "data": b"foo", "node_id": "A"}],
},
{
"name": "three_nodes_two_topics",
"supported_protocols": SUPPORTED_PROTOCOLS,
"adj_list": {
"A": ["B"],
"B": ["C"],
},
"topic_map": {
"topic1": ["B", "C"],
"topic2": ["B", "C"],
},
"adj_list": {"A": ["B"], "B": ["C"]},
"topic_map": {"topic1": ["B", "C"], "topic2": ["B", "C"]},
"messages": [
{
"topics": ["topic1"],
"data": b"foo",
"node_id": "A",
},
{
"topics": ["topic2"],
"data": b"Alex is tall",
"node_id": "A",
}
]
{"topics": ["topic1"], "data": b"foo", "node_id": "A"},
{"topics": ["topic2"], "data": b"Alex is tall", "node_id": "A"},
],
},
{
"name": "two_nodes_one_topic_single_subscriber_is_sender",
"supported_protocols": SUPPORTED_PROTOCOLS,
"adj_list": {
"A": ["B"],
},
"topic_map": {
"topic1": ["B"],
},
"messages": [
{
"topics": ["topic1"],
"data": b"Alex is tall",
"node_id": "B",
}
]
"adj_list": {"A": ["B"]},
"topic_map": {"topic1": ["B"]},
"messages": [{"topics": ["topic1"], "data": b"Alex is tall", "node_id": "B"}],
},
{
"name": "two_nodes_one_topic_two_msgs",
"supported_protocols": SUPPORTED_PROTOCOLS,
"adj_list": {
"A": ["B"],
},
"topic_map": {
"topic1": ["B"],
},
"adj_list": {"A": ["B"]},
"topic_map": {"topic1": ["B"]},
"messages": [
{
"topics": ["topic1"],
"data": b"Alex is tall",
"node_id": "B",
},
{
"topics": ["topic1"],
"data": b"foo",
"node_id": "A",
}
]
{"topics": ["topic1"], "data": b"Alex is tall", "node_id": "B"},
{"topics": ["topic1"], "data": b"foo", "node_id": "A"},
],
},
{
"name": "seven_nodes_tree_one_topics",
"supported_protocols": SUPPORTED_PROTOCOLS,
"adj_list": {
"1": ["2", "3"],
"2": ["4", "5"],
"3": ["6", "7"],
},
"topic_map": {
"astrophysics": ["2", "3", "4", "5", "6", "7"],
},
"messages": [
{
"topics": ["astrophysics"],
"data": b"e=mc^2",
"node_id": "1",
}
]
"adj_list": {"1": ["2", "3"], "2": ["4", "5"], "3": ["6", "7"]},
"topic_map": {"astrophysics": ["2", "3", "4", "5", "6", "7"]},
"messages": [{"topics": ["astrophysics"], "data": b"e=mc^2", "node_id": "1"}],
},
{
"name": "seven_nodes_tree_three_topics",
"supported_protocols": SUPPORTED_PROTOCOLS,
"adj_list": {
"1": ["2", "3"],
"2": ["4", "5"],
"3": ["6", "7"],
},
"adj_list": {"1": ["2", "3"], "2": ["4", "5"], "3": ["6", "7"]},
"topic_map": {
"astrophysics": ["2", "3", "4", "5", "6", "7"],
"space": ["2", "3", "4", "5", "6", "7"],
"onions": ["2", "3", "4", "5", "6", "7"],
},
"messages": [
{
"topics": ["astrophysics"],
"data": b"e=mc^2",
"node_id": "1",
},
{
"topics": ["space"],
"data": b"foobar",
"node_id": "1",
},
{
"topics": ["onions"],
"data": b"I am allergic",
"node_id": "1",
}
]
{"topics": ["astrophysics"], "data": b"e=mc^2", "node_id": "1"},
{"topics": ["space"], "data": b"foobar", "node_id": "1"},
{"topics": ["onions"], "data": b"I am allergic", "node_id": "1"},
],
},
{
"name": "seven_nodes_tree_three_topics_diff_origin",
"supported_protocols": SUPPORTED_PROTOCOLS,
"adj_list": {
"1": ["2", "3"],
"2": ["4", "5"],
"3": ["6", "7"],
},
"adj_list": {"1": ["2", "3"], "2": ["4", "5"], "3": ["6", "7"]},
"topic_map": {
"astrophysics": ["1", "2", "3", "4", "5", "6", "7"],
"space": ["1", "2", "3", "4", "5", "6", "7"],
"onions": ["1", "2", "3", "4", "5", "6", "7"],
},
"messages": [
{
"topics": ["astrophysics"],
"data": b"e=mc^2",
"node_id": "1",
},
{
"topics": ["space"],
"data": b"foobar",
"node_id": "4",
},
{
"topics": ["onions"],
"data": b"I am allergic",
"node_id": "7",
}
]
{"topics": ["astrophysics"], "data": b"e=mc^2", "node_id": "1"},
{"topics": ["space"], "data": b"foobar", "node_id": "4"},
{"topics": ["onions"], "data": b"I am allergic", "node_id": "7"},
],
},
{
"name": "three_nodes_clique_two_topic_diff_origin",
"supported_protocols": SUPPORTED_PROTOCOLS,
"adj_list": {
"1": ["2", "3"],
"2": ["3"],
},
"topic_map": {
"astrophysics": ["1", "2", "3"],
"school": ["1", "2", "3"],
},
"adj_list": {"1": ["2", "3"], "2": ["3"]},
"topic_map": {"astrophysics": ["1", "2", "3"], "school": ["1", "2", "3"]},
"messages": [
{
"topics": ["astrophysics"],
"data": b"e=mc^2",
"node_id": "1",
},
{
"topics": ["school"],
"data": b"foobar",
"node_id": "2",
},
{
"topics": ["astrophysics"],
"data": b"I am allergic",
"node_id": "1",
}
]
{"topics": ["astrophysics"], "data": b"e=mc^2", "node_id": "1"},
{"topics": ["school"], "data": b"foobar", "node_id": "2"},
{"topics": ["astrophysics"], "data": b"I am allergic", "node_id": "1"},
],
},
{
"name": "four_nodes_clique_two_topic_diff_origin_many_msgs",
@ -224,95 +110,33 @@ FLOODSUB_PROTOCOL_TEST_CASES = [
"school": ["1", "2", "3", "4"],
},
"messages": [
{
"topics": ["astrophysics"],
"data": b"e=mc^2",
"node_id": "1",
},
{
"topics": ["school"],
"data": b"foobar",
"node_id": "2",
},
{
"topics": ["astrophysics"],
"data": b"I am allergic",
"node_id": "1",
},
{
"topics": ["school"],
"data": b"foobar2",
"node_id": "2",
},
{
"topics": ["astrophysics"],
"data": b"I am allergic2",
"node_id": "1",
},
{
"topics": ["school"],
"data": b"foobar3",
"node_id": "2",
},
{
"topics": ["astrophysics"],
"data": b"I am allergic3",
"node_id": "1",
}
]
{"topics": ["astrophysics"], "data": b"e=mc^2", "node_id": "1"},
{"topics": ["school"], "data": b"foobar", "node_id": "2"},
{"topics": ["astrophysics"], "data": b"I am allergic", "node_id": "1"},
{"topics": ["school"], "data": b"foobar2", "node_id": "2"},
{"topics": ["astrophysics"], "data": b"I am allergic2", "node_id": "1"},
{"topics": ["school"], "data": b"foobar3", "node_id": "2"},
{"topics": ["astrophysics"], "data": b"I am allergic3", "node_id": "1"},
],
},
{
"name": "five_nodes_ring_two_topic_diff_origin_many_msgs",
"supported_protocols": SUPPORTED_PROTOCOLS,
"adj_list": {
"1": ["2"],
"2": ["3"],
"3": ["4"],
"4": ["5"],
"5": ["1"],
},
"adj_list": {"1": ["2"], "2": ["3"], "3": ["4"], "4": ["5"], "5": ["1"]},
"topic_map": {
"astrophysics": ["1", "2", "3", "4", "5"],
"school": ["1", "2", "3", "4", "5"],
},
"messages": [
{
"topics": ["astrophysics"],
"data": b"e=mc^2",
"node_id": "1",
},
{
"topics": ["school"],
"data": b"foobar",
"node_id": "2",
},
{
"topics": ["astrophysics"],
"data": b"I am allergic",
"node_id": "1",
},
{
"topics": ["school"],
"data": b"foobar2",
"node_id": "2",
},
{
"topics": ["astrophysics"],
"data": b"I am allergic2",
"node_id": "1",
},
{
"topics": ["school"],
"data": b"foobar3",
"node_id": "2",
},
{
"topics": ["astrophysics"],
"data": b"I am allergic3",
"node_id": "1",
}
]
}
{"topics": ["astrophysics"], "data": b"e=mc^2", "node_id": "1"},
{"topics": ["school"], "data": b"foobar", "node_id": "2"},
{"topics": ["astrophysics"], "data": b"I am allergic", "node_id": "1"},
{"topics": ["school"], "data": b"foobar2", "node_id": "2"},
{"topics": ["astrophysics"], "data": b"I am allergic2", "node_id": "1"},
{"topics": ["school"], "data": b"foobar3", "node_id": "2"},
{"topics": ["astrophysics"], "data": b"I am allergic3", "node_id": "1"},
],
},
]
# pylint: disable=invalid-name
@ -420,12 +244,7 @@ async def perform_test_from_obj(obj, router_factory):
# Publish message
# TODO: Should be single RPC package with several topics
for topic in topics:
tasks_publish.append(
pubsub_map[node_id].publish(
topic,
data,
)
)
tasks_publish.append(pubsub_map[node_id].publish(topic, data))
# For each topic in topics, add (topic, node_id, data) tuple to ordered test list
for topic in topics:

15
tests/pubsub/test_dummyaccount_demo.py
View File

@ -3,10 +3,7 @@ from threading import Thread
import pytest
from tests.utils import (
cleanup,
connect,
)
from tests.utils import cleanup, connect
from .dummy_account_node import DummyAccountNode
@ -16,6 +13,7 @@ from .dummy_account_node import DummyAccountNode
def create_setup_in_new_thread_func(dummy_node):
def setup_in_new_thread():
asyncio.ensure_future(dummy_node.setup_crypto_networking())
return setup_in_new_thread
@ -39,8 +37,9 @@ async def perform_test(num_nodes, adjacency_map, action_func, assertion_func):
for source_num in adjacency_map:
target_nums = adjacency_map[source_num]
for target_num in target_nums:
await connect(dummy_nodes[source_num].libp2p_node, \
dummy_nodes[target_num].libp2p_node)
await connect(
dummy_nodes[source_num].libp2p_node, dummy_nodes[target_num].libp2p_node
)
# Allow time for network creation to take place
await asyncio.sleep(0.25)
@ -142,6 +141,7 @@ async def test_set_then_send_from_root_seven_nodes_tree_topography():
await perform_test(num_nodes, adj_map, action_func, assertion_func)
@pytest.mark.asyncio
async def test_set_then_send_from_different_leafs_seven_nodes_tree_topography():
num_nodes = 7
@ -158,6 +158,7 @@ async def test_set_then_send_from_different_leafs_seven_nodes_tree_topography():
await perform_test(num_nodes, adj_map, action_func, assertion_func)
@pytest.mark.asyncio
async def test_simple_five_nodes_ring_topography():
num_nodes = 5
@ -171,6 +172,7 @@ async def test_simple_five_nodes_ring_topography():
await perform_test(num_nodes, adj_map, action_func, assertion_func)
@pytest.mark.asyncio
async def test_set_then_send_from_diff_nodes_five_nodes_ring_topography():
num_nodes = 5
@ -187,6 +189,7 @@ async def test_set_then_send_from_diff_nodes_five_nodes_ring_topography():
await perform_test(num_nodes, adj_map, action_func, assertion_func)
@pytest.mark.asyncio
async def test_set_then_send_from_five_diff_nodes_five_nodes_ring_topography():
num_nodes = 5

24
tests/pubsub/test_floodsub.py
View File

@ -7,15 +7,9 @@ from libp2p.peer.id import ID
from libp2p.pubsub.floodsub import FloodSub
from libp2p.pubsub.pubsub import Pubsub
from tests.utils import (
cleanup,
connect,
)
from tests.utils import cleanup, connect
from .configs import (
FLOODSUB_PROTOCOL_ID,
LISTEN_MADDR,
)
from .configs import FLOODSUB_PROTOCOL_ID, LISTEN_MADDR
from .floodsub_integration_test_settings import (
perform_test_from_obj,
floodsub_protocol_pytest_params,
@ -85,7 +79,9 @@ async def test_lru_cache_two_nodes(monkeypatch):
def get_msg_id(msg):
# Originally it is `(msg.seqno, msg.from_id)`
return (msg.data, msg.from_id)
import libp2p.pubsub.pubsub
monkeypatch.setattr(libp2p.pubsub.pubsub, "get_msg_id", get_msg_id)
# Initialize Pubsub with a cache_size of 4
@ -104,7 +100,7 @@ async def test_lru_cache_two_nodes(monkeypatch):
def _make_testing_data(i: int) -> bytes:
num_int_bytes = 4
if i >= 2**(num_int_bytes * 8):
if i >= 2 ** (num_int_bytes * 8):
raise ValueError("integer is too large to be serialized")
return b"data" + i.to_bytes(num_int_bytes, "big")
@ -121,13 +117,7 @@ async def test_lru_cache_two_nodes(monkeypatch):
await cleanup()
@pytest.mark.parametrize(
"test_case_obj",
floodsub_protocol_pytest_params,
)
@pytest.mark.parametrize("test_case_obj", floodsub_protocol_pytest_params)
@pytest.mark.asyncio
async def test_gossipsub_run_with_floodsub_tests(test_case_obj):
await perform_test_from_obj(
test_case_obj,
FloodSub,
)
await perform_test_from_obj(test_case_obj, FloodSub)

92
tests/pubsub/test_gossipsub.py
View File

@ -3,10 +3,7 @@ import random
import pytest
from tests.utils import (
cleanup,
connect,
)
from tests.utils import cleanup, connect
from .configs import GOSSIPSUB_PROTOCOL_ID
from .utils import (
@ -28,9 +25,9 @@ async def test_join():
libp2p_hosts = await create_libp2p_hosts(num_hosts)
# Create pubsub, gossipsub instances
pubsubs, gossipsubs = create_pubsub_and_gossipsub_instances(libp2p_hosts, \
SUPPORTED_PROTOCOLS, \
4, 3, 5, 30, 3, 5, 0.5)
pubsubs, gossipsubs = create_pubsub_and_gossipsub_instances(
libp2p_hosts, SUPPORTED_PROTOCOLS, 4, 3, 5, 30, 3, 5, 0.5
)
topic = "test_join"
central_node_index = 0
@ -69,10 +66,19 @@ async def test_join():
for i in hosts_indices:
if i in subscribed_peer_indices:
assert str(libp2p_hosts[i].get_id()) in gossipsubs[central_node_index].mesh[topic]
assert str(libp2p_hosts[central_node_index].get_id()) in gossipsubs[i].mesh[topic]
assert (
str(libp2p_hosts[i].get_id())
in gossipsubs[central_node_index].mesh[topic]
)
assert (
str(libp2p_hosts[central_node_index].get_id())
in gossipsubs[i].mesh[topic]
)
else:
assert str(libp2p_hosts[i].get_id()) not in gossipsubs[central_node_index].mesh[topic]
assert (
str(libp2p_hosts[i].get_id())
not in gossipsubs[central_node_index].mesh[topic]
)
assert topic not in gossipsubs[i].mesh
await cleanup()
@ -84,9 +90,9 @@ async def test_leave():
libp2p_hosts = await create_libp2p_hosts(num_hosts)
# Create pubsub, gossipsub instances
_, gossipsubs = create_pubsub_and_gossipsub_instances(libp2p_hosts, \
SUPPORTED_PROTOCOLS, \
10, 9, 11, 30, 3, 5, 0.5)
_, gossipsubs = create_pubsub_and_gossipsub_instances(
libp2p_hosts, SUPPORTED_PROTOCOLS, 10, 9, 11, 30, 3, 5, 0.5
)
gossipsub = gossipsubs[0]
topic = "test_leave"
@ -111,9 +117,9 @@ async def test_handle_graft(event_loop, monkeypatch):
libp2p_hosts = await create_libp2p_hosts(num_hosts)
# Create pubsub, gossipsub instances
_, gossipsubs = create_pubsub_and_gossipsub_instances(libp2p_hosts, \
SUPPORTED_PROTOCOLS, \
10, 9, 11, 30, 3, 5, 0.5)
_, gossipsubs = create_pubsub_and_gossipsub_instances(
libp2p_hosts, SUPPORTED_PROTOCOLS, 10, 9, 11, 30, 3, 5, 0.5
)
index_alice = 0
id_alice = str(libp2p_hosts[index_alice].get_id())
@ -131,10 +137,11 @@ async def test_handle_graft(event_loop, monkeypatch):
# Monkey patch bob's `emit_prune` function so we can
# check if it is called in `handle_graft`
event_emit_prune = asyncio.Event()
async def emit_prune(topic, sender_peer_id):
event_emit_prune.set()
monkeypatch.setattr(gossipsubs[index_bob], 'emit_prune', emit_prune)
monkeypatch.setattr(gossipsubs[index_bob], "emit_prune", emit_prune)
# Check that alice is bob's peer but not his mesh peer
assert id_alice in gossipsubs[index_bob].peers_gossipsub
@ -143,11 +150,7 @@ async def test_handle_graft(event_loop, monkeypatch):
await gossipsubs[index_alice].emit_graft(topic, id_bob)
# Check that `emit_prune` is called
await asyncio.wait_for(
event_emit_prune.wait(),
timeout=1,
loop=event_loop,
)
await asyncio.wait_for(event_emit_prune.wait(), timeout=1, loop=event_loop)
assert event_emit_prune.is_set()
# Check that bob is alice's peer but not her mesh peer
@ -171,9 +174,9 @@ async def test_handle_prune():
libp2p_hosts = await create_libp2p_hosts(num_hosts)
# Create pubsub, gossipsub instances
pubsubs, gossipsubs = create_pubsub_and_gossipsub_instances(libp2p_hosts, \
SUPPORTED_PROTOCOLS, \
10, 9, 11, 30, 3, 5, 3)
pubsubs, gossipsubs = create_pubsub_and_gossipsub_instances(
libp2p_hosts, SUPPORTED_PROTOCOLS, 10, 9, 11, 30, 3, 5, 3
)
index_alice = 0
id_alice = str(libp2p_hosts[index_alice].get_id())
@ -217,9 +220,9 @@ async def test_dense():
libp2p_hosts = await create_libp2p_hosts(num_hosts)
# Create pubsub, gossipsub instances
pubsubs, _ = create_pubsub_and_gossipsub_instances(libp2p_hosts, \
SUPPORTED_PROTOCOLS, \
10, 9, 11, 30, 3, 5, 0.5)
pubsubs, _ = create_pubsub_and_gossipsub_instances(
libp2p_hosts, SUPPORTED_PROTOCOLS, 10, 9, 11, 30, 3, 5, 0.5
)
# All pubsub subscribe to foobar
queues = []
@ -236,7 +239,7 @@ async def test_dense():
await asyncio.sleep(2)
for i in range(num_msgs):
msg_content = b"foo " + i.to_bytes(1, 'big')
msg_content = b"foo " + i.to_bytes(1, "big")
# randomly pick a message origin
origin_idx = random.randint(0, num_hosts - 1)
@ -260,9 +263,9 @@ async def test_fanout():
libp2p_hosts = await create_libp2p_hosts(num_hosts)
# Create pubsub, gossipsub instances
pubsubs, _ = create_pubsub_and_gossipsub_instances(libp2p_hosts, \
SUPPORTED_PROTOCOLS, \
10, 9, 11, 30, 3, 5, 0.5)
pubsubs, _ = create_pubsub_and_gossipsub_instances(
libp2p_hosts, SUPPORTED_PROTOCOLS, 10, 9, 11, 30, 3, 5, 0.5
)
# All pubsub subscribe to foobar except for `pubsubs[0]`
queues = []
@ -300,7 +303,7 @@ async def test_fanout():
# Send messages again
for i in range(num_msgs):
msg_content = b"bar " + i.to_bytes(1, 'big')
msg_content = b"bar " + i.to_bytes(1, "big")
# Pick the message origin to the node that is not subscribed to 'foobar'
origin_idx = 0
@ -316,6 +319,7 @@ async def test_fanout():
await cleanup()
@pytest.mark.asyncio
async def test_fanout_maintenance():
# Create libp2p hosts
@ -324,9 +328,9 @@ async def test_fanout_maintenance():
libp2p_hosts = await create_libp2p_hosts(num_hosts)
# Create pubsub, gossipsub instances
pubsubs, _ = create_pubsub_and_gossipsub_instances(libp2p_hosts, \
SUPPORTED_PROTOCOLS, \
10, 9, 11, 30, 3, 5, 0.5)
pubsubs, _ = create_pubsub_and_gossipsub_instances(
libp2p_hosts, SUPPORTED_PROTOCOLS, 10, 9, 11, 30, 3, 5, 0.5
)
# All pubsub subscribe to foobar
queues = []
@ -345,7 +349,7 @@ async def test_fanout_maintenance():
# Send messages with origin not subscribed
for i in range(num_msgs):
msg_content = b"foo " + i.to_bytes(1, 'big')
msg_content = b"foo " + i.to_bytes(1, "big")
# Pick the message origin to the node that is not subscribed to 'foobar'
origin_idx = 0
@ -377,7 +381,7 @@ async def test_fanout_maintenance():
# Check messages can still be sent
for i in range(num_msgs):
msg_content = b"bar " + i.to_bytes(1, 'big')
msg_content = b"bar " + i.to_bytes(1, "big")
# Pick the message origin to the node that is not subscribed to 'foobar'
origin_idx = 0
@ -402,22 +406,14 @@ async def test_gossip_propagation():
# Create pubsub, gossipsub instances
pubsubs, _ = create_pubsub_and_gossipsub_instances(
hosts,
SUPPORTED_PROTOCOLS,
1,
0,
2,
30,
50,
100,
0.5,
hosts, SUPPORTED_PROTOCOLS, 1, 0, 2, 30, 50, 100, 0.5
)
topic = "foo"
await pubsubs[0].subscribe(topic)
# node 0 publish to topic
msg_content = b'foo_msg'
msg_content = b"foo_msg"
# publish from the randomly chosen host
await pubsubs[0].publish(topic, msg_content)

18
tests/pubsub/test_gossipsub_backward_compatibility.py
View File

@ -8,10 +8,7 @@ from libp2p.pubsub.pubsub import Pubsub
from tests.utils import cleanup
from .configs import (
FLOODSUB_PROTOCOL_ID,
LISTEN_MADDR,
)
from .configs import FLOODSUB_PROTOCOL_ID, LISTEN_MADDR
from .floodsub_integration_test_settings import (
perform_test_from_obj,
floodsub_protocol_pytest_params,
@ -34,19 +31,12 @@ async def test_gossipsub_initialize_with_floodsub_protocol():
await cleanup()
@pytest.mark.parametrize(
"test_case_obj",
floodsub_protocol_pytest_params,
)
@pytest.mark.parametrize("test_case_obj", floodsub_protocol_pytest_params)
@pytest.mark.asyncio
async def test_gossipsub_run_with_floodsub_tests(test_case_obj):
await perform_test_from_obj(
test_case_obj,
functools.partial(
GossipSub,
degree=3,
degree_low=2,
degree_high=4,
time_to_live=30,
)
GossipSub, degree=3, degree_low=2, degree_high=4, time_to_live=30
),
)

6
tests/pubsub/test_mcache.py
View File

@ -33,7 +33,7 @@ async def test_mcache():
# successful read
assert get_msg == msg
gids = mcache.window('test')
gids = mcache.window("test")
assert len(gids) == 10
@ -55,7 +55,7 @@ async def test_mcache():
assert get_msg == msg
gids = mcache.window('test')
gids = mcache.window("test")
assert len(gids) == 20
@ -108,7 +108,7 @@ async def test_mcache():
assert get_msg == msg
gids = mcache.window('test')
gids = mcache.window("test")
assert len(gids) == 30

132
tests/pubsub/test_pubsub.py
View File

@ -7,23 +7,16 @@ import pytest
from libp2p.peer.id import ID
from libp2p.pubsub.pb import rpc_pb2
from tests.utils import (
connect,
)
from tests.utils import connect
from .utils import (
make_pubsub_msg,
)
from .utils import make_pubsub_msg
TESTING_TOPIC = "TEST_SUBSCRIBE"
TESTING_DATA = b"data"
@pytest.mark.parametrize(
"num_hosts",
(1,),
)
@pytest.mark.parametrize("num_hosts", (1,))
@pytest.mark.asyncio
async def test_subscribe_and_unsubscribe(pubsubs_fsub):
await pubsubs_fsub[0].subscribe(TESTING_TOPIC)
@ -33,10 +26,7 @@ async def test_subscribe_and_unsubscribe(pubsubs_fsub):
assert TESTING_TOPIC not in pubsubs_fsub[0].my_topics
@pytest.mark.parametrize(
"num_hosts",
(1,),
)
@pytest.mark.parametrize("num_hosts", (1,))
@pytest.mark.asyncio
async def test_re_subscribe(pubsubs_fsub):
await pubsubs_fsub[0].subscribe(TESTING_TOPIC)
@ -46,10 +36,7 @@ async def test_re_subscribe(pubsubs_fsub):
assert TESTING_TOPIC in pubsubs_fsub[0].my_topics
@pytest.mark.parametrize(
"num_hosts",
(1,),
)
@pytest.mark.parametrize("num_hosts", (1,))
@pytest.mark.asyncio
async def test_re_unsubscribe(pubsubs_fsub):
# Unsubscribe from topic we didn't even subscribe to
@ -80,19 +67,13 @@ async def test_peers_subscribe(pubsubs_fsub):
assert str(pubsubs_fsub[0].my_id) not in pubsubs_fsub[1].peer_topics[TESTING_TOPIC]
@pytest.mark.parametrize(
"num_hosts",
(1,),
)
@pytest.mark.parametrize("num_hosts", (1,))
@pytest.mark.asyncio
async def test_get_hello_packet(pubsubs_fsub):
def _get_hello_packet_topic_ids():
packet = rpc_pb2.RPC()
packet.ParseFromString(pubsubs_fsub[0].get_hello_packet())
return tuple(
sub.topicid
for sub in packet.subscriptions
)
return tuple(sub.topicid for sub in packet.subscriptions)
# pylint: disable=len-as-condition
# Test: No subscription, so there should not be any topic ids in the hello packet.
@ -100,10 +81,7 @@ async def test_get_hello_packet(pubsubs_fsub):
# Test: After subscriptions, topic ids should be in the hello packet.
topic_ids = ["t", "o", "p", "i", "c"]
await asyncio.gather(*[
pubsubs_fsub[0].subscribe(topic)
for topic in topic_ids
])
await asyncio.gather(*[pubsubs_fsub[0].subscribe(topic) for topic in topic_ids])
topic_ids_in_hello = _get_hello_packet_topic_ids()
for topic in topic_ids:
assert topic in topic_ids_in_hello
@ -128,14 +106,11 @@ class FakeNetStream:
async def write(self, data: bytes) -> int:
for i in data:
await self._queue.put(i.to_bytes(1, 'big'))
await self._queue.put(i.to_bytes(1, "big"))
return len(data)
@pytest.mark.parametrize(
"num_hosts",
(1,),
)
@pytest.mark.parametrize("num_hosts", (1,))
@pytest.mark.asyncio
async def test_continuously_read_stream(pubsubs_fsub, monkeypatch):
stream = FakeNetStream()
@ -156,7 +131,9 @@ async def test_continuously_read_stream(pubsubs_fsub, monkeypatch):
event_handle_rpc.set()
monkeypatch.setattr(pubsubs_fsub[0], "push_msg", mock_push_msg)
monkeypatch.setattr(pubsubs_fsub[0], "handle_subscription", mock_handle_subscription)
monkeypatch.setattr(
pubsubs_fsub[0], "handle_subscription", mock_handle_subscription
)
monkeypatch.setattr(pubsubs_fsub[0].router, "handle_rpc", mock_handle_rpc)
async def wait_for_event_occurring(event):
@ -176,9 +153,7 @@ async def test_continuously_read_stream(pubsubs_fsub, monkeypatch):
# Test: `push_msg` is called when publishing to a subscribed topic.
publish_subscribed_topic = rpc_pb2.RPC(
publish=[rpc_pb2.Message(
topicIDs=[TESTING_TOPIC]
)],
publish=[rpc_pb2.Message(topicIDs=[TESTING_TOPIC])]
)
await stream.write(publish_subscribed_topic.SerializeToString())
await wait_for_event_occurring(event_push_msg)
@ -190,18 +165,14 @@ async def test_continuously_read_stream(pubsubs_fsub, monkeypatch):
# Test: `push_msg` is not called when publishing to a topic-not-subscribed.
publish_not_subscribed_topic = rpc_pb2.RPC(
publish=[rpc_pb2.Message(
topicIDs=["NOT_SUBSCRIBED"]
)],
publish=[rpc_pb2.Message(topicIDs=["NOT_SUBSCRIBED"])]
)
await stream.write(publish_not_subscribed_topic.SerializeToString())
with pytest.raises(asyncio.TimeoutError):
await wait_for_event_occurring(event_push_msg)
# Test: `handle_subscription` is called when a subscription message is received.
subscription_msg = rpc_pb2.RPC(
subscriptions=[rpc_pb2.RPC.SubOpts()],
)
subscription_msg = rpc_pb2.RPC(subscriptions=[rpc_pb2.RPC.SubOpts()])
await stream.write(subscription_msg.SerializeToString())
await wait_for_event_occurring(event_handle_subscription)
# Make sure the other events are not emitted.
@ -229,23 +200,17 @@ async def test_continuously_read_stream(pubsubs_fsub, monkeypatch):
# - `test_handle_peer_queue`
@pytest.mark.parametrize(
"num_hosts",
(1,),
)
@pytest.mark.parametrize("num_hosts", (1,))
def test_handle_subscription(pubsubs_fsub):
assert len(pubsubs_fsub[0].peer_topics) == 0
sub_msg_0 = rpc_pb2.RPC.SubOpts(
subscribe=True,
topicid=TESTING_TOPIC,
)
peer_ids = [
ID(b"\x12\x20" + i.to_bytes(32, "big"))
for i in range(2)
]
sub_msg_0 = rpc_pb2.RPC.SubOpts(subscribe=True, topicid=TESTING_TOPIC)
peer_ids = [ID(b"\x12\x20" + i.to_bytes(32, "big")) for i in range(2)]
# Test: One peer is subscribed
pubsubs_fsub[0].handle_subscription(peer_ids[0], sub_msg_0)
assert len(pubsubs_fsub[0].peer_topics) == 1 and TESTING_TOPIC in pubsubs_fsub[0].peer_topics
assert (
len(pubsubs_fsub[0].peer_topics) == 1
and TESTING_TOPIC in pubsubs_fsub[0].peer_topics
)
assert len(pubsubs_fsub[0].peer_topics[TESTING_TOPIC]) == 1
assert str(peer_ids[0]) in pubsubs_fsub[0].peer_topics[TESTING_TOPIC]
# Test: Another peer is subscribed
@ -255,27 +220,18 @@ def test_handle_subscription(pubsubs_fsub):
assert str(peer_ids[1]) in pubsubs_fsub[0].peer_topics[TESTING_TOPIC]
# Test: Subscribe to another topic
another_topic = "ANOTHER_TOPIC"
sub_msg_1 = rpc_pb2.RPC.SubOpts(
subscribe=True,
topicid=another_topic,
)
sub_msg_1 = rpc_pb2.RPC.SubOpts(subscribe=True, topicid=another_topic)
pubsubs_fsub[0].handle_subscription(peer_ids[0], sub_msg_1)
assert len(pubsubs_fsub[0].peer_topics) == 2
assert another_topic in pubsubs_fsub[0].peer_topics
assert str(peer_ids[0]) in pubsubs_fsub[0].peer_topics[another_topic]
# Test: unsubscribe
unsub_msg = rpc_pb2.RPC.SubOpts(
subscribe=False,
topicid=TESTING_TOPIC,
)
unsub_msg = rpc_pb2.RPC.SubOpts(subscribe=False, topicid=TESTING_TOPIC)
pubsubs_fsub[0].handle_subscription(peer_ids[0], unsub_msg)
assert str(peer_ids[0]) not in pubsubs_fsub[0].peer_topics[TESTING_TOPIC]
@pytest.mark.parametrize(
"num_hosts",
(1,),
)
@pytest.mark.parametrize("num_hosts", (1,))
@pytest.mark.asyncio
async def test_handle_talk(pubsubs_fsub):
sub = await pubsubs_fsub[0].subscribe(TESTING_TOPIC)
@ -293,25 +249,19 @@ async def test_handle_talk(pubsubs_fsub):
seqno=b"\x11" * 8,
)
await pubsubs_fsub[0].handle_talk(msg_1)
assert len(pubsubs_fsub[0].my_topics) == 1 and sub == pubsubs_fsub[0].my_topics[TESTING_TOPIC]
assert (
len(pubsubs_fsub[0].my_topics) == 1
and sub == pubsubs_fsub[0].my_topics[TESTING_TOPIC]
)
assert sub.qsize() == 1
assert (await sub.get()) == msg_0
@pytest.mark.parametrize(
"num_hosts",
(1,),
)
@pytest.mark.parametrize("num_hosts", (1,))
@pytest.mark.asyncio
async def test_message_all_peers(pubsubs_fsub, monkeypatch):
peer_ids = [
ID(b"\x12\x20" + i.to_bytes(32, "big"))
for i in range(10)
]
mock_peers = {
str(peer_id): FakeNetStream()
for peer_id in peer_ids
}
peer_ids = [ID(b"\x12\x20" + i.to_bytes(32, "big")) for i in range(10)]
mock_peers = {str(peer_id): FakeNetStream() for peer_id in peer_ids}
monkeypatch.setattr(pubsubs_fsub[0], "peers", mock_peers)
empty_rpc = rpc_pb2.RPC()
@ -320,10 +270,7 @@ async def test_message_all_peers(pubsubs_fsub, monkeypatch):
assert (await stream.read()) == empty_rpc.SerializeToString()
@pytest.mark.parametrize(
"num_hosts",
(1,),
)
@pytest.mark.parametrize("num_hosts", (1,))
@pytest.mark.asyncio
async def test_publish(pubsubs_fsub, monkeypatch):
msg_forwarders = []
@ -332,20 +279,20 @@ async def test_publish(pubsubs_fsub, monkeypatch):
async def push_msg(msg_forwarder, msg):
msg_forwarders.append(msg_forwarder)
msgs.append(msg)
monkeypatch.setattr(pubsubs_fsub[0], "push_msg", push_msg)
await pubsubs_fsub[0].publish(TESTING_TOPIC, TESTING_DATA)
await pubsubs_fsub[0].publish(TESTING_TOPIC, TESTING_DATA)
assert len(msgs) == 2, "`push_msg` should be called every time `publish` is called"
assert (msg_forwarders[0] == msg_forwarders[1]) and (msg_forwarders[1] == pubsubs_fsub[0].my_id)
assert (msg_forwarders[0] == msg_forwarders[1]) and (
msg_forwarders[1] == pubsubs_fsub[0].my_id
)
assert msgs[0].seqno != msgs[1].seqno, "`seqno` should be different every time"
@pytest.mark.parametrize(
"num_hosts",
(1,),
)
@pytest.mark.parametrize("num_hosts", (1,))
@pytest.mark.asyncio
async def test_push_msg(pubsubs_fsub, monkeypatch):
# pylint: disable=protected-access
@ -360,6 +307,7 @@ async def test_push_msg(pubsubs_fsub, monkeypatch):
async def router_publish(*args, **kwargs):
event.set()
monkeypatch.setattr(pubsubs_fsub[0].router, "publish", router_publish)
# Test: `msg` is not seen before `push_msg`, and is seen after `push_msg`.

57
tests/pubsub/utils.py
View File

@ -1,8 +1,6 @@
import asyncio
import struct
from typing import (
Sequence,
)
from typing import Sequence
import multiaddr
@ -33,21 +31,16 @@ def message_id_generator(start_val):
val += 1
# Convert val to big endian
return struct.pack('>Q', val)
return struct.pack(">Q", val)
return generator
def make_pubsub_msg(
origin_id: ID,
topic_ids: Sequence[str],
data: bytes,
seqno: bytes) -> rpc_pb2.Message:
origin_id: ID, topic_ids: Sequence[str], data: bytes, seqno: bytes
) -> rpc_pb2.Message:
return rpc_pb2.Message(
from_id=origin_id.to_bytes(),
seqno=seqno,
data=data,
topicIDs=list(topic_ids),
from_id=origin_id.to_bytes(), seqno=seqno, data=data, topicIDs=list(topic_ids)
)
@ -61,13 +54,13 @@ def generate_RPC_packet(origin_id, topics, msg_content, msg_id):
"""
packet = rpc_pb2.RPC()
message = rpc_pb2.Message(
from_id=origin_id.encode('utf-8'),
from_id=origin_id.encode("utf-8"),
seqno=msg_id,
data=msg_content.encode('utf-8'),
data=msg_content.encode("utf-8"),
)
for topic in topics:
message.topicIDs.extend([topic.encode('utf-8')])
message.topicIDs.extend([topic.encode("utf-8")])
packet.publish.extend([message])
return packet
@ -95,22 +88,29 @@ async def create_libp2p_hosts(num_hosts):
def create_pubsub_and_gossipsub_instances(
libp2p_hosts,
supported_protocols,
degree,
degree_low,
degree_high,
time_to_live,
gossip_window,
gossip_history,
heartbeat_interval):
libp2p_hosts,
supported_protocols,
degree,
degree_low,
degree_high,
time_to_live,
gossip_window,
gossip_history,
heartbeat_interval,
):
pubsubs = []
gossipsubs = []
for node in libp2p_hosts:
gossipsub = GossipSub(supported_protocols, degree,
degree_low, degree_high, time_to_live,
gossip_window, gossip_history,
heartbeat_interval)
gossipsub = GossipSub(
supported_protocols,
degree,
degree_low,
degree_high,
time_to_live,
gossip_window,
gossip_history,
heartbeat_interval,
)
pubsub = Pubsub(node, gossipsub, node.get_id())
pubsubs.append(pubsub)
gossipsubs.append(gossipsub)
@ -121,6 +121,7 @@ def create_pubsub_and_gossipsub_instances(
# FIXME: There is no difference between `sparse_connect` and `dense_connect`,
# before `connect_some` is fixed.
async def sparse_connect(hosts):
await connect_some(hosts, 3)