From 6c5bac53d7ea511e3942ff006fedab2bb0e0469c Mon Sep 17 00:00:00 2001 From: Alex Haynes Date: Wed, 24 Apr 2019 22:11:54 -0400 Subject: [PATCH 1/7] refactored routedhost into router passed to swarm --- libp2p/__init__.py | 15 ++++++------- libp2p/host/basic_host.py | 21 ++++++++++--------- libp2p/kademlia/crawling.py | 2 +- libp2p/kademlia/kad_peerinfo.py | 6 +++--- libp2p/kademlia/network.py | 2 +- libp2p/kademlia/routed_host.py | 21 ------------------- libp2p/kademlia/routing.py | 6 +++--- libp2p/network/swarm.py | 10 +++++++-- libp2p/peer/id.py | 9 ++++++++ libp2p/routing/interfaces.py | 2 +- .../{kadmelia => kademlia}/__init__.py | 0 .../kademlia_content_router.py} | 0 .../kademlia_peer_router.py} | 4 +++- 13 files changed, 48 insertions(+), 50 deletions(-) delete mode 100644 libp2p/kademlia/routed_host.py rename libp2p/routing/{kadmelia => kademlia}/__init__.py (100%) rename libp2p/routing/{kadmelia/kadmelia_content_router.py => kademlia/kademlia_content_router.py} (100%) rename libp2p/routing/{kadmelia/kadmelia_peer_router.py => kademlia/kademlia_peer_router.py} (85%) diff --git a/libp2p/__init__.py b/libp2p/__init__.py index 2e1cf88b..faf8f13c 100644 --- a/libp2p/__init__.py +++ b/libp2p/__init__.py @@ -10,6 +10,7 @@ from .kademlia.routed_host import RoutedHost from .transport.upgrader import TransportUpgrader from .transport.tcp.tcp import TCP from .kademlia.network import KademliaServer +from .routing.kademlia.kademlia_peer_router import KadmeliaPeerRouter async def cleanup_done_tasks(): @@ -31,7 +32,7 @@ def generate_id(): # private_key = new_key.exportKey("PEM") return new_id -def initialize_default_kademlia( +def initialize_default_kademlia_router( ksize=20, alpha=3, id_opt=None, storage=None): """ initialize swam when no swarm is passed in @@ -46,8 +47,9 @@ def initialize_default_kademlia( id_opt = generate_id() node_id = id_opt.get_raw_id() - return KademliaServer(ksize=ksize, alpha=alpha, - node_id=node_id, storage=storage) + server = KademliaServer(ksize=ksize, alpha=alpha, + node_id=node_id, storage=storage) + return KadmeliaPeerRouter(server) def initialize_default_swarm( @@ -105,12 +107,11 @@ async def new_node( muxer_opt=muxer_opt, sec_opt=sec_opt, peerstore_opt=peerstore_opt) + swarm_opt.add_router(disc_opt) + # TODO enable support for other host type # TODO routing unimplemented - if not disc_opt: - host = BasicHost(swarm_opt) - else: - host = RoutedHost(swarm_opt, disc_opt) + host = BasicHost(swarm_opt) # Kick off cleanup job asyncio.ensure_future(cleanup_done_tasks()) diff --git a/libp2p/host/basic_host.py b/libp2p/host/basic_host.py index 0c3bf0eb..b85c4137 100644 --- a/libp2p/host/basic_host.py +++ b/libp2p/host/basic_host.py @@ -11,21 +11,22 @@ from .host_interface import IHost class BasicHost(IHost): # default options constructor - def __init__(self, _network): - self.network = _network - self.peerstore = self.network.peerstore + def __init__(self, network, router=None): + self._network = network + self.peerstore = self._network.peerstore + self._router = router def get_id(self): """ :return: peer_id of host """ - return self.network.get_peer_id() + return self._network.get_peer_id() def get_network(self): """ :return: network instance of host """ - return self.network + return self._network def get_peerstore(self): """ @@ -45,7 +46,7 @@ class BasicHost(IHost): p2p_part = multiaddr.Multiaddr('/p2p/{}'.format(self.get_id().pretty())) addrs = [] - for transport in self.network.listeners.values(): + for transport in self._network.listeners.values(): for addr in transport.get_addrs(): addrs.append(addr.encapsulate(p2p_part)) return addrs @@ -57,7 +58,7 @@ class BasicHost(IHost): :param stream_handler: a stream handler function :return: true if successful """ - return self.network.set_stream_handler(protocol_id, stream_handler) + return self._network.set_stream_handler(protocol_id, stream_handler) # protocol_id can be a list of protocol_ids # stream will decide which protocol_id to run on @@ -67,7 +68,7 @@ class BasicHost(IHost): :param protocol_id: protocol id that stream runs on :return: true if successful """ - stream = await self.network.new_stream(peer_id, protocol_ids) + stream = await self._network.new_stream(peer_id, protocol_ids) return stream async def connect(self, peer_info): @@ -84,7 +85,7 @@ class BasicHost(IHost): self.peerstore.add_addrs(peer_info.peer_id, peer_info.addrs, 10) # there is already a connection to this peer - if peer_info.peer_id in self.network.connections: + if peer_info.peer_id in self._network.connections: return - await self.network.dial_peer(peer_info.peer_id) + await self._network.dial_peer(peer_info.peer_id) diff --git a/libp2p/kademlia/crawling.py b/libp2p/kademlia/crawling.py index 6a3b5fe3..3fdfbc69 100644 --- a/libp2p/kademlia/crawling.py +++ b/libp2p/kademlia/crawling.py @@ -119,7 +119,7 @@ class ValueSpiderCrawl(SpiderCrawl): value_counts = Counter(values) if len(value_counts) != 1: log.warning("Got multiple values for key %i: %s", - self.node.long_id, str(values)) + self.node.xor_id, str(values)) value = value_counts.most_common(1)[0][0] peer = self.nearest_without_value.popleft() diff --git a/libp2p/kademlia/kad_peerinfo.py b/libp2p/kademlia/kad_peerinfo.py index fb40e3b7..0d8c233f 100644 --- a/libp2p/kademlia/kad_peerinfo.py +++ b/libp2p/kademlia/kad_peerinfo.py @@ -16,7 +16,7 @@ class KadPeerInfo(PeerInfo): super(KadPeerInfo, self).__init__(peer_id, peer_data) self.peer_id = peer_id.get_raw_id() - self.long_id = int(digest(peer_id.get_raw_id()).hex(), 16) + self.xor_id = peer_id.get_xor_id() self.addrs = peer_data.get_addrs() if peer_data else None @@ -34,7 +34,7 @@ class KadPeerInfo(PeerInfo): """ Get the distance between this node and another. """ - return self.long_id ^ node.long_id + return self.xor_id ^ node.xor_id def __iter__(self): """ @@ -43,7 +43,7 @@ class KadPeerInfo(PeerInfo): return iter([self.peer_id, self.ip, self.port]) def __repr__(self): - return repr([self.long_id, self.ip, self.port]) + return repr([self.xor_id, self.ip, self.port]) def __str__(self): return "%s:%s" % (self.ip, str(self.port)) diff --git a/libp2p/kademlia/network.py b/libp2p/kademlia/network.py index 86522ba2..836d34a2 100644 --- a/libp2p/kademlia/network.py +++ b/libp2p/kademlia/network.py @@ -67,7 +67,7 @@ class KademliaServer: listen = loop.create_datagram_endpoint(self._create_protocol, local_addr=(interface, port)) log.info("Node %i listening on %s:%i", - self.node.long_id, interface, port) + self.node.xor_id, interface, port) self.transport, self.protocol = await listen # finally, schedule refreshing table self.refresh_table() diff --git a/libp2p/kademlia/routed_host.py b/libp2p/kademlia/routed_host.py deleted file mode 100644 index 45fc0c6f..00000000 --- a/libp2p/kademlia/routed_host.py +++ /dev/null @@ -1,21 +0,0 @@ -from libp2p.host.basic_host import BasicHost - -class RoutedHost(BasicHost): - def __init__(self, _network, _kad_network): - super(RoutedHost, self).__init__(_network) - self.kad_network = _kad_network - - def get_kad_network(self): - return self.kad_network - - def routed_listen(self, port, interface='0.0.0.0'): - return self.kad_network.listen(port, interface) - - def routed_get(self, key): - return self.kad_network.get(key) - - def routed_set(self, key, value): - return self.kad_network.set(key, value) - - def routed_set_digest(self, dkey, value): - return self.kad_network.set_digest(dkey, value) diff --git a/libp2p/kademlia/routing.py b/libp2p/kademlia/routing.py index 7d97494e..b84717e9 100644 --- a/libp2p/kademlia/routing.py +++ b/libp2p/kademlia/routing.py @@ -33,7 +33,7 @@ class KBucket: one = KBucket(self.range[0], midpoint, self.ksize) two = KBucket(midpoint + 1, self.range[1], self.ksize) for node in self.nodes.values(): - bucket = one if node.long_id <= midpoint else two + bucket = one if node.xor_id <= midpoint else two bucket.nodes[node.peer_id] = node return (one, two) @@ -48,7 +48,7 @@ class KBucket: self.nodes[newnode.peer_id] = newnode def has_in_range(self, node): - return self.range[0] <= node.long_id <= self.range[1] + return self.range[0] <= node.xor_id <= self.range[1] def is_new_node(self, node): return node.peer_id not in self.nodes @@ -175,7 +175,7 @@ class RoutingTable: Get the index of the bucket that the given node would fall into. """ for index, bucket in enumerate(self.buckets): - if node.long_id < bucket.range[1]: + if node.xor_id < bucket.range[1]: return index # we should never be here, but make linter happy return None diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index b30567b5..4c0c1280 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -20,6 +20,7 @@ class Swarm(INetwork): self.listeners = dict() self.stream_handlers = dict() self.transport = None + self.router = None # Protocol muxing self.multiselect = Multiselect() @@ -57,8 +58,10 @@ class Swarm(INetwork): if not addrs: raise SwarmException("No known addresses to peer") - # TODO: define logic to choose which address to use, or try them all ? - multiaddr = addrs[0] + if not self.router: + multiaddr = addrs[0] + else: + multiaddr = self.router.find_peer(peer_id) if peer_id in self.connections: # If muxed connection already exists for peer_id, @@ -187,6 +190,9 @@ class Swarm(INetwork): # TODO: Support more than one transport self.transport = transport + def add_router(self, router): + self.router = router + def create_generic_protocol_handler(swarm): """ Create a generic protocol handler from the given swarm. We use swarm diff --git a/libp2p/peer/id.py b/libp2p/peer/id.py index 92b4db34..f3b96475 100644 --- a/libp2p/peer/id.py +++ b/libp2p/peer/id.py @@ -1,5 +1,6 @@ import base58 import multihash +import hashlib # MaxInlineKeyLength is the maximum length a key can be for it to be inlined in # the peer ID. @@ -21,6 +22,9 @@ class ID: def pretty(self): return base58.b58encode(self._id_str).decode() + def get_xor_id(self): + return int(digest(self.get_raw_id()).hex(), 16) + def __str__(self): pid = self.pretty() if len(pid) <= 10: @@ -67,3 +71,8 @@ def id_from_public_key(key): def id_from_private_key(key): return id_from_public_key(key.publickey()) + +def digest(string): + if not isinstance(string, bytes): + string = str(string).encode('utf8') + return hashlib.sha1(string).digest() diff --git a/libp2p/routing/interfaces.py b/libp2p/routing/interfaces.py index 1f29d48c..b519a096 100644 --- a/libp2p/routing/interfaces.py +++ b/libp2p/routing/interfaces.py @@ -28,4 +28,4 @@ class IPeerRouting(ABC): Find specific Peer FindPeer searches for a peer with given peer_id, returns a peer.PeerInfo with relevant addresses. - """ + """ \ No newline at end of file diff --git a/libp2p/routing/kadmelia/__init__.py b/libp2p/routing/kademlia/__init__.py similarity index 100% rename from libp2p/routing/kadmelia/__init__.py rename to libp2p/routing/kademlia/__init__.py diff --git a/libp2p/routing/kadmelia/kadmelia_content_router.py b/libp2p/routing/kademlia/kademlia_content_router.py similarity index 100% rename from libp2p/routing/kadmelia/kadmelia_content_router.py rename to libp2p/routing/kademlia/kademlia_content_router.py diff --git a/libp2p/routing/kadmelia/kadmelia_peer_router.py b/libp2p/routing/kademlia/kademlia_peer_router.py similarity index 85% rename from libp2p/routing/kadmelia/kadmelia_peer_router.py rename to libp2p/routing/kademlia/kademlia_peer_router.py index 27be67c7..3a13f459 100644 --- a/libp2p/routing/kadmelia/kadmelia_peer_router.py +++ b/libp2p/routing/kademlia/kademlia_peer_router.py @@ -16,7 +16,9 @@ class KadmeliaPeerRouter(IPeerRouting): FindPeer searches for a peer with given peer_id, returns a peer.PeerInfo with relevant addresses. """ - value = self.server.get(peer_id) + # switching peer_id to xor_id used by kademlia as node_id + xor_id = peer_id.get_xor_id() + value = self.server.get(xor_id) return decode_peerinfo(value) From 7962060cc05ef58ffc1811ee55d459f53484e3aa Mon Sep 17 00:00:00 2001 From: Alex Haynes Date: Wed, 24 Apr 2019 22:11:54 -0400 Subject: [PATCH 2/7] refactored routedhost into router passed to swarm --- libp2p/__init__.py | 16 +++++++------- libp2p/host/basic_host.py | 21 ++++++++++--------- libp2p/kademlia/crawling.py | 2 +- libp2p/kademlia/kad_peerinfo.py | 6 +++--- libp2p/kademlia/network.py | 2 +- libp2p/kademlia/routed_host.py | 21 ------------------- libp2p/kademlia/routing.py | 6 +++--- libp2p/network/swarm.py | 10 +++++++-- libp2p/peer/id.py | 9 ++++++++ libp2p/routing/interfaces.py | 2 +- .../{kadmelia => kademlia}/__init__.py | 0 .../kademlia_content_router.py} | 0 .../kademlia_peer_router.py} | 4 +++- 13 files changed, 48 insertions(+), 51 deletions(-) delete mode 100644 libp2p/kademlia/routed_host.py rename libp2p/routing/{kadmelia => kademlia}/__init__.py (100%) rename libp2p/routing/{kadmelia/kadmelia_content_router.py => kademlia/kademlia_content_router.py} (100%) rename libp2p/routing/{kadmelia/kadmelia_peer_router.py => kademlia/kademlia_peer_router.py} (85%) diff --git a/libp2p/__init__.py b/libp2p/__init__.py index 2e1cf88b..b114f2d5 100644 --- a/libp2p/__init__.py +++ b/libp2p/__init__.py @@ -6,10 +6,10 @@ from .peer.peerstore import PeerStore from .peer.id import id_from_public_key from .network.swarm import Swarm from .host.basic_host import BasicHost -from .kademlia.routed_host import RoutedHost from .transport.upgrader import TransportUpgrader from .transport.tcp.tcp import TCP from .kademlia.network import KademliaServer +from .routing.kademlia.kademlia_peer_router import KadmeliaPeerRouter async def cleanup_done_tasks(): @@ -31,7 +31,7 @@ def generate_id(): # private_key = new_key.exportKey("PEM") return new_id -def initialize_default_kademlia( +def initialize_default_kademlia_router( ksize=20, alpha=3, id_opt=None, storage=None): """ initialize swam when no swarm is passed in @@ -46,8 +46,9 @@ def initialize_default_kademlia( id_opt = generate_id() node_id = id_opt.get_raw_id() - return KademliaServer(ksize=ksize, alpha=alpha, - node_id=node_id, storage=storage) + server = KademliaServer(ksize=ksize, alpha=alpha, + node_id=node_id, storage=storage) + return KadmeliaPeerRouter(server) def initialize_default_swarm( @@ -105,12 +106,11 @@ async def new_node( muxer_opt=muxer_opt, sec_opt=sec_opt, peerstore_opt=peerstore_opt) + swarm_opt.add_router(disc_opt) + # TODO enable support for other host type # TODO routing unimplemented - if not disc_opt: - host = BasicHost(swarm_opt) - else: - host = RoutedHost(swarm_opt, disc_opt) + host = BasicHost(swarm_opt) # Kick off cleanup job asyncio.ensure_future(cleanup_done_tasks()) diff --git a/libp2p/host/basic_host.py b/libp2p/host/basic_host.py index 0c3bf0eb..b85c4137 100644 --- a/libp2p/host/basic_host.py +++ b/libp2p/host/basic_host.py @@ -11,21 +11,22 @@ from .host_interface import IHost class BasicHost(IHost): # default options constructor - def __init__(self, _network): - self.network = _network - self.peerstore = self.network.peerstore + def __init__(self, network, router=None): + self._network = network + self.peerstore = self._network.peerstore + self._router = router def get_id(self): """ :return: peer_id of host """ - return self.network.get_peer_id() + return self._network.get_peer_id() def get_network(self): """ :return: network instance of host """ - return self.network + return self._network def get_peerstore(self): """ @@ -45,7 +46,7 @@ class BasicHost(IHost): p2p_part = multiaddr.Multiaddr('/p2p/{}'.format(self.get_id().pretty())) addrs = [] - for transport in self.network.listeners.values(): + for transport in self._network.listeners.values(): for addr in transport.get_addrs(): addrs.append(addr.encapsulate(p2p_part)) return addrs @@ -57,7 +58,7 @@ class BasicHost(IHost): :param stream_handler: a stream handler function :return: true if successful """ - return self.network.set_stream_handler(protocol_id, stream_handler) + return self._network.set_stream_handler(protocol_id, stream_handler) # protocol_id can be a list of protocol_ids # stream will decide which protocol_id to run on @@ -67,7 +68,7 @@ class BasicHost(IHost): :param protocol_id: protocol id that stream runs on :return: true if successful """ - stream = await self.network.new_stream(peer_id, protocol_ids) + stream = await self._network.new_stream(peer_id, protocol_ids) return stream async def connect(self, peer_info): @@ -84,7 +85,7 @@ class BasicHost(IHost): self.peerstore.add_addrs(peer_info.peer_id, peer_info.addrs, 10) # there is already a connection to this peer - if peer_info.peer_id in self.network.connections: + if peer_info.peer_id in self._network.connections: return - await self.network.dial_peer(peer_info.peer_id) + await self._network.dial_peer(peer_info.peer_id) diff --git a/libp2p/kademlia/crawling.py b/libp2p/kademlia/crawling.py index 6a3b5fe3..3fdfbc69 100644 --- a/libp2p/kademlia/crawling.py +++ b/libp2p/kademlia/crawling.py @@ -119,7 +119,7 @@ class ValueSpiderCrawl(SpiderCrawl): value_counts = Counter(values) if len(value_counts) != 1: log.warning("Got multiple values for key %i: %s", - self.node.long_id, str(values)) + self.node.xor_id, str(values)) value = value_counts.most_common(1)[0][0] peer = self.nearest_without_value.popleft() diff --git a/libp2p/kademlia/kad_peerinfo.py b/libp2p/kademlia/kad_peerinfo.py index fb40e3b7..0d8c233f 100644 --- a/libp2p/kademlia/kad_peerinfo.py +++ b/libp2p/kademlia/kad_peerinfo.py @@ -16,7 +16,7 @@ class KadPeerInfo(PeerInfo): super(KadPeerInfo, self).__init__(peer_id, peer_data) self.peer_id = peer_id.get_raw_id() - self.long_id = int(digest(peer_id.get_raw_id()).hex(), 16) + self.xor_id = peer_id.get_xor_id() self.addrs = peer_data.get_addrs() if peer_data else None @@ -34,7 +34,7 @@ class KadPeerInfo(PeerInfo): """ Get the distance between this node and another. """ - return self.long_id ^ node.long_id + return self.xor_id ^ node.xor_id def __iter__(self): """ @@ -43,7 +43,7 @@ class KadPeerInfo(PeerInfo): return iter([self.peer_id, self.ip, self.port]) def __repr__(self): - return repr([self.long_id, self.ip, self.port]) + return repr([self.xor_id, self.ip, self.port]) def __str__(self): return "%s:%s" % (self.ip, str(self.port)) diff --git a/libp2p/kademlia/network.py b/libp2p/kademlia/network.py index 86522ba2..836d34a2 100644 --- a/libp2p/kademlia/network.py +++ b/libp2p/kademlia/network.py @@ -67,7 +67,7 @@ class KademliaServer: listen = loop.create_datagram_endpoint(self._create_protocol, local_addr=(interface, port)) log.info("Node %i listening on %s:%i", - self.node.long_id, interface, port) + self.node.xor_id, interface, port) self.transport, self.protocol = await listen # finally, schedule refreshing table self.refresh_table() diff --git a/libp2p/kademlia/routed_host.py b/libp2p/kademlia/routed_host.py deleted file mode 100644 index 45fc0c6f..00000000 --- a/libp2p/kademlia/routed_host.py +++ /dev/null @@ -1,21 +0,0 @@ -from libp2p.host.basic_host import BasicHost - -class RoutedHost(BasicHost): - def __init__(self, _network, _kad_network): - super(RoutedHost, self).__init__(_network) - self.kad_network = _kad_network - - def get_kad_network(self): - return self.kad_network - - def routed_listen(self, port, interface='0.0.0.0'): - return self.kad_network.listen(port, interface) - - def routed_get(self, key): - return self.kad_network.get(key) - - def routed_set(self, key, value): - return self.kad_network.set(key, value) - - def routed_set_digest(self, dkey, value): - return self.kad_network.set_digest(dkey, value) diff --git a/libp2p/kademlia/routing.py b/libp2p/kademlia/routing.py index 7d97494e..b84717e9 100644 --- a/libp2p/kademlia/routing.py +++ b/libp2p/kademlia/routing.py @@ -33,7 +33,7 @@ class KBucket: one = KBucket(self.range[0], midpoint, self.ksize) two = KBucket(midpoint + 1, self.range[1], self.ksize) for node in self.nodes.values(): - bucket = one if node.long_id <= midpoint else two + bucket = one if node.xor_id <= midpoint else two bucket.nodes[node.peer_id] = node return (one, two) @@ -48,7 +48,7 @@ class KBucket: self.nodes[newnode.peer_id] = newnode def has_in_range(self, node): - return self.range[0] <= node.long_id <= self.range[1] + return self.range[0] <= node.xor_id <= self.range[1] def is_new_node(self, node): return node.peer_id not in self.nodes @@ -175,7 +175,7 @@ class RoutingTable: Get the index of the bucket that the given node would fall into. """ for index, bucket in enumerate(self.buckets): - if node.long_id < bucket.range[1]: + if node.xor_id < bucket.range[1]: return index # we should never be here, but make linter happy return None diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index b30567b5..4c0c1280 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -20,6 +20,7 @@ class Swarm(INetwork): self.listeners = dict() self.stream_handlers = dict() self.transport = None + self.router = None # Protocol muxing self.multiselect = Multiselect() @@ -57,8 +58,10 @@ class Swarm(INetwork): if not addrs: raise SwarmException("No known addresses to peer") - # TODO: define logic to choose which address to use, or try them all ? - multiaddr = addrs[0] + if not self.router: + multiaddr = addrs[0] + else: + multiaddr = self.router.find_peer(peer_id) if peer_id in self.connections: # If muxed connection already exists for peer_id, @@ -187,6 +190,9 @@ class Swarm(INetwork): # TODO: Support more than one transport self.transport = transport + def add_router(self, router): + self.router = router + def create_generic_protocol_handler(swarm): """ Create a generic protocol handler from the given swarm. We use swarm diff --git a/libp2p/peer/id.py b/libp2p/peer/id.py index 92b4db34..f3b96475 100644 --- a/libp2p/peer/id.py +++ b/libp2p/peer/id.py @@ -1,5 +1,6 @@ import base58 import multihash +import hashlib # MaxInlineKeyLength is the maximum length a key can be for it to be inlined in # the peer ID. @@ -21,6 +22,9 @@ class ID: def pretty(self): return base58.b58encode(self._id_str).decode() + def get_xor_id(self): + return int(digest(self.get_raw_id()).hex(), 16) + def __str__(self): pid = self.pretty() if len(pid) <= 10: @@ -67,3 +71,8 @@ def id_from_public_key(key): def id_from_private_key(key): return id_from_public_key(key.publickey()) + +def digest(string): + if not isinstance(string, bytes): + string = str(string).encode('utf8') + return hashlib.sha1(string).digest() diff --git a/libp2p/routing/interfaces.py b/libp2p/routing/interfaces.py index 1f29d48c..b519a096 100644 --- a/libp2p/routing/interfaces.py +++ b/libp2p/routing/interfaces.py @@ -28,4 +28,4 @@ class IPeerRouting(ABC): Find specific Peer FindPeer searches for a peer with given peer_id, returns a peer.PeerInfo with relevant addresses. - """ + """ \ No newline at end of file diff --git a/libp2p/routing/kadmelia/__init__.py b/libp2p/routing/kademlia/__init__.py similarity index 100% rename from libp2p/routing/kadmelia/__init__.py rename to libp2p/routing/kademlia/__init__.py diff --git a/libp2p/routing/kadmelia/kadmelia_content_router.py b/libp2p/routing/kademlia/kademlia_content_router.py similarity index 100% rename from libp2p/routing/kadmelia/kadmelia_content_router.py rename to libp2p/routing/kademlia/kademlia_content_router.py diff --git a/libp2p/routing/kadmelia/kadmelia_peer_router.py b/libp2p/routing/kademlia/kademlia_peer_router.py similarity index 85% rename from libp2p/routing/kadmelia/kadmelia_peer_router.py rename to libp2p/routing/kademlia/kademlia_peer_router.py index 27be67c7..3a13f459 100644 --- a/libp2p/routing/kadmelia/kadmelia_peer_router.py +++ b/libp2p/routing/kademlia/kademlia_peer_router.py @@ -16,7 +16,9 @@ class KadmeliaPeerRouter(IPeerRouting): FindPeer searches for a peer with given peer_id, returns a peer.PeerInfo with relevant addresses. """ - value = self.server.get(peer_id) + # switching peer_id to xor_id used by kademlia as node_id + xor_id = peer_id.get_xor_id() + value = self.server.get(xor_id) return decode_peerinfo(value) From fdaa4f56b9b082af7a5f9e95b4e2898c66f9bf4d Mon Sep 17 00:00:00 2001 From: Alex Haynes Date: Sat, 27 Apr 2019 21:59:25 -0400 Subject: [PATCH 3/7] working on adding tests --- libp2p/kademlia/kad_peerinfo.py | 1 + .../routing/kademlia/kademlia_peer_router.py | 4 +-- tests/routing/__init__.py | 0 tests/routing/test_kad_peer_router.py | 25 +++++++++++++++++++ 4 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 tests/routing/__init__.py create mode 100644 tests/routing/test_kad_peer_router.py diff --git a/libp2p/kademlia/kad_peerinfo.py b/libp2p/kademlia/kad_peerinfo.py index 0d8c233f..195fbe01 100644 --- a/libp2p/kademlia/kad_peerinfo.py +++ b/libp2p/kademlia/kad_peerinfo.py @@ -15,6 +15,7 @@ class KadPeerInfo(PeerInfo): def __init__(self, peer_id, peer_data=None): super(KadPeerInfo, self).__init__(peer_id, peer_data) + self.peer_id_obj = peer_id self.peer_id = peer_id.get_raw_id() self.xor_id = peer_id.get_xor_id() diff --git a/libp2p/routing/kademlia/kademlia_peer_router.py b/libp2p/routing/kademlia/kademlia_peer_router.py index 3a13f459..958066cf 100644 --- a/libp2p/routing/kademlia/kademlia_peer_router.py +++ b/libp2p/routing/kademlia/kademlia_peer_router.py @@ -10,7 +10,7 @@ class KadmeliaPeerRouter(IPeerRouting): def __init__(self, dht_server): self.server = dht_server - def find_peer(self, peer_id): + async def find_peer(self, peer_id): """ Find specific Peer FindPeer searches for a peer with given peer_id, returns a peer.PeerInfo @@ -18,7 +18,7 @@ class KadmeliaPeerRouter(IPeerRouting): """ # switching peer_id to xor_id used by kademlia as node_id xor_id = peer_id.get_xor_id() - value = self.server.get(xor_id) + value = await self.server.get(xor_id) return decode_peerinfo(value) diff --git a/tests/routing/__init__.py b/tests/routing/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/routing/test_kad_peer_router.py b/tests/routing/test_kad_peer_router.py new file mode 100644 index 00000000..9c7f402f --- /dev/null +++ b/tests/routing/test_kad_peer_router.py @@ -0,0 +1,25 @@ +import asyncio +import pytest + +from libp2p.kademlia.network import KademliaServer +from libp2p.routing.kademlia.kademlia_peer_router import KadmeliaPeerRouter +from libp2p.peer.id import id_b58_encode + +@pytest.mark.asyncio +async def test_example(): + node_a = KademliaServer() + await node_a.listen(5678) + + node_b = KademliaServer() + await node_b.listen(5679) + + value = await node_b.bootstrap([("127.0.0.1", 5678)]) + peer_info = value[0] + peer_id = peer_info.peer_id_obj + print(id_b58_encode(peer_id)) + # await node_a.set(peer_info.xor_id, str(peer_info.ip) + "/" + str(peer_info.port)) + # router = KadmeliaPeerRouter(node_b) + # value = await router.find_peer(peer_id) + # print("value vvvv") + # print(value.xor_) + # assert value == str(peer_info.ip) + "/" + str(peer_info.port) From 3b1a3fb198688c5340a579d49ee213b9bf003d44 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sun, 28 Apr 2019 13:49:10 -0400 Subject: [PATCH 4/7] fix tests --- .../routing/kademlia/kademlia_peer_router.py | 21 ++++---- tests/routing/test_kad_peer_router.py | 48 ++++++++++++++----- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/libp2p/routing/kademlia/kademlia_peer_router.py b/libp2p/routing/kademlia/kademlia_peer_router.py index 958066cf..3a0a3ba3 100644 --- a/libp2p/routing/kademlia/kademlia_peer_router.py +++ b/libp2p/routing/kademlia/kademlia_peer_router.py @@ -19,15 +19,14 @@ class KadmeliaPeerRouter(IPeerRouting): # switching peer_id to xor_id used by kademlia as node_id xor_id = peer_id.get_xor_id() value = await self.server.get(xor_id) - return decode_peerinfo(value) + return value - -def decode_peerinfo(encoded): - if isinstance(encoded, bytes): - encoded = encoded.decode() - lines = encoded.splitlines() - peer_id = lines[0] - addrs = lines[1:] - peer_data = PeerData() - peer_data.add_addrs(addrs) - return PeerInfo(peer_id, addrs) +# def decode_peerinfo(encoded): +# if isinstance(encoded, bytes): +# encoded = encoded.decode() +# lines = encoded.splitlines() +# peer_id = lines[0] +# addrs = lines[1:] +# peer_data = PeerData() +# peer_data.add_addrs(addrs) +# return PeerInfo(peer_id, addrs) diff --git a/tests/routing/test_kad_peer_router.py b/tests/routing/test_kad_peer_router.py index 9c7f402f..146f6d22 100644 --- a/tests/routing/test_kad_peer_router.py +++ b/tests/routing/test_kad_peer_router.py @@ -1,25 +1,47 @@ -import asyncio import pytest from libp2p.kademlia.network import KademliaServer from libp2p.routing.kademlia.kademlia_peer_router import KadmeliaPeerRouter -from libp2p.peer.id import id_b58_encode @pytest.mark.asyncio -async def test_example(): +async def test_simple_two_nodes(): node_a = KademliaServer() await node_a.listen(5678) node_b = KademliaServer() await node_b.listen(5679) - value = await node_b.bootstrap([("127.0.0.1", 5678)]) - peer_info = value[0] - peer_id = peer_info.peer_id_obj - print(id_b58_encode(peer_id)) - # await node_a.set(peer_info.xor_id, str(peer_info.ip) + "/" + str(peer_info.port)) - # router = KadmeliaPeerRouter(node_b) - # value = await router.find_peer(peer_id) - # print("value vvvv") - # print(value.xor_) - # assert value == str(peer_info.ip) + "/" + str(peer_info.port) + node_a_value = await node_b.bootstrap([("127.0.0.1", 5678)]) + node_a_kad_peerinfo = node_a_value[0] + + await node_a.set(node_a_kad_peerinfo.xor_id, + str(node_a_kad_peerinfo.ip)\ + + "/" + str(node_a_kad_peerinfo.port)) + + router = KadmeliaPeerRouter(node_b) + returned_info = await router.find_peer(node_a_kad_peerinfo.peer_id_obj) + assert returned_info == str(node_a_kad_peerinfo.ip)\ + + "/" + str(node_a_kad_peerinfo.port) + +@pytest.mark.asyncio +async def test_simple_three_nodes(): + node_a = KademliaServer() + await node_a.listen(5701) + + node_b = KademliaServer() + await node_b.listen(5702) + + node_c = KademliaServer() + await node_c.listen(5703) + + node_a_value = await node_b.bootstrap([("127.0.0.1", 5701)]) + node_a_kad_peerinfo = node_a_value[0] + + await node_c.bootstrap([("127.0.0.1", 5702)]) + await node_a.set(node_a_kad_peerinfo.xor_id, + str(node_a_kad_peerinfo.ip)\ + + "/" + str(node_a_kad_peerinfo.port)) + + router = KadmeliaPeerRouter(node_c) + returned_info = await router.find_peer(node_a_kad_peerinfo.peer_id_obj) + assert returned_info == str(node_a_kad_peerinfo.ip) + "/" + str(node_a_kad_peerinfo.port) From 8ab387ac05ff36056c8b30571c9fbab0e0333713 Mon Sep 17 00:00:00 2001 From: zixuanzh Date: Sun, 28 Apr 2019 15:00:43 -0400 Subject: [PATCH 5/7] refactor add_router in swarm --- libp2p/__init__.py | 9 +++++---- libp2p/network/swarm.py | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/libp2p/__init__.py b/libp2p/__init__.py index 3e133b64..9d779f9d 100644 --- a/libp2p/__init__.py +++ b/libp2p/__init__.py @@ -52,8 +52,8 @@ def initialize_default_kademlia_router( def initialize_default_swarm( - id_opt=None, transport_opt=None, - muxer_opt=None, sec_opt=None, peerstore_opt=None): + id_opt=None, transport_opt=None, muxer_opt=None, + sec_opt=None, peerstore_opt=None, disc_opt=None): """ initialize swarm when no swarm is passed in :param id_opt: optional id for host @@ -79,7 +79,8 @@ def initialize_default_swarm( upgrader = TransportUpgrader(sec, muxer) peerstore = peerstore_opt or PeerStore() - swarm_opt = Swarm(id_opt, peerstore, upgrader, transport) + swarm_opt = Swarm(id_opt, peerstore,\ + upgrader, transport, disc_opt) return swarm_opt @@ -106,7 +107,7 @@ async def new_node( swarm_opt = initialize_default_swarm( id_opt=id_opt, transport_opt=transport_opt, muxer_opt=muxer_opt, sec_opt=sec_opt, - peerstore_opt=peerstore_opt) + peerstore_opt=peerstore_opt, disc_opt=disc_opt) swarm_opt.add_router(disc_opt) diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index 8e6ba732..9b37b50a 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -10,17 +10,17 @@ from .stream.net_stream import NetStream from .connection.raw_connection import RawConnection class Swarm(INetwork): - # pylint: disable=too-many-instance-attributes, cell-var-from-loop + # pylint: disable=too-many-instance-attributes,cell-var-from-loop,too-many-arguments - def __init__(self, peer_id, peerstore, upgrader, transport): + def __init__(self, peer_id, peerstore, upgrader, transport, router): self.self_id = peer_id self.peerstore = peerstore self.upgrader = upgrader self.transport = transport + self.router = router self.connections = dict() self.listeners = dict() self.stream_handlers = dict() - self.router = None # Protocol muxing self.multiselect = Multiselect() From 2efc0d3b5c6a4f7682048dbbdeec67a541fcf001 Mon Sep 17 00:00:00 2001 From: Alex Haynes Date: Sun, 28 Apr 2019 17:38:40 -0400 Subject: [PATCH 6/7] updated peerinfo encoding stored in DHT --- libp2p/kademlia/kad_peerinfo.py | 6 ++- .../routing/kademlia/kademlia_peer_router.py | 31 ++++++++------ tests/routing/test_kad_peer_router.py | 42 +++++++++++++++---- 3 files changed, 57 insertions(+), 22 deletions(-) diff --git a/libp2p/kademlia/kad_peerinfo.py b/libp2p/kademlia/kad_peerinfo.py index 195fbe01..a5ef5dbd 100644 --- a/libp2p/kademlia/kad_peerinfo.py +++ b/libp2p/kademlia/kad_peerinfo.py @@ -44,11 +44,15 @@ class KadPeerInfo(PeerInfo): return iter([self.peer_id, self.ip, self.port]) def __repr__(self): - return repr([self.xor_id, self.ip, self.port]) + return repr([self.xor_id, self.ip, self.port, self.peer_id]) def __str__(self): return "%s:%s" % (self.ip, str(self.port)) + def encode(self): + return str(self.peer_id) + "\n" + \ + str("/ip4/" + str(self.ip) + "/udp/" + str(self.port)) + class KadPeerHeap: """ A heap of peers ordered by distance to a given node. diff --git a/libp2p/routing/kademlia/kademlia_peer_router.py b/libp2p/routing/kademlia/kademlia_peer_router.py index 3a0a3ba3..00de28af 100644 --- a/libp2p/routing/kademlia/kademlia_peer_router.py +++ b/libp2p/routing/kademlia/kademlia_peer_router.py @@ -1,7 +1,7 @@ +import ast + from libp2p.routing.interfaces import IPeerRouting -from libp2p.kademlia.utils import digest -from libp2p.peer.peerinfo import PeerInfo -from libp2p.peer.peerdata import PeerData +from libp2p.kademlia.kad_peerinfo import create_kad_peerinfo class KadmeliaPeerRouter(IPeerRouting): @@ -19,14 +19,19 @@ class KadmeliaPeerRouter(IPeerRouting): # switching peer_id to xor_id used by kademlia as node_id xor_id = peer_id.get_xor_id() value = await self.server.get(xor_id) - return value + return decode_peerinfo(value) -# def decode_peerinfo(encoded): -# if isinstance(encoded, bytes): -# encoded = encoded.decode() -# lines = encoded.splitlines() -# peer_id = lines[0] -# addrs = lines[1:] -# peer_data = PeerData() -# peer_data.add_addrs(addrs) -# return PeerInfo(peer_id, addrs) +def decode_peerinfo(encoded): + if isinstance(encoded, bytes): + encoded = encoded.decode() + print(encoded) + try: + lines = ast.literal_eval(encoded) + except SyntaxError: + return None + # xor_id = lines[0] + ip = lines[1] # pylint: disable=invalid-name + port = lines[2] + peer_id = lines[3] + peer_info = create_kad_peerinfo(peer_id, ip, port) + return peer_info diff --git a/tests/routing/test_kad_peer_router.py b/tests/routing/test_kad_peer_router.py index 146f6d22..7f639fbc 100644 --- a/tests/routing/test_kad_peer_router.py +++ b/tests/routing/test_kad_peer_router.py @@ -13,15 +13,14 @@ async def test_simple_two_nodes(): node_a_value = await node_b.bootstrap([("127.0.0.1", 5678)]) node_a_kad_peerinfo = node_a_value[0] - await node_a.set(node_a_kad_peerinfo.xor_id, - str(node_a_kad_peerinfo.ip)\ - + "/" + str(node_a_kad_peerinfo.port)) + repr(node_a_kad_peerinfo)) router = KadmeliaPeerRouter(node_b) returned_info = await router.find_peer(node_a_kad_peerinfo.peer_id_obj) - assert returned_info == str(node_a_kad_peerinfo.ip)\ - + "/" + str(node_a_kad_peerinfo.port) + print(repr(returned_info)) + print(repr(node_a_kad_peerinfo)) + assert repr(returned_info) == repr(node_a_kad_peerinfo) @pytest.mark.asyncio async def test_simple_three_nodes(): @@ -39,9 +38,36 @@ async def test_simple_three_nodes(): await node_c.bootstrap([("127.0.0.1", 5702)]) await node_a.set(node_a_kad_peerinfo.xor_id, - str(node_a_kad_peerinfo.ip)\ - + "/" + str(node_a_kad_peerinfo.port)) + repr(node_a_kad_peerinfo)) router = KadmeliaPeerRouter(node_c) returned_info = await router.find_peer(node_a_kad_peerinfo.peer_id_obj) - assert returned_info == str(node_a_kad_peerinfo.ip) + "/" + str(node_a_kad_peerinfo.port) + assert str(returned_info) == str(node_a_kad_peerinfo) + +@pytest.mark.asyncio +async def test_simple_four_nodes(): + node_a = KademliaServer() + await node_a.listen(5801) + + node_b = KademliaServer() + await node_b.listen(5802) + + node_c = KademliaServer() + await node_c.listen(5803) + + node_d = KademliaServer() + await node_d.listen(5804) + + node_a_value = await node_b.bootstrap([("127.0.0.1", 5801)]) + node_a_kad_peerinfo = node_a_value[0] + + await node_c.bootstrap([("127.0.0.1", 5802)]) + + await node_d.bootstrap([("127.0.0.1", 5803)]) + + await node_b.set(node_a_kad_peerinfo.xor_id, + repr(node_a_kad_peerinfo)) + + router = KadmeliaPeerRouter(node_d) + returned_info = await router.find_peer(node_a_kad_peerinfo.peer_id_obj) + assert str(returned_info) == str(node_a_kad_peerinfo) From ea4fd8e81f7b29473d762b36f1bf07efd5ed015b Mon Sep 17 00:00:00 2001 From: Alex Haynes Date: Sun, 5 May 2019 14:32:41 -0400 Subject: [PATCH 7/7] requested changes --- libp2p/__init__.py | 2 -- libp2p/host/basic_host.py | 2 +- libp2p/routing/kademlia/kademlia_peer_router.py | 8 +++----- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/libp2p/__init__.py b/libp2p/__init__.py index 9d779f9d..2772bed7 100644 --- a/libp2p/__init__.py +++ b/libp2p/__init__.py @@ -109,8 +109,6 @@ async def new_node( muxer_opt=muxer_opt, sec_opt=sec_opt, peerstore_opt=peerstore_opt, disc_opt=disc_opt) - swarm_opt.add_router(disc_opt) - # TODO enable support for other host type # TODO routing unimplemented host = BasicHost(swarm_opt) diff --git a/libp2p/host/basic_host.py b/libp2p/host/basic_host.py index b85c4137..40292dfc 100644 --- a/libp2p/host/basic_host.py +++ b/libp2p/host/basic_host.py @@ -13,8 +13,8 @@ class BasicHost(IHost): # default options constructor def __init__(self, network, router=None): self._network = network - self.peerstore = self._network.peerstore self._router = router + self.peerstore = self._network.peerstore def get_id(self): """ diff --git a/libp2p/routing/kademlia/kademlia_peer_router.py b/libp2p/routing/kademlia/kademlia_peer_router.py index 00de28af..45c43c82 100644 --- a/libp2p/routing/kademlia/kademlia_peer_router.py +++ b/libp2p/routing/kademlia/kademlia_peer_router.py @@ -12,9 +12,9 @@ class KadmeliaPeerRouter(IPeerRouting): async def find_peer(self, peer_id): """ - Find specific Peer - FindPeer searches for a peer with given peer_id, returns a peer.PeerInfo - with relevant addresses. + Find a specific peer + :param peer_id: peer to search for + :return: KadPeerInfo of specified peer """ # switching peer_id to xor_id used by kademlia as node_id xor_id = peer_id.get_xor_id() @@ -24,12 +24,10 @@ class KadmeliaPeerRouter(IPeerRouting): def decode_peerinfo(encoded): if isinstance(encoded, bytes): encoded = encoded.decode() - print(encoded) try: lines = ast.literal_eval(encoded) except SyntaxError: return None - # xor_id = lines[0] ip = lines[1] # pylint: disable=invalid-name port = lines[2] peer_id = lines[3]