varun-r-mallya/py-libp2p
1
0
Fork 0
mirror of https://github.com/varun-r-mallya/py-libp2p.git synced 2025-12-31 20:36:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Run black over repo

Browse Source
This commit is contained in:
Alex Stokes
2019-07-31 15:00:12 -07:00
parent a2133d8c7c
commit 0ae9840928
69 changed files with 791 additions and 1095 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
libp2p/__init__.py
View File

@ -26,14 +26,15 @@ async def cleanup_done_tasks():
# Some sleep necessary to context switch
await asyncio.sleep(3)
def generate_id():
new_key = RSA.generate(2048, e=65537)
new_id = id_from_public_key(new_key.publickey())
# private_key = new_key.exportKey("PEM")
return new_id
def initialize_default_kademlia_router(
ksize=20, alpha=3, id_opt=None, storage=None):
def initialize_default_kademlia_router(ksize=20, alpha=3, id_opt=None, storage=None):
"""
initialize kadmelia router when no kademlia router is passed in
:param ksize: The k parameter from the paper
@ -47,14 +48,18 @@ def initialize_default_kademlia_router(
id_opt = generate_id()
node_id = id_opt.get_raw_id()
server = KademliaServer(ksize=ksize, alpha=alpha,
node_id=node_id, storage=storage)
server = KademliaServer(ksize=ksize, alpha=alpha, node_id=node_id, storage=storage)
return KadmeliaPeerRouter(server)
def initialize_default_swarm(
id_opt=None, transport_opt=None, muxer_opt=None,
sec_opt=None, peerstore_opt=None, disc_opt=None):
id_opt=None,
transport_opt=None,
muxer_opt=None,
sec_opt=None,
peerstore_opt=None,
disc_opt=None,
):
"""
initialize swarm when no swarm is passed in
:param id_opt: optional id for host
@ -82,16 +87,20 @@ def initialize_default_swarm(
peerstore = peerstore_opt or PeerStore()
# TODO: Initialize discovery if not presented
swarm_opt = Swarm(id_opt, peerstore,\
upgrader, transport, disc_opt)
swarm_opt = Swarm(id_opt, peerstore, upgrader, transport, disc_opt)
return swarm_opt
async def new_node(
swarm_opt=None, id_opt=None, transport_opt=None,
muxer_opt=None, sec_opt=None, peerstore_opt=None,
disc_opt=None):
swarm_opt=None,
id_opt=None,
transport_opt=None,
muxer_opt=None,
sec_opt=None,
peerstore_opt=None,
disc_opt=None,
):
"""
create new libp2p node
:param swarm_opt: optional swarm
@ -110,9 +119,13 @@ async def new_node(
if not swarm_opt:
swarm_opt = initialize_default_swarm(
id_opt=id_opt, transport_opt=transport_opt,
muxer_opt=muxer_opt, sec_opt=sec_opt,
peerstore_opt=peerstore_opt, disc_opt=disc_opt)
id_opt=id_opt,
transport_opt=transport_opt,
muxer_opt=muxer_opt,
sec_opt=sec_opt,
peerstore_opt=peerstore_opt,
disc_opt=disc_opt,
)
# TODO enable support for other host type
# TODO routing unimplemented

14
libp2p/host/basic_host.py
View File

@ -1,10 +1,4 @@
from typing import (
Any,
Awaitable,
Callable,
List,
Sequence,
)
from typing import Any, Awaitable, Callable, List, Sequence
import multiaddr
@ -67,7 +61,7 @@ class BasicHost(IHost):
"""
:return: all the multiaddr addresses this host is listening too
"""
p2p_part = multiaddr.Multiaddr('/p2p/{}'.format(self.get_id().pretty()))
p2p_part = multiaddr.Multiaddr("/p2p/{}".format(self.get_id().pretty()))
addrs: List[multiaddr.Multiaddr] = []
for transport in self._network.listeners.values():
@ -75,7 +69,9 @@ class BasicHost(IHost):
addrs.append(addr.encapsulate(p2p_part))
return addrs
def set_stream_handler(self, protocol_id: str, stream_handler: StreamHandlerFn) -> bool:
def set_stream_handler(
self, protocol_id: str, stream_handler: StreamHandlerFn
) -> bool:
"""
set stream handler for host
:param protocol_id: protocol id used on stream

17
libp2p/host/host_interface.py
View File

@ -1,11 +1,5 @@
from abc import ABC, abstractmethod
from typing import (
Any,
Awaitable,
Callable,
List,
Sequence,
)
from typing import Any, Awaitable, Callable, List, Sequence
import multiaddr
@ -20,7 +14,6 @@ StreamHandlerFn = Callable[[INetStream], Awaitable[None]]
class IHost(ABC):
@abstractmethod
def get_id(self) -> ID:
"""
@ -47,7 +40,9 @@ class IHost(ABC):
"""
@abstractmethod
def set_stream_handler(self, protocol_id: str, stream_handler: StreamHandlerFn) -> bool:
def set_stream_handler(
self, protocol_id: str, stream_handler: StreamHandlerFn
) -> bool:
"""
set stream handler for host
:param protocol_id: protocol id used on stream
@ -58,9 +53,7 @@ class IHost(ABC):
# protocol_id can be a list of protocol_ids
# stream will decide which protocol_id to run on
@abstractmethod
async def new_stream(self,
peer_id: ID,
protocol_ids: Sequence[str]) -> INetStream:
async def new_stream(self, peer_id: ID, protocol_ids: Sequence[str]) -> INetStream:
"""
:param peer_id: peer_id that host is connecting
:param protocol_ids: protocol ids that stream can run on

7
libp2p/kademlia/crawling.py
View File

@ -118,8 +118,9 @@ class ValueSpiderCrawl(SpiderCrawl):
"""
value_counts = Counter(values)
if len(value_counts) != 1:
log.warning("Got multiple values for key %i: %s",
self.node.xor_id, str(values))
log.warning(
"Got multiple values for key %i: %s", self.node.xor_id, str(values)
)
value = value_counts.most_common(1)[0][0]
peer = self.nearest_without_value.popleft()
@ -175,7 +176,7 @@ class RPCFindResponse:
return isinstance(self.response[1], dict)
def get_value(self):
return self.response[1]['value']
return self.response[1]["value"]
def get_node_list(self):
"""

27
libp2p/kademlia/kad_peerinfo.py
View File

@ -11,6 +11,7 @@ from .utils import digest
P_IP = "ip4"
P_UDP = "udp"
class KadPeerInfo(PeerInfo):
def __init__(self, peer_id, peer_data=None):
super(KadPeerInfo, self).__init__(peer_id, peer_data)
@ -22,11 +23,8 @@ class KadPeerInfo(PeerInfo):
self.addrs = peer_data.get_addrs() if peer_data else None
# pylint: disable=invalid-name
self.ip = self.addrs[0].value_for_protocol(P_IP)\
if peer_data else None
self.port = int(self.addrs[0].value_for_protocol(P_UDP))\
if peer_data else None
self.ip = self.addrs[0].value_for_protocol(P_IP) if peer_data else None
self.port = int(self.addrs[0].value_for_protocol(P_UDP)) if peer_data else None
def same_home_as(self, node):
return sorted(self.addrs) == sorted(node.addrs)
@ -50,13 +48,18 @@ class KadPeerInfo(PeerInfo):
return "%s:%s" % (self.ip, str(self.port))
def encode(self):
return str(self.peer_id) + "\n" + \
str("/ip4/" + str(self.ip) + "/udp/" + str(self.port))
return (
str(self.peer_id)
+ "\n"
+ str("/ip4/" + str(self.ip) + "/udp/" + str(self.port))
)
class KadPeerHeap:
"""
A heap of peers ordered by distance to a given node.
"""
def __init__(self, node, maxsize):
"""
Constructor.
@ -134,13 +137,17 @@ class KadPeerHeap:
def get_uncontacted(self):
return [n for n in self if n.peer_id not in self.contacted]
def create_kad_peerinfo(raw_node_id=None, sender_ip=None, sender_port=None):
node_id = ID(raw_node_id) if raw_node_id else ID(digest(random.getrandbits(255)))
peer_data = None
if sender_ip and sender_port:
peer_data = PeerData() #pylint: disable=no-value-for-parameter
addr = [Multiaddr("/"+ P_IP +"/" + str(sender_ip) + "/"\
+ P_UDP + "/" + str(sender_port))]
peer_data = PeerData() # pylint: disable=no-value-for-parameter
addr = [
Multiaddr(
"/" + P_IP + "/" + str(sender_ip) + "/" + P_UDP + "/" + str(sender_port)
)
]
peer_data.add_addrs(addr)
return KadPeerInfo(node_id, peer_data)

76
libp2p/kademlia/network.py
View File

@ -57,17 +57,17 @@ class KademliaServer:
def _create_protocol(self):
return self.protocol_class(self.node, self.storage, self.ksize)
async def listen(self, port, interface='0.0.0.0'):
async def listen(self, port, interface="0.0.0.0"):
"""
Start listening on the given port.
Provide interface="::" to accept ipv6 address
"""
loop = asyncio.get_event_loop()
listen = loop.create_datagram_endpoint(self._create_protocol,
local_addr=(interface, port))
log.info("Node %i listening on %s:%i",
self.node.xor_id, interface, port)
listen = loop.create_datagram_endpoint(
self._create_protocol, local_addr=(interface, port)
)
log.info("Node %i listening on %s:%i", self.node.xor_id, interface, port)
self.transport, self.protocol = await listen
# finally, schedule refreshing table
self.refresh_table()
@ -87,8 +87,9 @@ class KademliaServer:
for node_id in self.protocol.get_refresh_ids():
node = create_kad_peerinfo(node_id)
nearest = self.protocol.router.find_neighbors(node, self.alpha)
spider = NodeSpiderCrawl(self.protocol, node, nearest,
self.ksize, self.alpha)
spider = NodeSpiderCrawl(
self.protocol, node, nearest, self.ksize, self.alpha
)
results.append(spider.find())
# do our crawling
@ -119,13 +120,13 @@ class KademliaServer:
addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP
addresses are acceptable - hostnames will cause an error.
"""
log.debug("Attempting to bootstrap node with %i initial contacts",
len(addrs))
log.debug("Attempting to bootstrap node with %i initial contacts", len(addrs))
cos = list(map(self.bootstrap_node, addrs))
gathered = await asyncio.gather(*cos)
nodes = [node for node in gathered if node is not None]
spider = NodeSpiderCrawl(self.protocol, self.node, nodes,
self.ksize, self.alpha)
spider = NodeSpiderCrawl(
self.protocol, self.node, nodes, self.ksize, self.alpha
)
return await spider.find()
async def bootstrap_node(self, addr):
@ -150,8 +151,7 @@ class KademliaServer:
if not nearest:
log.warning("There are no known neighbors to get key %s", key)
return None
spider = ValueSpiderCrawl(self.protocol, node, nearest,
self.ksize, self.alpha)
spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return await spider.find()
async def set(self, key, value):
@ -159,9 +159,7 @@ class KademliaServer:
Set the given string key to the given value in the network.
"""
if not check_dht_value_type(value):
raise TypeError(
"Value must be of type int, float, bool, str, or bytes"
)
raise TypeError("Value must be of type int, float, bool, str, or bytes")
log.info("setting '%s' = '%s' on network", key, value)
dkey = digest(key)
return await self.set_digest(dkey, value)
@ -171,7 +169,10 @@ class KademliaServer:
publish to the network that it provides for a particular key
"""
neighbors = self.protocol.router.find_neighbors(self.node)
return [await self.protocol.call_add_provider(n, key, self.node.peer_id) for n in neighbors]
return [
await self.protocol.call_add_provider(n, key, self.node.peer_id)
for n in neighbors
]
async def get_providers(self, key):
"""
@ -189,12 +190,10 @@ class KademliaServer:
nearest = self.protocol.router.find_neighbors(node)
if not nearest:
log.warning("There are no known neighbors to set key %s",
dkey.hex())
log.warning("There are no known neighbors to set key %s", dkey.hex())
return False
spider = NodeSpiderCrawl(self.protocol, node, nearest,
self.ksize, self.alpha)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
nodes = await spider.find()
log.info("setting '%s' on %s", dkey.hex(), list(map(str, nodes)))
@ -213,15 +212,15 @@ class KademliaServer:
"""
log.info("Saving state to %s", fname)
data = {
'ksize': self.ksize,
'alpha': self.alpha,
'id': self.node.peer_id,
'neighbors': self.bootstrappable_neighbors()
"ksize": self.ksize,
"alpha": self.alpha,
"id": self.node.peer_id,
"neighbors": self.bootstrappable_neighbors(),
}
if not data['neighbors']:
if not data["neighbors"]:
log.warning("No known neighbors, so not writing to cache.")
return
with open(fname, 'wb') as file:
with open(fname, "wb") as file:
pickle.dump(data, file)
@classmethod
@ -231,11 +230,11 @@ class KademliaServer:
from a cache file with the given fname.
"""
log.info("Loading state from %s", fname)
with open(fname, 'rb') as file:
with open(fname, "rb") as file:
data = pickle.load(file)
svr = KademliaServer(data['ksize'], data['alpha'], data['id'])
if data['neighbors']:
svr.bootstrap(data['neighbors'])
svr = KademliaServer(data["ksize"], data["alpha"], data["id"])
if data["neighbors"]:
svr.bootstrap(data["neighbors"])
return svr
def save_state_regularly(self, fname, frequency=600):
@ -250,10 +249,9 @@ class KademliaServer:
"""
self.save_state(fname)
loop = asyncio.get_event_loop()
self.save_state_loop = loop.call_later(frequency,
self.save_state_regularly,
fname,
frequency)
self.save_state_loop = loop.call_later(
frequency, self.save_state_regularly, fname, frequency
)
def check_dht_value_type(value):
@ -261,11 +259,5 @@ def check_dht_value_type(value):
Checks to see if the type of the value is a valid type for
placing in the dht.
"""
typeset = [
int,
float,
bool,
str,
bytes
]
typeset = [int, float, bool, str, bytes]
return type(value) in typeset # pylint: disable=unidiomatic-typecheck

34
libp2p/kademlia/protocol.py
View File

@ -20,6 +20,7 @@ class KademliaProtocol(RPCProtocol):
(ip, udp_port, node_id) for k closest nodes to target
FIND_VALUE behaves like FIND_NODE unless a value is stored
"""
def __init__(self, source_node, storage, ksize):
RPCProtocol.__init__(self)
self.router = RoutingTable(self, ksize, source_node)
@ -32,7 +33,7 @@ class KademliaProtocol(RPCProtocol):
"""
ids = []
for bucket in self.router.lonely_buckets():
rid = random.randint(*bucket.range).to_bytes(20, byteorder='big')
rid = random.randint(*bucket.range).to_bytes(20, byteorder="big")
ids.append(rid)
return ids
@ -49,14 +50,14 @@ class KademliaProtocol(RPCProtocol):
source = create_kad_peerinfo(nodeid, sender[0], sender[1])
self.welcome_if_new(source)
log.debug("got a store request from %s, storing '%s'='%s'",
sender, key.hex(), value)
log.debug(
"got a store request from %s, storing '%s'='%s'", sender, key.hex(), value
)
self.storage[key] = value
return True
def rpc_find_node(self, sender, nodeid, key):
log.info("finding neighbors of %i in local table",
int(nodeid.hex(), 16))
log.info("finding neighbors of %i in local table", int(nodeid.hex(), 16))
source = create_kad_peerinfo(nodeid, sender[0], sender[1])
self.welcome_if_new(source)
@ -71,7 +72,7 @@ class KademliaProtocol(RPCProtocol):
value = self.storage.get(key, None)
if value is None:
return self.rpc_find_node(sender, nodeid, key)
return {'value': value}
return {"value": value}
def rpc_add_provider(self, sender, nodeid, key, provider_id):
# pylint: disable=unused-argument
@ -82,8 +83,9 @@ class KademliaProtocol(RPCProtocol):
we store a map of content_id to peer_id (non xor)
"""
if nodeid == provider_id:
log.info("adding provider %s for key %s in local table",
provider_id, str(key))
log.info(
"adding provider %s for key %s in local table", provider_id, str(key)
)
self.storage[key] = provider_id
return True
return False
@ -111,14 +113,16 @@ class KademliaProtocol(RPCProtocol):
async def call_find_node(self, node_to_ask, node_to_find):
address = (node_to_ask.ip, node_to_ask.port)
result = await self.find_node(address, self.source_node.peer_id,
node_to_find.peer_id)
result = await self.find_node(
address, self.source_node.peer_id, node_to_find.peer_id
)
return self.handle_call_response(result, node_to_ask)
async def call_find_value(self, node_to_ask, node_to_find):
address = (node_to_ask.ip, node_to_ask.port)
result = await self.find_value(address, self.source_node.peer_id,
node_to_find.peer_id)
result = await self.find_value(
address, self.source_node.peer_id, node_to_find.peer_id
)
return self.handle_call_response(result, node_to_ask)
async def call_ping(self, node_to_ask):
@ -133,9 +137,9 @@ class KademliaProtocol(RPCProtocol):
async def call_add_provider(self, node_to_ask, key, provider_id):
address = (node_to_ask.ip, node_to_ask.port)
result = await self.add_provider(address,
self.source_node.peer_id,
key, provider_id)
result = await self.add_provider(
address, self.source_node.peer_id, key, provider_id
)
return self.handle_call_response(result, node_to_ask)

3
libp2p/kademlia/routing.py
View File

@ -15,6 +15,7 @@ class KBucket:
each k-bucket implements a last seen eviction
policy except that live nodes are never removed
"""
def __init__(self, rangeLower, rangeUpper, ksize):
self.range = (rangeLower, rangeUpper)
self.nodes = OrderedDict()
@ -92,7 +93,7 @@ class TableTraverser:
table.buckets[index].touch_last_updated()
self.current_nodes = table.buckets[index].get_nodes()
self.left_buckets = table.buckets[:index]
self.right_buckets = table.buckets[(index + 1):]
self.right_buckets = table.buckets[(index + 1) :]
self.left = True
def __iter__(self):

4
libp2p/kademlia/utils.py
View File

@ -14,7 +14,7 @@ async def gather_dict(dic):
def digest(string):
if not isinstance(string, bytes):
string = str(string).encode('utf8')
string = str(string).encode("utf8")
return hashlib.sha1(string).digest()
@ -53,5 +53,5 @@ def shared_prefix(args):
def bytes_to_bit_string(bites):
bits = [bin(bite)[2:].rjust(8, '0') for bite in bites]
bits = [bin(bite)[2:].rjust(8, "0") for bite in bites]
return "".join(bits)

17
libp2p/network/connection/raw_connection.py
View File

@ -2,6 +2,7 @@ import asyncio
from .raw_connection_interface import IRawConnection
class RawConnection(IRawConnection):
conn_ip: str
@ -11,12 +12,14 @@ class RawConnection(IRawConnection):
_next_id: int
initiator: bool
def __init__(self,
ip: str,
port: str,
reader: asyncio.StreamReader,
writer: asyncio.StreamWriter,
initiator: bool) -> None:
def __init__(
self,
ip: str,
port: str,
reader: asyncio.StreamReader,
writer: asyncio.StreamWriter,
initiator: bool,
) -> None:
# pylint: disable=too-many-arguments
self.conn_ip = ip
self.conn_port = port
@ -32,7 +35,7 @@ class RawConnection(IRawConnection):
async def read(self) -> bytes:
line = await self.reader.readline()
adjusted_line = line.decode().rstrip('\n')
adjusted_line = line.decode().rstrip("\n")
# TODO: figure out a way to remove \n without going back and forth with
# encoding and decoding

23
libp2p/network/network_interface.py
View File

@ -1,14 +1,5 @@
from abc import (
ABC,
abstractmethod,
)
from typing import (
Awaitable,
Callable,
Dict,
Sequence,
TYPE_CHECKING,
)
from abc import ABC, abstractmethod
from typing import Awaitable, Callable, Dict, Sequence, TYPE_CHECKING
from multiaddr import Multiaddr
@ -49,7 +40,9 @@ class INetwork(ABC):
"""
@abstractmethod
def set_stream_handler(self, protocol_id: str, stream_handler: StreamHandlerFn) -> bool:
def set_stream_handler(
self, protocol_id: str, stream_handler: StreamHandlerFn
) -> bool:
"""
:param protocol_id: protocol id used on stream
:param stream_handler: a stream handler instance
@ -57,9 +50,7 @@ class INetwork(ABC):
"""
@abstractmethod
async def new_stream(self,
peer_id: ID,
protocol_ids: Sequence[str]) -> INetStream:
async def new_stream(self, peer_id: ID, protocol_ids: Sequence[str]) -> INetStream:
"""
:param peer_id: peer_id of destination
:param protocol_ids: available protocol ids to use for stream
@ -74,7 +65,7 @@ class INetwork(ABC):
"""
@abstractmethod
def notify(self, notifee: 'INotifee') -> bool:
def notify(self, notifee: "INotifee") -> bool:
"""
:param notifee: object implementing Notifee interface
:return: true if notifee registered successfully, false otherwise

18
libp2p/network/notifee_interface.py
View File

@ -1,7 +1,4 @@
from abc import (
ABC,
abstractmethod,
)
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING
from multiaddr import Multiaddr
@ -15,44 +12,43 @@ if TYPE_CHECKING:
class INotifee(ABC):
@abstractmethod
async def opened_stream(self, network: 'INetwork', stream: INetStream) -> None:
async def opened_stream(self, network: "INetwork", stream: INetStream) -> None:
"""
:param network: network the stream was opened on
:param stream: stream that was opened
"""
@abstractmethod
async def closed_stream(self, network: 'INetwork', stream: INetStream) -> None:
async def closed_stream(self, network: "INetwork", stream: INetStream) -> None:
"""
:param network: network the stream was closed on
:param stream: stream that was closed
"""
@abstractmethod
async def connected(self, network: 'INetwork', conn: IMuxedConn) -> None:
async def connected(self, network: "INetwork", conn: IMuxedConn) -> None:
"""
:param network: network the connection was opened on
:param conn: connection that was opened
"""
@abstractmethod
async def disconnected(self, network: 'INetwork', conn: IMuxedConn) -> None:
async def disconnected(self, network: "INetwork", conn: IMuxedConn) -> None:
"""
:param network: network the connection was closed on
:param conn: connection that was closed
"""
@abstractmethod
async def listen(self, network: 'INetwork', multiaddr: Multiaddr) -> None:
async def listen(self, network: "INetwork", multiaddr: Multiaddr) -> None:
"""
:param network: network the listener is listening on
:param multiaddr: multiaddress listener is listening on
"""
@abstractmethod
async def listen_close(self, network: 'INetwork', multiaddr: Multiaddr) -> None:
async def listen_close(self, network: "INetwork", multiaddr: Multiaddr) -> None:
"""
:param network: network the connection was opened on
:param multiaddr: multiaddress listener is no longer listening on

62
libp2p/network/swarm.py
View File

@ -1,18 +1,9 @@
import asyncio
from typing import (
Awaitable,
Callable,
Dict,
List,
Sequence,
)
from typing import Awaitable, Callable, Dict, List, Sequence
from multiaddr import Multiaddr
from libp2p.peer.id import (
ID,
id_b58_decode,
)
from libp2p.peer.id import ID, id_b58_decode
from libp2p.peer.peerstore import PeerStore
from libp2p.protocol_muxer.multiselect import Multiselect
from libp2p.protocol_muxer.multiselect_client import MultiselectClient
@ -51,12 +42,14 @@ class Swarm(INetwork):
notifees: List[INotifee]
def __init__(self,
peer_id: ID,
peerstore: PeerStore,
upgrader: TransportUpgrader,
transport: ITransport,
router: IPeerRouting):
def __init__(
self,
peer_id: ID,
peerstore: PeerStore,
upgrader: TransportUpgrader,
transport: ITransport,
router: IPeerRouting,
):
self.self_id = peer_id
self.peerstore = peerstore
self.upgrader = upgrader
@ -79,7 +72,9 @@ class Swarm(INetwork):
def get_peer_id(self) -> ID:
return self.self_id
def set_stream_handler(self, protocol_id: str, stream_handler: StreamHandlerFn) -> bool:
def set_stream_handler(
self, protocol_id: str, stream_handler: StreamHandlerFn
) -> bool:
"""
:param protocol_id: protocol id used on stream
:param stream_handler: a stream handler instance
@ -119,8 +114,9 @@ class Swarm(INetwork):
# Per, https://discuss.libp2p.io/t/multistream-security/130, we first secure
# the conn and then mux the conn
secured_conn = await self.upgrader.upgrade_security(raw_conn, peer_id, True)
muxed_conn = self.upgrader.upgrade_connection(secured_conn, \
self.generic_protocol_handler, peer_id)
muxed_conn = self.upgrader.upgrade_connection(
secured_conn, self.generic_protocol_handler, peer_id
)
# Store muxed connection in connections
self.connections[peer_id] = muxed_conn
@ -154,8 +150,7 @@ class Swarm(INetwork):
# Perform protocol muxing to determine protocol to use
selected_protocol = await self.multiselect_client.select_one_of(
list(protocol_ids),
muxed_stream,
list(protocol_ids), muxed_stream
)
# Create a net stream with the selected protocol
@ -186,8 +181,9 @@ class Swarm(INetwork):
if str(multiaddr) in self.listeners:
return True
async def conn_handler(reader: asyncio.StreamReader,
writer: asyncio.StreamWriter) -> None:
async def conn_handler(
reader: asyncio.StreamReader, writer: asyncio.StreamWriter
) -> None:
# Read in first message (should be peer_id of initiator) and ack
peer_id = id_b58_decode((await reader.read(1024)).decode())
@ -196,14 +192,22 @@ class Swarm(INetwork):
# Upgrade reader/write to a net_stream and pass \
# to appropriate stream handler (using multiaddr)
raw_conn = RawConnection(multiaddr.value_for_protocol('ip4'),
multiaddr.value_for_protocol('tcp'), reader, writer, False)
raw_conn = RawConnection(
multiaddr.value_for_protocol("ip4"),
multiaddr.value_for_protocol("tcp"),
reader,
writer,
False,
)
# Per, https://discuss.libp2p.io/t/multistream-security/130, we first secure
# the conn and then mux the conn
secured_conn = await self.upgrader.upgrade_security(raw_conn, peer_id, False)
muxed_conn = self.upgrader.upgrade_connection(secured_conn, \
self.generic_protocol_handler, peer_id)
secured_conn = await self.upgrader.upgrade_security(
raw_conn, peer_id, False
)
muxed_conn = self.upgrader.upgrade_connection(
secured_conn, self.generic_protocol_handler, peer_id
)
# Store muxed_conn with peer id
self.connections[peer_id] = muxed_conn

6
libp2p/peer/addrbook_interface.py
View File

@ -1,8 +1,5 @@
from abc import ABC, abstractmethod
from typing import (
List,
Sequence,
)
from typing import List, Sequence
from multiaddr import Multiaddr
@ -11,7 +8,6 @@ from .id import ID
class IAddrBook(ABC):
def __init__(self) -> None:
pass

11
libp2p/peer/id.py
View File

@ -1,7 +1,5 @@
import hashlib
from typing import (
Union,
)
from typing import Union
import base58
@ -44,7 +42,7 @@ class ID:
__repr__ = __str__
def __eq__(self, other: object) -> bool:
#pylint: disable=protected-access
# pylint: disable=protected-access
if not isinstance(other, ID):
return NotImplemented
return self._id_str == other._id_str
@ -57,7 +55,7 @@ def id_b58_encode(peer_id: ID) -> str:
"""
return a b58-encoded string
"""
#pylint: disable=protected-access
# pylint: disable=protected-access
return base58.b58encode(peer_id.get_raw_id()).decode()
@ -84,7 +82,8 @@ def id_from_public_key(key: RsaKey) -> ID:
def id_from_private_key(key: RsaKey) -> ID:
return id_from_public_key(key.publickey())
def digest(data: Union[str, bytes]) -> bytes:
if isinstance(data, str):
data = data.encode('utf8')
data = data.encode("utf8")
return hashlib.sha1(data).digest()

7
libp2p/peer/peerdata.py
View File

@ -1,9 +1,4 @@
from typing import (
Any,
Dict,
List,
Sequence,
)
from typing import Any, Dict, List, Sequence
from multiaddr import Multiaddr

7
libp2p/peer/peerdata_interface.py
View File

@ -1,9 +1,5 @@
from abc import ABC, abstractmethod
from typing import (
Any,
List,
Sequence,
)
from typing import Any, List, Sequence
from multiaddr import Multiaddr
@ -11,7 +7,6 @@ from .peermetadata_interface import IPeerMetadata
class IPeerData(ABC):
@abstractmethod
def get_protocols(self) -> List[str]:
"""

13
libp2p/peer/peerinfo.py
View File

@ -1,13 +1,8 @@
from typing import (
List,
)
from typing import List
import multiaddr
from .id import (
ID,
id_b58_decode,
)
from .id import ID, id_b58_decode
from .peerdata import PeerData
@ -31,7 +26,9 @@ def info_from_p2p_addr(addr: multiaddr.Multiaddr) -> PeerInfo:
parts = addr.split()
if not parts:
raise InvalidAddrError(f"`parts`={parts} should at least have a protocol `P_P2P`")
raise InvalidAddrError(
f"`parts`={parts} should at least have a protocol `P_P2P`"
)
p2p_part = parts[-1]
last_protocol_code = p2p_part.protocols()[0].code

9
libp2p/peer/peermetadata_interface.py
View File

@ -1,15 +1,10 @@
from abc import ABC, abstractmethod
from typing import (
Any,
)
from typing import Any
from .id import (
ID,
)
from .id import ID
class IPeerMetadata(ABC):
def __init__(self) -> None:
pass

8
libp2p/peer/peerstore.py
View File

@ -1,10 +1,4 @@
from typing import (
Any,
Dict,
List,
Optional,
Sequence,
)
from typing import Any, Dict, List, Optional, Sequence
from multiaddr import Multiaddr

6
libp2p/peer/peerstore_interface.py
View File

@ -1,8 +1,5 @@
from abc import abstractmethod
from typing import (
List,
Sequence,
)
from typing import List, Sequence
from .addrbook_interface import IAddrBook
@ -12,7 +9,6 @@ from .peermetadata_interface import IPeerMetadata
class IPeerStore(IAddrBook, IPeerMetadata):
def __init__(self) -> None:
IPeerMetadata.__init__(self)
IAddrBook.__init__(self)

1
libp2p/protocol_muxer/multiselect_client.py
View File

@ -118,5 +118,6 @@ def validate_handshake(handshake_contents):
# is added
return handshake_contents == MULTISELECT_PROTOCOL_ID
class MultiselectClientError(ValueError):
"""Raised when an error occurs in protocol selection process"""

21
libp2p/pubsub/floodsub.py
View File

@ -1,13 +1,6 @@
from typing import (
Iterable,
List,
Sequence,
)
from typing import Iterable, List, Sequence
from libp2p.peer.id import (
ID,
id_b58_decode,
)
from libp2p.peer.id import ID, id_b58_decode
from .pb import rpc_pb2
from .pubsub import Pubsub
@ -78,9 +71,7 @@ class FloodSub(IPubsubRouter):
msg_forwarder=msg_forwarder,
origin=ID(pubsub_msg.from_id),
)
rpc_msg = rpc_pb2.RPC(
publish=[pubsub_msg],
)
rpc_msg = rpc_pb2.RPC(publish=[pubsub_msg])
for peer_id in peers_gen:
stream = self.pubsub.peers[str(peer_id)]
# FIXME: We should add a `WriteMsg` similar to write delimited messages.
@ -103,10 +94,8 @@ class FloodSub(IPubsubRouter):
"""
def _get_peers_to_send(
self,
topic_ids: Iterable[str],
msg_forwarder: ID,
origin: ID) -> Iterable[ID]:
self, topic_ids: Iterable[str], msg_forwarder: ID, origin: ID
) -> Iterable[ID]:
"""
Get the eligible peers to send the data to.
:param msg_forwarder: peer ID of the peer who forwards the message to us.

139
libp2p/pubsub/gossipsub.py
View File

@ -1,19 +1,9 @@
from ast import literal_eval
import asyncio
import random
from typing import (
Any,
Dict,
Iterable,
List,
Set,
Sequence,
)
from typing import Any, Dict, Iterable, List, Set, Sequence
from libp2p.peer.id import (
ID,
id_b58_decode,
)
from libp2p.peer.id import ID, id_b58_decode
from .mcache import MessageCache
from .pb import rpc_pb2
@ -45,24 +35,26 @@ class GossipSub(IPubsubRouter):
time_since_last_publish: Dict[str, int]
#FIXME: Should be changed to List[ID]
# FIXME: Should be changed to List[ID]
peers_gossipsub: List[str]
#FIXME: Should be changed to List[ID]
# FIXME: Should be changed to List[ID]
peers_floodsub: List[str]
mcache: MessageCache
heartbeat_interval: int
def __init__(self,
protocols: Sequence[str],
degree: int,
degree_low: int,
degree_high: int,
time_to_live: int,
gossip_window: int = 3,
gossip_history: int = 5,
heartbeat_interval: int = 120) -> None:
def __init__(
self,
protocols: Sequence[str],
degree: int,
degree_low: int,
degree_high: int,
time_to_live: int,
gossip_window: int = 3,
gossip_history: int = 5,
heartbeat_interval: int = 120,
) -> None:
# pylint: disable=too-many-arguments
self.protocols = list(protocols)
self.pubsub = None
@ -181,9 +173,7 @@ class GossipSub(IPubsubRouter):
msg_forwarder=msg_forwarder,
origin=ID(pubsub_msg.from_id),
)
rpc_msg = rpc_pb2.RPC(
publish=[pubsub_msg],
)
rpc_msg = rpc_pb2.RPC(publish=[pubsub_msg])
for peer_id in peers_gen:
stream = self.pubsub.peers[str(peer_id)]
# FIXME: We should add a `WriteMsg` similar to write delimited messages.
@ -192,10 +182,8 @@ class GossipSub(IPubsubRouter):
await stream.write(rpc_msg.SerializeToString())
def _get_peers_to_send(
self,
topic_ids: Iterable[str],
msg_forwarder: ID,
origin: ID) -> Iterable[ID]:
self, topic_ids: Iterable[str], msg_forwarder: ID, origin: ID
) -> Iterable[ID]:
"""
Get the eligible peers to send the data to.
:param msg_forwarder: the peer id of the peer who forwards the message to me.
@ -231,9 +219,7 @@ class GossipSub(IPubsubRouter):
if (topic not in self.fanout) or (len(self.fanout[topic]) == 0):
# If no peers in fanout, choose some peers from gossipsub peers in topic.
self.fanout[topic] = self._get_in_topic_gossipsub_peers_from_minus(
topic,
self.degree,
[],
topic, self.degree, []
)
in_topic_gossipsub_peers = self.fanout[topic]
for peer_id_str in in_topic_gossipsub_peers:
@ -264,9 +250,7 @@ class GossipSub(IPubsubRouter):
# Selects the remaining number of peers (D-x) from peers.gossipsub[topic].
if topic in self.pubsub.peer_topics:
selected_peers = self._get_in_topic_gossipsub_peers_from_minus(
topic,
self.degree - fanout_size,
fanout_peers,
topic, self.degree - fanout_size, fanout_peers
)
# Combine fanout peers with selected peers
fanout_peers += selected_peers
@ -308,11 +292,13 @@ class GossipSub(IPubsubRouter):
# FIXME: type of `peers` should be changed to `List[ID]`
# FIXME: type of `msg_sender` and `origin_id` should be changed to `ID`
async def deliver_messages_to_peers(self,
peers: List[str],
msg_sender: str,
origin_id: str,
serialized_packet: bytes) -> None:
async def deliver_messages_to_peers(
self,
peers: List[str],
msg_sender: str,
origin_id: str,
serialized_packet: bytes,
) -> None:
for peer_id_in_topic in peers:
# Forward to all peers that are not the
# message sender and are not the message origin
@ -349,16 +335,12 @@ class GossipSub(IPubsubRouter):
if num_mesh_peers_in_topic < self.degree_low:
# Select D - |mesh[topic]| peers from peers.gossipsub[topic] - mesh[topic]
selected_peers = self._get_in_topic_gossipsub_peers_from_minus(
topic,
self.degree - num_mesh_peers_in_topic,
self.mesh[topic],
topic, self.degree - num_mesh_peers_in_topic, self.mesh[topic]
)
# FIXME: Should be changed to `List[ID]`
fanout_peers_not_in_mesh: List[str] = [
peer
for peer in selected_peers
if peer not in self.mesh[topic]
peer for peer in selected_peers if peer not in self.mesh[topic]
]
for peer in fanout_peers_not_in_mesh:
# Add peer to mesh[topic]
@ -371,9 +353,7 @@ class GossipSub(IPubsubRouter):
# Select |mesh[topic]| - D peers from mesh[topic]
# FIXME: Should be changed to `List[ID]`
selected_peers = GossipSub.select_from_minus(
num_mesh_peers_in_topic - self.degree,
self.mesh[topic],
[],
num_mesh_peers_in_topic - self.degree, self.mesh[topic], []
)
for peer in selected_peers:
# Remove peer from mesh[topic]
@ -415,15 +395,16 @@ class GossipSub(IPubsubRouter):
if topic in self.pubsub.peer_topics:
# Select D peers from peers.gossipsub[topic]
peers_to_emit_ihave_to = self._get_in_topic_gossipsub_peers_from_minus(
topic,
self.degree,
[],
topic, self.degree, []
)
for peer in peers_to_emit_ihave_to:
# TODO: this line is a monster, can hopefully be simplified
if (topic not in self.mesh or (peer not in self.mesh[topic]))\
and (topic not in self.fanout or (peer not in self.fanout[topic])):
if (
topic not in self.mesh or (peer not in self.mesh[topic])
) and (
topic not in self.fanout or (peer not in self.fanout[topic])
):
msg_id_strs = [str(msg_id) for msg_id in msg_ids]
await self.emit_ihave(topic, msg_id_strs, peer)
@ -438,12 +419,13 @@ class GossipSub(IPubsubRouter):
if topic in self.pubsub.peer_topics:
# Select D peers from peers.gossipsub[topic]
peers_to_emit_ihave_to = self._get_in_topic_gossipsub_peers_from_minus(
topic,
self.degree,
[],
topic, self.degree, []
)
for peer in peers_to_emit_ihave_to:
if peer not in self.mesh[topic] and peer not in self.fanout[topic]:
if (
peer not in self.mesh[topic]
and peer not in self.fanout[topic]
):
msg_id_strs = [str(msg) for msg in msg_ids]
await self.emit_ihave(topic, msg_id_strs, peer)
@ -451,9 +433,9 @@ class GossipSub(IPubsubRouter):
self.mcache.shift()
@staticmethod
def select_from_minus(num_to_select: int,
pool: Sequence[Any],
minus: Sequence[Any]) -> List[Any]:
def select_from_minus(
num_to_select: int, pool: Sequence[Any], minus: Sequence[Any]
) -> List[Any]:
"""
Select at most num_to_select subset of elements from the set (pool - minus) randomly.
:param num_to_select: number of elements to randomly select
@ -482,24 +464,22 @@ class GossipSub(IPubsubRouter):
# FIXME: type of `minus` should be changed to type `Sequence[ID]`
# FIXME: return type should be changed to type `List[ID]`
def _get_in_topic_gossipsub_peers_from_minus(
self,
topic: str,
num_to_select: int,
minus: Sequence[str]) -> List[str]:
self, topic: str, num_to_select: int, minus: Sequence[str]
) -> List[str]:
gossipsub_peers_in_topic = [
peer_str
for peer_str in self.pubsub.peer_topics[topic]
if peer_str in self.peers_gossipsub
]
return self.select_from_minus(
num_to_select,
gossipsub_peers_in_topic,
list(minus),
num_to_select, gossipsub_peers_in_topic, list(minus)
)
# RPC handlers
async def handle_ihave(self, ihave_msg: rpc_pb2.Message, sender_peer_id: str) -> None:
async def handle_ihave(
self, ihave_msg: rpc_pb2.Message, sender_peer_id: str
) -> None:
"""
Checks the seen set and requests unknown messages with an IWANT message.
"""
@ -509,8 +489,7 @@ class GossipSub(IPubsubRouter):
# Get list of all seen (seqnos, from) from the (seqno, from) tuples in seen_messages cache
seen_seqnos_and_peers = [
seqno_and_from
for seqno_and_from in self.pubsub.seen_messages.keys()
seqno_and_from for seqno_and_from in self.pubsub.seen_messages.keys()
]
# Add all unknown message ids (ids that appear in ihave_msg but not in seen_seqnos) to list
@ -526,7 +505,9 @@ class GossipSub(IPubsubRouter):
if msg_ids_wanted:
await self.emit_iwant(msg_ids_wanted, from_id_str)
async def handle_iwant(self, iwant_msg: rpc_pb2.Message, sender_peer_id: str) -> None:
async def handle_iwant(
self, iwant_msg: rpc_pb2.Message, sender_peer_id: str
) -> None:
"""
Forwards all request messages that are present in mcache to the requesting peer.
"""
@ -564,7 +545,9 @@ class GossipSub(IPubsubRouter):
# 4) And write the packet to the stream
await peer_stream.write(rpc_msg)
async def handle_graft(self, graft_msg: rpc_pb2.Message, sender_peer_id: str) -> None:
async def handle_graft(
self, graft_msg: rpc_pb2.Message, sender_peer_id: str
) -> None:
topic: str = graft_msg.topicID
from_id_str = sender_peer_id
@ -577,7 +560,9 @@ class GossipSub(IPubsubRouter):
# Respond with PRUNE if not subscribed to the topic
await self.emit_prune(topic, sender_peer_id)
async def handle_prune(self, prune_msg: rpc_pb2.Message, sender_peer_id: str) -> None:
async def handle_prune(
self, prune_msg: rpc_pb2.Message, sender_peer_id: str
) -> None:
topic: str = prune_msg.topicID
from_id_str = sender_peer_id
@ -641,7 +626,9 @@ class GossipSub(IPubsubRouter):
await self.emit_control_message(control_msg, to_peer)
async def emit_control_message(self, control_msg: rpc_pb2.ControlMessage, to_peer: str) -> None:
async def emit_control_message(
self, control_msg: rpc_pb2.ControlMessage, to_peer: str
) -> None:
# Add control message to packet
packet: rpc_pb2.RPC = rpc_pb2.RPC()
packet.control.CopyFrom(control_msg)

15
libp2p/pubsub/mcache.py
View File

@ -1,10 +1,4 @@
from typing import (
Dict,
List,
Optional,
Sequence,
Tuple,
)
from typing import Dict, List, Optional, Sequence, Tuple
from .pb import rpc_pb2
@ -18,6 +12,7 @@ class CacheEntry:
"""
A logical representation of an entry in the mcache's _history_.
"""
def __init__(self, mid: Tuple[bytes, bytes], topics: Sequence[str]) -> None:
"""
Constructor.
@ -30,7 +25,6 @@ class CacheEntry:
class MessageCache:
window_size: int
history_size: int
@ -53,10 +47,7 @@ class MessageCache:
# max length of history_size. each item is a list of CacheEntry.
# messages lost upon shift().
self.history = [
[]
for _ in range(history_size)
]
self.history = [[] for _ in range(history_size)]
def put(self, msg: rpc_pb2.Message) -> None:
"""

53
libp2p/pubsub/pubsub.py
View File

@ -1,13 +1,7 @@
# pylint: disable=no-name-in-module
import asyncio
import time
from typing import (
Any,
Dict,
List,
Tuple,
TYPE_CHECKING,
)
from typing import Any, Dict, List, Tuple, TYPE_CHECKING
from lru import LRU
@ -34,18 +28,18 @@ class Pubsub:
host: IHost
my_id: ID
router: 'IPubsubRouter'
router: "IPubsubRouter"
peer_queue: 'asyncio.Queue[ID]'
peer_queue: "asyncio.Queue[ID]"
protocols: List[str]
incoming_msgs_from_peers: 'asyncio.Queue[rpc_pb2.Message]'
outgoing_messages: 'asyncio.Queue[rpc_pb2.Message]'
incoming_msgs_from_peers: "asyncio.Queue[rpc_pb2.Message]"
outgoing_messages: "asyncio.Queue[rpc_pb2.Message]"
seen_messages: LRU
my_topics: Dict[str, 'asyncio.Queue[rpc_pb2.Message]']
my_topics: Dict[str, "asyncio.Queue[rpc_pb2.Message]"]
# FIXME: Should be changed to `Dict[str, List[ID]]`
peer_topics: Dict[str, List[str]]
@ -55,11 +49,9 @@ class Pubsub:
# NOTE: Be sure it is increased atomically everytime.
counter: int # uint64
def __init__(self,
host: IHost,
router: 'IPubsubRouter',
my_id: ID,
cache_size: int = None) -> None:
def __init__(
self, host: IHost, router: "IPubsubRouter", my_id: ID, cache_size: int = None
) -> None:
"""
Construct a new Pubsub object, which is responsible for handling all
Pubsub-related messages and relaying messages as appropriate to the
@ -120,12 +112,9 @@ class Pubsub:
"""
packet = rpc_pb2.RPC()
for topic_id in self.my_topics:
packet.subscriptions.extend([
rpc_pb2.RPC.SubOpts(
subscribe=True,
topicid=topic_id,
)
])
packet.subscriptions.extend(
[rpc_pb2.RPC.SubOpts(subscribe=True, topicid=topic_id)]
)
return packet.SerializeToString()
async def continuously_read_stream(self, stream: INetStream) -> None:
@ -262,7 +251,7 @@ class Pubsub:
# for each topic
await self.my_topics[topic].put(publish_message)
async def subscribe(self, topic_id: str) -> 'asyncio.Queue[rpc_pb2.Message]':
async def subscribe(self, topic_id: str) -> "asyncio.Queue[rpc_pb2.Message]":
"""
Subscribe ourself to a topic
:param topic_id: topic_id to subscribe to
@ -277,10 +266,9 @@ class Pubsub:
# Create subscribe message
packet: rpc_pb2.RPC = rpc_pb2.RPC()
packet.subscriptions.extend([rpc_pb2.RPC.SubOpts(
subscribe=True,
topicid=topic_id.encode('utf-8')
)])
packet.subscriptions.extend(
[rpc_pb2.RPC.SubOpts(subscribe=True, topicid=topic_id.encode("utf-8"))]
)
# Send out subscribe message to all peers
await self.message_all_peers(packet.SerializeToString())
@ -305,10 +293,9 @@ class Pubsub:
# Create unsubscribe message
packet: rpc_pb2.RPC = rpc_pb2.RPC()
packet.subscriptions.extend([rpc_pb2.RPC.SubOpts(
subscribe=False,
topicid=topic_id.encode('utf-8')
)])
packet.subscriptions.extend(
[rpc_pb2.RPC.SubOpts(subscribe=False, topicid=topic_id.encode("utf-8"))]
)
# Send out unsubscribe message to all peers
await self.message_all_peers(packet.SerializeToString())
@ -371,7 +358,7 @@ class Pubsub:
Make the next message sequence id.
"""
self.counter += 1
return self.counter.to_bytes(8, 'big')
return self.counter.to_bytes(8, "big")
def _is_msg_seen(self, msg: rpc_pb2.Message) -> bool:
msg_id = get_msg_id(msg)

8
libp2p/pubsub/pubsub_notifee.py
View File

@ -1,6 +1,4 @@
from typing import (
TYPE_CHECKING,
)
from typing import TYPE_CHECKING
from multiaddr import Multiaddr
@ -18,9 +16,9 @@ if TYPE_CHECKING:
class PubsubNotifee(INotifee):
# pylint: disable=too-many-instance-attributes, cell-var-from-loop, unsubscriptable-object
initiator_peers_queue: 'asyncio.Queue[ID]'
initiator_peers_queue: "asyncio.Queue[ID]"
def __init__(self, initiator_peers_queue: 'asyncio.Queue[ID]') -> None:
def __init__(self, initiator_peers_queue: "asyncio.Queue[ID]") -> None:
"""
:param initiator_peers_queue: queue to add new peers to so that pubsub
can process new peers after we connect to them

9
libp2p/pubsub/pubsub_router_interface.py
View File

@ -1,8 +1,5 @@
from abc import ABC, abstractmethod
from typing import (
List,
TYPE_CHECKING,
)
from typing import List, TYPE_CHECKING
from libp2p.peer.id import ID
@ -11,8 +8,8 @@ from .pb import rpc_pb2
if TYPE_CHECKING:
from .pubsub import Pubsub
class IPubsubRouter(ABC):
class IPubsubRouter(ABC):
@abstractmethod
def get_protocols(self) -> List[str]:
"""
@ -20,7 +17,7 @@ class IPubsubRouter(ABC):
"""
@abstractmethod
def attach(self, pubsub: 'Pubsub') -> None:
def attach(self, pubsub: "Pubsub") -> None:
"""
Attach is invoked by the PubSub constructor to attach the router to a
freshly initialized PubSub instance.

8
libp2p/routing/interfaces.py
View File

@ -1,16 +1,13 @@
from abc import (
ABC,
abstractmethod,
)
from abc import ABC, abstractmethod
from typing import Iterable
from libp2p.peer.id import ID
from libp2p.peer.peerinfo import PeerInfo
# pylint: disable=too-few-public-methods
class IContentRouting(ABC):
@abstractmethod
def provide(self, cid: bytes, announce: bool = True) -> None:
"""
@ -28,7 +25,6 @@ class IContentRouting(ABC):
class IPeerRouting(ABC):
@abstractmethod
async def find_peer(self, peer_id: ID) -> PeerInfo:
"""

5
libp2p/routing/kademlia/kademlia_content_router.py
View File

@ -1,13 +1,10 @@
from typing import (
Iterable,
)
from typing import Iterable
from libp2p.peer.peerinfo import PeerInfo
from libp2p.routing.interfaces import IContentRouting
class KadmeliaContentRouter(IContentRouting):
def provide(self, cid: bytes, announce: bool = True) -> None:
"""
Provide adds the given cid to the content routing system. If announce is True,

12
libp2p/routing/kademlia/kademlia_peer_router.py
View File

@ -1,12 +1,7 @@
import ast
from typing import (
Union,
)
from typing import Union
from libp2p.kademlia.kad_peerinfo import (
KadPeerInfo,
create_kad_peerinfo,
)
from libp2p.kademlia.kad_peerinfo import KadPeerInfo, create_kad_peerinfo
from libp2p.kademlia.network import KademliaServer
from libp2p.peer.id import ID
from libp2p.routing.interfaces import IPeerRouting
@ -31,6 +26,7 @@ class KadmeliaPeerRouter(IPeerRouting):
value = await self.server.get(xor_id)
return decode_peerinfo(value)
def decode_peerinfo(encoded: Union[bytes, str]) -> KadPeerInfo:
if isinstance(encoded, bytes):
encoded = encoded.decode()
@ -38,7 +34,7 @@ def decode_peerinfo(encoded: Union[bytes, str]) -> KadPeerInfo:
lines = ast.literal_eval(encoded)
except SyntaxError:
return None
ip = lines[1] # pylint: disable=invalid-name
ip = lines[1] # pylint: disable=invalid-name
port = lines[2]
peer_id = lines[3]
peer_info = create_kad_peerinfo(peer_id, ip, port)

4
libp2p/security/insecure_security.py
View File

@ -1,8 +1,8 @@
from libp2p.security.secure_transport_interface import ISecureTransport
from libp2p.security.secure_conn_interface import ISecureConn
class InsecureTransport(ISecureTransport):
class InsecureTransport(ISecureTransport):
def __init__(self, transport_id):
self.transport_id = transport_id
@ -24,8 +24,8 @@ class InsecureTransport(ISecureTransport):
insecure_conn = InsecureConn(conn, self.transport_id)
return insecure_conn
class InsecureConn(ISecureConn):
class InsecureConn(ISecureConn):
def __init__(self, conn, conn_id):
self.conn = conn
self.details = {}

3
libp2p/security/secure_conn_interface.py
View File

@ -8,8 +8,9 @@ involved in the secured connection
Relevant go repo: https://github.com/libp2p/go-conn-security/blob/master/interface.go
"""
class ISecureConn(ABC):
class ISecureConn(ABC):
@abstractmethod
def get_conn(self):
"""

3
libp2p/security/secure_transport_interface.py
View File

@ -8,8 +8,9 @@ chosen by a security transport multistream module.
Relevant go repo: https://github.com/libp2p/go-conn-security/blob/master/interface.go
"""
class ISecureTransport(ABC):
class ISecureTransport(ABC):
@abstractmethod
async def secure_inbound(self, conn):
"""

11
libp2p/security/security_multistream.py
View File

@ -10,8 +10,9 @@ involved in the secured connection
Relevant go repo: https://github.com/libp2p/go-conn-security/blob/master/interface.go
"""
class SecurityMultistream(ABC):
class SecurityMultistream(ABC):
def __init__(self):
# Map protocol to secure transport
self.transports = {}
@ -31,7 +32,6 @@ class SecurityMultistream(ABC):
# we only care about selecting the protocol, not any handler function
self.multiselect.add_handler(protocol, None)
async def secure_inbound(self, conn):
"""
Secure the connection, either locally or by communicating with opposing node via conn,
@ -47,7 +47,6 @@ class SecurityMultistream(ABC):
return secure_conn
async def secure_outbound(self, conn, peer_id):
"""
Secure the connection, either locally or by communicating with opposing node via conn,
@ -63,7 +62,6 @@ class SecurityMultistream(ABC):
return secure_conn
async def select_transport(self, conn, initiator):
"""
Select a transport that both us and the node on the
@ -79,8 +77,9 @@ class SecurityMultistream(ABC):
protocol = None
if initiator:
# Select protocol if initiator
protocol = \
await self.multiselect_client.select_one_of(list(self.transports.keys()), conn)
protocol = await self.multiselect_client.select_one_of(
list(self.transports.keys()), conn
)
else:
# Select protocol if non-initiator
protocol, _ = await self.multiselect.negotiate(conn)

12
libp2p/security/simple_security.py
View File

@ -2,8 +2,8 @@ import asyncio
from libp2p.security.secure_transport_interface import ISecureTransport
from libp2p.security.secure_conn_interface import ISecureConn
class SimpleSecurityTransport(ISecureTransport):
class SimpleSecurityTransport(ISecureTransport):
def __init__(self, key_phrase):
self.key_phrase = key_phrase
@ -17,7 +17,9 @@ class SimpleSecurityTransport(ISecureTransport):
incoming = (await conn.read()).decode()
if incoming != self.key_phrase:
raise Exception("Key phrase differed between nodes. Expected " + self.key_phrase)
raise Exception(
"Key phrase differed between nodes. Expected " + self.key_phrase
)
secure_conn = SimpleSecureConn(conn, self.key_phrase)
return secure_conn
@ -36,13 +38,15 @@ class SimpleSecurityTransport(ISecureTransport):
await asyncio.sleep(0)
if incoming != self.key_phrase:
raise Exception("Key phrase differed between nodes. Expected " + self.key_phrase)
raise Exception(
"Key phrase differed between nodes. Expected " + self.key_phrase
)
secure_conn = SimpleSecureConn(conn, self.key_phrase)
return secure_conn
class SimpleSecureConn(ISecureConn):
class SimpleSecureConn(ISecureConn):
def __init__(self, conn, key_phrase):
self.conn = conn
self.details = {}

7
libp2p/stream_muxer/mplex/constants.py
View File

@ -1,6 +1 @@
HEADER_TAGS = {
"NEW_STREAM": 0,
"MESSAGE": 2,
"CLOSE": 4,
"RESET": 6
}
HEADER_TAGS = {"NEW_STREAM": 0, "MESSAGE": 2, "CLOSE": 4, "RESET": 6}

4
libp2p/stream_muxer/mplex/mplex.py
View File

@ -158,7 +158,9 @@ class Mplex(IMuxedConn):
try:
header = await decode_uvarint_from_stream(self.raw_conn.reader, timeout)
length = await decode_uvarint_from_stream(self.raw_conn.reader, timeout)
message = await asyncio.wait_for(self.raw_conn.reader.read(length), timeout=timeout)
message = await asyncio.wait_for(
self.raw_conn.reader.read(length), timeout=timeout
)
except asyncio.TimeoutError:
return None, None, None

10
libp2p/stream_muxer/mplex/mplex_stream.py
View File

@ -40,7 +40,8 @@ class MplexStream(IMuxedStream):
:return: number of bytes written
"""
return await self.mplex_conn.send_message(
get_flag(self.initiator, "MESSAGE"), data, self.stream_id)
get_flag(self.initiator, "MESSAGE"), data, self.stream_id
)
async def close(self):
"""
@ -50,7 +51,9 @@ class MplexStream(IMuxedStream):
"""
# TODO error handling with timeout
# TODO understand better how mutexes are used from go repo
await self.mplex_conn.send_message(get_flag(self.initiator, "CLOSE"), None, self.stream_id)
await self.mplex_conn.send_message(
get_flag(self.initiator, "CLOSE"), None, self.stream_id
)
remote_lock = ""
async with self.stream_lock:
@ -79,7 +82,8 @@ class MplexStream(IMuxedStream):
if not self.remote_closed:
await self.mplex_conn.send_message(
get_flag(self.initiator, "RESET"), None, self.stream_id)
get_flag(self.initiator, "RESET"), None, self.stream_id
)
self.local_closed = True
self.remote_closed = True

16
libp2p/stream_muxer/mplex/utils.py
View File

@ -5,14 +5,14 @@ from .constants import HEADER_TAGS
def encode_uvarint(number):
"""Pack `number` into varint bytes"""
buf = b''
buf = b""
while True:
towrite = number & 0x7f
towrite = number & 0x7F
number >>= 7
if number:
buf += bytes((towrite | 0x80, ))
buf += bytes((towrite | 0x80,))
else:
buf += bytes((towrite, ))
buf += bytes((towrite,))
break
return buf
@ -22,7 +22,7 @@ def decode_uvarint(buff, index):
result = 0
while True:
i = buff[index]
result |= (i & 0x7f) << shift
result |= (i & 0x7F) << shift
shift += 7
if not i & 0x80:
break
@ -30,19 +30,21 @@ def decode_uvarint(buff, index):
return result, index + 1
async def decode_uvarint_from_stream(reader, timeout):
shift = 0
result = 0
while True:
byte = await asyncio.wait_for(reader.read(1), timeout=timeout)
i = struct.unpack('>H', b'\x00' + byte)[0]
result |= (i & 0x7f) << shift
i = struct.unpack(">H", b"\x00" + byte)[0]
result |= (i & 0x7F) << shift
shift += 7
if not i & 0x80:
break
return result
def get_flag(initiator, action):
"""
get header flag based on action for mplex

5
libp2p/stream_muxer/muxed_stream_interface.py
View File

@ -1,7 +1,4 @@
from abc import (
ABC,
abstractmethod,
)
from abc import ABC, abstractmethod
from libp2p.stream_muxer.muxed_connection_interface import IMuxedConn

1
libp2p/transport/listener_interface.py
View File

@ -2,7 +2,6 @@ from abc import ABC, abstractmethod
class IListener(ABC):
@abstractmethod
def listen(self, maddr):
"""

10
libp2p/transport/tcp/tcp.py
View File

@ -10,12 +10,10 @@ from ..transport_interface import ITransport
class TCP(ITransport):
def __init__(self):
self.listener = self.Listener()
class Listener(IListener):
def __init__(self, handler_function=None):
self.multiaddrs = []
self.server = None
@ -29,8 +27,8 @@ class TCP(ITransport):
"""
self.server = await asyncio.start_server(
self.handler,
maddr.value_for_protocol('ip4'),
maddr.value_for_protocol('tcp'),
maddr.value_for_protocol("ip4"),
maddr.value_for_protocol("tcp"),
)
socket = self.server.sockets[0]
self.multiaddrs.append(_multiaddr_from_socket(socket))
@ -70,8 +68,8 @@ class TCP(ITransport):
:param options: optional object
:return: True if successful
"""
host = maddr.value_for_protocol('ip4')
port = int(maddr.value_for_protocol('tcp'))
host = maddr.value_for_protocol("ip4")
port = int(maddr.value_for_protocol("tcp"))
reader, writer = await asyncio.open_connection(host, port)

1
libp2p/transport/transport_interface.py
View File

@ -2,7 +2,6 @@ from abc import ABC, abstractmethod
class ITransport(ABC):
@abstractmethod
def dial(self, maddr, self_id, options=None):
"""