varun-r-mallya/py-libp2p
1
0
Fork 0
mirror of https://github.com/varun-r-mallya/py-libp2p.git synced 2026-02-10 23:20:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Run black over repo

Browse Source
This commit is contained in:
Alex Stokes
2019-07-31 15:00:12 -07:00
parent a2133d8c7c
commit 0ae9840928
69 changed files with 791 additions and 1095 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
libp2p/kademlia/crawling.py
View File

@ -118,8 +118,9 @@ class ValueSpiderCrawl(SpiderCrawl):
"""
value_counts = Counter(values)
if len(value_counts) != 1:
log.warning("Got multiple values for key %i: %s",
self.node.xor_id, str(values))
log.warning(
"Got multiple values for key %i: %s", self.node.xor_id, str(values)
)
value = value_counts.most_common(1)[0][0]
peer = self.nearest_without_value.popleft()
@ -175,7 +176,7 @@ class RPCFindResponse:
return isinstance(self.response[1], dict)
def get_value(self):
return self.response[1]['value']
return self.response[1]["value"]
def get_node_list(self):
"""

27
libp2p/kademlia/kad_peerinfo.py
View File

@ -11,6 +11,7 @@ from .utils import digest
P_IP = "ip4"
P_UDP = "udp"
class KadPeerInfo(PeerInfo):
def __init__(self, peer_id, peer_data=None):
super(KadPeerInfo, self).__init__(peer_id, peer_data)
@ -22,11 +23,8 @@ class KadPeerInfo(PeerInfo):
self.addrs = peer_data.get_addrs() if peer_data else None
# pylint: disable=invalid-name
self.ip = self.addrs[0].value_for_protocol(P_IP)\
if peer_data else None
self.port = int(self.addrs[0].value_for_protocol(P_UDP))\
if peer_data else None
self.ip = self.addrs[0].value_for_protocol(P_IP) if peer_data else None
self.port = int(self.addrs[0].value_for_protocol(P_UDP)) if peer_data else None
def same_home_as(self, node):
return sorted(self.addrs) == sorted(node.addrs)
@ -50,13 +48,18 @@ class KadPeerInfo(PeerInfo):
return "%s:%s" % (self.ip, str(self.port))
def encode(self):
return str(self.peer_id) + "\n" + \
str("/ip4/" + str(self.ip) + "/udp/" + str(self.port))
return (
str(self.peer_id)
+ "\n"
+ str("/ip4/" + str(self.ip) + "/udp/" + str(self.port))
)
class KadPeerHeap:
"""
A heap of peers ordered by distance to a given node.
"""
def __init__(self, node, maxsize):
"""
Constructor.
@ -134,13 +137,17 @@ class KadPeerHeap:
def get_uncontacted(self):
return [n for n in self if n.peer_id not in self.contacted]
def create_kad_peerinfo(raw_node_id=None, sender_ip=None, sender_port=None):
node_id = ID(raw_node_id) if raw_node_id else ID(digest(random.getrandbits(255)))
peer_data = None
if sender_ip and sender_port:
peer_data = PeerData() #pylint: disable=no-value-for-parameter
addr = [Multiaddr("/"+ P_IP +"/" + str(sender_ip) + "/"\
+ P_UDP + "/" + str(sender_port))]
peer_data = PeerData() # pylint: disable=no-value-for-parameter
addr = [
Multiaddr(
"/" + P_IP + "/" + str(sender_ip) + "/" + P_UDP + "/" + str(sender_port)
)
]
peer_data.add_addrs(addr)
return KadPeerInfo(node_id, peer_data)

76
libp2p/kademlia/network.py
View File

@ -57,17 +57,17 @@ class KademliaServer:
def _create_protocol(self):
return self.protocol_class(self.node, self.storage, self.ksize)
async def listen(self, port, interface='0.0.0.0'):
async def listen(self, port, interface="0.0.0.0"):
"""
Start listening on the given port.
Provide interface="::" to accept ipv6 address
"""
loop = asyncio.get_event_loop()
listen = loop.create_datagram_endpoint(self._create_protocol,
local_addr=(interface, port))
log.info("Node %i listening on %s:%i",
self.node.xor_id, interface, port)
listen = loop.create_datagram_endpoint(
self._create_protocol, local_addr=(interface, port)
)
log.info("Node %i listening on %s:%i", self.node.xor_id, interface, port)
self.transport, self.protocol = await listen
# finally, schedule refreshing table
self.refresh_table()
@ -87,8 +87,9 @@ class KademliaServer:
for node_id in self.protocol.get_refresh_ids():
node = create_kad_peerinfo(node_id)
nearest = self.protocol.router.find_neighbors(node, self.alpha)
spider = NodeSpiderCrawl(self.protocol, node, nearest,
self.ksize, self.alpha)
spider = NodeSpiderCrawl(
self.protocol, node, nearest, self.ksize, self.alpha
)
results.append(spider.find())
# do our crawling
@ -119,13 +120,13 @@ class KademliaServer:
addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP
addresses are acceptable - hostnames will cause an error.
"""
log.debug("Attempting to bootstrap node with %i initial contacts",
len(addrs))
log.debug("Attempting to bootstrap node with %i initial contacts", len(addrs))
cos = list(map(self.bootstrap_node, addrs))
gathered = await asyncio.gather(*cos)
nodes = [node for node in gathered if node is not None]
spider = NodeSpiderCrawl(self.protocol, self.node, nodes,
self.ksize, self.alpha)
spider = NodeSpiderCrawl(
self.protocol, self.node, nodes, self.ksize, self.alpha
)
return await spider.find()
async def bootstrap_node(self, addr):
@ -150,8 +151,7 @@ class KademliaServer:
if not nearest:
log.warning("There are no known neighbors to get key %s", key)
return None
spider = ValueSpiderCrawl(self.protocol, node, nearest,
self.ksize, self.alpha)
spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return await spider.find()
async def set(self, key, value):
@ -159,9 +159,7 @@ class KademliaServer:
Set the given string key to the given value in the network.
"""
if not check_dht_value_type(value):
raise TypeError(
"Value must be of type int, float, bool, str, or bytes"
)
raise TypeError("Value must be of type int, float, bool, str, or bytes")
log.info("setting '%s' = '%s' on network", key, value)
dkey = digest(key)
return await self.set_digest(dkey, value)
@ -171,7 +169,10 @@ class KademliaServer:
publish to the network that it provides for a particular key
"""
neighbors = self.protocol.router.find_neighbors(self.node)
return [await self.protocol.call_add_provider(n, key, self.node.peer_id) for n in neighbors]
return [
await self.protocol.call_add_provider(n, key, self.node.peer_id)
for n in neighbors
]
async def get_providers(self, key):
"""
@ -189,12 +190,10 @@ class KademliaServer:
nearest = self.protocol.router.find_neighbors(node)
if not nearest:
log.warning("There are no known neighbors to set key %s",
dkey.hex())
log.warning("There are no known neighbors to set key %s", dkey.hex())
return False
spider = NodeSpiderCrawl(self.protocol, node, nearest,
self.ksize, self.alpha)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
nodes = await spider.find()
log.info("setting '%s' on %s", dkey.hex(), list(map(str, nodes)))
@ -213,15 +212,15 @@ class KademliaServer:
"""
log.info("Saving state to %s", fname)
data = {
'ksize': self.ksize,
'alpha': self.alpha,
'id': self.node.peer_id,
'neighbors': self.bootstrappable_neighbors()
"ksize": self.ksize,
"alpha": self.alpha,
"id": self.node.peer_id,
"neighbors": self.bootstrappable_neighbors(),
}
if not data['neighbors']:
if not data["neighbors"]:
log.warning("No known neighbors, so not writing to cache.")
return
with open(fname, 'wb') as file:
with open(fname, "wb") as file:
pickle.dump(data, file)
@classmethod
@ -231,11 +230,11 @@ class KademliaServer:
from a cache file with the given fname.
"""
log.info("Loading state from %s", fname)
with open(fname, 'rb') as file:
with open(fname, "rb") as file:
data = pickle.load(file)
svr = KademliaServer(data['ksize'], data['alpha'], data['id'])
if data['neighbors']:
svr.bootstrap(data['neighbors'])
svr = KademliaServer(data["ksize"], data["alpha"], data["id"])
if data["neighbors"]:
svr.bootstrap(data["neighbors"])
return svr
def save_state_regularly(self, fname, frequency=600):
@ -250,10 +249,9 @@ class KademliaServer:
"""
self.save_state(fname)
loop = asyncio.get_event_loop()
self.save_state_loop = loop.call_later(frequency,
self.save_state_regularly,
fname,
frequency)
self.save_state_loop = loop.call_later(
frequency, self.save_state_regularly, fname, frequency
)
def check_dht_value_type(value):
@ -261,11 +259,5 @@ def check_dht_value_type(value):
Checks to see if the type of the value is a valid type for
placing in the dht.
"""
typeset = [
int,
float,
bool,
str,
bytes
]
typeset = [int, float, bool, str, bytes]
return type(value) in typeset # pylint: disable=unidiomatic-typecheck

34
libp2p/kademlia/protocol.py
View File

@ -20,6 +20,7 @@ class KademliaProtocol(RPCProtocol):
(ip, udp_port, node_id) for k closest nodes to target
FIND_VALUE behaves like FIND_NODE unless a value is stored
"""
def __init__(self, source_node, storage, ksize):
RPCProtocol.__init__(self)
self.router = RoutingTable(self, ksize, source_node)
@ -32,7 +33,7 @@ class KademliaProtocol(RPCProtocol):
"""
ids = []
for bucket in self.router.lonely_buckets():
rid = random.randint(*bucket.range).to_bytes(20, byteorder='big')
rid = random.randint(*bucket.range).to_bytes(20, byteorder="big")
ids.append(rid)
return ids
@ -49,14 +50,14 @@ class KademliaProtocol(RPCProtocol):
source = create_kad_peerinfo(nodeid, sender[0], sender[1])
self.welcome_if_new(source)
log.debug("got a store request from %s, storing '%s'='%s'",
sender, key.hex(), value)
log.debug(
"got a store request from %s, storing '%s'='%s'", sender, key.hex(), value
)
self.storage[key] = value
return True
def rpc_find_node(self, sender, nodeid, key):
log.info("finding neighbors of %i in local table",
int(nodeid.hex(), 16))
log.info("finding neighbors of %i in local table", int(nodeid.hex(), 16))
source = create_kad_peerinfo(nodeid, sender[0], sender[1])
self.welcome_if_new(source)
@ -71,7 +72,7 @@ class KademliaProtocol(RPCProtocol):
value = self.storage.get(key, None)
if value is None:
return self.rpc_find_node(sender, nodeid, key)
return {'value': value}
return {"value": value}
def rpc_add_provider(self, sender, nodeid, key, provider_id):
# pylint: disable=unused-argument
@ -82,8 +83,9 @@ class KademliaProtocol(RPCProtocol):
we store a map of content_id to peer_id (non xor)
"""
if nodeid == provider_id:
log.info("adding provider %s for key %s in local table",
provider_id, str(key))
log.info(
"adding provider %s for key %s in local table", provider_id, str(key)
)
self.storage[key] = provider_id
return True
return False
@ -111,14 +113,16 @@ class KademliaProtocol(RPCProtocol):
async def call_find_node(self, node_to_ask, node_to_find):
address = (node_to_ask.ip, node_to_ask.port)
result = await self.find_node(address, self.source_node.peer_id,
node_to_find.peer_id)
result = await self.find_node(
address, self.source_node.peer_id, node_to_find.peer_id
)
return self.handle_call_response(result, node_to_ask)
async def call_find_value(self, node_to_ask, node_to_find):
address = (node_to_ask.ip, node_to_ask.port)
result = await self.find_value(address, self.source_node.peer_id,
node_to_find.peer_id)
result = await self.find_value(
address, self.source_node.peer_id, node_to_find.peer_id
)
return self.handle_call_response(result, node_to_ask)
async def call_ping(self, node_to_ask):
@ -133,9 +137,9 @@ class KademliaProtocol(RPCProtocol):
async def call_add_provider(self, node_to_ask, key, provider_id):
address = (node_to_ask.ip, node_to_ask.port)
result = await self.add_provider(address,
self.source_node.peer_id,
key, provider_id)
result = await self.add_provider(
address, self.source_node.peer_id, key, provider_id
)
return self.handle_call_response(result, node_to_ask)

3
libp2p/kademlia/routing.py
View File

@ -15,6 +15,7 @@ class KBucket:
each k-bucket implements a last seen eviction
policy except that live nodes are never removed
"""
def __init__(self, rangeLower, rangeUpper, ksize):
self.range = (rangeLower, rangeUpper)
self.nodes = OrderedDict()
@ -92,7 +93,7 @@ class TableTraverser:
table.buckets[index].touch_last_updated()
self.current_nodes = table.buckets[index].get_nodes()
self.left_buckets = table.buckets[:index]
self.right_buckets = table.buckets[(index + 1):]
self.right_buckets = table.buckets[(index + 1) :]
self.left = True
def __iter__(self):

4
libp2p/kademlia/utils.py
View File

@ -14,7 +14,7 @@ async def gather_dict(dic):
def digest(string):
if not isinstance(string, bytes):
string = str(string).encode('utf8')
string = str(string).encode("utf8")
return hashlib.sha1(string).digest()
@ -53,5 +53,5 @@ def shared_prefix(args):
def bytes_to_bit_string(bites):
bits = [bin(bite)[2:].rjust(8, '0') for bite in bites]
bits = [bin(bite)[2:].rjust(8, "0") for bite in bites]
return "".join(bits)