varun-r-mallya/py-libp2p
1
0
Fork 0
mirror of https://github.com/varun-r-mallya/py-libp2p.git synced 2026-02-12 16:10:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add automatic docstring formatter and apply

Browse Source
This commit is contained in:
Dominik Muhs
2019-10-24 08:41:10 +02:00
parent 30aeb35122
commit eef505f2d9
74 changed files with 565 additions and 760 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
libp2p/kademlia/__init__.py
View File

@ -1,5 +1,3 @@
"""
Kademlia is a Python implementation of the Kademlia protocol which
utilizes the asyncio library.
"""
"""Kademlia is a Python implementation of the Kademlia protocol which utilizes
the asyncio library."""
__version__ = "2.0"

49
libp2p/kademlia/crawling.py
View File

@ -8,13 +8,10 @@ log = logging.getLogger(__name__)
class SpiderCrawl:
"""
Crawl the network and look for given 160-bit keys.
"""
"""Crawl the network and look for given 160-bit keys."""
def __init__(self, protocol, node, peers, ksize, alpha):
"""
Create a new C{SpiderCrawl}er.
"""Create a new C{SpiderCrawl}er.
Args:
protocol: A :class:`~kademlia.protocol.KademliaProtocol` instance.
@ -35,8 +32,7 @@ class SpiderCrawl:
self.nearest.push(peers)
async def _find(self, rpcmethod):
"""
Get either a value or list of nodes.
"""Get either a value or list of nodes.
Args:
rpcmethod: The protocol's callfindValue or call_find_node.
@ -75,15 +71,11 @@ class ValueSpiderCrawl(SpiderCrawl):
self.nearest_without_value = KadPeerHeap(self.node, 1)
async def find(self):
"""
Find either the closest nodes or the value requested.
"""
"""Find either the closest nodes or the value requested."""
return await self._find(self.protocol.call_find_value)
async def _nodes_found(self, responses):
"""
Handle the result of an iteration in _find.
"""
"""Handle the result of an iteration in _find."""
toremove = []
found_values = []
for peerid, response in responses.items():
@ -106,11 +98,11 @@ class ValueSpiderCrawl(SpiderCrawl):
return await self.find()
async def _handle_found_values(self, values):
"""
We got some values! Exciting. But let's make sure
they're all the same or freak out a little bit. Also,
make sure we tell the nearest node that *didn't* have
the value to store it.
"""We got some values!
Exciting. But let's make sure they're all the same or freak out
a little bit. Also, make sure we tell the nearest node that
*didn't* have the value to store it.
"""
value_counts = Counter(values)
if len(value_counts) != 1:
@ -127,15 +119,11 @@ class ValueSpiderCrawl(SpiderCrawl):
class NodeSpiderCrawl(SpiderCrawl):
async def find(self):
"""
Find the closest nodes.
"""
"""Find the closest nodes."""
return await self._find(self.protocol.call_find_node)
async def _nodes_found(self, responses):
"""
Handle the result of an iteration in _find.
"""
"""Handle the result of an iteration in _find."""
toremove = []
for peerid, response in responses.items():
response = RPCFindResponse(response)
@ -152,8 +140,7 @@ class NodeSpiderCrawl(SpiderCrawl):
class RPCFindResponse:
def __init__(self, response):
"""
A wrapper for the result of a RPC find.
"""A wrapper for the result of a RPC find.
Args:
response: This will be a tuple of (<response received>, <value>)
@ -163,9 +150,7 @@ class RPCFindResponse:
self.response = response
def happened(self):
"""
Did the other host actually respond?
"""
"""Did the other host actually respond?"""
return self.response[0]
def has_value(self):
@ -175,9 +160,9 @@ class RPCFindResponse:
return self.response[1]["value"]
def get_node_list(self):
"""
Get the node list in the response. If there's no value, this should
be set.
"""Get the node list in the response.
If there's no value, this should be set.
"""
nodelist = self.response[1] or []
return [create_kad_peerinfo(*nodeple) for nodeple in nodelist]

27
libp2p/kademlia/kad_peerinfo.py
View File

@ -30,9 +30,7 @@ class KadPeerInfo(PeerInfo):
return sorted(self.addrs) == sorted(node.addrs)
def distance_to(self, node):
"""
Get the distance between this node and another.
"""
"""Get the distance between this node and another."""
return self.xor_id ^ node.xor_id
def __iter__(self):
@ -56,13 +54,10 @@ class KadPeerInfo(PeerInfo):
class KadPeerHeap:
"""
A heap of peers ordered by distance to a given node.
"""
"""A heap of peers ordered by distance to a given node."""
def __init__(self, node, maxsize):
"""
Constructor.
"""Constructor.
@param node: The node to measure all distnaces from.
@param maxsize: The maximum size that this heap can grow to.
@ -73,12 +68,13 @@ class KadPeerHeap:
self.maxsize = maxsize
def remove(self, peers):
"""
Remove a list of peer ids from this heap. Note that while this
heap retains a constant visible size (based on the iterator), it's
actual size may be quite a bit larger than what's exposed. Therefore,
removal of nodes may not change the visible size as previously added
nodes suddenly become visible.
"""Remove a list of peer ids from this heap.
Note that while this heap retains a constant visible size (based
on the iterator), it's actual size may be quite a bit larger
than what's exposed. Therefore, removal of nodes may not change
the visible size as previously added nodes suddenly become
visible.
"""
peers = set(peers)
if not peers:
@ -108,8 +104,7 @@ class KadPeerHeap:
return heapq.heappop(self.heap)[1] if self else None
def push(self, nodes):
"""
Push nodes onto heap.
"""Push nodes onto heap.
@param nodes: This can be a single item or a C{list}.
"""

73
libp2p/kademlia/network.py
View File

@ -1,6 +1,4 @@
"""
Package for interacting on the network at a high level.
"""
"""Package for interacting on the network at a high level."""
import asyncio
import logging
import pickle
@ -15,16 +13,17 @@ log = logging.getLogger(__name__)
class KademliaServer:
"""
High level view of a node instance. This is the object that should be
created to start listening as an active node on the network.
"""High level view of a node instance.
This is the object that should be created to start listening as an
active node on the network.
"""
protocol_class = KademliaProtocol
def __init__(self, ksize=20, alpha=3, node_id=None, storage=None):
"""
Create a server instance. This will start listening on the given port.
"""Create a server instance. This will start listening on the given
port.
Args:
ksize (int): The k parameter from the paper
@ -56,8 +55,7 @@ class KademliaServer:
return self.protocol_class(self.node, self.storage, self.ksize)
async def listen(self, port, interface="0.0.0.0"):
"""
Start listening on the given port.
"""Start listening on the given port.
Provide interface="::" to accept ipv6 address
"""
@ -77,10 +75,8 @@ class KademliaServer:
self.refresh_loop = loop.call_later(3600, self.refresh_table)
async def _refresh_table(self):
"""
Refresh buckets that haven't had any lookups in the last hour
(per section 2.3 of the paper).
"""
"""Refresh buckets that haven't had any lookups in the last hour (per
section 2.3 of the paper)."""
results = []
for node_id in self.protocol.get_refresh_ids():
node = create_kad_peerinfo(node_id)
@ -98,8 +94,7 @@ class KademliaServer:
await self.set_digest(dkey, value)
def bootstrappable_neighbors(self):
"""
Get a :class:`list` of (ip, port) :class:`tuple` pairs suitable for
"""Get a :class:`list` of (ip, port) :class:`tuple` pairs suitable for
use as an argument to the bootstrap method.
The server should have been bootstrapped
@ -111,8 +106,8 @@ class KademliaServer:
return [tuple(n)[-2:] for n in neighbors]
async def bootstrap(self, addrs):
"""
Bootstrap the server by connecting to other known nodes in the network.
"""Bootstrap the server by connecting to other known nodes in the
network.
Args:
addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP
@ -132,8 +127,7 @@ class KademliaServer:
return create_kad_peerinfo(result[1], addr[0], addr[1]) if result[0] else None
async def get(self, key):
"""
Get a key if the network has it.
"""Get a key if the network has it.
Returns:
:class:`None` if not found, the value otherwise.
@ -153,9 +147,7 @@ class KademliaServer:
return await spider.find()
async def set(self, key, value):
"""
Set the given string key to the given value in the network.
"""
"""Set the given string key to the given value in the network."""
if not check_dht_value_type(value):
raise TypeError("Value must be of type int, float, bool, str, or bytes")
log.info("setting '%s' = '%s' on network", key, value)
@ -163,9 +155,7 @@ class KademliaServer:
return await self.set_digest(dkey, value)
async def provide(self, key):
"""
publish to the network that it provides for a particular key
"""
"""publish to the network that it provides for a particular key."""
neighbors = self.protocol.router.find_neighbors(self.node)
return [
await self.protocol.call_add_provider(n, key, self.node.peer_id_bytes)
@ -173,17 +163,13 @@ class KademliaServer:
]
async def get_providers(self, key):
"""
get the list of providers for a key
"""
"""get the list of providers for a key."""
neighbors = self.protocol.router.find_neighbors(self.node)
return [await self.protocol.call_get_providers(n, key) for n in neighbors]
async def set_digest(self, dkey, value):
"""
Set the given SHA1 digest key (bytes) to the given value in the
network.
"""
"""Set the given SHA1 digest key (bytes) to the given value in the
network."""
node = create_kad_peerinfo(dkey)
nearest = self.protocol.router.find_neighbors(node)
@ -204,10 +190,8 @@ class KademliaServer:
return any(await asyncio.gather(*results))
def save_state(self, fname):
"""
Save the state of this node (the alpha/ksize/id/immediate neighbors)
to a cache file with the given fname.
"""
"""Save the state of this node (the alpha/ksize/id/immediate neighbors)
to a cache file with the given fname."""
log.info("Saving state to %s", fname)
data = {
"ksize": self.ksize,
@ -223,10 +207,8 @@ class KademliaServer:
@classmethod
def load_state(cls, fname):
"""
Load the state of this node (the alpha/ksize/id/immediate neighbors)
from a cache file with the given fname.
"""
"""Load the state of this node (the alpha/ksize/id/immediate neighbors)
from a cache file with the given fname."""
log.info("Loading state from %s", fname)
with open(fname, "rb") as file:
data = pickle.load(file)
@ -236,8 +218,7 @@ class KademliaServer:
return svr
def save_state_regularly(self, fname, frequency=600):
"""
Save the state of node with a given regularity to the given
"""Save the state of node with a given regularity to the given
filename.
Args:
@ -253,9 +234,7 @@ class KademliaServer:
def check_dht_value_type(value):
"""
Checks to see if the type of the value is a valid type for
placing in the dht.
"""
"""Checks to see if the type of the value is a valid type for placing in
the dht."""
typeset = [int, float, bool, str, bytes]
return type(value) in typeset

46
libp2p/kademlia/protocol.py
View File

@ -11,15 +11,11 @@ log = logging.getLogger(__name__)
class KademliaProtocol(RPCProtocol):
"""
There are four main RPCs in the Kademlia protocol
PING, STORE, FIND_NODE, FIND_VALUE
PING probes if a node is still online
STORE instructs a node to store (key, value)
FIND_NODE takes a 160-bit ID and gets back
(ip, udp_port, node_id) for k closest nodes to target
FIND_VALUE behaves like FIND_NODE unless a value is stored
"""
"""There are four main RPCs in the Kademlia protocol PING, STORE,
FIND_NODE, FIND_VALUE PING probes if a node is still online STORE instructs
a node to store (key, value) FIND_NODE takes a 160-bit ID and gets back
(ip, udp_port, node_id) for k closest nodes to target FIND_VALUE behaves
like FIND_NODE unless a value is stored."""
def __init__(self, source_node, storage, ksize):
RPCProtocol.__init__(self)
@ -28,9 +24,7 @@ class KademliaProtocol(RPCProtocol):
self.source_node = source_node
def get_refresh_ids(self):
"""
Get ids to search for to keep old buckets up to date.
"""
"""Get ids to search for to keep old buckets up to date."""
ids = []
for bucket in self.router.lonely_buckets():
rid = random.randint(*bucket.range).to_bytes(20, byteorder="big")
@ -75,12 +69,10 @@ class KademliaProtocol(RPCProtocol):
return {"value": value}
def rpc_add_provider(self, sender, nodeid, key, provider_id):
"""
rpc when receiving an add_provider call
should validate received PeerInfo matches sender nodeid
if it does, receipient must store a record in its datastore
we store a map of content_id to peer_id (non xor)
"""
"""rpc when receiving an add_provider call should validate received
PeerInfo matches sender nodeid if it does, receipient must store a
record in its datastore we store a map of content_id to peer_id (non
xor)"""
if nodeid == provider_id:
log.info(
"adding provider %s for key %s in local table", provider_id, str(key)
@ -90,11 +82,9 @@ class KademliaProtocol(RPCProtocol):
return False
def rpc_get_providers(self, sender, key):
"""
rpc when receiving a get_providers call
should look up key in data store and respond with records
plus a list of closer peers in its routing table
"""
"""rpc when receiving a get_providers call should look up key in data
store and respond with records plus a list of closer peers in its
routing table."""
providers = []
record = self.storage.get(key, None)
@ -147,8 +137,7 @@ class KademliaProtocol(RPCProtocol):
return self.handle_call_response(result, node_to_ask)
def welcome_if_new(self, node):
"""
Given a new node, send it all the keys/values it should be storing,
"""Given a new node, send it all the keys/values it should be storing,
then add it to the routing table.
@param node: A new node that just joined (or that we just found out
@ -177,9 +166,10 @@ class KademliaProtocol(RPCProtocol):
self.router.add_contact(node)
def handle_call_response(self, result, node):
"""
If we get a response, add the node to the routing table. If
we get no response, make sure it's removed from the routing table.
"""If we get a response, add the node to the routing table.
If we get no response, make sure it's removed from the routing
table.
"""
if not result[0]:
log.warning("no response from %s, removing from router", node)

29
libp2p/kademlia/routing.py
View File

@ -8,13 +8,10 @@ from .utils import OrderedSet, bytes_to_bit_string, shared_prefix
class KBucket:
"""
each node keeps a list of (ip, udp_port, node_id)
for nodes of distance between 2^i and 2^(i+1)
this list that every node keeps is a k-bucket
each k-bucket implements a last seen eviction
policy except that live nodes are never removed
"""
"""each node keeps a list of (ip, udp_port, node_id) for nodes of distance
between 2^i and 2^(i+1) this list that every node keeps is a k-bucket each
k-bucket implements a last seen eviction policy except that live nodes are
never removed."""
def __init__(self, rangeLower, rangeUpper, ksize):
self.range = (rangeLower, rangeUpper)
@ -55,9 +52,8 @@ class KBucket:
return node.peer_id_bytes not in self.nodes
def add_node(self, node):
"""
Add a C{Node} to the C{KBucket}. Return True if successful,
False if the bucket is full.
"""Add a C{Node} to the C{KBucket}. Return True if successful, False
if the bucket is full.
If the bucket is full, keep track of node in a replacement list,
per section 4.1 of the paper.
@ -100,9 +96,7 @@ class TableTraverser:
return self
def __next__(self):
"""
Pop an item from the left subtree, then right, then left, etc.
"""
"""Pop an item from the left subtree, then right, then left, etc."""
if self.current_nodes:
return self.current_nodes.pop()
@ -140,10 +134,7 @@ class RoutingTable:
self.buckets.insert(index + 1, two)
def lonely_buckets(self):
"""
Get all of the buckets that haven't been updated in over
an hour.
"""
"""Get all of the buckets that haven't been updated in over an hour."""
hrago = time.monotonic() - 3600
return [b for b in self.buckets if b.last_updated < hrago]
@ -172,9 +163,7 @@ class RoutingTable:
asyncio.ensure_future(self.protocol.call_ping(bucket.head()))
def get_bucket_for(self, node):
"""
Get the index of the bucket that the given node would fall into.
"""
"""Get the index of the bucket that the given node would fall into."""
for index, bucket in enumerate(self.buckets):
if node.xor_id < bucket.range[1]:
return index

36
libp2p/kademlia/storage.py
View File

@ -6,48 +6,44 @@ import time
class IStorage(ABC):
"""
Local storage for this node.
IStorage implementations of get must return the same type as put in by set
"""Local storage for this node.
IStorage implementations of get must return the same type as put in
by set
"""
@abstractmethod
def __setitem__(self, key, value):
"""
Set a key to the given value.
"""
"""Set a key to the given value."""
@abstractmethod
def __getitem__(self, key):
"""
Get the given key. If item doesn't exist, raises C{KeyError}
"""Get the given key.
If item doesn't exist, raises C{KeyError}
"""
@abstractmethod
def get(self, key, default=None):
"""
Get given key. If not found, return default.
"""Get given key.
If not found, return default.
"""
@abstractmethod
def iter_older_than(self, seconds_old):
"""
Return the an iterator over (key, value) tuples for items older
than the given seconds_old.
"""
"""Return the an iterator over (key, value) tuples for items older than
the given seconds_old."""
@abstractmethod
def __iter__(self):
"""
Get the iterator for this storage, should yield tuple of (key, value)
"""
"""Get the iterator for this storage, should yield tuple of (key,
value)"""
class ForgetfulStorage(IStorage):
def __init__(self, ttl=604800):
"""
By default, max age is a week.
"""
"""By default, max age is a week."""
self.data = OrderedDict()
self.ttl = ttl

11
libp2p/kademlia/utils.py
View File

@ -1,6 +1,4 @@
"""
General catchall for functions that don't make sense as methods.
"""
"""General catchall for functions that don't make sense as methods."""
import asyncio
import hashlib
import operator
@ -19,8 +17,8 @@ def digest(string):
class OrderedSet(list):
"""
Acts like a list in all ways, except in the behavior of the
"""Acts like a list in all ways, except in the behavior of the.
:meth:`push` method.
"""
@ -35,8 +33,7 @@ class OrderedSet(list):
def shared_prefix(args):
"""
Find the shared prefix between the strings.
"""Find the shared prefix between the strings.
For instance: