added changes from future commits to last passing bmuller commit

.travis.yml

@@ -13,7 +13,7 @@ install:
 
 script:
   - pytest --cov=./libp2p tests/
-  - pylint --rcfile=.pylintrc libp2p/!(kademlia) tests
+  - pylint --rcfile=.pylintrc libp2p tests
 
 after_success:
   - codecov

libp2p/kademlia/crawling.py

@@ -15,6 +15,7 @@ class SpiderCrawl:
     """
 
     def __init__(self, protocol, node, peers, ksize, alpha):
+        # pylint: disable=too-many-arguments
         """
         Create a new C{SpiderCrawl}er.
 
@@ -71,6 +72,7 @@ class SpiderCrawl:
 
 class ValueSpiderCrawl(SpiderCrawl):
     def __init__(self, protocol, node, peers, ksize, alpha):
+        # pylint: disable=too-many-arguments
         SpiderCrawl.__init__(self, protocol, node, peers, ksize, alpha)
         # keep track of the single nearest node without value - per
         # section 2.3 so we can set the key there if found

libp2p/kademlia/protocol.py

@@ -12,6 +12,15 @@ log = logging.getLogger(__name__)  # pylint: disable=invalid-name
 
 
 class KademliaProtocol(RPCProtocol):
+    """
+    There are four main RPCs in the Kademlia protocol
+    PING, STORE, FIND_NODE, FIND_VALUE
+    PING probes if a node is still online
+    STORE instructs a node to store (key, value)
+    FIND_NODE takes a 160-bit ID and gets back
+    (ip, udp_port, node_id) for k closest nodes to target
+    FIND_VALUE behaves like FIND_NODE unless a value is stored
+    """
     def __init__(self, source_node, storage, ksize):
         RPCProtocol.__init__(self)
         self.router = RoutingTable(self, ksize, source_node)
@@ -28,6 +37,12 @@ class KademliaProtocol(RPCProtocol):
             ids.append(rid)
         return ids
 
+    def rpc_add_provider(self, sender, nodeid, key):
+        pass
+
+    def rpc_get_providers(self, sender, nodeid, key):
+        pass
+
     def rpc_stun(self, sender):  # pylint: disable=no-self-use
         return sender
 

libp2p/kademlia/routing.py

@@ -8,6 +8,13 @@ from .utils import OrderedSet, shared_prefix, bytes_to_bit_string
 
 
 class KBucket:
+    """
+    each node keeps a list of (ip, udp_port, node_id)
+    for nodes of distance between 2^i and 2^(i+1)
+    this list that every node keeps is a k-bucket
+    each k-bucket implements a last seen eviction
+    policy except that live nodes are never removed
+    """
     def __init__(self, rangeLower, rangeUpper, ksize):
         self.range = (rangeLower, rangeUpper)
         self.nodes = OrderedDict()

libp2p/kademlia/rpc.proto

@@ -0,0 +1,78 @@
+// Record represents a dht record that contains a value
+// for a key value pair
+message Record {
+	// The key that references this record
+	bytes key = 1;
+
+	// The actual value this record is storing
+	bytes value = 2;
+
+	// Note: These fields were removed from the Record message
+	// hash of the authors public key
+	//optional string author = 3;
+	// A PKI signature for the key+value+author
+	//optional bytes signature = 4;
+
+	// Time the record was received, set by receiver
+	string timeReceived = 5;
+};
+
+message Message {
+	enum MessageType {
+		PUT_VALUE = 0;
+		GET_VALUE = 1;
+		ADD_PROVIDER = 2;
+		GET_PROVIDERS = 3;
+		FIND_NODE = 4;
+		PING = 5;
+	}
+
+	enum ConnectionType {
+		// sender does not have a connection to peer, and no extra information (default)
+		NOT_CONNECTED = 0;
+
+		// sender has a live connection to peer
+		CONNECTED = 1;
+
+		// sender recently connected to peer
+		CAN_CONNECT = 2;
+
+		// sender recently tried to connect to peer repeatedly but failed to connect
+		// ("try" here is loose, but this should signal "made strong effort, failed")
+		CANNOT_CONNECT = 3;
+	}
+
+	message Peer {
+		// ID of a given peer.
+		bytes id = 1;
+
+		// multiaddrs for a given peer
+		repeated bytes addrs = 2;
+
+		// used to signal the sender's connection capabilities to the peer
+		ConnectionType connection = 3;
+	}
+
+	// defines what type of message it is.
+	MessageType type = 1;
+
+	// defines what coral cluster level this query/response belongs to.
+	// in case we want to implement coral's cluster rings in the future.
+	int32 clusterLevelRaw = 10; // NOT USED
+
+	// Used to specify the key associated with this message.
+	// PUT_VALUE, GET_VALUE, ADD_PROVIDER, GET_PROVIDERS
+	bytes key = 2;
+
+	// Used to return a value
+	// PUT_VALUE, GET_VALUE
+	Record record = 3;
+
+	// Used to return peers closer to a key in a query
+	// GET_VALUE, GET_PROVIDERS, FIND_NODE
+	repeated Peer closerPeers = 8;
+
+	// Used to return Providers
+	// GET_VALUE, ADD_PROVIDER, GET_PROVIDERS
+	repeated Peer providerPeers = 9;
+} 

libp2p/kademlia/storage.py

@@ -33,7 +33,7 @@ class IStorage(ABC):
     def iter_older_than(self, seconds_old):
         """
         Return the an iterator over (key, value) tuples for items older
-        than the given secondsOld.
+        than the given seconds_old.
         """
 
     @abstractmethod

libp2p/routing/kadmelia/kadmelia_content_router.py

@@ -9,6 +9,8 @@ class KadmeliaContentRouter(IContentRouting):
         it also announces it, otherwise it is just kept in the local
         accounting of which objects are being provided.
         """
+        # the DHT finds the closest peers to `key` using the `FIND_NODE` RPC
+        # then sends a `ADD_PROVIDER` RPC with its own `PeerInfo` to each of these peers.
         pass
 
     def find_provider_iter(self, cid, count):

libp2p/routing/kadmelia/kadmelia_peer_router.py

@@ -5,6 +5,7 @@ from libp2p.peer.peerdata import PeerData
 
 
 class KadmeliaPeerRouter(IPeerRouting):
+    # pylint: disable=too-few-public-methods
 
     def __init__(self, dht_server):
         self.server = dht_server

tests/kademlia/test_basic.py

@@ -1,5 +1,6 @@
 import pytest
 from libp2p.kademlia.network import Server
+import math
 
 
 @pytest.mark.asyncio
@@ -28,16 +29,15 @@ async def test_example():
 @pytest.mark.parametrize("nodes_nr", [(2**i) for i in range(2, 5)])
 @pytest.mark.asyncio
 async def test_multiple_nodes_bootstrap_set_get(nodes_nr):
-    nodes_nr = 25
 
     node_bootstrap = Server()
-    await node_bootstrap.listen(5678)
+    await node_bootstrap.listen(1000 + nodes_nr * 2)
 
     nodes = []
     for i in range(nodes_nr):
         node = Server()
-        addrs = [("127.0.0.1", 5678)]
+        addrs = [("127.0.0.1", 1000 + nodes_nr * 2)]
-        await node.listen(5679 + i)
+        await node.listen(1001 + i + nodes_nr * 2)
         await node.bootstrap(addrs)
         nodes.append(node)
 
@@ -57,16 +57,14 @@ async def test_multiple_nodes_bootstrap_set_get(nodes_nr):
 @pytest.mark.parametrize("nodes_nr", [(2**i) for i in range(2, 5)])
 @pytest.mark.asyncio
 async def test_multiple_nodes_set_bootstrap_get(nodes_nr):
-    nodes_nr = 25
-
     node_bootstrap = Server()
-    await node_bootstrap.listen(5678)
+    await node_bootstrap.listen(2000 + nodes_nr * 2)
     
     nodes = []
     for i in range(nodes_nr):
         node = Server()
-        addrs = [("127.0.0.1", 5678)]
+        addrs = [("127.0.0.1", 2000 + nodes_nr * 2)]
-        await node.listen(5679 + i)
+        await node.listen(2001 + i + nodes_nr * 2)
         await node.bootstrap(addrs)
 
         value = "my awesome value %d" % i