From 8a4ebd4cbb89222e8ccb08bba1d0fc723641120d Mon Sep 17 00:00:00 2001 From: mhchia Date: Sat, 15 Feb 2020 12:12:12 +0800 Subject: [PATCH 01/22] Support read/write of noise msg and payload --- libp2p/security/noise/exceptions.py | 9 ++ libp2p/security/noise/io.py | 108 +++++++++++++++++++ libp2p/security/noise/patterns.py | 47 +++----- tests/security/noise/test_msg_read_writer.py | 27 +++++ tests/security/{ => noise}/test_noise.py | 0 5 files changed, 158 insertions(+), 33 deletions(-) create mode 100644 libp2p/security/noise/exceptions.py create mode 100644 libp2p/security/noise/io.py create mode 100644 tests/security/noise/test_msg_read_writer.py rename tests/security/{ => noise}/test_noise.py (100%) diff --git a/libp2p/security/noise/exceptions.py b/libp2p/security/noise/exceptions.py new file mode 100644 index 00000000..5e6040e7 --- /dev/null +++ b/libp2p/security/noise/exceptions.py @@ -0,0 +1,9 @@ +from libp2p.security.exceptions import HandshakeFailure + + +class NoiseFailure(HandshakeFailure): + pass + + +class HandshakeHasNotFinished(NoiseFailure): + pass diff --git a/libp2p/security/noise/io.py b/libp2p/security/noise/io.py new file mode 100644 index 00000000..6a6f7be7 --- /dev/null +++ b/libp2p/security/noise/io.py @@ -0,0 +1,108 @@ +from abc import ABC, abstractmethod +from typing import cast + +from noise.connection import NoiseConnection as NoiseState + +from libp2p.io.abc import ReadWriter +from libp2p.io.utils import read_exactly +from libp2p.network.connection.raw_connection_interface import IRawConnection + +SIZE_NOISE_MESSAGE_LEN = 2 +MAX_NOISE_MESSAGE_LEN = 2 ** (8 * SIZE_NOISE_MESSAGE_LEN) - 1 +SIZE_NOISE_MESSAGE_BODY_LEN = 2 +MAX_NOISE_MESSAGE_BODY_LEN = MAX_NOISE_MESSAGE_LEN - SIZE_NOISE_MESSAGE_BODY_LEN +BYTE_ORDER = "big" + +# | Noise packet | +# < 2 bytes -><- 65535 -> +# | noise msg len | noise msg | +# | body len | body | padding | +# <-2 bytes-><- max=65533 bytes -> + + +def encode_data(data: bytes, size_len: int) -> bytes: + len_data = len(data) + try: + len_bytes = len_data.to_bytes(size_len, BYTE_ORDER) + except OverflowError as e: + raise ValueError from e + return len_bytes + data + + +class MsgReader(ABC): + @abstractmethod + async def read_msg(self) -> bytes: + ... + + +class MsgWriter(ABC): + @abstractmethod + async def write_msg(self, msg: bytes) -> None: + ... + + +class MsgReadWriter(MsgReader, MsgWriter): + pass + + +# TODO: Add comments +class NoisePacketReadWriter(MsgReadWriter): + """Encode and decode the low level noise messages.""" + + read_writer: ReadWriter + + def __init__(self, read_writer: ReadWriter) -> None: + self.read_writer = read_writer + + async def read_msg(self) -> bytes: + len_bytes = await read_exactly(self.read_writer, SIZE_NOISE_MESSAGE_LEN) + len_int = int.from_bytes(len_bytes, BYTE_ORDER) + return await read_exactly(self.read_writer, len_int) + + async def write_msg(self, msg: bytes) -> None: + encoded_data = encode_data(msg, SIZE_NOISE_MESSAGE_LEN) + await self.read_writer.write(encoded_data) + + +# TODO: Add comments +def encode_msg_body(msg_body: bytes) -> bytes: + encoded_msg_body = encode_data(msg_body, SIZE_NOISE_MESSAGE_BODY_LEN) + if len(encoded_msg_body) > MAX_NOISE_MESSAGE_BODY_LEN: + raise ValueError( + f"msg_body is too long: {len(msg_body)}, " + f"maximum={MAX_NOISE_MESSAGE_BODY_LEN}" + ) + # NOTE: Improvements: + # 1. Senders *SHOULD* use a source of random data to populate the padding field. + # 2. and *may* use any length of padding that does not cause the total length of + # the Noise message to exceed 65535 bytes. + # Ref: https://github.com/libp2p/specs/tree/master/noise#encrypted-payloads + return encoded_msg_body # + padding + + +def decode_msg_body(noise_msg: bytes) -> bytes: + len_body = int.from_bytes(noise_msg[:SIZE_NOISE_MESSAGE_BODY_LEN], BYTE_ORDER) + # Just ignore the padding + return noise_msg[ + SIZE_NOISE_MESSAGE_BODY_LEN : (SIZE_NOISE_MESSAGE_BODY_LEN + len_body) + ] + + +# TODO: Add comments +class NoiseHandshakeReadWriter(MsgReadWriter): + read_writer: MsgReadWriter + noise_state: NoiseState + + def __init__(self, conn: IRawConnection, noise_state: NoiseState) -> None: + self.read_writer = NoisePacketReadWriter(cast(ReadWriter, conn)) + self.noise_state = noise_state + + async def write_msg(self, data: bytes) -> None: + noise_msg = encode_msg_body(data) + data_encrypted = self.noise_state.write_message(noise_msg) + await self.read_writer.write_msg(data_encrypted) + + async def read_msg(self) -> bytes: + noise_msg_encrypted = await self.read_writer.read_msg() + noise_msg = self.noise_state.read_message(noise_msg_encrypted) + return decode_msg_body(noise_msg) diff --git a/libp2p/security/noise/patterns.py b/libp2p/security/noise/patterns.py index dee9a619..a05e8070 100644 --- a/libp2p/security/noise/patterns.py +++ b/libp2p/security/noise/patterns.py @@ -9,24 +9,8 @@ from libp2p.peer.id import ID from libp2p.security.secure_conn_interface import ISecureConn from .connection import NoiseConnection - -# FIXME: Choose a serious bound number. -NUM_BYTES_TO_READ = 2048 - - -# TODO: Merged into `BasePattern`? -class PreHandshakeConnection: - conn: IRawConnection - - def __init__(self, conn: IRawConnection) -> None: - self.conn = conn - - async def write_msg(self, data: bytes) -> None: - # TODO: - await self.conn.write(data) - - async def read_msg(self) -> bytes: - return await self.conn.read(NUM_BYTES_TO_READ) +from .exceptions import HandshakeHasNotFinished +from .io import NoiseHandshakeReadWriter class IPattern(ABC): @@ -66,25 +50,24 @@ class PatternXX(BasePattern): async def handshake_inbound(self, conn: IRawConnection) -> ISecureConn: noise_state = self.create_noise_state() - handshake_conn = PreHandshakeConnection(conn) noise_state.set_as_responder() noise_state.start_handshake() - msg_0_encrypted = await handshake_conn.read_msg() + read_writer = NoiseHandshakeReadWriter(conn, noise_state) # TODO: Parse and save the payload from the other side. - _ = noise_state.read_message(msg_0_encrypted) + _ = await read_writer.read_msg() # TODO: Send our payload. our_payload = b"server" - msg_1_encrypted = noise_state.write_message(our_payload) - await handshake_conn.write_msg(msg_1_encrypted) + await read_writer.write_msg(our_payload) - msg_2_encrypted = await handshake_conn.read_msg() # TODO: Parse and save another payload from the other side. - _ = noise_state.read_message(msg_2_encrypted) + _ = await read_writer.read_msg() # TODO: Add a specific exception if not noise_state.handshake_finished: - raise Exception + raise HandshakeHasNotFinished( + "handshake done but it is not marked as finished in `noise_state`" + ) # FIXME: `remote_peer` should be derived from the messages. return NoiseConnection(self.local_peer, self.libp2p_privkey, None, conn, False) @@ -93,19 +76,17 @@ class PatternXX(BasePattern): self, conn: IRawConnection, remote_peer: ID ) -> ISecureConn: noise_state = self.create_noise_state() - handshake_conn = PreHandshakeConnection(conn) + read_writer = NoiseHandshakeReadWriter(conn, noise_state) noise_state.set_as_initiator() noise_state.start_handshake() - msg_0 = noise_state.write_message() - await handshake_conn.write_msg(msg_0) - msg_1_encrypted = await handshake_conn.read_msg() + await read_writer.write_msg(b"") + # TODO: Parse and save the payload from the other side. - _ = noise_state.read_message(msg_1_encrypted) + _ = await read_writer.read_msg() # TODO: Send our payload. our_payload = b"client" - msg_2_encrypted = noise_state.write_message(our_payload) - await handshake_conn.write_msg(msg_2_encrypted) + await read_writer.write_msg(our_payload) # TODO: Add a specific exception if not noise_state.handshake_finished: diff --git a/tests/security/noise/test_msg_read_writer.py b/tests/security/noise/test_msg_read_writer.py new file mode 100644 index 00000000..47e9afac --- /dev/null +++ b/tests/security/noise/test_msg_read_writer.py @@ -0,0 +1,27 @@ +import pytest + +from libp2p.security.noise.io import MAX_NOISE_MESSAGE_LEN, NoisePacketReadWriter +from libp2p.tools.factories import raw_conn_factory + + +@pytest.mark.parametrize( + "noise_msg", + (b"", b"data", pytest.param(b"A" * MAX_NOISE_MESSAGE_LEN, id="maximum length")), +) +@pytest.mark.trio +async def test_noise_msg_read_write_round_trip(nursery, noise_msg): + async with raw_conn_factory(nursery) as conns: + reader, writer = ( + NoisePacketReadWriter(conns[0]), + NoisePacketReadWriter(conns[1]), + ) + await writer.write_msg(noise_msg) + assert (await reader.read_msg()) == noise_msg + + +@pytest.mark.trio +async def test_noise_msg_write_too_long(nursery): + async with raw_conn_factory(nursery) as conns: + writer = NoisePacketReadWriter(conns[0]) + with pytest.raises(ValueError): + await writer.write_msg(b"1" * (MAX_NOISE_MESSAGE_LEN + 1)) diff --git a/tests/security/test_noise.py b/tests/security/noise/test_noise.py similarity index 100% rename from tests/security/test_noise.py rename to tests/security/noise/test_noise.py From d0290d2b5ad6c4a290c98916d15711086bb2c55c Mon Sep 17 00:00:00 2001 From: mhchia Date: Sun, 16 Feb 2020 00:42:49 +0800 Subject: [PATCH 02/22] Noise: complete handshake process TODO - Figure out why `state.rs` is erased at some moment(even handshake is not done). - Refactor - Add tests --- libp2p/security/noise/exceptions.py | 13 ++++ libp2p/security/noise/messages.py | 56 +++++++++++++++++ libp2p/security/noise/patterns.py | 97 ++++++++++++++++++++++------- libp2p/security/noise/transport.py | 7 ++- libp2p/tools/factories.py | 17 ++++- tests/security/noise/test_noise.py | 10 ++- 6 files changed, 176 insertions(+), 24 deletions(-) create mode 100644 libp2p/security/noise/messages.py diff --git a/libp2p/security/noise/exceptions.py b/libp2p/security/noise/exceptions.py index 5e6040e7..85cf3f29 100644 --- a/libp2p/security/noise/exceptions.py +++ b/libp2p/security/noise/exceptions.py @@ -7,3 +7,16 @@ class NoiseFailure(HandshakeFailure): class HandshakeHasNotFinished(NoiseFailure): pass + + +class InvalidSignature(NoiseFailure): + pass + + +class NoiseStateError(NoiseFailure): + """Raised when anything goes wrong in the noise state in `noiseprotocol` + package.""" + + +class PeerIDMismatchesPubkey(NoiseFailure): + pass diff --git a/libp2p/security/noise/messages.py b/libp2p/security/noise/messages.py new file mode 100644 index 00000000..feb27666 --- /dev/null +++ b/libp2p/security/noise/messages.py @@ -0,0 +1,56 @@ +from dataclasses import dataclass + +from libp2p.crypto.keys import PrivateKey, PublicKey +from libp2p.crypto.serialization import deserialize_public_key + +from .pb import noise_pb2 as noise_pb + +SIGNED_DATA_PREFIX = "noise-libp2p-static-key:" + + +@dataclass +class NoiseHandshakePayload: + id_pubkey: PublicKey + id_sig: bytes + early_data: bytes = None + + def serialize(self) -> bytes: + msg = noise_pb.NoiseHandshakePayload( + identity_key=self.id_pubkey.serialize(), identity_sig=self.id_sig + ) + if self.early_data is not None: + msg.data = self.early_data + return msg.SerializeToString() + + @classmethod + def deserialize(cls, protobuf_bytes: bytes) -> "NoiseHandshakePayload": + msg = noise_pb.NoiseHandshakePayload.FromString(protobuf_bytes) + return cls( + id_pubkey=deserialize_public_key(msg.identity_key), + id_sig=msg.identity_sig, + early_data=msg.data if msg.data != b"" else None, + ) + + +def make_data_to_be_signed(noise_static_pubkey: PublicKey) -> bytes: + prefix_bytes = SIGNED_DATA_PREFIX.encode("utf-8") + return prefix_bytes + noise_static_pubkey.to_bytes() + + +def make_handshake_payload_sig( + id_privkey: PrivateKey, noise_static_pubkey: PublicKey +) -> bytes: + data = make_data_to_be_signed(noise_static_pubkey) + return id_privkey.sign(data) + + +def verify_handshake_payload_sig( + payload: NoiseHandshakePayload, noise_static_pubkey: PublicKey +) -> bool: + """ + Verify if the signature + 1. is composed of the data `SIGNED_DATA_PREFIX`++`noise_static_pubkey` and + 2. signed by the private key corresponding to `id_pubkey` + """ + expected_data = make_data_to_be_signed(noise_static_pubkey) + return payload.id_pubkey.verify(expected_data, payload.id_sig) diff --git a/libp2p/security/noise/patterns.py b/libp2p/security/noise/patterns.py index a05e8070..4590b73e 100644 --- a/libp2p/security/noise/patterns.py +++ b/libp2p/security/noise/patterns.py @@ -3,14 +3,25 @@ from abc import ABC, abstractmethod from noise.connection import Keypair as NoiseKeypair from noise.connection import NoiseConnection as NoiseState +from libp2p.crypto.ed25519 import Ed25519PublicKey from libp2p.crypto.keys import PrivateKey from libp2p.network.connection.raw_connection_interface import IRawConnection from libp2p.peer.id import ID from libp2p.security.secure_conn_interface import ISecureConn from .connection import NoiseConnection -from .exceptions import HandshakeHasNotFinished +from .exceptions import ( + HandshakeHasNotFinished, + InvalidSignature, + NoiseStateError, + PeerIDMismatchesPubkey, +) from .io import NoiseHandshakeReadWriter +from .messages import ( + NoiseHandshakePayload, + make_handshake_payload_sig, + verify_handshake_payload_sig, +) class IPattern(ABC): @@ -30,6 +41,7 @@ class BasePattern(IPattern): noise_static_key: PrivateKey local_peer: ID libp2p_privkey: PrivateKey + early_data: bytes def create_noise_state(self) -> NoiseState: noise_state = NoiseState.from_name(self.protocol_name) @@ -38,59 +50,102 @@ class BasePattern(IPattern): ) return noise_state + def make_handshake_payload(self) -> NoiseHandshakePayload: + signature = make_handshake_payload_sig( + self.libp2p_privkey, self.noise_static_key.get_public_key() + ) + return NoiseHandshakePayload(self.libp2p_privkey.get_public_key(), signature) + class PatternXX(BasePattern): def __init__( - self, local_peer: ID, libp2p_privkey: PrivateKey, noise_static_key: PrivateKey + self, + local_peer: ID, + libp2p_privkey: PrivateKey, + noise_static_key: PrivateKey, + early_data: bytes = None, ) -> None: self.protocol_name = b"Noise_XX_25519_ChaChaPoly_SHA256" self.local_peer = local_peer self.libp2p_privkey = libp2p_privkey self.noise_static_key = noise_static_key + self.early_data = early_data async def handshake_inbound(self, conn: IRawConnection) -> ISecureConn: noise_state = self.create_noise_state() noise_state.set_as_responder() noise_state.start_handshake() + state = noise_state.noise_protocol.handshake_state read_writer = NoiseHandshakeReadWriter(conn, noise_state) - # TODO: Parse and save the payload from the other side. - _ = await read_writer.read_msg() - # TODO: Send our payload. - our_payload = b"server" - await read_writer.write_msg(our_payload) + # Consume msg#1 + await read_writer.read_msg() - # TODO: Parse and save another payload from the other side. - _ = await read_writer.read_msg() + # Send msg#2, which should include our handshake payload. + our_payload = self.make_handshake_payload() + msg_2 = our_payload.serialize() + await read_writer.write_msg(msg_2) + + # Receive msg#3 + msg_3 = await read_writer.read_msg() + peer_handshake_payload = NoiseHandshakePayload.deserialize(msg_3) + + if state.rs is None: + raise NoiseStateError + remote_pubkey = Ed25519PublicKey.from_bytes(state.rs.public_bytes) + if not verify_handshake_payload_sig(peer_handshake_payload, remote_pubkey): + raise InvalidSignature + remote_peer_id_from_pubkey = ID.from_pubkey(peer_handshake_payload.id_pubkey) - # TODO: Add a specific exception if not noise_state.handshake_finished: raise HandshakeHasNotFinished( - "handshake done but it is not marked as finished in `noise_state`" + "handshake is done but it is not marked as finished in `noise_state`" ) - # FIXME: `remote_peer` should be derived from the messages. - return NoiseConnection(self.local_peer, self.libp2p_privkey, None, conn, False) + return NoiseConnection( + self.local_peer, + self.libp2p_privkey, + remote_peer_id_from_pubkey, + conn, + False, + ) async def handshake_outbound( self, conn: IRawConnection, remote_peer: ID ) -> ISecureConn: noise_state = self.create_noise_state() + read_writer = NoiseHandshakeReadWriter(conn, noise_state) noise_state.set_as_initiator() noise_state.start_handshake() - await read_writer.write_msg(b"") + state = noise_state.noise_protocol.handshake_state - # TODO: Parse and save the payload from the other side. - _ = await read_writer.read_msg() + msg_1 = b"" + await read_writer.write_msg(msg_1) - # TODO: Send our payload. - our_payload = b"client" - await read_writer.write_msg(our_payload) + msg_2 = await read_writer.read_msg() + peer_handshake_payload = NoiseHandshakePayload.deserialize(msg_2) + if state.rs is None: + raise NoiseStateError + remote_pubkey = Ed25519PublicKey.from_bytes(state.rs.public_bytes) + if not verify_handshake_payload_sig(peer_handshake_payload, remote_pubkey): + raise InvalidSignature + remote_peer_id_from_pubkey = ID.from_pubkey(peer_handshake_payload.id_pubkey) + if remote_peer_id_from_pubkey != remote_peer: + raise PeerIDMismatchesPubkey( + "peer id does not correspond to the received pubkey: " + f"remote_peer={remote_peer}, " + f"remote_peer_id_from_pubkey={remote_peer_id_from_pubkey}" + ) + + our_payload = self.make_handshake_payload() + msg_3 = our_payload.serialize() + await read_writer.write_msg(msg_3) - # TODO: Add a specific exception if not noise_state.handshake_finished: - raise Exception + raise HandshakeHasNotFinished( + "handshake is done but it is not marked as finished in `noise_state`" + ) return NoiseConnection( self.local_peer, self.libp2p_privkey, remote_peer, conn, False diff --git a/libp2p/security/noise/transport.py b/libp2p/security/noise/transport.py index 37e15e81..5ee3ba52 100644 --- a/libp2p/security/noise/transport.py +++ b/libp2p/security/noise/transport.py @@ -38,7 +38,12 @@ class Transport(ISecureTransport): if self.with_noise_pipes: raise NotImplementedError else: - return PatternXX(self.local_peer, self.libp2p_privkey, self.noise_privkey) + return PatternXX( + self.local_peer, + self.libp2p_privkey, + self.noise_privkey, + self.early_data, + ) async def secure_inbound(self, conn: IRawConnection) -> ISecureConn: # TODO: SecureInbound attempts to complete a noise-libp2p handshake initiated diff --git a/libp2p/tools/factories.py b/libp2p/tools/factories.py index 75a37a32..7fad8360 100644 --- a/libp2p/tools/factories.py +++ b/libp2p/tools/factories.py @@ -29,6 +29,10 @@ from libp2p.pubsub.gossipsub import GossipSub from libp2p.pubsub.pubsub import Pubsub from libp2p.routing.interfaces import IPeerRouting from libp2p.security.insecure.transport import PLAINTEXT_PROTOCOL_ID, InsecureTransport +from libp2p.security.noise.messages import ( + NoiseHandshakePayload, + make_handshake_payload_sig, +) from libp2p.security.noise.transport import Transport as NoiseTransport import libp2p.security.secio.transport as secio from libp2p.security.secure_conn_interface import ISecureConn @@ -73,6 +77,17 @@ def noise_static_key_factory() -> PrivateKey: return create_ed25519_key_pair().private_key +def noise_handshake_payload_factory() -> NoiseHandshakePayload: + libp2p_keypair = create_secp256k1_key_pair() + noise_static_privkey = noise_static_key_factory() + return NoiseHandshakePayload( + libp2p_keypair.public_key, + make_handshake_payload_sig( + libp2p_keypair.private_key, noise_static_privkey.get_public_key() + ), + ) + + def noise_transport_factory() -> NoiseTransport: return NoiseTransport( libp2p_keypair=create_secp256k1_key_pair(), @@ -118,7 +133,7 @@ async def noise_conn_factory( async def upgrade_local_conn() -> None: nonlocal local_secure_conn local_secure_conn = await local_transport.secure_outbound( - local_conn, local_transport.local_peer + local_conn, remote_transport.local_peer ) async def upgrade_remote_conn() -> None: diff --git a/tests/security/noise/test_noise.py b/tests/security/noise/test_noise.py index c60f83cc..1c5eebbf 100644 --- a/tests/security/noise/test_noise.py +++ b/tests/security/noise/test_noise.py @@ -1,6 +1,7 @@ import pytest -from libp2p.tools.factories import noise_conn_factory +from libp2p.security.noise.messages import NoiseHandshakePayload +from libp2p.tools.factories import noise_conn_factory, noise_handshake_payload_factory DATA = b"testing_123" @@ -18,3 +19,10 @@ async def test_noise_connection(nursery): await local_conn.write(DATA) read_data = await remote_conn.read(len(DATA)) assert read_data == DATA + + +def test_noise_handshake_payload(): + payload = noise_handshake_payload_factory() + payload_serialized = payload.serialize() + payload_deserialized = NoiseHandshakePayload.deserialize(payload_serialized) + assert payload == payload_deserialized From 6ea96e93139267560b1252956175c0eb9aca9592 Mon Sep 17 00:00:00 2001 From: mhchia Date: Sun, 16 Feb 2020 18:23:16 +0800 Subject: [PATCH 03/22] Add comments in pattern xx --- libp2p/security/noise/patterns.py | 34 ++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/libp2p/security/noise/patterns.py b/libp2p/security/noise/patterns.py index 4590b73e..783c101e 100644 --- a/libp2p/security/noise/patterns.py +++ b/libp2p/security/noise/patterns.py @@ -75,10 +75,10 @@ class PatternXX(BasePattern): noise_state = self.create_noise_state() noise_state.set_as_responder() noise_state.start_handshake() - state = noise_state.noise_protocol.handshake_state + handshake_state = noise_state.noise_protocol.handshake_state read_writer = NoiseHandshakeReadWriter(conn, noise_state) - # Consume msg#1 + # Consume msg#1. await read_writer.read_msg() # Send msg#2, which should include our handshake payload. @@ -86,13 +86,18 @@ class PatternXX(BasePattern): msg_2 = our_payload.serialize() await read_writer.write_msg(msg_2) - # Receive msg#3 + # Receive and consume msg#3. msg_3 = await read_writer.read_msg() peer_handshake_payload = NoiseHandshakePayload.deserialize(msg_3) - if state.rs is None: - raise NoiseStateError - remote_pubkey = Ed25519PublicKey.from_bytes(state.rs.public_bytes) + if handshake_state.rs is None: + raise NoiseStateError( + "something is wrong in the underlying noise `handshake_state`: " + "we received and consumed msg#3, which should have included the" + " remote static public key, but it is not present in the handshake_state" + ) + # Use `Ed25519PublicKey` since 25519 is used in our pattern. + remote_pubkey = Ed25519PublicKey.from_bytes(handshake_state.rs.public_bytes) if not verify_handshake_payload_sig(peer_handshake_payload, remote_pubkey): raise InvalidSignature remote_peer_id_from_pubkey = ID.from_pubkey(peer_handshake_payload.id_pubkey) @@ -118,16 +123,24 @@ class PatternXX(BasePattern): read_writer = NoiseHandshakeReadWriter(conn, noise_state) noise_state.set_as_initiator() noise_state.start_handshake() - state = noise_state.noise_protocol.handshake_state + handshake_state = noise_state.noise_protocol.handshake_state + # Send msg#1, which is *not* encrypted. msg_1 = b"" await read_writer.write_msg(msg_1) + # Read msg#2 from the remote, which contains the public key of the peer. msg_2 = await read_writer.read_msg() peer_handshake_payload = NoiseHandshakePayload.deserialize(msg_2) - if state.rs is None: - raise NoiseStateError - remote_pubkey = Ed25519PublicKey.from_bytes(state.rs.public_bytes) + + if handshake_state.rs is None: + raise NoiseStateError( + "something is wrong in the underlying noise `handshake_state`: " + "we received and consumed msg#3, which should have included the" + " remote static public key, but it is not present in the handshake_state" + ) + # Use `Ed25519PublicKey` since 25519 is used in our pattern. + remote_pubkey = Ed25519PublicKey.from_bytes(handshake_state.rs.public_bytes) if not verify_handshake_payload_sig(peer_handshake_payload, remote_pubkey): raise InvalidSignature remote_peer_id_from_pubkey = ID.from_pubkey(peer_handshake_payload.id_pubkey) @@ -138,6 +151,7 @@ class PatternXX(BasePattern): f"remote_peer_id_from_pubkey={remote_peer_id_from_pubkey}" ) + # Send msg#3, which includes our encrypted payload and our noise static key. our_payload = self.make_handshake_payload() msg_3 = our_payload.serialize() await read_writer.write_msg(msg_3) From f8739268e2ffd668d0ed4f3b666dcf75c69bbe8b Mon Sep 17 00:00:00 2001 From: mhchia Date: Mon, 17 Feb 2020 16:30:44 +0800 Subject: [PATCH 04/22] Noise: encrypt and decrypt in `NoiseConnection` TODO: Add a buffer to read only `n` bytes in `read(n)` --- libp2p/security/noise/connection.py | 18 +++++++++++++----- libp2p/security/noise/io.py | 20 +++++++++++++++++++- libp2p/security/noise/patterns.py | 3 ++- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/libp2p/security/noise/connection.py b/libp2p/security/noise/connection.py index 940d807f..8f94a0f1 100644 --- a/libp2p/security/noise/connection.py +++ b/libp2p/security/noise/connection.py @@ -1,11 +1,15 @@ +from noise.connection import NoiseConnection as NoiseState + from libp2p.crypto.keys import PrivateKey from libp2p.network.connection.raw_connection_interface import IRawConnection from libp2p.peer.id import ID from libp2p.security.base_session import BaseSession +from libp2p.security.noise.io import MsgReadWriter, NoiseTransportReadWriter class NoiseConnection(BaseSession): - conn: IRawConnection + read_writer: IRawConnection + noise_state: NoiseState def __init__( self, @@ -14,17 +18,21 @@ class NoiseConnection(BaseSession): remote_peer: ID, conn: IRawConnection, is_initiator: bool, + noise_state: NoiseState, ) -> None: super().__init__(local_peer, local_private_key, is_initiator, remote_peer) self.conn = conn + self.noise_state = noise_state + + def get_msg_read_writer(self) -> MsgReadWriter: + return NoiseTransportReadWriter(self.conn, self.noise_state) async def read(self, n: int = None) -> bytes: - # TODO: Add decryption logic here - return await self.conn.read(n) + # TODO: Use a buffer to handle buffered messages. + return await self.get_msg_read_writer().read_msg() async def write(self, data: bytes) -> None: - # TODO: Add encryption logic here - await self.conn.write(data) + await self.get_msg_read_writer().write_msg(data) async def close(self) -> None: await self.conn.close() diff --git a/libp2p/security/noise/io.py b/libp2p/security/noise/io.py index 6a6f7be7..25335307 100644 --- a/libp2p/security/noise/io.py +++ b/libp2p/security/noise/io.py @@ -88,7 +88,6 @@ def decode_msg_body(noise_msg: bytes) -> bytes: ] -# TODO: Add comments class NoiseHandshakeReadWriter(MsgReadWriter): read_writer: MsgReadWriter noise_state: NoiseState @@ -106,3 +105,22 @@ class NoiseHandshakeReadWriter(MsgReadWriter): noise_msg_encrypted = await self.read_writer.read_msg() noise_msg = self.noise_state.read_message(noise_msg_encrypted) return decode_msg_body(noise_msg) + + +class NoiseTransportReadWriter(MsgReadWriter): + read_writer: MsgReadWriter + noise_state: NoiseState + + def __init__(self, conn: IRawConnection, noise_state: NoiseState) -> None: + self.read_writer = NoisePacketReadWriter(cast(ReadWriter, conn)) + self.noise_state = noise_state + + async def write_msg(self, data: bytes) -> None: + noise_msg = encode_msg_body(data) + data_encrypted = self.noise_state.encrypt(noise_msg) + await self.read_writer.write_msg(data_encrypted) + + async def read_msg(self) -> bytes: + noise_msg_encrypted = await self.read_writer.read_msg() + noise_msg = self.noise_state.decrypt(noise_msg_encrypted) + return decode_msg_body(noise_msg) diff --git a/libp2p/security/noise/patterns.py b/libp2p/security/noise/patterns.py index 783c101e..91ce350a 100644 --- a/libp2p/security/noise/patterns.py +++ b/libp2p/security/noise/patterns.py @@ -113,6 +113,7 @@ class PatternXX(BasePattern): remote_peer_id_from_pubkey, conn, False, + noise_state, ) async def handshake_outbound( @@ -162,5 +163,5 @@ class PatternXX(BasePattern): ) return NoiseConnection( - self.local_peer, self.libp2p_privkey, remote_peer, conn, False + self.local_peer, self.libp2p_privkey, remote_peer, conn, False, noise_state ) From ea645f0bd66fa177591ce9e5a1a41fa919b04840 Mon Sep 17 00:00:00 2001 From: mhchia Date: Mon, 17 Feb 2020 17:06:00 +0800 Subject: [PATCH 05/22] Copied read/write from secio --- libp2p/security/noise/connection.py | 58 +++++++++++++++++++++++++++-- tests/security/noise/test_noise.py | 13 +++++-- 2 files changed, 63 insertions(+), 8 deletions(-) diff --git a/libp2p/security/noise/connection.py b/libp2p/security/noise/connection.py index 8f94a0f1..29bbc8bc 100644 --- a/libp2p/security/noise/connection.py +++ b/libp2p/security/noise/connection.py @@ -1,3 +1,5 @@ +import io + from noise.connection import NoiseConnection as NoiseState from libp2p.crypto.keys import PrivateKey @@ -8,6 +10,10 @@ from libp2p.security.noise.io import MsgReadWriter, NoiseTransportReadWriter class NoiseConnection(BaseSession): + buf: io.BytesIO + low_watermark: int + high_watermark: int + read_writer: IRawConnection noise_state: NoiseState @@ -19,20 +25,64 @@ class NoiseConnection(BaseSession): conn: IRawConnection, is_initiator: bool, noise_state: NoiseState, + # remote_permanent_pubkey ) -> None: super().__init__(local_peer, local_private_key, is_initiator, remote_peer) self.conn = conn self.noise_state = noise_state + self._reset_internal_buffer() def get_msg_read_writer(self) -> MsgReadWriter: return NoiseTransportReadWriter(self.conn, self.noise_state) + async def close(self) -> None: + await self.conn.close() + + def _reset_internal_buffer(self) -> None: + self.buf = io.BytesIO() + self.low_watermark = 0 + self.high_watermark = 0 + + def _drain(self, n: int) -> bytes: + if self.low_watermark == self.high_watermark: + return bytes() + + data = self.buf.getbuffer()[self.low_watermark : self.high_watermark] + + if n is None: + n = len(data) + result = data[:n].tobytes() + self.low_watermark += len(result) + + if self.low_watermark == self.high_watermark: + del data # free the memoryview so we can free the underlying BytesIO + self.buf.close() + self._reset_internal_buffer() + return result + async def read(self, n: int = None) -> bytes: - # TODO: Use a buffer to handle buffered messages. + if n == 0: + return bytes() + + data_from_buffer = self._drain(n) + if len(data_from_buffer) > 0: + return data_from_buffer + + msg = await self.read_msg() + + if n < len(msg): + self.buf.write(msg) + self.low_watermark = 0 + self.high_watermark = len(msg) + return self._drain(n) + else: + return msg + + async def read_msg(self) -> bytes: return await self.get_msg_read_writer().read_msg() async def write(self, data: bytes) -> None: - await self.get_msg_read_writer().write_msg(data) + await self.write_msg(data) - async def close(self) -> None: - await self.conn.close() + async def write_msg(self, msg: bytes) -> None: + await self.get_msg_read_writer().write_msg(msg) diff --git a/tests/security/noise/test_noise.py b/tests/security/noise/test_noise.py index 1c5eebbf..f1d208ca 100644 --- a/tests/security/noise/test_noise.py +++ b/tests/security/noise/test_noise.py @@ -3,7 +3,9 @@ import pytest from libp2p.security.noise.messages import NoiseHandshakePayload from libp2p.tools.factories import noise_conn_factory, noise_handshake_payload_factory -DATA = b"testing_123" +DATA_0 = b"data_0" +DATA_1 = b"1" * 1000 +DATA_2 = b"data_2" @pytest.mark.trio @@ -16,9 +18,12 @@ async def test_noise_transport(nursery): async def test_noise_connection(nursery): async with noise_conn_factory(nursery) as conns: local_conn, remote_conn = conns - await local_conn.write(DATA) - read_data = await remote_conn.read(len(DATA)) - assert read_data == DATA + await local_conn.write(DATA_0) + await local_conn.write(DATA_1) + assert DATA_0 == (await remote_conn.read(len(DATA_0))) + assert DATA_1 == (await remote_conn.read(len(DATA_1))) + await local_conn.write(DATA_2) + assert DATA_2 == (await remote_conn.read(len(DATA_2))) def test_noise_handshake_payload(): From 874c6bbca4fe65f6bb2784121039dc97681f0c80 Mon Sep 17 00:00:00 2001 From: mhchia Date: Mon, 17 Feb 2020 17:30:29 +0800 Subject: [PATCH 06/22] Refactor MsgIOReadWriter - Abstract it as `MsgReadWriter` - `MsgIOReadWriter` as a subclass of `MsgReadWriter` --- libp2p/io/abc.py | 20 ++++++++++++++ libp2p/io/msgio.py | 67 +++++++++++++++++----------------------------- 2 files changed, 44 insertions(+), 43 deletions(-) diff --git a/libp2p/io/abc.py b/libp2p/io/abc.py index 265663ac..be97288d 100644 --- a/libp2p/io/abc.py +++ b/libp2p/io/abc.py @@ -32,3 +32,23 @@ class ReadWriter(Reader, Writer): class ReadWriteCloser(Reader, Writer, Closer): pass + + +class MsgReader(ABC): + @abstractmethod + async def read_msg(self) -> bytes: + ... + + @abstractmethod + async def next_msg_len(self) -> int: + ... + + +class MsgWriter(ABC): + @abstractmethod + async def write_msg(self, msg: bytes) -> None: + ... + + +class MsgReadWriter(MsgReader, MsgWriter): + pass diff --git a/libp2p/io/msgio.py b/libp2p/io/msgio.py index 82c55ac4..65ee8b0e 100644 --- a/libp2p/io/msgio.py +++ b/libp2p/io/msgio.py @@ -6,60 +6,39 @@ from that repo: "a simple package to r/w length-delimited slices." NOTE: currently missing the capability to indicate lengths by "varint" method. """ # TODO unify w/ https://github.com/libp2p/py-libp2p/blob/1aed52856f56a4b791696bbcbac31b5f9c2e88c9/libp2p/utils.py#L85-L99 # noqa: E501 -from typing import Optional, cast +from typing import Optional -from libp2p.io.abc import Closer, ReadCloser, Reader, ReadWriteCloser, WriteCloser +from libp2p.io.abc import MsgReadWriter, Reader, ReadWriteCloser from libp2p.io.utils import read_exactly -SIZE_LEN_BYTES = 4 +SIZE_NOISE_LEN_BYTES = 2 +SIZE_SECIO_LEN_BYTES = 4 BYTE_ORDER = "big" -async def read_length(reader: Reader) -> int: - length_bytes = await read_exactly(reader, SIZE_LEN_BYTES) +async def read_length(reader: Reader, size_len_bytes: int) -> int: + length_bytes = await read_exactly(reader, size_len_bytes) return int.from_bytes(length_bytes, byteorder=BYTE_ORDER) -def encode_msg_with_length(msg_bytes: bytes) -> bytes: - len_prefix = len(msg_bytes).to_bytes(SIZE_LEN_BYTES, "big") +def encode_msg_with_length(msg_bytes: bytes, size_len_bytes: int) -> bytes: + len_prefix = len(msg_bytes).to_bytes(size_len_bytes, byteorder=BYTE_ORDER) return len_prefix + msg_bytes -class MsgIOWriter(WriteCloser): - write_closer: WriteCloser - - def __init__(self, write_closer: WriteCloser) -> None: - self.write_closer = write_closer - - async def write(self, data: bytes) -> None: - await self.write_msg(data) - - async def write_msg(self, msg: bytes) -> None: - data = encode_msg_with_length(msg) - await self.write_closer.write(data) - - async def close(self) -> None: - await self.write_closer.close() - - -class MsgIOReader(ReadCloser): - read_closer: ReadCloser +class BaseMsgReadWriter(MsgReadWriter): next_length: Optional[int] + read_write_closer: ReadWriteCloser + size_len_bytes: int - def __init__(self, read_closer: ReadCloser) -> None: - # NOTE: the following line is required to satisfy the - # multiple inheritance but `mypy` does not like it... - super().__init__(read_closer) # type: ignore - self.read_closer = read_closer + def __init__(self, read_write_closer: ReadWriteCloser) -> None: + self.read_write_closer = read_write_closer self.next_length = None - async def read(self, n: int = None) -> bytes: - return await self.read_msg() - async def read_msg(self) -> bytes: length = await self.next_msg_len() - data = await read_exactly(self.read_closer, length) + data = await read_exactly(self.read_write_closer, length) if len(data) < length: self.next_length = length - len(data) else: @@ -68,16 +47,18 @@ class MsgIOReader(ReadCloser): async def next_msg_len(self) -> int: if self.next_length is None: - self.next_length = await read_length(self.read_closer) + self.next_length = await read_length( + self.read_write_closer, self.size_len_bytes + ) return self.next_length async def close(self) -> None: - await self.read_closer.close() + await self.read_write_closer.close() + + async def write_msg(self, msg: bytes) -> None: + data = encode_msg_with_length(msg, self.size_len_bytes) + await self.read_write_closer.write(data) -class MsgIOReadWriter(MsgIOReader, MsgIOWriter, Closer): - def __init__(self, read_write_closer: ReadWriteCloser) -> None: - super().__init__(cast(ReadCloser, read_write_closer)) - - async def close(self) -> None: - await self.read_closer.close() +class MsgIOReadWriter(BaseMsgReadWriter): + size_len_bytes = SIZE_SECIO_LEN_BYTES From 2df47a943c9342be1f65c216b5c893cac2a515e9 Mon Sep 17 00:00:00 2001 From: mhchia Date: Mon, 17 Feb 2020 19:02:18 +0800 Subject: [PATCH 07/22] Refactor --- libp2p/io/abc.py | 20 +++++-- libp2p/io/msgio.py | 15 +++--- libp2p/security/noise/io.py | 84 ++++++++---------------------- libp2p/security/noise/patterns.py | 12 ++++- libp2p/security/secio/transport.py | 7 ++- 5 files changed, 63 insertions(+), 75 deletions(-) diff --git a/libp2p/io/abc.py b/libp2p/io/abc.py index be97288d..f2e5020d 100644 --- a/libp2p/io/abc.py +++ b/libp2p/io/abc.py @@ -39,9 +39,9 @@ class MsgReader(ABC): async def read_msg(self) -> bytes: ... - @abstractmethod - async def next_msg_len(self) -> int: - ... + # @abstractmethod + # async def next_msg_len(self) -> int: + # ... class MsgWriter(ABC): @@ -52,3 +52,17 @@ class MsgWriter(ABC): class MsgReadWriter(MsgReader, MsgWriter): pass + + +class Encrypter(ABC): + @abstractmethod + def encrypt(self, data: bytes) -> bytes: + ... + + @abstractmethod + def decrypt(self, data: bytes) -> bytes: + ... + + +class EncryptedMsgReadWriter(MsgReadWriter, Encrypter): + pass diff --git a/libp2p/io/msgio.py b/libp2p/io/msgio.py index 65ee8b0e..a9610a45 100644 --- a/libp2p/io/msgio.py +++ b/libp2p/io/msgio.py @@ -11,8 +11,7 @@ from typing import Optional from libp2p.io.abc import MsgReadWriter, Reader, ReadWriteCloser from libp2p.io.utils import read_exactly -SIZE_NOISE_LEN_BYTES = 2 -SIZE_SECIO_LEN_BYTES = 4 + BYTE_ORDER = "big" @@ -22,7 +21,13 @@ async def read_length(reader: Reader, size_len_bytes: int) -> int: def encode_msg_with_length(msg_bytes: bytes, size_len_bytes: int) -> bytes: - len_prefix = len(msg_bytes).to_bytes(size_len_bytes, byteorder=BYTE_ORDER) + try: + len_prefix = len(msg_bytes).to_bytes(size_len_bytes, byteorder=BYTE_ORDER) + except OverflowError: + raise ValueError( + "msg_bytes is too large for `size_len_bytes` bytes length: " + f"msg_bytes={msg_bytes}, size_len_bytes={size_len_bytes}" + ) return len_prefix + msg_bytes @@ -58,7 +63,3 @@ class BaseMsgReadWriter(MsgReadWriter): async def write_msg(self, msg: bytes) -> None: data = encode_msg_with_length(msg, self.size_len_bytes) await self.read_write_closer.write(data) - - -class MsgIOReadWriter(BaseMsgReadWriter): - size_len_bytes = SIZE_SECIO_LEN_BYTES diff --git a/libp2p/security/noise/io.py b/libp2p/security/noise/io.py index 25335307..4b01f9d6 100644 --- a/libp2p/security/noise/io.py +++ b/libp2p/security/noise/io.py @@ -3,10 +3,12 @@ from typing import cast from noise.connection import NoiseConnection as NoiseState -from libp2p.io.abc import ReadWriter +from libp2p.io.abc import ReadWriteCloser, MsgReadWriter, EncryptedMsgReadWriter +from libp2p.io.msgio import BaseMsgReadWriter, encode_msg_with_length from libp2p.io.utils import read_exactly from libp2p.network.connection.raw_connection_interface import IRawConnection + SIZE_NOISE_MESSAGE_LEN = 2 MAX_NOISE_MESSAGE_LEN = 2 ** (8 * SIZE_NOISE_MESSAGE_LEN) - 1 SIZE_NOISE_MESSAGE_BODY_LEN = 2 @@ -20,53 +22,12 @@ BYTE_ORDER = "big" # <-2 bytes-><- max=65533 bytes -> -def encode_data(data: bytes, size_len: int) -> bytes: - len_data = len(data) - try: - len_bytes = len_data.to_bytes(size_len, BYTE_ORDER) - except OverflowError as e: - raise ValueError from e - return len_bytes + data +class NoisePacketReadWriter(BaseMsgReadWriter): + size_len_bytes = SIZE_NOISE_MESSAGE_LEN -class MsgReader(ABC): - @abstractmethod - async def read_msg(self) -> bytes: - ... - - -class MsgWriter(ABC): - @abstractmethod - async def write_msg(self, msg: bytes) -> None: - ... - - -class MsgReadWriter(MsgReader, MsgWriter): - pass - - -# TODO: Add comments -class NoisePacketReadWriter(MsgReadWriter): - """Encode and decode the low level noise messages.""" - - read_writer: ReadWriter - - def __init__(self, read_writer: ReadWriter) -> None: - self.read_writer = read_writer - - async def read_msg(self) -> bytes: - len_bytes = await read_exactly(self.read_writer, SIZE_NOISE_MESSAGE_LEN) - len_int = int.from_bytes(len_bytes, BYTE_ORDER) - return await read_exactly(self.read_writer, len_int) - - async def write_msg(self, msg: bytes) -> None: - encoded_data = encode_data(msg, SIZE_NOISE_MESSAGE_LEN) - await self.read_writer.write(encoded_data) - - -# TODO: Add comments def encode_msg_body(msg_body: bytes) -> bytes: - encoded_msg_body = encode_data(msg_body, SIZE_NOISE_MESSAGE_BODY_LEN) + encoded_msg_body = encode_msg_with_length(msg_body, SIZE_NOISE_MESSAGE_BODY_LEN) if len(encoded_msg_body) > MAX_NOISE_MESSAGE_BODY_LEN: raise ValueError( f"msg_body is too long: {len(msg_body)}, " @@ -88,39 +49,36 @@ def decode_msg_body(noise_msg: bytes) -> bytes: ] -class NoiseHandshakeReadWriter(MsgReadWriter): +class BaseNoiseMsgReadWriter(EncryptedMsgReadWriter): read_writer: MsgReadWriter noise_state: NoiseState def __init__(self, conn: IRawConnection, noise_state: NoiseState) -> None: - self.read_writer = NoisePacketReadWriter(cast(ReadWriter, conn)) + self.read_writer = NoisePacketReadWriter(cast(ReadWriteCloser, conn)) self.noise_state = noise_state async def write_msg(self, data: bytes) -> None: noise_msg = encode_msg_body(data) - data_encrypted = self.noise_state.write_message(noise_msg) + data_encrypted = self.encrypt(noise_msg) await self.read_writer.write_msg(data_encrypted) async def read_msg(self) -> bytes: noise_msg_encrypted = await self.read_writer.read_msg() - noise_msg = self.noise_state.read_message(noise_msg_encrypted) + noise_msg = self.decrypt(noise_msg_encrypted) return decode_msg_body(noise_msg) -class NoiseTransportReadWriter(MsgReadWriter): - read_writer: MsgReadWriter - noise_state: NoiseState +class NoiseHandshakeReadWriter(BaseNoiseMsgReadWriter): + def encrypt(self, data: bytes) -> bytes: + return self.noise_state.write_message(data) - def __init__(self, conn: IRawConnection, noise_state: NoiseState) -> None: - self.read_writer = NoisePacketReadWriter(cast(ReadWriter, conn)) - self.noise_state = noise_state + def decrypt(self, data: bytes) -> bytes: + return self.noise_state.read_message(data) - async def write_msg(self, data: bytes) -> None: - noise_msg = encode_msg_body(data) - data_encrypted = self.noise_state.encrypt(noise_msg) - await self.read_writer.write_msg(data_encrypted) - async def read_msg(self) -> bytes: - noise_msg_encrypted = await self.read_writer.read_msg() - noise_msg = self.noise_state.decrypt(noise_msg_encrypted) - return decode_msg_body(noise_msg) +class NoiseTransportReadWriter(BaseNoiseMsgReadWriter): + def encrypt(self, data: bytes) -> bytes: + return self.noise_state.encrypt(data) + + def decrypt(self, data: bytes) -> bytes: + return self.noise_state.decrypt(data) diff --git a/libp2p/security/noise/patterns.py b/libp2p/security/noise/patterns.py index 91ce350a..213b1571 100644 --- a/libp2p/security/noise/patterns.py +++ b/libp2p/security/noise/patterns.py @@ -16,7 +16,7 @@ from .exceptions import ( NoiseStateError, PeerIDMismatchesPubkey, ) -from .io import NoiseHandshakeReadWriter +from .io import encode_msg_body, decode_msg_body, NoiseHandshakeReadWriter from .messages import ( NoiseHandshakePayload, make_handshake_payload_sig, @@ -56,6 +56,16 @@ class BasePattern(IPattern): ) return NoiseHandshakePayload(self.libp2p_privkey.get_public_key(), signature) + async def write_msg(self, conn: IRawConnection, data: bytes) -> None: + noise_msg = encode_msg_body(data) + data_encrypted = self.noise_state.write_message(noise_msg) + await self.read_writer.write_msg(data_encrypted) + + async def read_msg(self) -> bytes: + noise_msg_encrypted = await self.read_writer.read_msg() + noise_msg = self.noise_state.read_message(noise_msg_encrypted) + return decode_msg_body(noise_msg) + class PatternXX(BasePattern): def __init__( diff --git a/libp2p/security/secio/transport.py b/libp2p/security/secio/transport.py index d8216b80..109f488c 100644 --- a/libp2p/security/secio/transport.py +++ b/libp2p/security/secio/transport.py @@ -19,7 +19,7 @@ from libp2p.crypto.key_exchange import create_ephemeral_key_pair from libp2p.crypto.keys import PrivateKey, PublicKey from libp2p.crypto.serialization import deserialize_public_key from libp2p.io.exceptions import DecryptionFailedException, IOException -from libp2p.io.msgio import MsgIOReadWriter +from libp2p.io.msgio import BaseMsgReadWriter from libp2p.network.connection.raw_connection_interface import IRawConnection from libp2p.peer.id import ID as PeerID from libp2p.security.base_session import BaseSession @@ -41,6 +41,7 @@ from .pb.spipe_pb2 import Exchange, Propose ID = TProtocol("/secio/1.0.0") NONCE_SIZE = 16 # bytes +SIZE_SECIO_LEN_BYTES = 4 # NOTE: the following is only a subset of allowable parameters according to the # `secio` specification. @@ -49,6 +50,10 @@ DEFAULT_SUPPORTED_CIPHERS = "AES-128" DEFAULT_SUPPORTED_HASHES = "SHA256" +class MsgIOReadWriter(BaseMsgReadWriter): + size_len_bytes = SIZE_SECIO_LEN_BYTES + + class SecureSession(BaseSession): buf: io.BytesIO low_watermark: int From 3c2e835725af48ad497baeb40d0cdf28b4ebb7b1 Mon Sep 17 00:00:00 2001 From: mhchia Date: Mon, 17 Feb 2020 23:33:45 +0800 Subject: [PATCH 08/22] Security: `SecureSession` Make security sessions(secio, noise) share the same implementation `BaseSession` to avoid duplicate implementation of buffered read. --- libp2p/io/abc.py | 11 +- libp2p/io/msgio.py | 7 +- libp2p/security/insecure/io.py | 19 +++ libp2p/security/insecure/transport.py | 116 +++++++++--------- libp2p/security/noise/io.py | 17 ++- libp2p/security/noise/patterns.py | 30 ++--- libp2p/security/secio/transport.py | 104 ++++------------ .../connection.py => secure_session.py} | 42 +++---- 8 files changed, 150 insertions(+), 196 deletions(-) create mode 100644 libp2p/security/insecure/io.py rename libp2p/security/{noise/connection.py => secure_session.py} (63%) diff --git a/libp2p/io/abc.py b/libp2p/io/abc.py index f2e5020d..b7be31f2 100644 --- a/libp2p/io/abc.py +++ b/libp2p/io/abc.py @@ -2,6 +2,7 @@ from abc import ABC, abstractmethod class Closer(ABC): + @abstractmethod async def close(self) -> None: ... @@ -39,10 +40,6 @@ class MsgReader(ABC): async def read_msg(self) -> bytes: ... - # @abstractmethod - # async def next_msg_len(self) -> int: - # ... - class MsgWriter(ABC): @abstractmethod @@ -50,7 +47,7 @@ class MsgWriter(ABC): ... -class MsgReadWriter(MsgReader, MsgWriter): +class MsgReadWriteCloser(MsgReader, MsgWriter, Closer): pass @@ -64,5 +61,5 @@ class Encrypter(ABC): ... -class EncryptedMsgReadWriter(MsgReadWriter, Encrypter): - pass +class EncryptedMsgReadWriter(MsgReadWriteCloser, Encrypter): + """Read/write message with encryption/decryption.""" diff --git a/libp2p/io/msgio.py b/libp2p/io/msgio.py index a9610a45..38710ff5 100644 --- a/libp2p/io/msgio.py +++ b/libp2p/io/msgio.py @@ -8,10 +8,9 @@ NOTE: currently missing the capability to indicate lengths by "varint" method. # TODO unify w/ https://github.com/libp2p/py-libp2p/blob/1aed52856f56a4b791696bbcbac31b5f9c2e88c9/libp2p/utils.py#L85-L99 # noqa: E501 from typing import Optional -from libp2p.io.abc import MsgReadWriter, Reader, ReadWriteCloser +from libp2p.io.abc import MsgReadWriteCloser, Reader, ReadWriteCloser from libp2p.io.utils import read_exactly - BYTE_ORDER = "big" @@ -26,12 +25,12 @@ def encode_msg_with_length(msg_bytes: bytes, size_len_bytes: int) -> bytes: except OverflowError: raise ValueError( "msg_bytes is too large for `size_len_bytes` bytes length: " - f"msg_bytes={msg_bytes}, size_len_bytes={size_len_bytes}" + f"msg_bytes={msg_bytes!r}, size_len_bytes={size_len_bytes}" ) return len_prefix + msg_bytes -class BaseMsgReadWriter(MsgReadWriter): +class BaseMsgReadWriter(MsgReadWriteCloser): next_length: Optional[int] read_write_closer: ReadWriteCloser size_len_bytes: int diff --git a/libp2p/security/insecure/io.py b/libp2p/security/insecure/io.py new file mode 100644 index 00000000..1cbff365 --- /dev/null +++ b/libp2p/security/insecure/io.py @@ -0,0 +1,19 @@ +from libp2p.io.abc import MsgReadWriteCloser, ReadWriteCloser +from libp2p.utils import encode_fixedint_prefixed, read_fixedint_prefixed + + +class PlaintextHandshakeReadWriter(MsgReadWriteCloser): + conn: ReadWriteCloser + + def __init__(self, conn: ReadWriteCloser) -> None: + self.conn = conn + + async def read_msg(self) -> bytes: + return await read_fixedint_prefixed(self.conn) + + async def write_msg(self, msg: bytes) -> None: + encoded_msg_bytes = encode_fixedint_prefixed(msg) + await self.conn.write(encoded_msg_bytes) + + async def close(self) -> None: + await self.conn.close() diff --git a/libp2p/security/insecure/transport.py b/libp2p/security/insecure/transport.py index f452e53c..28e76412 100644 --- a/libp2p/security/insecure/transport.py +++ b/libp2p/security/insecure/transport.py @@ -13,8 +13,8 @@ from libp2p.security.base_transport import BaseSecureTransport from libp2p.security.exceptions import HandshakeFailure from libp2p.security.secure_conn_interface import ISecureConn from libp2p.typing import TProtocol -from libp2p.utils import encode_fixedint_prefixed, read_fixedint_prefixed +from .io import PlaintextHandshakeReadWriter from .pb import plaintext_pb2 # Reference: https://github.com/libp2p/go-libp2p-core/blob/master/sec/insecure/insecure.go @@ -44,60 +44,66 @@ class InsecureSession(BaseSession): async def close(self) -> None: await self.conn.close() - async def run_handshake(self) -> None: - """Raise `HandshakeFailure` when handshake failed.""" - msg = make_exchange_message(self.local_private_key.get_public_key()) - msg_bytes = msg.SerializeToString() - encoded_msg_bytes = encode_fixedint_prefixed(msg_bytes) - try: - await self.write(encoded_msg_bytes) - except RawConnError as e: - raise HandshakeFailure("connection closed") from e - try: - remote_msg_bytes = await read_fixedint_prefixed(self.conn) - except RawConnError as e: - raise HandshakeFailure("connection closed") from e - remote_msg = plaintext_pb2.Exchange() - remote_msg.ParseFromString(remote_msg_bytes) - received_peer_id = ID(remote_msg.id) +async def run_handshake( + local_peer: ID, + local_private_key: PrivateKey, + conn: IRawConnection, + is_initiator: bool, + remote_peer_id: ID, +) -> ISecureConn: + """Raise `HandshakeFailure` when handshake failed.""" + msg = make_exchange_message(local_private_key.get_public_key()) + msg_bytes = msg.SerializeToString() + read_writer = PlaintextHandshakeReadWriter(conn) + try: + await read_writer.write_msg(msg_bytes) + except RawConnError as e: + raise HandshakeFailure("connection closed") from e - # Verify if the receive `ID` matches the one we originally initialize the session. - # We only need to check it when we are the initiator, because only in that condition - # we possibly knows the `ID` of the remote. - if self.is_initiator and self.remote_peer_id != received_peer_id: - raise HandshakeFailure( - "remote peer sent unexpected peer ID. " - f"expected={self.remote_peer_id} received={received_peer_id}" - ) + try: + remote_msg_bytes = await read_writer.read_msg() + except RawConnError as e: + raise HandshakeFailure("connection closed") from e + remote_msg = plaintext_pb2.Exchange() + remote_msg.ParseFromString(remote_msg_bytes) + received_peer_id = ID(remote_msg.id) - # Verify if the given `pubkey` matches the given `peer_id` - try: - received_pubkey = deserialize_public_key( - remote_msg.pubkey.SerializeToString() - ) - except ValueError as e: - raise HandshakeFailure( - f"unknown `key_type` of remote_msg.pubkey={remote_msg.pubkey}" - ) from e - except MissingDeserializerError as error: - raise HandshakeFailure() from error - peer_id_from_received_pubkey = ID.from_pubkey(received_pubkey) - if peer_id_from_received_pubkey != received_peer_id: - raise HandshakeFailure( - "peer id and pubkey from the remote mismatch: " - f"received_peer_id={received_peer_id}, remote_pubkey={received_pubkey}, " - f"peer_id_from_received_pubkey={peer_id_from_received_pubkey}" - ) + # Verify if the receive `ID` matches the one we originally initialize the session. + # We only need to check it when we are the initiator, because only in that condition + # we possibly knows the `ID` of the remote. + if is_initiator and remote_peer_id != received_peer_id: + raise HandshakeFailure( + "remote peer sent unexpected peer ID. " + f"expected={remote_peer_id} received={received_peer_id}" + ) - # Nothing is wrong. Store the `pubkey` and `peer_id` in the session. - self.remote_permanent_pubkey = received_pubkey - # Only need to set peer's id when we don't know it before, - # i.e. we are not the connection initiator. - if not self.is_initiator: - self.remote_peer_id = received_peer_id + # Verify if the given `pubkey` matches the given `peer_id` + try: + received_pubkey = deserialize_public_key(remote_msg.pubkey.SerializeToString()) + except ValueError as e: + raise HandshakeFailure( + f"unknown `key_type` of remote_msg.pubkey={remote_msg.pubkey}" + ) from e + except MissingDeserializerError as error: + raise HandshakeFailure() from error + peer_id_from_received_pubkey = ID.from_pubkey(received_pubkey) + if peer_id_from_received_pubkey != received_peer_id: + raise HandshakeFailure( + "peer id and pubkey from the remote mismatch: " + f"received_peer_id={received_peer_id}, remote_pubkey={received_pubkey}, " + f"peer_id_from_received_pubkey={peer_id_from_received_pubkey}" + ) - # TODO: Store `pubkey` and `peer_id` to `PeerStore` + secure_conn = InsecureSession( + local_peer, local_private_key, conn, is_initiator, received_peer_id + ) + + # Nothing is wrong. Store the `pubkey` and `peer_id` in the session. + secure_conn.remote_permanent_pubkey = received_pubkey + # TODO: Store `pubkey` and `peer_id` to `PeerStore` + + return secure_conn class InsecureTransport(BaseSecureTransport): @@ -113,9 +119,9 @@ class InsecureTransport(BaseSecureTransport): :return: secure connection object (that implements secure_conn_interface) """ - session = InsecureSession(self.local_peer, self.local_private_key, conn, False) - await session.run_handshake() - return session + return await run_handshake( + self.local_peer, self.local_private_key, conn, False, None + ) async def secure_outbound(self, conn: IRawConnection, peer_id: ID) -> ISecureConn: """ @@ -124,11 +130,9 @@ class InsecureTransport(BaseSecureTransport): :return: secure connection object (that implements secure_conn_interface) """ - session = InsecureSession( + return await run_handshake( self.local_peer, self.local_private_key, conn, True, peer_id ) - await session.run_handshake() - return session def make_exchange_message(pubkey: PublicKey) -> plaintext_pb2.Exchange: diff --git a/libp2p/security/noise/io.py b/libp2p/security/noise/io.py index 4b01f9d6..5ffeafe1 100644 --- a/libp2p/security/noise/io.py +++ b/libp2p/security/noise/io.py @@ -1,14 +1,11 @@ -from abc import ABC, abstractmethod from typing import cast from noise.connection import NoiseConnection as NoiseState -from libp2p.io.abc import ReadWriteCloser, MsgReadWriter, EncryptedMsgReadWriter +from libp2p.io.abc import EncryptedMsgReadWriter, MsgReadWriteCloser, ReadWriteCloser from libp2p.io.msgio import BaseMsgReadWriter, encode_msg_with_length -from libp2p.io.utils import read_exactly from libp2p.network.connection.raw_connection_interface import IRawConnection - SIZE_NOISE_MESSAGE_LEN = 2 MAX_NOISE_MESSAGE_LEN = 2 ** (8 * SIZE_NOISE_MESSAGE_LEN) - 1 SIZE_NOISE_MESSAGE_BODY_LEN = 2 @@ -50,7 +47,14 @@ def decode_msg_body(noise_msg: bytes) -> bytes: class BaseNoiseMsgReadWriter(EncryptedMsgReadWriter): - read_writer: MsgReadWriter + """ + The base implementation of noise message reader/writer. + + `encrypt` and `decrypt` are not implemented here, which should be + implemented by the subclasses. + """ + + read_writer: MsgReadWriteCloser noise_state: NoiseState def __init__(self, conn: IRawConnection, noise_state: NoiseState) -> None: @@ -67,6 +71,9 @@ class BaseNoiseMsgReadWriter(EncryptedMsgReadWriter): noise_msg = self.decrypt(noise_msg_encrypted) return decode_msg_body(noise_msg) + async def close(self) -> None: + await self.read_writer.close() + class NoiseHandshakeReadWriter(BaseNoiseMsgReadWriter): def encrypt(self, data: bytes) -> bytes: diff --git a/libp2p/security/noise/patterns.py b/libp2p/security/noise/patterns.py index 213b1571..64d49656 100644 --- a/libp2p/security/noise/patterns.py +++ b/libp2p/security/noise/patterns.py @@ -8,15 +8,15 @@ from libp2p.crypto.keys import PrivateKey from libp2p.network.connection.raw_connection_interface import IRawConnection from libp2p.peer.id import ID from libp2p.security.secure_conn_interface import ISecureConn +from libp2p.security.secure_session import SecureSession -from .connection import NoiseConnection from .exceptions import ( HandshakeHasNotFinished, InvalidSignature, NoiseStateError, PeerIDMismatchesPubkey, ) -from .io import encode_msg_body, decode_msg_body, NoiseHandshakeReadWriter +from .io import NoiseHandshakeReadWriter, NoiseTransportReadWriter from .messages import ( NoiseHandshakePayload, make_handshake_payload_sig, @@ -56,16 +56,6 @@ class BasePattern(IPattern): ) return NoiseHandshakePayload(self.libp2p_privkey.get_public_key(), signature) - async def write_msg(self, conn: IRawConnection, data: bytes) -> None: - noise_msg = encode_msg_body(data) - data_encrypted = self.noise_state.write_message(noise_msg) - await self.read_writer.write_msg(data_encrypted) - - async def read_msg(self) -> bytes: - noise_msg_encrypted = await self.read_writer.read_msg() - noise_msg = self.noise_state.read_message(noise_msg_encrypted) - return decode_msg_body(noise_msg) - class PatternXX(BasePattern): def __init__( @@ -116,14 +106,13 @@ class PatternXX(BasePattern): raise HandshakeHasNotFinished( "handshake is done but it is not marked as finished in `noise_state`" ) - - return NoiseConnection( + transport_read_writer = NoiseTransportReadWriter(conn, noise_state) + return SecureSession( self.local_peer, self.libp2p_privkey, remote_peer_id_from_pubkey, - conn, + transport_read_writer, False, - noise_state, ) async def handshake_outbound( @@ -171,7 +160,12 @@ class PatternXX(BasePattern): raise HandshakeHasNotFinished( "handshake is done but it is not marked as finished in `noise_state`" ) + transport_read_writer = NoiseTransportReadWriter(conn, noise_state) - return NoiseConnection( - self.local_peer, self.libp2p_privkey, remote_peer, conn, False, noise_state + return SecureSession( + self.local_peer, + self.libp2p_privkey, + remote_peer, + transport_read_writer, + False, ) diff --git a/libp2p/security/secio/transport.py b/libp2p/security/secio/transport.py index 109f488c..ba774d99 100644 --- a/libp2p/security/secio/transport.py +++ b/libp2p/security/secio/transport.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -import io import itertools from typing import Optional, Tuple @@ -18,13 +17,14 @@ from libp2p.crypto.exceptions import MissingDeserializerError from libp2p.crypto.key_exchange import create_ephemeral_key_pair from libp2p.crypto.keys import PrivateKey, PublicKey from libp2p.crypto.serialization import deserialize_public_key +from libp2p.io.abc import EncryptedMsgReadWriter from libp2p.io.exceptions import DecryptionFailedException, IOException from libp2p.io.msgio import BaseMsgReadWriter from libp2p.network.connection.raw_connection_interface import IRawConnection from libp2p.peer.id import ID as PeerID -from libp2p.security.base_session import BaseSession from libp2p.security.base_transport import BaseSecureTransport from libp2p.security.secure_conn_interface import ISecureConn +from libp2p.security.secure_session import SecureSession from libp2p.typing import TProtocol from .exceptions import ( @@ -54,30 +54,20 @@ class MsgIOReadWriter(BaseMsgReadWriter): size_len_bytes = SIZE_SECIO_LEN_BYTES -class SecureSession(BaseSession): - buf: io.BytesIO - low_watermark: int - high_watermark: int +class SecioMsgReadWriter(EncryptedMsgReadWriter): + read_writer: MsgIOReadWriter def __init__( self, - local_peer: PeerID, - local_private_key: PrivateKey, local_encryption_parameters: AuthenticatedEncryptionParameters, - remote_peer: PeerID, remote_encryption_parameters: AuthenticatedEncryptionParameters, - conn: MsgIOReadWriter, - is_initiator: bool, + read_writer: MsgIOReadWriter, ) -> None: - super().__init__(local_peer, local_private_key, is_initiator, remote_peer) - self.conn = conn - self.local_encryption_parameters = local_encryption_parameters self.remote_encryption_parameters = remote_encryption_parameters self._initialize_authenticated_encryption_for_local_peer() self._initialize_authenticated_encryption_for_remote_peer() - - self._reset_internal_buffer() + self.read_writer = read_writer def _initialize_authenticated_encryption_for_local_peer(self) -> None: self.local_encrypter = Encrypter(self.local_encryption_parameters) @@ -85,68 +75,28 @@ class SecureSession(BaseSession): def _initialize_authenticated_encryption_for_remote_peer(self) -> None: self.remote_encrypter = Encrypter(self.remote_encryption_parameters) - async def next_msg_len(self) -> int: - return await self.conn.next_msg_len() + def encrypt(self, data: bytes) -> bytes: + encrypted_data = self.local_encrypter.encrypt(data) + tag = self.local_encrypter.authenticate(encrypted_data) + return encrypted_data + tag - def _reset_internal_buffer(self) -> None: - self.buf = io.BytesIO() - self.low_watermark = 0 - self.high_watermark = 0 - - def _drain(self, n: int) -> bytes: - if self.low_watermark == self.high_watermark: - return bytes() - - data = self.buf.getbuffer()[self.low_watermark : self.high_watermark] - - if n is None: - n = len(data) - result = data[:n].tobytes() - self.low_watermark += len(result) - - if self.low_watermark == self.high_watermark: - del data # free the memoryview so we can free the underlying BytesIO - self.buf.close() - self._reset_internal_buffer() - return result - - async def _fill(self) -> None: - msg = await self.read_msg() - self.buf.write(msg) - self.low_watermark = 0 - self.high_watermark = len(msg) - - async def read(self, n: int = None) -> bytes: - if n == 0: - return bytes() - - data_from_buffer = self._drain(n) - if len(data_from_buffer) > 0: - return data_from_buffer - - next_length = await self.next_msg_len() - - if n < next_length: - await self._fill() - return self._drain(n) - else: - return await self.read_msg() - - async def read_msg(self) -> bytes: - msg = await self.conn.read_msg() + def decrypt(self, data: bytes) -> bytes: try: - decrypted_msg = self.remote_encrypter.decrypt_if_valid(msg) + decrypted_data = self.remote_encrypter.decrypt_if_valid(data) except InvalidMACException as e: raise DecryptionFailedException() from e - return decrypted_msg - - async def write(self, data: bytes) -> None: - await self.write_msg(data) + return decrypted_data async def write_msg(self, msg: bytes) -> None: - encrypted_data = self.local_encrypter.encrypt(msg) - tag = self.local_encrypter.authenticate(encrypted_data) - await self.conn.write_msg(encrypted_data + tag) + data_encrypted = self.encrypt(msg) + await self.read_writer.write_msg(data_encrypted) + + async def read_msg(self) -> bytes: + msg_encrypted = await self.read_writer.read_msg() + return self.decrypt(msg_encrypted) + + async def close(self) -> None: + await self.read_writer.close() @dataclass(frozen=True) @@ -387,22 +337,20 @@ def _mk_session_from( if session_parameters.order < 0: key_set1, key_set2 = key_set2, key_set1 - + secio_read_writer = SecioMsgReadWriter(key_set1, key_set2, conn) session = SecureSession( session_parameters.local_peer, local_private_key, - key_set1, session_parameters.remote_peer, - key_set2, - conn, + secio_read_writer, is_initiator, ) return session async def _finish_handshake(session: SecureSession, remote_nonce: bytes) -> bytes: - await session.write_msg(remote_nonce) - return await session.read_msg() + await session.conn.write_msg(remote_nonce) + return await session.conn.read_msg() async def create_secure_session( diff --git a/libp2p/security/noise/connection.py b/libp2p/security/secure_session.py similarity index 63% rename from libp2p/security/noise/connection.py rename to libp2p/security/secure_session.py index 29bbc8bc..9bbc00a2 100644 --- a/libp2p/security/noise/connection.py +++ b/libp2p/security/secure_session.py @@ -1,43 +1,29 @@ import io -from noise.connection import NoiseConnection as NoiseState - from libp2p.crypto.keys import PrivateKey -from libp2p.network.connection.raw_connection_interface import IRawConnection +from libp2p.io.abc import EncryptedMsgReadWriter from libp2p.peer.id import ID from libp2p.security.base_session import BaseSession -from libp2p.security.noise.io import MsgReadWriter, NoiseTransportReadWriter -class NoiseConnection(BaseSession): +class SecureSession(BaseSession): buf: io.BytesIO low_watermark: int high_watermark: int - read_writer: IRawConnection - noise_state: NoiseState - def __init__( self, local_peer: ID, local_private_key: PrivateKey, remote_peer: ID, - conn: IRawConnection, + conn: EncryptedMsgReadWriter, is_initiator: bool, - noise_state: NoiseState, - # remote_permanent_pubkey ) -> None: super().__init__(local_peer, local_private_key, is_initiator, remote_peer) self.conn = conn - self.noise_state = noise_state + self._reset_internal_buffer() - def get_msg_read_writer(self) -> MsgReadWriter: - return NoiseTransportReadWriter(self.conn, self.noise_state) - - async def close(self) -> None: - await self.conn.close() - def _reset_internal_buffer(self) -> None: self.buf = io.BytesIO() self.low_watermark = 0 @@ -60,6 +46,11 @@ class NoiseConnection(BaseSession): self._reset_internal_buffer() return result + def _fill(self, msg: bytes) -> None: + self.buf.write(msg) + self.low_watermark = 0 + self.high_watermark = len(msg) + async def read(self, n: int = None) -> bytes: if n == 0: return bytes() @@ -68,21 +59,16 @@ class NoiseConnection(BaseSession): if len(data_from_buffer) > 0: return data_from_buffer - msg = await self.read_msg() + msg = await self.conn.read_msg() if n < len(msg): - self.buf.write(msg) - self.low_watermark = 0 - self.high_watermark = len(msg) + self._fill(msg) return self._drain(n) else: return msg - async def read_msg(self) -> bytes: - return await self.get_msg_read_writer().read_msg() - async def write(self, data: bytes) -> None: - await self.write_msg(data) + await self.conn.write_msg(data) - async def write_msg(self, msg: bytes) -> None: - await self.get_msg_read_writer().write_msg(msg) + async def close(self) -> None: + await self.conn.close() From f0df2d189f5c25eb93e87431e7adcea002033200 Mon Sep 17 00:00:00 2001 From: mhchia Date: Mon, 17 Feb 2020 23:43:00 +0800 Subject: [PATCH 09/22] Plaintext: use existing msgio reader --- libp2p/io/msgio.py | 2 +- libp2p/security/insecure/io.py | 19 ------------------- libp2p/security/insecure/transport.py | 8 +++++++- libp2p/utils.py | 17 ----------------- 4 files changed, 8 insertions(+), 38 deletions(-) delete mode 100644 libp2p/security/insecure/io.py diff --git a/libp2p/io/msgio.py b/libp2p/io/msgio.py index 38710ff5..837f642c 100644 --- a/libp2p/io/msgio.py +++ b/libp2p/io/msgio.py @@ -5,7 +5,7 @@ from that repo: "a simple package to r/w length-delimited slices." NOTE: currently missing the capability to indicate lengths by "varint" method. """ -# TODO unify w/ https://github.com/libp2p/py-libp2p/blob/1aed52856f56a4b791696bbcbac31b5f9c2e88c9/libp2p/utils.py#L85-L99 # noqa: E501 + from typing import Optional from libp2p.io.abc import MsgReadWriteCloser, Reader, ReadWriteCloser diff --git a/libp2p/security/insecure/io.py b/libp2p/security/insecure/io.py deleted file mode 100644 index 1cbff365..00000000 --- a/libp2p/security/insecure/io.py +++ /dev/null @@ -1,19 +0,0 @@ -from libp2p.io.abc import MsgReadWriteCloser, ReadWriteCloser -from libp2p.utils import encode_fixedint_prefixed, read_fixedint_prefixed - - -class PlaintextHandshakeReadWriter(MsgReadWriteCloser): - conn: ReadWriteCloser - - def __init__(self, conn: ReadWriteCloser) -> None: - self.conn = conn - - async def read_msg(self) -> bytes: - return await read_fixedint_prefixed(self.conn) - - async def write_msg(self, msg: bytes) -> None: - encoded_msg_bytes = encode_fixedint_prefixed(msg) - await self.conn.write(encoded_msg_bytes) - - async def close(self) -> None: - await self.conn.close() diff --git a/libp2p/security/insecure/transport.py b/libp2p/security/insecure/transport.py index 28e76412..052d342c 100644 --- a/libp2p/security/insecure/transport.py +++ b/libp2p/security/insecure/transport.py @@ -5,6 +5,7 @@ from libp2p.crypto.keys import PrivateKey, PublicKey from libp2p.crypto.pb import crypto_pb2 from libp2p.crypto.serialization import deserialize_public_key from libp2p.io.abc import ReadWriteCloser +from libp2p.io.msgio import BaseMsgReadWriter from libp2p.network.connection.exceptions import RawConnError from libp2p.network.connection.raw_connection_interface import IRawConnection from libp2p.peer.id import ID @@ -14,7 +15,6 @@ from libp2p.security.exceptions import HandshakeFailure from libp2p.security.secure_conn_interface import ISecureConn from libp2p.typing import TProtocol -from .io import PlaintextHandshakeReadWriter from .pb import plaintext_pb2 # Reference: https://github.com/libp2p/go-libp2p-core/blob/master/sec/insecure/insecure.go @@ -22,6 +22,12 @@ from .pb import plaintext_pb2 PLAINTEXT_PROTOCOL_ID = TProtocol("/plaintext/2.0.0") +SIZE_PLAINTEXT_LEN_BYTES = 4 + + +class PlaintextHandshakeReadWriter(BaseMsgReadWriter): + size_len_bytes = SIZE_PLAINTEXT_LEN_BYTES + class InsecureSession(BaseSession): def __init__( diff --git a/libp2p/utils.py b/libp2p/utils.py index 3d0794a1..64937829 100644 --- a/libp2p/utils.py +++ b/libp2p/utils.py @@ -78,20 +78,3 @@ async def read_delim(reader: Reader) -> bytes: f'`msg_bytes` is not delimited by b"\\n": `msg_bytes`={msg_bytes!r}' ) return msg_bytes[:-1] - - -SIZE_LEN_BYTES = 4 - -# Fixed-prefixed read/write, used by "/plaintext/2.0.0". -# Reference: https://github.com/libp2p/go-msgio/blob/d5bbf59d3c4240266b1d2e5df9dc993454c42011/num.go#L11-L33 # noqa: E501 # noqa: E501 - - -def encode_fixedint_prefixed(msg_bytes: bytes) -> bytes: - len_prefix = len(msg_bytes).to_bytes(SIZE_LEN_BYTES, "big") - return len_prefix + msg_bytes - - -async def read_fixedint_prefixed(reader: Reader) -> bytes: - len_bytes = await reader.read(SIZE_LEN_BYTES) - len_int = int.from_bytes(len_bytes, "big") - return await reader.read(len_int) From e02bba93b1f58baee249bdeabaa10cef2eeaa7f4 Mon Sep 17 00:00:00 2001 From: mhchia Date: Mon, 17 Feb 2020 23:55:13 +0800 Subject: [PATCH 10/22] Noise: fix docs `connection.py` is removed. --- docs/libp2p.security.noise.rst | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/docs/libp2p.security.noise.rst b/docs/libp2p.security.noise.rst index a6aaa493..65ae79ad 100644 --- a/docs/libp2p.security.noise.rst +++ b/docs/libp2p.security.noise.rst @@ -11,10 +11,26 @@ Subpackages Submodules ---------- -libp2p.security.noise.connection module +libp2p.security.noise.exceptions module --------------------------------------- -.. automodule:: libp2p.security.noise.connection +.. automodule:: libp2p.security.noise.exceptions + :members: + :undoc-members: + :show-inheritance: + +libp2p.security.noise.io module +------------------------------- + +.. automodule:: libp2p.security.noise.io + :members: + :undoc-members: + :show-inheritance: + +libp2p.security.noise.messages module +------------------------------------- + +.. automodule:: libp2p.security.noise.messages :members: :undoc-members: :show-inheritance: From 48a7c1a9699103904d0440f23373bcf580492828 Mon Sep 17 00:00:00 2001 From: mhchia Date: Tue, 18 Feb 2020 22:43:29 +0800 Subject: [PATCH 11/22] Security: ensure remote pubkeys are stored --- libp2p/security/base_session.py | 12 +++++++----- libp2p/security/insecure/transport.py | 25 +++++++++++++++++-------- libp2p/security/noise/patterns.py | 23 ++++++++++++----------- libp2p/security/secio/transport.py | 14 +++++++++----- libp2p/security/secure_session.py | 14 +++++++++++--- 5 files changed, 56 insertions(+), 32 deletions(-) diff --git a/libp2p/security/base_session.py b/libp2p/security/base_session.py index cce1b6cf..abc3abb6 100644 --- a/libp2p/security/base_session.py +++ b/libp2p/security/base_session.py @@ -11,20 +11,22 @@ class BaseSession(ISecureConn): local_peer: ID local_private_key: PrivateKey - remote_peer_id: ID + remote_peer: ID remote_permanent_pubkey: PublicKey def __init__( self, + *, local_peer: ID, local_private_key: PrivateKey, + remote_peer: ID, + remote_permanent_pubkey: PublicKey, is_initiator: bool, - peer_id: Optional[ID] = None, ) -> None: self.local_peer = local_peer self.local_private_key = local_private_key - self.remote_peer_id = peer_id - self.remote_permanent_pubkey = None + self.remote_peer = remote_peer + self.remote_permanent_pubkey = remote_permanent_pubkey self.is_initiator = is_initiator def get_local_peer(self) -> ID: @@ -34,7 +36,7 @@ class BaseSession(ISecureConn): return self.local_private_key def get_remote_peer(self) -> ID: - return self.remote_peer_id + return self.remote_peer def get_remote_public_key(self) -> Optional[PublicKey]: return self.remote_permanent_pubkey diff --git a/libp2p/security/insecure/transport.py b/libp2p/security/insecure/transport.py index 052d342c..861ca716 100644 --- a/libp2p/security/insecure/transport.py +++ b/libp2p/security/insecure/transport.py @@ -1,5 +1,3 @@ -from typing import Optional - from libp2p.crypto.exceptions import MissingDeserializerError from libp2p.crypto.keys import PrivateKey, PublicKey from libp2p.crypto.pb import crypto_pb2 @@ -32,13 +30,21 @@ class PlaintextHandshakeReadWriter(BaseMsgReadWriter): class InsecureSession(BaseSession): def __init__( self, + *, local_peer: ID, local_private_key: PrivateKey, - conn: ReadWriteCloser, + remote_peer: ID, + remote_permanent_pubkey: PublicKey, is_initiator: bool, - peer_id: Optional[ID] = None, + conn: ReadWriteCloser, ) -> None: - super().__init__(local_peer, local_private_key, is_initiator, peer_id) + super().__init__( + local_peer=local_peer, + local_private_key=local_private_key, + remote_peer=remote_peer, + remote_permanent_pubkey=remote_permanent_pubkey, + is_initiator=is_initiator, + ) self.conn = conn async def write(self, data: bytes) -> None: @@ -102,11 +108,14 @@ async def run_handshake( ) secure_conn = InsecureSession( - local_peer, local_private_key, conn, is_initiator, received_peer_id + local_peer=local_peer, + local_private_key=local_private_key, + remote_peer=received_peer_id, + remote_permanent_pubkey=received_pubkey, + is_initiator=is_initiator, + conn=conn, ) - # Nothing is wrong. Store the `pubkey` and `peer_id` in the session. - secure_conn.remote_permanent_pubkey = received_pubkey # TODO: Store `pubkey` and `peer_id` to `PeerStore` return secure_conn diff --git a/libp2p/security/noise/patterns.py b/libp2p/security/noise/patterns.py index 64d49656..cb5e7bfa 100644 --- a/libp2p/security/noise/patterns.py +++ b/libp2p/security/noise/patterns.py @@ -108,11 +108,12 @@ class PatternXX(BasePattern): ) transport_read_writer = NoiseTransportReadWriter(conn, noise_state) return SecureSession( - self.local_peer, - self.libp2p_privkey, - remote_peer_id_from_pubkey, - transport_read_writer, - False, + local_peer=self.local_peer, + local_private_key=self.libp2p_privkey, + remote_peer=remote_peer_id_from_pubkey, + remote_permanent_pubkey=remote_pubkey, + is_initiator=False, + conn=transport_read_writer, ) async def handshake_outbound( @@ -161,11 +162,11 @@ class PatternXX(BasePattern): "handshake is done but it is not marked as finished in `noise_state`" ) transport_read_writer = NoiseTransportReadWriter(conn, noise_state) - return SecureSession( - self.local_peer, - self.libp2p_privkey, - remote_peer, - transport_read_writer, - False, + local_peer=self.local_peer, + local_private_key=self.libp2p_privkey, + remote_peer=remote_peer_id_from_pubkey, + remote_permanent_pubkey=remote_pubkey, + is_initiator=True, + conn=transport_read_writer, ) diff --git a/libp2p/security/secio/transport.py b/libp2p/security/secio/transport.py index ba774d99..4759ccc3 100644 --- a/libp2p/security/secio/transport.py +++ b/libp2p/security/secio/transport.py @@ -338,12 +338,16 @@ def _mk_session_from( if session_parameters.order < 0: key_set1, key_set2 = key_set2, key_set1 secio_read_writer = SecioMsgReadWriter(key_set1, key_set2, conn) + remote_permanent_pubkey = ( + session_parameters.remote_encryption_parameters.permanent_public_key + ) session = SecureSession( - session_parameters.local_peer, - local_private_key, - session_parameters.remote_peer, - secio_read_writer, - is_initiator, + local_peer=session_parameters.local_peer, + local_private_key=local_private_key, + remote_peer=session_parameters.remote_peer, + remote_permanent_pubkey=remote_permanent_pubkey, + is_initiator=is_initiator, + conn=secio_read_writer, ) return session diff --git a/libp2p/security/secure_session.py b/libp2p/security/secure_session.py index 9bbc00a2..dbabd1a7 100644 --- a/libp2p/security/secure_session.py +++ b/libp2p/security/secure_session.py @@ -1,6 +1,6 @@ import io -from libp2p.crypto.keys import PrivateKey +from libp2p.crypto.keys import PrivateKey, PublicKey from libp2p.io.abc import EncryptedMsgReadWriter from libp2p.peer.id import ID from libp2p.security.base_session import BaseSession @@ -13,13 +13,21 @@ class SecureSession(BaseSession): def __init__( self, + *, local_peer: ID, local_private_key: PrivateKey, remote_peer: ID, - conn: EncryptedMsgReadWriter, + remote_permanent_pubkey: PublicKey, is_initiator: bool, + conn: EncryptedMsgReadWriter, ) -> None: - super().__init__(local_peer, local_private_key, is_initiator, remote_peer) + super().__init__( + local_peer=local_peer, + local_private_key=local_private_key, + remote_peer=remote_peer, + remote_permanent_pubkey=remote_permanent_pubkey, + is_initiator=is_initiator, + ) self.conn = conn self._reset_internal_buffer() From 1d2a976597b02692faeba9242bb23d8458c0110a Mon Sep 17 00:00:00 2001 From: mhchia Date: Tue, 18 Feb 2020 22:55:48 +0800 Subject: [PATCH 12/22] Remove TODOs --- libp2p/security/noise/transport.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/libp2p/security/noise/transport.py b/libp2p/security/noise/transport.py index 5ee3ba52..80bb6ee1 100644 --- a/libp2p/security/noise/transport.py +++ b/libp2p/security/noise/transport.py @@ -16,6 +16,10 @@ class Transport(ISecureTransport): local_peer: ID early_data: bytes with_noise_pipes: bool + + # NOTE: Implementations that support Noise Pipes must decide whether to use + # an XX or IK handshake based on whether they possess a cached static + # Noise key for the remote peer. # TODO: A storage of seen noise static keys for pattern IK? def __init__( @@ -46,15 +50,9 @@ class Transport(ISecureTransport): ) async def secure_inbound(self, conn: IRawConnection) -> ISecureConn: - # TODO: SecureInbound attempts to complete a noise-libp2p handshake initiated - # by a remote peer over the given InsecureConnection. pattern = self.get_pattern() return await pattern.handshake_inbound(conn) async def secure_outbound(self, conn: IRawConnection, peer_id: ID) -> ISecureConn: - # TODO: Validate libp2p pubkey with `peer_id`. Abort if not correct. - # NOTE: Implementations that support Noise Pipes must decide whether to use - # an XX or IK handshake based on whether they possess a cached static - # Noise key for the remote peer. pattern = self.get_pattern() return await pattern.handshake_outbound(conn, peer_id) From 13e8f496a7bd5c8d166fde4f34214baccca1a298 Mon Sep 17 00:00:00 2001 From: mhchia Date: Wed, 19 Feb 2020 23:15:03 +0800 Subject: [PATCH 13/22] Noise: add noise option in the factories and tests --- libp2p/security/security_multistream.py | 1 - libp2p/tools/factories.py | 197 ++++++++++++++------ libp2p/tools/interop/daemon.py | 10 +- tests/conftest.py | 10 +- tests/examples/test_examples.py | 7 +- tests/host/test_ping.py | 14 +- tests/host/test_routed_host.py | 7 +- tests/identity/identify/test_protocol.py | 7 +- tests/libp2p/test_libp2p.py | 42 +++-- tests/network/conftest.py | 16 +- tests/network/test_notify.py | 4 +- tests/network/test_swarm.py | 18 +- tests/protocol_muxer/test_protocol_muxer.py | 26 +-- tests/pubsub/test_floodsub.py | 5 +- tests/pubsub/test_pubsub.py | 16 +- tests/security/test_security_multistream.py | 93 +++------ tests/stream_muxer/conftest.py | 12 +- tests_interop/conftest.py | 27 ++- tests_interop/test_bindings.py | 6 +- tests_interop/test_echo.py | 21 ++- tests_interop/test_pubsub.py | 4 +- 21 files changed, 331 insertions(+), 212 deletions(-) diff --git a/libp2p/security/security_multistream.py b/libp2p/security/security_multistream.py index 0507a524..c83a2aa4 100644 --- a/libp2p/security/security_multistream.py +++ b/libp2p/security/security_multistream.py @@ -27,7 +27,6 @@ class SecurityMultistream(ABC): Go implementation: github.com/libp2p/go-conn-security-multistream/ssms.go """ - # NOTE: Can be changed to `typing.OrderedDict` since Python 3.7.2. transports: "OrderedDict[TProtocol, ISecureTransport]" multiselect: Multiselect multiselect_client: MultiselectClient diff --git a/libp2p/tools/factories.py b/libp2p/tools/factories.py index 7fad8360..a97c8889 100644 --- a/libp2p/tools/factories.py +++ b/libp2p/tools/factories.py @@ -1,4 +1,4 @@ -from typing import Any, AsyncIterator, Dict, List, Sequence, Tuple, cast +from typing import Any, AsyncIterator, Callable, Dict, List, Sequence, Tuple, cast from async_exit_stack import AsyncExitStack from async_generator import asynccontextmanager @@ -33,6 +33,7 @@ from libp2p.security.noise.messages import ( NoiseHandshakePayload, make_handshake_payload_sig, ) +from libp2p.security.noise.transport import PROTOCOL_ID as NOISE_PROTOCOL_ID from libp2p.security.noise.transport import Transport as NoiseTransport import libp2p.security.secio.transport as secio from libp2p.security.secure_conn_interface import ISecureConn @@ -41,20 +42,26 @@ from libp2p.stream_muxer.mplex.mplex import MPLEX_PROTOCOL_ID, Mplex from libp2p.stream_muxer.mplex.mplex_stream import MplexStream from libp2p.tools.constants import GOSSIPSUB_PARAMS from libp2p.transport.tcp.tcp import TCP -from libp2p.transport.typing import TMuxerOptions +from libp2p.transport.typing import TMuxerOptions, TSecurityOptions from libp2p.transport.upgrader import TransportUpgrader from libp2p.typing import TProtocol from .constants import FLOODSUB_PROTOCOL_ID, GOSSIPSUB_PROTOCOL_ID, LISTEN_MADDR from .utils import connect, connect_swarm +DEFAULT_SECURITY_PROTOCOL_ID = PLAINTEXT_PROTOCOL_ID + + +def default_key_pair_factory() -> KeyPair: + return generate_new_rsa_identity() + class IDFactory(factory.Factory): class Meta: model = ID peer_id_bytes = factory.LazyFunction( - lambda: generate_peer_id_from(generate_new_rsa_identity()) + lambda: generate_peer_id_from(default_key_pair_factory()) ) @@ -64,15 +71,6 @@ def initialize_peerstore_with_our_keypair(self_id: ID, key_pair: KeyPair) -> Pee return peer_store -def security_transport_factory( - is_secure: bool, key_pair: KeyPair -) -> Dict[TProtocol, ISecureTransport]: - if not is_secure: - return {PLAINTEXT_PROTOCOL_ID: InsecureTransport(key_pair)} - else: - return {secio.ID: secio.Transport(key_pair)} - - def noise_static_key_factory() -> PrivateKey: return create_ed25519_key_pair().private_key @@ -88,15 +86,52 @@ def noise_handshake_payload_factory() -> NoiseHandshakePayload: ) -def noise_transport_factory() -> NoiseTransport: +def plaintext_transport_factory(key_pair: KeyPair) -> ISecureTransport: + return InsecureTransport(key_pair) + + +def secio_transport_factory(key_pair: KeyPair) -> ISecureTransport: + return secio.Transport(key_pair) + + +def noise_transport_factory(key_pair: KeyPair) -> ISecureTransport: return NoiseTransport( - libp2p_keypair=create_secp256k1_key_pair(), + libp2p_keypair=key_pair, noise_privkey=noise_static_key_factory(), early_data=None, with_noise_pipes=False, ) +def security_options_factory_factory( + protocol_id: TProtocol = None +) -> Callable[[KeyPair], TSecurityOptions]: + if protocol_id is None: + protocol_id = DEFAULT_SECURITY_PROTOCOL_ID + + def security_options_factory(key_pair: KeyPair) -> TSecurityOptions: + transport_factory: Callable[[KeyPair], ISecureTransport] + if protocol_id == PLAINTEXT_PROTOCOL_ID: + transport_factory = plaintext_transport_factory + elif protocol_id == secio.ID: + transport_factory = secio_transport_factory + elif protocol_id == NOISE_PROTOCOL_ID: + transport_factory = noise_transport_factory + else: + raise Exception(f"security transport {protocol_id} is not supported") + return {protocol_id: transport_factory(key_pair)} + + return security_options_factory + + +def mplex_transport_factory() -> TMuxerOptions: + return {MPLEX_PROTOCOL_ID: Mplex} + + +def default_muxer_transport_factory() -> TMuxerOptions: + return mplex_transport_factory() + + @asynccontextmanager async def raw_conn_factory( nursery: trio.Nursery @@ -124,8 +159,12 @@ async def raw_conn_factory( async def noise_conn_factory( nursery: trio.Nursery ) -> AsyncIterator[Tuple[ISecureConn, ISecureConn]]: - local_transport = noise_transport_factory() - remote_transport = noise_transport_factory() + local_transport = cast( + NoiseTransport, noise_transport_factory(create_secp256k1_key_pair()) + ) + remote_transport = cast( + NoiseTransport, noise_transport_factory(create_secp256k1_key_pair()) + ) local_secure_conn: ISecureConn = None remote_secure_conn: ISecureConn = None @@ -158,9 +197,9 @@ class SwarmFactory(factory.Factory): model = Swarm class Params: - is_secure = False - key_pair = factory.LazyFunction(generate_new_rsa_identity) - muxer_opt = {MPLEX_PROTOCOL_ID: Mplex} + key_pair = factory.LazyFunction(default_key_pair_factory) + security_protocol = DEFAULT_SECURITY_PROTOCOL_ID + muxer_opt = factory.LazyFunction(default_muxer_transport_factory) peer_id = factory.LazyAttribute(lambda o: generate_peer_id_from(o.key_pair)) peerstore = factory.LazyAttribute( @@ -168,7 +207,8 @@ class SwarmFactory(factory.Factory): ) upgrader = factory.LazyAttribute( lambda o: TransportUpgrader( - security_transport_factory(o.is_secure, o.key_pair), o.muxer_opt + (security_options_factory_factory(o.security_protocol))(o.key_pair), + o.muxer_opt, ) ) transport = factory.LazyFunction(TCP) @@ -176,7 +216,10 @@ class SwarmFactory(factory.Factory): @classmethod @asynccontextmanager async def create_and_listen( - cls, is_secure: bool, key_pair: KeyPair = None, muxer_opt: TMuxerOptions = None + cls, + key_pair: KeyPair = None, + security_protocol: TProtocol = None, + muxer_opt: TMuxerOptions = None, ) -> AsyncIterator[Swarm]: # `factory.Factory.__init__` does *not* prepare a *default value* if we pass # an argument explicitly with `None`. If an argument is `None`, we don't pass it to @@ -184,9 +227,11 @@ class SwarmFactory(factory.Factory): optional_kwargs: Dict[str, Any] = {} if key_pair is not None: optional_kwargs["key_pair"] = key_pair + if security_protocol is not None: + optional_kwargs["security_protocol"] = security_protocol if muxer_opt is not None: optional_kwargs["muxer_opt"] = muxer_opt - swarm = cls(is_secure=is_secure, **optional_kwargs) + swarm = cls(**optional_kwargs) async with background_trio_service(swarm): await swarm.listen(LISTEN_MADDR) yield swarm @@ -194,12 +239,17 @@ class SwarmFactory(factory.Factory): @classmethod @asynccontextmanager async def create_batch_and_listen( - cls, is_secure: bool, number: int, muxer_opt: TMuxerOptions = None + cls, + number: int, + security_protocol: TProtocol = None, + muxer_opt: TMuxerOptions = None, ) -> AsyncIterator[Tuple[Swarm, ...]]: async with AsyncExitStack() as stack: ctx_mgrs = [ await stack.enter_async_context( - cls.create_and_listen(is_secure=is_secure, muxer_opt=muxer_opt) + cls.create_and_listen( + security_protocol=security_protocol, muxer_opt=muxer_opt + ) ) for _ in range(number) ] @@ -211,17 +261,27 @@ class HostFactory(factory.Factory): model = BasicHost class Params: - is_secure = False - key_pair = factory.LazyFunction(generate_new_rsa_identity) + key_pair = factory.LazyFunction(default_key_pair_factory) + security_protocol: TProtocol = None + muxer_opt = factory.LazyFunction(default_muxer_transport_factory) - network = factory.LazyAttribute(lambda o: SwarmFactory(is_secure=o.is_secure)) + network = factory.LazyAttribute( + lambda o: SwarmFactory( + security_protocol=o.security_protocol, muxer_opt=o.muxer_opt + ) + ) @classmethod @asynccontextmanager async def create_batch_and_listen( - cls, is_secure: bool, number: int + cls, + number: int, + security_protocol: TProtocol = None, + muxer_opt: TMuxerOptions = None, ) -> AsyncIterator[Tuple[BasicHost, ...]]: - async with SwarmFactory.create_batch_and_listen(is_secure, number) as swarms: + async with SwarmFactory.create_batch_and_listen( + number, security_protocol=security_protocol, muxer_opt=muxer_opt + ) as swarms: hosts = tuple(BasicHost(swarm) for swarm in swarms) yield hosts @@ -245,20 +305,29 @@ class RoutedHostFactory(factory.Factory): model = RoutedHost class Params: - is_secure = False + key_pair = factory.LazyFunction(default_key_pair_factory) + security_protocol: TProtocol = None + muxer_opt = factory.LazyFunction(default_muxer_transport_factory) network = factory.LazyAttribute( - lambda o: HostFactory(is_secure=o.is_secure).get_network() + lambda o: HostFactory( + security_protocol=o.security_protocol, muxer_opt=o.muxer_opt + ).get_network() ) router = factory.LazyFunction(DummyRouter) @classmethod @asynccontextmanager async def create_batch_and_listen( - cls, is_secure: bool, number: int + cls, + number: int, + security_protocol: TProtocol = None, + muxer_opt: TMuxerOptions = None, ) -> AsyncIterator[Tuple[RoutedHost, ...]]: routing_table = DummyRouter() - async with HostFactory.create_batch_and_listen(is_secure, number) as hosts: + async with HostFactory.create_batch_and_listen( + number, security_protocol=security_protocol, muxer_opt=muxer_opt + ) as hosts: for host in hosts: routing_table._add_peer(host.get_id(), host.get_addrs()) routed_hosts = tuple( @@ -319,11 +388,14 @@ class PubsubFactory(factory.Factory): cls, number: int, routers: Sequence[IPubsubRouter], - is_secure: bool = False, cache_size: int = None, strict_signing: bool = False, + security_protocol: TProtocol = None, + muxer_opt: TMuxerOptions = None, ) -> AsyncIterator[Tuple[Pubsub, ...]]: - async with HostFactory.create_batch_and_listen(is_secure, number) as hosts: + async with HostFactory.create_batch_and_listen( + number, security_protocol=security_protocol, muxer_opt=muxer_opt + ) as hosts: # Pubsubs should exit before hosts async with AsyncExitStack() as stack: pubsubs = [ @@ -339,17 +411,23 @@ class PubsubFactory(factory.Factory): async def create_batch_with_floodsub( cls, number: int, - is_secure: bool = False, cache_size: int = None, strict_signing: bool = False, protocols: Sequence[TProtocol] = None, + security_protocol: TProtocol = None, + muxer_opt: TMuxerOptions = None, ) -> AsyncIterator[Tuple[Pubsub, ...]]: if protocols is not None: floodsubs = FloodsubFactory.create_batch(number, protocols=list(protocols)) else: floodsubs = FloodsubFactory.create_batch(number) async with cls._create_batch_with_router( - number, floodsubs, is_secure, cache_size, strict_signing + number, + floodsubs, + cache_size, + strict_signing, + security_protocol=security_protocol, + muxer_opt=muxer_opt, ) as pubsubs: yield pubsubs @@ -359,7 +437,6 @@ class PubsubFactory(factory.Factory): cls, number: int, *, - is_secure: bool = False, cache_size: int = None, strict_signing: bool = False, protocols: Sequence[TProtocol] = None, @@ -371,6 +448,8 @@ class PubsubFactory(factory.Factory): gossip_history: int = GOSSIPSUB_PARAMS.gossip_history, heartbeat_interval: float = GOSSIPSUB_PARAMS.heartbeat_interval, heartbeat_initial_delay: float = GOSSIPSUB_PARAMS.heartbeat_initial_delay, + security_protocol: TProtocol = None, + muxer_opt: TMuxerOptions = None, ) -> AsyncIterator[Tuple[Pubsub, ...]]: if protocols is not None: gossipsubs = GossipsubFactory.create_batch( @@ -395,7 +474,12 @@ class PubsubFactory(factory.Factory): ) async with cls._create_batch_with_router( - number, gossipsubs, is_secure, cache_size, strict_signing + number, + gossipsubs, + cache_size, + strict_signing, + security_protocol=security_protocol, + muxer_opt=muxer_opt, ) as pubsubs: async with AsyncExitStack() as stack: for router in gossipsubs: @@ -405,10 +489,10 @@ class PubsubFactory(factory.Factory): @asynccontextmanager async def swarm_pair_factory( - is_secure: bool, muxer_opt: TMuxerOptions = None + security_protocol: TProtocol = None, muxer_opt: TMuxerOptions = None ) -> AsyncIterator[Tuple[Swarm, Swarm]]: async with SwarmFactory.create_batch_and_listen( - is_secure, 2, muxer_opt=muxer_opt + 2, security_protocol=security_protocol, muxer_opt=muxer_opt ) as swarms: await connect_swarm(swarms[0], swarms[1]) yield swarms[0], swarms[1] @@ -416,18 +500,22 @@ async def swarm_pair_factory( @asynccontextmanager async def host_pair_factory( - is_secure: bool + security_protocol: TProtocol = None, muxer_opt: TMuxerOptions = None ) -> AsyncIterator[Tuple[BasicHost, BasicHost]]: - async with HostFactory.create_batch_and_listen(is_secure, 2) as hosts: + async with HostFactory.create_batch_and_listen( + 2, security_protocol=security_protocol, muxer_opt=muxer_opt + ) as hosts: await connect(hosts[0], hosts[1]) yield hosts[0], hosts[1] @asynccontextmanager async def swarm_conn_pair_factory( - is_secure: bool, muxer_opt: TMuxerOptions = None + security_protocol: TProtocol = None, muxer_opt: TMuxerOptions = None ) -> AsyncIterator[Tuple[SwarmConn, SwarmConn]]: - async with swarm_pair_factory(is_secure) as swarms: + async with swarm_pair_factory( + security_protocol=security_protocol, muxer_opt=muxer_opt + ) as swarms: conn_0 = swarms[0].connections[swarms[1].get_peer_id()] conn_1 = swarms[1].connections[swarms[0].get_peer_id()] yield cast(SwarmConn, conn_0), cast(SwarmConn, conn_1) @@ -435,10 +523,11 @@ async def swarm_conn_pair_factory( @asynccontextmanager async def mplex_conn_pair_factory( - is_secure: bool + security_protocol: TProtocol = None ) -> AsyncIterator[Tuple[Mplex, Mplex]]: - muxer_opt = {MPLEX_PROTOCOL_ID: Mplex} - async with swarm_conn_pair_factory(is_secure, muxer_opt=muxer_opt) as swarm_pair: + async with swarm_conn_pair_factory( + security_protocol=security_protocol, muxer_opt=default_muxer_transport_factory() + ) as swarm_pair: yield ( cast(Mplex, swarm_pair[0].muxed_conn), cast(Mplex, swarm_pair[1].muxed_conn), @@ -447,9 +536,11 @@ async def mplex_conn_pair_factory( @asynccontextmanager async def mplex_stream_pair_factory( - is_secure: bool + security_protocol: TProtocol = None ) -> AsyncIterator[Tuple[MplexStream, MplexStream]]: - async with mplex_conn_pair_factory(is_secure) as mplex_conn_pair_info: + async with mplex_conn_pair_factory( + security_protocol=security_protocol + ) as mplex_conn_pair_info: mplex_conn_0, mplex_conn_1 = mplex_conn_pair_info stream_0 = cast(MplexStream, await mplex_conn_0.open_stream()) await trio.sleep(0.01) @@ -463,7 +554,7 @@ async def mplex_stream_pair_factory( @asynccontextmanager async def net_stream_pair_factory( - is_secure: bool + security_protocol: TProtocol = None, muxer_opt: TMuxerOptions = None ) -> AsyncIterator[Tuple[INetStream, INetStream]]: protocol_id = TProtocol("/example/id/1") @@ -478,7 +569,9 @@ async def net_stream_pair_factory( stream_1 = stream await event_handler_finished.wait() - async with host_pair_factory(is_secure) as hosts: + async with host_pair_factory( + security_protocol=security_protocol, muxer_opt=muxer_opt + ) as hosts: hosts[1].set_stream_handler(protocol_id, handler) stream_0 = await hosts[0].new_stream(hosts[1].get_id(), [protocol_id]) diff --git a/libp2p/tools/interop/daemon.py b/libp2p/tools/interop/daemon.py index 3344255a..4dfcfb63 100644 --- a/libp2p/tools/interop/daemon.py +++ b/libp2p/tools/interop/daemon.py @@ -8,6 +8,8 @@ import trio from libp2p.peer.id import ID from libp2p.peer.peerinfo import PeerInfo, info_from_p2p_addr +from libp2p.security.insecure.transport import PLAINTEXT_PROTOCOL_ID +from libp2p.typing import TProtocol from .constants import LOCALHOST_IP from .envs import GO_BIN_PATH @@ -20,7 +22,7 @@ class P2PDProcess(BaseInteractiveProcess): def __init__( self, control_maddr: Multiaddr, - is_secure: bool, + security_protocol: TProtocol, is_pubsub_enabled: bool = True, is_gossipsub: bool = True, is_pubsub_signing: bool = False, @@ -28,7 +30,7 @@ class P2PDProcess(BaseInteractiveProcess): ) -> None: args = [f"-listen={control_maddr!s}"] # NOTE: To support `-insecure`, we need to hack `go-libp2p-daemon`. - if not is_secure: + if security_protocol == PLAINTEXT_PROTOCOL_ID: args.append("-insecure=true") if is_pubsub_enabled: args.append("-pubsub") @@ -85,7 +87,7 @@ class Daemon: async def make_p2pd( daemon_control_port: int, client_callback_port: int, - is_secure: bool, + security_protocol: TProtocol, is_pubsub_enabled: bool = True, is_gossipsub: bool = True, is_pubsub_signing: bool = False, @@ -94,7 +96,7 @@ async def make_p2pd( control_maddr = Multiaddr(f"/ip4/{LOCALHOST_IP}/tcp/{daemon_control_port}") p2pd_proc = P2PDProcess( control_maddr, - is_secure, + security_protocol, is_pubsub_enabled, is_gossipsub, is_pubsub_signing, diff --git a/tests/conftest.py b/tests/conftest.py index 48d705c2..c1a2b132 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,8 +4,8 @@ from libp2p.tools.factories import HostFactory @pytest.fixture -def is_host_secure(): - return False +def security_protocol(): + return None @pytest.fixture @@ -14,6 +14,8 @@ def num_hosts(): @pytest.fixture -async def hosts(num_hosts, is_host_secure, nursery): - async with HostFactory.create_batch_and_listen(is_host_secure, num_hosts) as _hosts: +async def hosts(num_hosts, security_protocol, nursery): + async with HostFactory.create_batch_and_listen( + num_hosts, security_protocol=security_protocol + ) as _hosts: yield _hosts diff --git a/tests/examples/test_examples.py b/tests/examples/test_examples.py index c8ce9ed8..df59bd24 100644 --- a/tests/examples/test_examples.py +++ b/tests/examples/test_examples.py @@ -92,8 +92,11 @@ async def no_common_protocol(host_a, host_b): "test", [(hello_world), (connect_write), (connect_read), (no_common_protocol)] ) @pytest.mark.trio -async def test_chat(test, is_host_secure): - async with HostFactory.create_batch_and_listen(is_host_secure, 2) as hosts: +async def test_chat(test, security_protocol): + print("!@# ", security_protocol) + async with HostFactory.create_batch_and_listen( + 2, security_protocol=security_protocol + ) as hosts: addr = hosts[0].get_addrs()[0] info = info_from_p2p_addr(addr) await hosts[1].connect(info) diff --git a/tests/host/test_ping.py b/tests/host/test_ping.py index 7a0f8db5..79285bc8 100644 --- a/tests/host/test_ping.py +++ b/tests/host/test_ping.py @@ -8,8 +8,11 @@ from libp2p.tools.factories import host_pair_factory @pytest.mark.trio -async def test_ping_once(is_host_secure): - async with host_pair_factory(is_host_secure) as (host_a, host_b): +async def test_ping_once(security_protocol): + async with host_pair_factory(security_protocol=security_protocol) as ( + host_a, + host_b, + ): stream = await host_b.new_stream(host_a.get_id(), (ID,)) some_ping = secrets.token_bytes(PING_LENGTH) await stream.write(some_ping) @@ -23,8 +26,11 @@ SOME_PING_COUNT = 3 @pytest.mark.trio -async def test_ping_several(is_host_secure): - async with host_pair_factory(is_host_secure) as (host_a, host_b): +async def test_ping_several(security_protocol): + async with host_pair_factory(security_protocol=security_protocol) as ( + host_a, + host_b, + ): stream = await host_b.new_stream(host_a.get_id(), (ID,)) for _ in range(SOME_PING_COUNT): some_ping = secrets.token_bytes(PING_LENGTH) diff --git a/tests/host/test_routed_host.py b/tests/host/test_routed_host.py index 4cfed6bf..7dfa8e7f 100644 --- a/tests/host/test_routed_host.py +++ b/tests/host/test_routed_host.py @@ -7,7 +7,7 @@ from libp2p.tools.factories import HostFactory, RoutedHostFactory @pytest.mark.trio async def test_host_routing_success(): - async with RoutedHostFactory.create_batch_and_listen(False, 2) as hosts: + async with RoutedHostFactory.create_batch_and_listen(2) as hosts: # forces to use routing as no addrs are provided await hosts[0].connect(PeerInfo(hosts[1].get_id(), [])) await hosts[1].connect(PeerInfo(hosts[0].get_id(), [])) @@ -15,10 +15,9 @@ async def test_host_routing_success(): @pytest.mark.trio async def test_host_routing_fail(): - is_secure = False async with RoutedHostFactory.create_batch_and_listen( - is_secure, 2 - ) as routed_hosts, HostFactory.create_batch_and_listen(is_secure, 1) as basic_hosts: + 2 + ) as routed_hosts, HostFactory.create_batch_and_listen(1) as basic_hosts: # routing fails because host_c does not use routing with pytest.raises(ConnectionFailure): await routed_hosts[0].connect(PeerInfo(basic_hosts[0].get_id(), [])) diff --git a/tests/identity/identify/test_protocol.py b/tests/identity/identify/test_protocol.py index 4bbdbcba..0d398ff9 100644 --- a/tests/identity/identify/test_protocol.py +++ b/tests/identity/identify/test_protocol.py @@ -6,8 +6,11 @@ from libp2p.tools.factories import host_pair_factory @pytest.mark.trio -async def test_identify_protocol(is_host_secure): - async with host_pair_factory(is_host_secure) as (host_a, host_b): +async def test_identify_protocol(security_protocol): + async with host_pair_factory(security_protocol=security_protocol) as ( + host_a, + host_b, + ): stream = await host_b.new_stream(host_a.get_id(), (ID,)) response = await stream.read() await stream.close() diff --git a/tests/libp2p/test_libp2p.py b/tests/libp2p/test_libp2p.py index 99a60bd5..8cf8e069 100644 --- a/tests/libp2p/test_libp2p.py +++ b/tests/libp2p/test_libp2p.py @@ -19,8 +19,10 @@ ACK_STR_3 = "ack_3:" @pytest.mark.trio -async def test_simple_messages(is_host_secure): - async with HostFactory.create_batch_and_listen(is_host_secure, 2) as hosts: +async def test_simple_messages(security_protocol): + async with HostFactory.create_batch_and_listen( + 2, security_protocol=security_protocol + ) as hosts: hosts[1].set_stream_handler( PROTOCOL_ID_0, create_echo_stream_handler(ACK_STR_0) ) @@ -38,8 +40,10 @@ async def test_simple_messages(is_host_secure): @pytest.mark.trio -async def test_double_response(is_host_secure): - async with HostFactory.create_batch_and_listen(is_host_secure, 2) as hosts: +async def test_double_response(security_protocol): + async with HostFactory.create_batch_and_listen( + 2, security_protocol=security_protocol + ) as hosts: async def double_response_stream_handler(stream): while True: @@ -78,11 +82,13 @@ async def test_double_response(is_host_secure): @pytest.mark.trio -async def test_multiple_streams(is_host_secure): +async def test_multiple_streams(security_protocol): # hosts[0] should be able to open a stream with hosts[1] and then vice versa. # Stream IDs should be generated uniquely so that the stream state is not overwritten - async with HostFactory.create_batch_and_listen(is_host_secure, 2) as hosts: + async with HostFactory.create_batch_and_listen( + 2, security_protocol=security_protocol + ) as hosts: hosts[0].set_stream_handler( PROTOCOL_ID_0, create_echo_stream_handler(ACK_STR_0) ) @@ -115,8 +121,10 @@ async def test_multiple_streams(is_host_secure): @pytest.mark.trio -async def test_multiple_streams_same_initiator_different_protocols(is_host_secure): - async with HostFactory.create_batch_and_listen(is_host_secure, 2) as hosts: +async def test_multiple_streams_same_initiator_different_protocols(security_protocol): + async with HostFactory.create_batch_and_listen( + 2, security_protocol=security_protocol + ) as hosts: hosts[1].set_stream_handler( PROTOCOL_ID_0, create_echo_stream_handler(ACK_STR_0) @@ -161,8 +169,10 @@ async def test_multiple_streams_same_initiator_different_protocols(is_host_secur @pytest.mark.trio -async def test_multiple_streams_two_initiators(is_host_secure): - async with HostFactory.create_batch_and_listen(is_host_secure, 2) as hosts: +async def test_multiple_streams_two_initiators(security_protocol): + async with HostFactory.create_batch_and_listen( + 2, security_protocol=security_protocol + ) as hosts: hosts[0].set_stream_handler( PROTOCOL_ID_2, create_echo_stream_handler(ACK_STR_2) ) @@ -217,8 +227,10 @@ async def test_multiple_streams_two_initiators(is_host_secure): @pytest.mark.trio -async def test_triangle_nodes_connection(is_host_secure): - async with HostFactory.create_batch_and_listen(is_host_secure, 3) as hosts: +async def test_triangle_nodes_connection(security_protocol): + async with HostFactory.create_batch_and_listen( + 3, security_protocol=security_protocol + ) as hosts: hosts[0].set_stream_handler( PROTOCOL_ID_0, create_echo_stream_handler(ACK_STR_0) @@ -268,8 +280,10 @@ async def test_triangle_nodes_connection(is_host_secure): @pytest.mark.trio -async def test_host_connect(is_host_secure): - async with HostFactory.create_batch_and_listen(is_host_secure, 2) as hosts: +async def test_host_connect(security_protocol): + async with HostFactory.create_batch_and_listen( + 2, security_protocol=security_protocol + ) as hosts: assert len(hosts[0].get_peerstore().peer_ids()) == 1 await connect(hosts[0], hosts[1]) diff --git a/tests/network/conftest.py b/tests/network/conftest.py index 5aad36c9..42ea8730 100644 --- a/tests/network/conftest.py +++ b/tests/network/conftest.py @@ -8,18 +8,22 @@ from libp2p.tools.factories import ( @pytest.fixture -async def net_stream_pair(is_host_secure): - async with net_stream_pair_factory(is_host_secure) as net_stream_pair: +async def net_stream_pair(security_protocol): + async with net_stream_pair_factory( + security_protocol=security_protocol + ) as net_stream_pair: yield net_stream_pair @pytest.fixture -async def swarm_pair(is_host_secure): - async with swarm_pair_factory(is_host_secure) as swarms: +async def swarm_pair(security_protocol): + async with swarm_pair_factory(security_protocol=security_protocol) as swarms: yield swarms @pytest.fixture -async def swarm_conn_pair(is_host_secure): - async with swarm_conn_pair_factory(is_host_secure) as swarm_conn_pair: +async def swarm_conn_pair(security_protocol): + async with swarm_conn_pair_factory( + security_protocol=security_protocol + ) as swarm_conn_pair: yield swarm_conn_pair diff --git a/tests/network/test_notify.py b/tests/network/test_notify.py index 328ff128..5be4c082 100644 --- a/tests/network/test_notify.py +++ b/tests/network/test_notify.py @@ -55,8 +55,8 @@ class MyNotifee(INotifee): @pytest.mark.trio -async def test_notify(is_host_secure): - swarms = [SwarmFactory(is_secure=is_host_secure) for _ in range(2)] +async def test_notify(security_protocol): + swarms = [SwarmFactory(security_protocol=security_protocol) for _ in range(2)] events_0_0 = [] events_1_0 = [] diff --git a/tests/network/test_swarm.py b/tests/network/test_swarm.py index 70b82477..42b7db1c 100644 --- a/tests/network/test_swarm.py +++ b/tests/network/test_swarm.py @@ -9,8 +9,10 @@ from libp2p.tools.utils import connect_swarm @pytest.mark.trio -async def test_swarm_dial_peer(is_host_secure): - async with SwarmFactory.create_batch_and_listen(is_host_secure, 3) as swarms: +async def test_swarm_dial_peer(security_protocol): + async with SwarmFactory.create_batch_and_listen( + 3, security_protocol=security_protocol + ) as swarms: # Test: No addr found. with pytest.raises(SwarmException): await swarms[0].dial_peer(swarms[1].get_peer_id()) @@ -38,8 +40,10 @@ async def test_swarm_dial_peer(is_host_secure): @pytest.mark.trio -async def test_swarm_close_peer(is_host_secure): - async with SwarmFactory.create_batch_and_listen(is_host_secure, 3) as swarms: +async def test_swarm_close_peer(security_protocol): + async with SwarmFactory.create_batch_and_listen( + 3, security_protocol=security_protocol + ) as swarms: # 0 <> 1 <> 2 await connect_swarm(swarms[0], swarms[1]) await connect_swarm(swarms[1], swarms[2]) @@ -90,8 +94,10 @@ async def test_swarm_remove_conn(swarm_pair): @pytest.mark.trio -async def test_swarm_multiaddr(is_host_secure): - async with SwarmFactory.create_batch_and_listen(is_host_secure, 3) as swarms: +async def test_swarm_multiaddr(security_protocol): + async with SwarmFactory.create_batch_and_listen( + 3, security_protocol=security_protocol + ) as swarms: def clear(): swarms[0].peerstore.clear_addrs(swarms[1].get_peer_id()) diff --git a/tests/protocol_muxer/test_protocol_muxer.py b/tests/protocol_muxer/test_protocol_muxer.py index cd82652c..0537c4ea 100644 --- a/tests/protocol_muxer/test_protocol_muxer.py +++ b/tests/protocol_muxer/test_protocol_muxer.py @@ -16,9 +16,11 @@ async def perform_simple_test( expected_selected_protocol, protocols_for_client, protocols_with_handlers, - is_host_secure, + security_protocol, ): - async with HostFactory.create_batch_and_listen(is_host_secure, 2) as hosts: + async with HostFactory.create_batch_and_listen( + 2, security_protocol=security_protocol + ) as hosts: for protocol in protocols_with_handlers: hosts[1].set_stream_handler( protocol, create_echo_stream_handler(ACK_PREFIX) @@ -38,28 +40,28 @@ async def perform_simple_test( @pytest.mark.trio -async def test_single_protocol_succeeds(is_host_secure): +async def test_single_protocol_succeeds(security_protocol): expected_selected_protocol = PROTOCOL_ECHO await perform_simple_test( expected_selected_protocol, [expected_selected_protocol], [expected_selected_protocol], - is_host_secure, + security_protocol, ) @pytest.mark.trio -async def test_single_protocol_fails(is_host_secure): +async def test_single_protocol_fails(security_protocol): with pytest.raises(StreamFailure): await perform_simple_test( - "", [PROTOCOL_ECHO], [PROTOCOL_POTATO], is_host_secure + "", [PROTOCOL_ECHO], [PROTOCOL_POTATO], security_protocol ) # Cleanup not reached on error @pytest.mark.trio -async def test_multiple_protocol_first_is_valid_succeeds(is_host_secure): +async def test_multiple_protocol_first_is_valid_succeeds(security_protocol): expected_selected_protocol = PROTOCOL_ECHO protocols_for_client = [PROTOCOL_ECHO, PROTOCOL_POTATO] protocols_for_listener = [PROTOCOL_FOO, PROTOCOL_ECHO] @@ -67,12 +69,12 @@ async def test_multiple_protocol_first_is_valid_succeeds(is_host_secure): expected_selected_protocol, protocols_for_client, protocols_for_listener, - is_host_secure, + security_protocol, ) @pytest.mark.trio -async def test_multiple_protocol_second_is_valid_succeeds(is_host_secure): +async def test_multiple_protocol_second_is_valid_succeeds(security_protocol): expected_selected_protocol = PROTOCOL_FOO protocols_for_client = [PROTOCOL_ROCK, PROTOCOL_FOO] protocols_for_listener = [PROTOCOL_FOO, PROTOCOL_ECHO] @@ -80,15 +82,15 @@ async def test_multiple_protocol_second_is_valid_succeeds(is_host_secure): expected_selected_protocol, protocols_for_client, protocols_for_listener, - is_host_secure, + security_protocol, ) @pytest.mark.trio -async def test_multiple_protocol_fails(is_host_secure): +async def test_multiple_protocol_fails(security_protocol): protocols_for_client = [PROTOCOL_ROCK, PROTOCOL_FOO, "/bar/1.0.0"] protocols_for_listener = ["/aspyn/1.0.0", "/rob/1.0.0", "/zx/1.0.0", "/alex/1.0.0"] with pytest.raises(StreamFailure): await perform_simple_test( - "", protocols_for_client, protocols_for_listener, is_host_secure + "", protocols_for_client, protocols_for_listener, security_protocol ) diff --git a/tests/pubsub/test_floodsub.py b/tests/pubsub/test_floodsub.py index 148c001b..36bf6fcd 100644 --- a/tests/pubsub/test_floodsub.py +++ b/tests/pubsub/test_floodsub.py @@ -82,10 +82,11 @@ async def test_lru_cache_two_nodes(monkeypatch): @pytest.mark.parametrize("test_case_obj", floodsub_protocol_pytest_params) @pytest.mark.trio @pytest.mark.slow -async def test_gossipsub_run_with_floodsub_tests(test_case_obj, is_host_secure): +async def test_gossipsub_run_with_floodsub_tests(test_case_obj, security_protocol): await perform_test_from_obj( test_case_obj, functools.partial( - PubsubFactory.create_batch_with_floodsub, is_secure=is_host_secure + PubsubFactory.create_batch_with_floodsub, + security_protocol=security_protocol, ), ) diff --git a/tests/pubsub/test_pubsub.py b/tests/pubsub/test_pubsub.py index d6c29310..2c6fe7dc 100644 --- a/tests/pubsub/test_pubsub.py +++ b/tests/pubsub/test_pubsub.py @@ -236,7 +236,7 @@ async def test_validate_msg(is_topic_1_val_passed, is_topic_2_val_passed): @pytest.mark.trio -async def test_continuously_read_stream(monkeypatch, nursery, is_host_secure): +async def test_continuously_read_stream(monkeypatch, nursery, security_protocol): async def wait_for_event_occurring(event): await trio.hazmat.checkpoint() with trio.fail_after(0.1): @@ -271,8 +271,10 @@ async def test_continuously_read_stream(monkeypatch, nursery, is_host_secure): yield Events(event_push_msg, event_handle_subscription, event_handle_rpc) async with PubsubFactory.create_batch_with_floodsub( - 1, is_secure=is_host_secure - ) as pubsubs_fsub, net_stream_pair_factory(is_secure=is_host_secure) as stream_pair: + 1, security_protocol=security_protocol + ) as pubsubs_fsub, net_stream_pair_factory( + security_protocol=security_protocol + ) as stream_pair: await pubsubs_fsub[0].subscribe(TESTING_TOPIC) # Kick off the task `continuously_read_stream` nursery.start_soon(pubsubs_fsub[0].continuously_read_stream, stream_pair[0]) @@ -394,10 +396,12 @@ async def test_handle_talk(): @pytest.mark.trio -async def test_message_all_peers(monkeypatch, is_host_secure): +async def test_message_all_peers(monkeypatch, security_protocol): async with PubsubFactory.create_batch_with_floodsub( - 1, is_secure=is_host_secure - ) as pubsubs_fsub, net_stream_pair_factory(is_secure=is_host_secure) as stream_pair: + 1, security_protocol=security_protocol + ) as pubsubs_fsub, net_stream_pair_factory( + security_protocol=security_protocol + ) as stream_pair: peer_id = IDFactory() mock_peers = {peer_id: stream_pair[0]} with monkeypatch.context() as m: diff --git a/tests/security/test_security_multistream.py b/tests/security/test_security_multistream.py index cd968ac2..32534cfb 100644 --- a/tests/security/test_security_multistream.py +++ b/tests/security/test_security_multistream.py @@ -1,88 +1,49 @@ import pytest -import trio -from libp2p import new_host from libp2p.crypto.rsa import create_new_key_pair -from libp2p.security.insecure.transport import InsecureSession, InsecureTransport -from libp2p.tools.constants import LISTEN_MADDR -from libp2p.tools.utils import connect - -# TODO: Add tests for multiple streams being opened on different -# protocols through the same connection - - -def peer_id_for_node(node): - return node.get_id() - +from libp2p.security.insecure.transport import PLAINTEXT_PROTOCOL_ID, InsecureSession +from libp2p.security.noise.transport import PROTOCOL_ID as NOISE_PROTOCOL_ID +from libp2p.security.secio.transport import ID as SECIO_PROTOCOL_ID +from libp2p.security.secure_session import SecureSession +from libp2p.tools.factories import host_pair_factory initiator_key_pair = create_new_key_pair() noninitiator_key_pair = create_new_key_pair() -async def perform_simple_test( - assertion_func, transports_for_initiator, transports_for_noninitiator -): - # Create libp2p nodes and connect them, then secure the connection, then check - # the proper security was chosen - # TODO: implement -- note we need to introduce the notion of communicating over a raw connection - # for testing, we do NOT want to communicate over a stream so we can't just create two nodes - # and use their conn because our mplex will internally relay messages to a stream - - node1 = new_host(key_pair=initiator_key_pair, sec_opt=transports_for_initiator) - node2 = new_host( - key_pair=noninitiator_key_pair, sec_opt=transports_for_noninitiator - ) - async with node1.run(listen_addrs=[LISTEN_MADDR]), node2.run( - listen_addrs=[LISTEN_MADDR] - ): - await connect(node1, node2) - - # Wait a very short period to allow conns to be stored (since the functions - # storing the conns are async, they may happen at slightly different times - # on each node) - await trio.sleep(0.1) - - # Get conns - node1_conn = node1.get_network().connections[peer_id_for_node(node2)] - node2_conn = node2.get_network().connections[peer_id_for_node(node1)] +async def perform_simple_test(assertion_func, security_protocol): + async with host_pair_factory(security_protocol=security_protocol) as hosts: + conn_0 = hosts[0].get_network().connections[hosts[1].get_id()] + conn_1 = hosts[1].get_network().connections[hosts[0].get_id()] # Perform assertion - assertion_func(node1_conn.muxed_conn.secured_conn) - assertion_func(node2_conn.muxed_conn.secured_conn) + assertion_func(conn_0.muxed_conn.secured_conn) + assertion_func(conn_1.muxed_conn.secured_conn) @pytest.mark.trio -async def test_single_insecure_security_transport_succeeds(): - transports_for_initiator = {"foo": InsecureTransport(initiator_key_pair)} - transports_for_noninitiator = {"foo": InsecureTransport(noninitiator_key_pair)} - +@pytest.mark.parametrize( + "security_protocol, transport_type", + ( + (PLAINTEXT_PROTOCOL_ID, InsecureSession), + (SECIO_PROTOCOL_ID, SecureSession), + (NOISE_PROTOCOL_ID, SecureSession), + ), +) +@pytest.mark.trio +async def test_single_insecure_security_transport_succeeds( + security_protocol, transport_type +): def assertion_func(conn): - assert isinstance(conn, InsecureSession) + assert isinstance(conn, transport_type) - await perform_simple_test( - assertion_func, transports_for_initiator, transports_for_noninitiator - ) + await perform_simple_test(assertion_func, security_protocol) @pytest.mark.trio async def test_default_insecure_security(): - transports_for_initiator = None - transports_for_noninitiator = None - - conn1 = None - conn2 = None - def assertion_func(conn): - nonlocal conn1 - nonlocal conn2 - if not conn1: - conn1 = conn - elif not conn2: - conn2 = conn - else: - assert conn1 == conn2 + assert isinstance(conn, InsecureSession) - await perform_simple_test( - assertion_func, transports_for_initiator, transports_for_noninitiator - ) + await perform_simple_test(assertion_func, None) diff --git a/tests/stream_muxer/conftest.py b/tests/stream_muxer/conftest.py index 248422d9..44606d59 100644 --- a/tests/stream_muxer/conftest.py +++ b/tests/stream_muxer/conftest.py @@ -4,14 +4,18 @@ from libp2p.tools.factories import mplex_conn_pair_factory, mplex_stream_pair_fa @pytest.fixture -async def mplex_conn_pair(is_host_secure): - async with mplex_conn_pair_factory(is_host_secure) as mplex_conn_pair: +async def mplex_conn_pair(security_protocol): + async with mplex_conn_pair_factory( + security_protocol=security_protocol + ) as mplex_conn_pair: assert mplex_conn_pair[0].is_initiator assert not mplex_conn_pair[1].is_initiator yield mplex_conn_pair[0], mplex_conn_pair[1] @pytest.fixture -async def mplex_stream_pair(is_host_secure): - async with mplex_stream_pair_factory(is_host_secure) as mplex_stream_pair: +async def mplex_stream_pair(security_protocol): + async with mplex_stream_pair_factory( + security_protocol=security_protocol + ) as mplex_stream_pair: yield mplex_stream_pair diff --git a/tests_interop/conftest.py b/tests_interop/conftest.py index 067db83f..b14f91cc 100644 --- a/tests_interop/conftest.py +++ b/tests_interop/conftest.py @@ -6,14 +6,15 @@ import pytest import trio from libp2p.io.abc import ReadWriteCloser +from libp2p.security.insecure.transport import PLAINTEXT_PROTOCOL_ID from libp2p.tools.factories import HostFactory, PubsubFactory from libp2p.tools.interop.daemon import make_p2pd from libp2p.tools.interop.utils import connect @pytest.fixture -def is_host_secure(): - return False +def security_protocol(): + return PLAINTEXT_PROTOCOL_ID @pytest.fixture @@ -38,7 +39,11 @@ def is_pubsub_signing_strict(): @pytest.fixture async def p2pds( - num_p2pds, is_host_secure, is_gossipsub, is_pubsub_signing, is_pubsub_signing_strict + num_p2pds, + security_protocol, + is_gossipsub, + is_pubsub_signing, + is_pubsub_signing_strict, ): async with AsyncExitStack() as stack: p2pds = [ @@ -46,7 +51,7 @@ async def p2pds( make_p2pd( get_unused_tcp_port(), get_unused_tcp_port(), - is_host_secure, + security_protocol, is_gossipsub=is_gossipsub, is_pubsub_signing=is_pubsub_signing, is_pubsub_signing_strict=is_pubsub_signing_strict, @@ -62,14 +67,16 @@ async def p2pds( @pytest.fixture -async def pubsubs(num_hosts, is_host_secure, is_gossipsub, is_pubsub_signing_strict): +async def pubsubs(num_hosts, security_protocol, is_gossipsub, is_pubsub_signing_strict): if is_gossipsub: yield PubsubFactory.create_batch_with_gossipsub( - num_hosts, is_secure=is_host_secure, strict_signing=is_pubsub_signing_strict + num_hosts, + security_protocol=security_protocol, + strict_signing=is_pubsub_signing_strict, ) else: yield PubsubFactory.create_batch_with_floodsub( - num_hosts, is_host_secure, strict_signing=is_pubsub_signing_strict + num_hosts, security_protocol, strict_signing=is_pubsub_signing_strict ) @@ -97,8 +104,10 @@ async def is_to_fail_daemon_stream(): @pytest.fixture -async def py_to_daemon_stream_pair(p2pds, is_host_secure, is_to_fail_daemon_stream): - async with HostFactory.create_batch_and_listen(is_host_secure, 1) as hosts: +async def py_to_daemon_stream_pair(p2pds, security_protocol, is_to_fail_daemon_stream): + async with HostFactory.create_batch_and_listen( + 1, security_protocol=security_protocol + ) as hosts: assert len(p2pds) >= 1 host = hosts[0] p2pd = p2pds[0] diff --git a/tests_interop/test_bindings.py b/tests_interop/test_bindings.py index 87cdbb1b..d7a169cb 100644 --- a/tests_interop/test_bindings.py +++ b/tests_interop/test_bindings.py @@ -6,8 +6,10 @@ from libp2p.tools.interop.utils import connect @pytest.mark.trio -async def test_connect(is_host_secure, p2pds): - async with HostFactory.create_batch_and_listen(is_host_secure, 1) as hosts: +async def test_connect(security_protocol, p2pds): + async with HostFactory.create_batch_and_listen( + 1, security_protocol=security_protocol + ) as hosts: p2pd = p2pds[0] host = hosts[0] assert len(await p2pd.control.list_peers()) == 0 diff --git a/tests_interop/test_echo.py b/tests_interop/test_echo.py index 85810a7f..eac8fd30 100644 --- a/tests_interop/test_echo.py +++ b/tests_interop/test_echo.py @@ -6,6 +6,7 @@ import pytest import trio from libp2p.peer.peerinfo import PeerInfo, info_from_p2p_addr +from libp2p.security.insecure.transport import PLAINTEXT_PROTOCOL_ID from libp2p.tools.factories import HostFactory from libp2p.tools.interop.envs import GO_BIN_PATH from libp2p.tools.interop.process import BaseInteractiveProcess @@ -20,10 +21,10 @@ class EchoProcess(BaseInteractiveProcess): _peer_info: PeerInfo def __init__( - self, port: int, is_secure: bool, destination: Multiaddr = None + self, port: int, security_protocol: TProtocol, destination: Multiaddr = None ) -> None: args = [f"-l={port}"] - if not is_secure: + if security_protocol == PLAINTEXT_PROTOCOL_ID: args.append("-insecure") if destination is not None: args.append(f"-d={str(destination)}") @@ -61,9 +62,11 @@ class EchoProcess(BaseInteractiveProcess): @pytest.mark.trio -async def test_insecure_conn_py_to_go(is_host_secure): - async with HostFactory.create_batch_and_listen(is_host_secure, 1) as hosts: - go_proc = EchoProcess(get_unused_tcp_port(), is_host_secure) +async def test_insecure_conn_py_to_go(security_protocol): + async with HostFactory.create_batch_and_listen( + 1, security_protocol=security_protocol + ) as hosts: + go_proc = EchoProcess(get_unused_tcp_port(), security_protocol) await go_proc.start() host = hosts[0] @@ -78,8 +81,10 @@ async def test_insecure_conn_py_to_go(is_host_secure): @pytest.mark.trio -async def test_insecure_conn_go_to_py(is_host_secure): - async with HostFactory.create_batch_and_listen(is_host_secure, 1) as hosts: +async def test_insecure_conn_go_to_py(security_protocol): + async with HostFactory.create_batch_and_listen( + 1, security_protocol=security_protocol + ) as hosts: host = hosts[0] expected_data = "Hello, world!\n" reply_data = "Replyooo!\n" @@ -94,6 +99,6 @@ async def test_insecure_conn_go_to_py(is_host_secure): host.set_stream_handler(ECHO_PROTOCOL_ID, _handle_echo) py_maddr = host.get_addrs()[0] - go_proc = EchoProcess(get_unused_tcp_port(), is_host_secure, py_maddr) + go_proc = EchoProcess(get_unused_tcp_port(), security_protocol, py_maddr) await go_proc.start() await event_handler_finished.wait() diff --git a/tests_interop/test_pubsub.py b/tests_interop/test_pubsub.py index 793f2446..cc7fa0dd 100644 --- a/tests_interop/test_pubsub.py +++ b/tests_interop/test_pubsub.py @@ -54,7 +54,7 @@ def validate_pubsub_msg(msg: rpc_pb2.Message, data: bytes, from_peer_id: ID) -> @pytest.mark.parametrize("num_p2pds", (2,)) @pytest.mark.trio async def test_pubsub( - p2pds, is_gossipsub, is_host_secure, is_pubsub_signing_strict, nursery + p2pds, is_gossipsub, security_protocol, is_pubsub_signing_strict, nursery ): pubsub_factory = None if is_gossipsub: @@ -63,7 +63,7 @@ async def test_pubsub( pubsub_factory = PubsubFactory.create_batch_with_floodsub async with pubsub_factory( - 1, is_secure=is_host_secure, strict_signing=is_pubsub_signing_strict + 1, security_protocol=security_protocol, strict_signing=is_pubsub_signing_strict ) as pubsubs: # # Test: Recognize pubsub peers on connection. From a9f4f285eeeec874bdd294a2123762847f4ff2c6 Mon Sep 17 00:00:00 2001 From: mhchia Date: Wed, 19 Feb 2020 23:54:12 +0800 Subject: [PATCH 14/22] Noise: clean up dup code --- libp2p/security/noise/patterns.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/libp2p/security/noise/patterns.py b/libp2p/security/noise/patterns.py index cb5e7bfa..fd6efc56 100644 --- a/libp2p/security/noise/patterns.py +++ b/libp2p/security/noise/patterns.py @@ -1,10 +1,11 @@ from abc import ABC, abstractmethod -from noise.connection import Keypair as NoiseKeypair +from noise.backends.default.keypairs import KeyPair as NoiseKeyPair +from noise.connection import Keypair as NoiseKeypairEnum from noise.connection import NoiseConnection as NoiseState from libp2p.crypto.ed25519 import Ed25519PublicKey -from libp2p.crypto.keys import PrivateKey +from libp2p.crypto.keys import PrivateKey, PublicKey from libp2p.network.connection.raw_connection_interface import IRawConnection from libp2p.peer.id import ID from libp2p.security.secure_conn_interface import ISecureConn @@ -46,7 +47,7 @@ class BasePattern(IPattern): def create_noise_state(self) -> NoiseState: noise_state = NoiseState.from_name(self.protocol_name) noise_state.set_keypair_from_private_bytes( - NoiseKeypair.STATIC, self.noise_static_key.to_bytes() + NoiseKeypairEnum.STATIC, self.noise_static_key.to_bytes() ) return noise_state @@ -96,8 +97,8 @@ class PatternXX(BasePattern): "we received and consumed msg#3, which should have included the" " remote static public key, but it is not present in the handshake_state" ) - # Use `Ed25519PublicKey` since 25519 is used in our pattern. - remote_pubkey = Ed25519PublicKey.from_bytes(handshake_state.rs.public_bytes) + remote_pubkey = self._get_pubkey_from_noise_keypair(handshake_state.rs) + if not verify_handshake_payload_sig(peer_handshake_payload, remote_pubkey): raise InvalidSignature remote_peer_id_from_pubkey = ID.from_pubkey(peer_handshake_payload.id_pubkey) @@ -140,8 +141,8 @@ class PatternXX(BasePattern): "we received and consumed msg#3, which should have included the" " remote static public key, but it is not present in the handshake_state" ) - # Use `Ed25519PublicKey` since 25519 is used in our pattern. - remote_pubkey = Ed25519PublicKey.from_bytes(handshake_state.rs.public_bytes) + remote_pubkey = self._get_pubkey_from_noise_keypair(handshake_state.rs) + if not verify_handshake_payload_sig(peer_handshake_payload, remote_pubkey): raise InvalidSignature remote_peer_id_from_pubkey = ID.from_pubkey(peer_handshake_payload.id_pubkey) @@ -170,3 +171,13 @@ class PatternXX(BasePattern): is_initiator=True, conn=transport_read_writer, ) + + @staticmethod + def _get_pubkey_from_noise_keypair(key_pair: NoiseKeyPair) -> PublicKey: + # Use `Ed25519PublicKey` since 25519 is used in our pattern. + # NOTE: Ignore the warning for now, since it is also not fixed in `noiseprotocol`. + # "CryptographyDeprecationWarning: public_bytes now requires + # encoding and format arguments. Support for calling without arguments will be + # removed in cryptography 2.7" + raw_bytes = key_pair.public.public_bytes() + return Ed25519PublicKey.from_bytes(raw_bytes) From 88f660a9c5cd998acc9b70b389b98629f9d794eb Mon Sep 17 00:00:00 2001 From: mhchia Date: Thu, 20 Feb 2020 17:38:44 +0800 Subject: [PATCH 15/22] Noise: try to use noise in go --- tests_interop/go_pkgs/examples/echo/main.go | 4 +- tests_interop/go_pkgs/examples/go.mod | 11 +- tests_interop/go_pkgs/examples/go.sum | 108 +++++++++++++++++++ tests_interop/go_pkgs/examples/utils/host.go | 27 +++-- tests_interop/test_echo.py | 4 +- 5 files changed, 138 insertions(+), 16 deletions(-) diff --git a/tests_interop/go_pkgs/examples/echo/main.go b/tests_interop/go_pkgs/examples/echo/main.go index e9ec5844..3bf6f281 100644 --- a/tests_interop/go_pkgs/examples/echo/main.go +++ b/tests_interop/go_pkgs/examples/echo/main.go @@ -28,7 +28,7 @@ func main() { // Parse options from the command line listenF := flag.Int("l", 0, "wait for incoming connections") target := flag.String("d", "", "target peer to dial") - insecure := flag.Bool("insecure", false, "use an unencrypted connection") + protocolID := flag.String("security", "", "security protocol used for secure channel") seed := flag.Int64("seed", 0, "set random seed for id generation") flag.Parse() @@ -37,7 +37,7 @@ func main() { } // Make a host that listens on the given multiaddress - ha, err := utils.MakeBasicHost(*listenF, *insecure, *seed) + ha, err := utils.MakeBasicHost(*listenF, *protocolID, *seed) if err != nil { log.Fatal(err) } diff --git a/tests_interop/go_pkgs/examples/go.mod b/tests_interop/go_pkgs/examples/go.mod index e3625e45..78d092f8 100644 --- a/tests_interop/go_pkgs/examples/go.mod +++ b/tests_interop/go_pkgs/examples/go.mod @@ -4,8 +4,11 @@ go 1.12 require ( github.com/ipfs/go-log v0.0.1 - github.com/libp2p/go-libp2p v0.3.1 - github.com/libp2p/go-libp2p-core v0.2.2 - github.com/multiformats/go-multiaddr v0.0.4 - github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc + github.com/libp2p/go-libp2p v0.5.1 + github.com/libp2p/go-libp2p-core v0.3.0 + github.com/libp2p/go-libp2p-noise v0.0.0-20200203154915-813ed1ee6a46 + github.com/libp2p/go-libp2p-secio v0.2.1 + github.com/multiformats/go-multiaddr v0.2.0 + github.com/whyrusleeping/go-logging v0.0.1 + github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f // indirect ) diff --git a/tests_interop/go_pkgs/examples/go.sum b/tests_interop/go_pkgs/examples/go.sum index ea85590b..ce296704 100644 --- a/tests_interop/go_pkgs/examples/go.sum +++ b/tests_interop/go_pkgs/examples/go.sum @@ -2,12 +2,16 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/ChainSafe/log15 v1.0.0/go.mod h1:5v1+ALHtdW0NfAeeoYyKmzCAMcAeqkdhIg4uxXWIgOg= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c h1:aEbSeNALREWXk0G7UdNhR3ayBV7tZ4M2PNmnrCAph6Q= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= +github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= +github.com/btcsuite/btcd v0.20.1-beta h1:Ik4hyJqN8Jfyv3S4AGBOmyouMsYE3EdYODkMbQjwPGw= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= @@ -28,27 +32,38 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -59,11 +74,18 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2 h1:tuuKaZPU1M6HcejsO3AcYWW8sZ8MTvyxfc4uqB4eFE8= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.4 h1:UlfXKrZx1DjZoBhQHmNHLC1fK1dUJDN20Y28A7s+gJ8= +github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= +github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= +github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaHzfGTzuE3s= +github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= +github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= @@ -85,11 +107,14 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b h1:wxtKgYHEncAU00muMD06dzLiahtGM1eouRNOzVV7tdQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= +github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d h1:68u9r4wEvL3gYg2jvAOgROwZ3H+Y3hIDk4tbbmIjcYQ= +github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -105,23 +130,37 @@ github.com/libp2p/go-eventbus v0.1.0 h1:mlawomSAjjkk97QnYiEmHsLu7E136+2oCWSHRUvM github.com/libp2p/go-eventbus v0.1.0/go.mod h1:vROgu5cs5T7cv7POWlWxBaVLxfSegC5UGQf8A2eEmx4= github.com/libp2p/go-flow-metrics v0.0.1 h1:0gxuFd2GuK7IIP5pKljLwps6TvcuYgvG7Atqi3INF5s= github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZxBdp967ls1g+k8= +github.com/libp2p/go-flow-metrics v0.0.3 h1:8tAs/hSdNvUiLgtlSy3mxwxWP4I9y/jlkPFT7epKdeM= +github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-libp2p v0.3.1 h1:opd8/1Sm9zFG37LzNQsIzMTMeBabhlcX5VlvLrNZPV0= github.com/libp2p/go-libp2p v0.3.1/go.mod h1:e6bwxbdYH1HqWTz8faTChKGR0BjPc8p+6SyP8GTTR7Y= +github.com/libp2p/go-libp2p v0.5.1 h1:kZ9jg+2B9IIptRcltBHKBrQdhXNNSrjCoztvrMx7tqI= +github.com/libp2p/go-libp2p v0.5.1/go.mod h1:Os7a5Z3B+ErF4v7zgIJ7nBHNu2LYt8ZMLkTQUB3G/wA= github.com/libp2p/go-libp2p-autonat v0.1.0 h1:aCWAu43Ri4nU0ZPO7NyLzUvvfqd0nE3dX0R/ZGYVgOU= github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= +github.com/libp2p/go-libp2p-autonat v0.1.1 h1:WLBZcIRsjZlWdAZj9CiBSvU2wQXoUOiS1Zk1tM7DTJI= +github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/djNA3fdpCWloIudE= github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro= github.com/libp2p/go-libp2p-blankhost v0.1.3/go.mod h1:KML1//wiKR8vuuJO0y3LUd1uLv+tlkGTAr3jC0S5cLg= +github.com/libp2p/go-libp2p-blankhost v0.1.4/go.mod h1:oJF0saYsAXQCSfDq254GMNmLNz6ZTHTOvtF4ZydUvwU= github.com/libp2p/go-libp2p-circuit v0.1.1 h1:eopfG9fAg6rEHWQO1TSrLosXDgYbbbu/RTva/tBANus= github.com/libp2p/go-libp2p-circuit v0.1.1/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= +github.com/libp2p/go-libp2p-circuit v0.1.4 h1:Phzbmrg3BkVzbqd4ZZ149JxCuUWu2wZcXf/Kr6hZJj8= +github.com/libp2p/go-libp2p-circuit v0.1.4/go.mod h1:CY67BrEjKNDhdTk8UgBX1Y/H5c3xkAcs3gnksxY7osU= github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGcGip57wjTU4fco= github.com/libp2p/go-libp2p-core v0.0.4/go.mod h1:jyuCQP356gzfCFtRKyvAbNkyeuxb7OlyhWZ3nls5d2I= github.com/libp2p/go-libp2p-core v0.0.6/go.mod h1:0d9xmaYAVY5qmbp/fcgxHT3ZJsLjYeYPMJAUKpaCHrE= github.com/libp2p/go-libp2p-core v0.2.0/go.mod h1:X0eyB0Gy93v0DZtSYbEM7RnMChm9Uv3j7yRXjO77xSI= github.com/libp2p/go-libp2p-core v0.2.2 h1:Sv1ggdoMx9c7v7FOFkR7agraHCnAgqYsXrU1ARSRUMs= github.com/libp2p/go-libp2p-core v0.2.2/go.mod h1:8fcwTbsG2B+lTgRJ1ICZtiM5GWCWZVoVrLaDRvIRng0= +github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g= +github.com/libp2p/go-libp2p-core v0.3.0 h1:F7PqduvrztDtFsAa/bcheQ3azmNo+Nq7m8hQY5GiUW8= +github.com/libp2p/go-libp2p-core v0.3.0/go.mod h1:ACp3DmS3/N64c2jDzcV429ukDpicbL6+TrrxANBjPGw= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= github.com/libp2p/go-libp2p-discovery v0.1.0 h1:j+R6cokKcGbnZLf4kcNwpx6mDEUPF3N6SrqMymQhmvs= github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g= +github.com/libp2p/go-libp2p-discovery v0.2.0 h1:1p3YSOq7VsgaL+xVHPi8XAmtGyas6D2J6rWBEfz/aiY= +github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= @@ -129,21 +168,32 @@ github.com/libp2p/go-libp2p-mplex v0.2.1 h1:E1xaJBQnbSiTHGI1gaBKmKhu1TUKkErKJnE8 github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE= github.com/libp2p/go-libp2p-nat v0.0.4 h1:+KXK324yaY701On8a0aGjTnw8467kW3ExKcqW2wwmyw= github.com/libp2p/go-libp2p-nat v0.0.4/go.mod h1:N9Js/zVtAXqaeT99cXgTV9e75KpnWCvVOiGzlcHmBbY= +github.com/libp2p/go-libp2p-nat v0.0.5 h1:/mH8pXFVKleflDL1YwqMg27W9GD8kjEx7NY0P6eGc98= +github.com/libp2p/go-libp2p-nat v0.0.5/go.mod h1:1qubaE5bTZMJE+E/uu2URroMbzdubFz1ChgiN79yKPE= github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU= +github.com/libp2p/go-libp2p-noise v0.0.0-20200203154915-813ed1ee6a46 h1:Ix0osa0VFHTGiRTpgp1IoIlkCKoE18wHHl/6dXjRKmA= +github.com/libp2p/go-libp2p-noise v0.0.0-20200203154915-813ed1ee6a46/go.mod h1:sh2qxZXxvWv/bGcO6rZR4ElbEo3G0K5zCDHqcwOOJ4Y= github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= github.com/libp2p/go-libp2p-peerstore v0.1.3 h1:wMgajt1uM2tMiqf4M+4qWKVyyFc8SfA+84VV9glZq1M= github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= +github.com/libp2p/go-libp2p-peerstore v0.1.4 h1:d23fvq5oYMJ/lkkbO4oTwBp/JP+I/1m5gZJobNXCE/k= +github.com/libp2p/go-libp2p-peerstore v0.1.4/go.mod h1:+4BDbDiiKf4PzpANZDAT+knVdLxvqh7hXOujessqdzs= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-secio v0.2.0 h1:ywzZBsWEEz2KNTn5RtzauEDq5RFEefPsttXYwAWqHng= github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= +github.com/libp2p/go-libp2p-secio v0.2.1 h1:eNWbJTdyPA7NxhP7J3c5lT97DC5d+u+IldkgCYFTPVA= +github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8= github.com/libp2p/go-libp2p-swarm v0.1.0/go.mod h1:wQVsCdjsuZoc730CgOvh5ox6K8evllckjebkdiY5ta4= github.com/libp2p/go-libp2p-swarm v0.2.1 h1:9A8oQqPIZvbaRyrjViHeDYS7fE7fNtP7BRWdJrBHbe8= github.com/libp2p/go-libp2p-swarm v0.2.1/go.mod h1:x07b4zkMFo2EvgPV2bMTlNmdQc8i+74Jjio7xGvsTgU= +github.com/libp2p/go-libp2p-swarm v0.2.2 h1:T4hUpgEs2r371PweU3DuH7EOmBIdTBCwWs+FLcgx3bQ= +github.com/libp2p/go-libp2p-swarm v0.2.2/go.mod h1:fvmtQ0T1nErXym1/aa1uJEyN7JzaTNyBcHImCxRpPKU= github.com/libp2p/go-libp2p-testing v0.0.2/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.3/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.0.4/go.mod h1:gvchhf3FQOtBdr+eFUABet5a4MBLK8jM3V4Zghvmi+E= github.com/libp2p/go-libp2p-testing v0.1.0/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0= +github.com/libp2p/go-libp2p-testing v0.1.1/go.mod h1:xaZWMJrPUM5GlDBxCeGUi7kI4eqnjVyavGroI2nxEM0= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1 h1:PZMS9lhjK9VytzMCW3tWHAXtKXmlURSc3ZdvwEcKCzw= github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA= github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8= @@ -160,7 +210,13 @@ github.com/libp2p/go-msgio v0.0.4 h1:agEFehY3zWJFUHK6SEMR7UYmk2z6kC3oeCM7ybLhguA github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-nat v0.0.3 h1:l6fKV+p0Xa354EqQOQP+d8CivdLM4kl5GxC1hSc/UeI= github.com/libp2p/go-nat v0.0.3/go.mod h1:88nUEt0k0JD45Bk93NIwDqjlhiOwOoV36GchpcVc1yI= +github.com/libp2p/go-nat v0.0.4 h1:KbizNnq8YIf7+Hn7+VFL/xE0eDrkPru2zIO9NMwL8UQ= +github.com/libp2p/go-nat v0.0.4/go.mod h1:Nmw50VAvKuk38jUBcmNh6p9lUJLoODbJRvYAa/+KSDo= +github.com/libp2p/go-openssl v0.0.2 h1:9pP2d3Ubaxkv7ZisLjx9BFwgOGnQdQYnfcH29HNY3ls= github.com/libp2p/go-openssl v0.0.2/go.mod h1:v8Zw2ijCSWBQi8Pq5GAixw6DbFfa9u6VIYDXnvOXkc0= +github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= +github.com/libp2p/go-openssl v0.0.4 h1:d27YZvLoTyMhIN4njrkr8zMDOM4lfpHIp6A+TK9fovg= +github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-reuseport v0.0.1 h1:7PhkfH73VXfPJYKQ6JwS5I/eVcoyYi9IMNGc6FWpFLw= github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= github.com/libp2p/go-reuseport-transport v0.0.2 h1:WglMwyXyBu61CMkjCCtnmqNqnjib0GIEjMiHTwR/KN4= @@ -170,8 +226,12 @@ github.com/libp2p/go-stream-muxer-multistream v0.2.0 h1:714bRJ4Zy9mdhyTLJ+ZKiROm github.com/libp2p/go-stream-muxer-multistream v0.2.0/go.mod h1:j9eyPol/LLRqT+GPLSxvimPhNph4sfYfMoDPd7HkzIc= github.com/libp2p/go-tcp-transport v0.1.0 h1:IGhowvEqyMFknOar4FWCKSWE0zL36UFKQtiRQD60/8o= github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc= +github.com/libp2p/go-tcp-transport v0.1.1 h1:yGlqURmqgNA2fvzjSgZNlHcsd/IulAnKM8Ncu+vlqnw= +github.com/libp2p/go-tcp-transport v0.1.1/go.mod h1:3HzGvLbx6etZjnFlERyakbaYPdfjg2pWP97dFZworkY= github.com/libp2p/go-ws-transport v0.1.0 h1:F+0OvvdmPTDsVc4AjPHjV7L7Pk1B7D5QwtDcKE2oag4= github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw7yT74kj3raBFuo= +github.com/libp2p/go-ws-transport v0.2.0 h1:MJCw2OrPA9+76YNRvdo1wMnSOxb9Bivj6sVFY1Xrj6w= +github.com/libp2p/go-ws-transport v0.2.0/go.mod h1:9BHJz/4Q5A9ludYWKoGCFC5gUElzlHoKzu0yY9p/klM= github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.2.3 h1:xX8A36vpXb59frIzWFdEgptLMsOANMFq2K7fPRlunYI= github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= @@ -180,9 +240,13 @@ github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= @@ -191,32 +255,53 @@ github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+ github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.0 h1:U41/2erhAKcmSI14xh/ZTUdBPOzDOIfS93ibzUSl8KM= github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.1/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= github.com/mr-tron/base58 v1.1.2 h1:ZEw4I2EgPKDJ2iEw0cNmLB3ROrEmkOtXIkaG7wZg+78= github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.1.3 h1:v+sk57XuaCKGXpWtVBX8YJzO7hMGx4Aajh4TQbdEFdc= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiformats/go-base32 v0.0.3 h1:tw5+NhuwaOjJCC5Pp82QuXbrmLzWg7uxlMFp8Nq/kkI= github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= github.com/multiformats/go-multiaddr v0.0.1/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.0.2/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= github.com/multiformats/go-multiaddr v0.0.4 h1:WgMSI84/eRLdbptXMkMWDXPjPq7SPLIgGUVm2eroyU4= github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= +github.com/multiformats/go-multiaddr v0.1.0/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lgmoS58qz/pzqmAxV44= +github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= +github.com/multiformats/go-multiaddr v0.2.0 h1:lR52sFwcTCuQb6bTfnXF6zA2XfyYvyd+5a9qECv/J90= +github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2 h1:/Bbsgsy3R6e3jf2qBahzNHzww6usYaZ0NhNH3sqdFS8= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= +github.com/multiformats/go-multiaddr-dns v0.2.0 h1:YWJoIDwLePniH7OU5hBnDZV6SWuvJqJ0YtN6pLeH9zA= +github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= github.com/multiformats/go-multiaddr-fmt v0.0.1 h1:5YjeOIzbX8OTKVaN72aOzGIYW7PnrZrnkDyOfAWRSMA= github.com/multiformats/go-multiaddr-fmt v0.0.1/go.mod h1:aBYjqL4T/7j4Qx+R73XSv/8JsgnRFlf0w2KGLCmXl3Q= +github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= +github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= github.com/multiformats/go-multiaddr-net v0.0.1 h1:76O59E3FavvHqNg7jvzWzsPSW5JSi/ek0E4eiDVbg9g= github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d1tnvM7ihcFwHMSstNAVUU= +github.com/multiformats/go-multiaddr-net v0.1.0/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ= +github.com/multiformats/go-multiaddr-net v0.1.1 h1:jFFKUuXTXv+3ARyHZi3XUqQO+YWMKgBdhEvuGRfnL6s= +github.com/multiformats/go-multiaddr-net v0.1.1/go.mod h1:5JNbcfBOP4dnhoZOv10JJVkJO0pCCEf8mTnipAo2UZQ= github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5 h1:1wxmCvTXAifAepIMyF39vZinRw5sbqjPs/UIi93+uik= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= +github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.10 h1:lMoNbh2Ssd9PUF74Nz008KGzGPlfeV6wH3rit5IIGCM= +github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multistream v0.1.0 h1:UpO6jrsjqs46mqAK3n6wKRYFhugss9ArzbyUzU+4wkQ= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= +github.com/multiformats/go-varint v0.0.1 h1:TR/0rdQtnNxuN2IhiB639xC3tWM4IUi7DkTBVTdGW/M= +github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -232,6 +317,7 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= +github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -246,22 +332,29 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= +github.com/whyrusleeping/go-logging v0.0.1 h1:fwpzlmT0kRC/Fmd0MdmGgJG/CXIZ6gFq46FQZjprUcc= +github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f h1:M/lL30eFZTKnomXY6huvM6G0+gVquFNf6mxghaWlFUg= github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f/go.mod h1:cZNvX9cFybI01GriPRMXDtczuvUhgbcYr9iCGaNlRv8= github.com/whyrusleeping/mafmt v1.2.8 h1:TCghSl5kkwEE0j+sU/gudyhVMRlpBin8fMBBHg59EbA= github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA= github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= +github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:E9S12nwJwEOXe2d6gT6qxdvqMnNq+VnSsKPgm2ZZNds= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= +go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -270,11 +363,15 @@ golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443 h1:IcSOAf4PyMp3U3XbIEj1/xJ2BjNN2jWv7JoyOsMxXUU= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -298,20 +395,31 @@ golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181130052023-1c3d964395ce/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/tests_interop/go_pkgs/examples/utils/host.go b/tests_interop/go_pkgs/examples/utils/host.go index 4024da51..b4ca2925 100644 --- a/tests_interop/go_pkgs/examples/utils/host.go +++ b/tests_interop/go_pkgs/examples/utils/host.go @@ -11,13 +11,16 @@ import ( "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/host" + plaintext "github.com/libp2p/go-libp2p-core/sec/insecure" + noise "github.com/libp2p/go-libp2p-noise" + secio "github.com/libp2p/go-libp2p-secio" ma "github.com/multiformats/go-multiaddr" ) // MakeBasicHost creates a LibP2P host with a random peer ID listening on the // given multiaddress. It won't encrypt the connection if insecure is true. -func MakeBasicHost(listenPort int, insecure bool, randseed int64) (host.Host, error) { +func MakeBasicHost(listenPort int, protocolID string, randseed int64) (host.Host, error) { // If the seed is zero, use real cryptographic randomness. Otherwise, use a // deterministic randomness source to make generated keys stay the same @@ -42,8 +45,22 @@ func MakeBasicHost(listenPort int, insecure bool, randseed int64) (host.Host, er libp2p.DisableRelay(), } - if insecure { + if protocolID == plaintext.ID { opts = append(opts, libp2p.NoSecurity) + } else if protocolID == noise.ID { + tpt, err := noise.New(priv, noise.NoiseKeyPair(nil)) + if err != nil { + return nil, err + } + opts = append(opts, libp2p.Security(protocolID, tpt)) + } else if protocolID == secio.ID { + tpt, err := secio.New(priv) + if err != nil { + return nil, err + } + opts = append(opts, libp2p.Security(protocolID, tpt)) + } else { + return nil, fmt.Errorf("security protocolID '%s' is not supported", protocolID) } basicHost, err := libp2p.New(context.Background(), opts...) @@ -59,11 +76,7 @@ func MakeBasicHost(listenPort int, insecure bool, randseed int64) (host.Host, er addr := basicHost.Addrs()[0] fullAddr := addr.Encapsulate(hostAddr) log.Printf("I am %s\n", fullAddr) - if insecure { - log.Printf("Now run \"./echo -l %d -d %s -insecure\" on a different terminal\n", listenPort+1, fullAddr) - } else { - log.Printf("Now run \"./echo -l %d -d %s\" on a different terminal\n", listenPort+1, fullAddr) - } + log.Printf("Now run \"./echo -l %d -d %s -security %s\" on a different terminal\n", listenPort+1, fullAddr, protocolID) return basicHost, nil } diff --git a/tests_interop/test_echo.py b/tests_interop/test_echo.py index eac8fd30..6a96ac76 100644 --- a/tests_interop/test_echo.py +++ b/tests_interop/test_echo.py @@ -23,9 +23,7 @@ class EchoProcess(BaseInteractiveProcess): def __init__( self, port: int, security_protocol: TProtocol, destination: Multiaddr = None ) -> None: - args = [f"-l={port}"] - if security_protocol == PLAINTEXT_PROTOCOL_ID: - args.append("-insecure") + args = [f"-l={port}", f"-security={security_protocol}"] if destination is not None: args.append(f"-d={str(destination)}") From 6016ea731b9830bce82539e0eead96b282807527 Mon Sep 17 00:00:00 2001 From: mhchia Date: Thu, 20 Feb 2020 21:48:03 +0800 Subject: [PATCH 16/22] `BaseMsgReadWriter` - Change `BaseMsgReadWriter` to encode/decode messages with abstract method, which can be implemented by the subclasses. This allows us to create subclasses `FixedSizeLenMsgReadWriter` and `VarIntLenMsgReadWriter`. --- libp2p/io/exceptions.py | 4 ++ libp2p/io/msgio.py | 61 +++++++++++++++++++-------- libp2p/security/insecure/transport.py | 4 +- libp2p/security/noise/io.py | 4 +- libp2p/security/secio/transport.py | 16 +++---- 5 files changed, 59 insertions(+), 30 deletions(-) diff --git a/libp2p/io/exceptions.py b/libp2p/io/exceptions.py index 0f2230f7..2c237c93 100644 --- a/libp2p/io/exceptions.py +++ b/libp2p/io/exceptions.py @@ -23,3 +23,7 @@ class MissingMessageException(MsgioException): class DecryptionFailedException(MsgioException): pass + + +class MessageTooLarge(MsgioException): + pass diff --git a/libp2p/io/msgio.py b/libp2p/io/msgio.py index 837f642c..457f0553 100644 --- a/libp2p/io/msgio.py +++ b/libp2p/io/msgio.py @@ -5,11 +5,13 @@ from that repo: "a simple package to r/w length-delimited slices." NOTE: currently missing the capability to indicate lengths by "varint" method. """ - -from typing import Optional +from abc import abstractmethod from libp2p.io.abc import MsgReadWriteCloser, Reader, ReadWriteCloser from libp2p.io.utils import read_exactly +from libp2p.utils import decode_uvarint_from_stream, encode_varint_prefixed + +from .exceptions import MessageTooLarge BYTE_ORDER = "big" @@ -31,34 +33,57 @@ def encode_msg_with_length(msg_bytes: bytes, size_len_bytes: int) -> bytes: class BaseMsgReadWriter(MsgReadWriteCloser): - next_length: Optional[int] read_write_closer: ReadWriteCloser size_len_bytes: int def __init__(self, read_write_closer: ReadWriteCloser) -> None: self.read_write_closer = read_write_closer - self.next_length = None async def read_msg(self) -> bytes: length = await self.next_msg_len() + return await read_exactly(self.read_write_closer, length) - data = await read_exactly(self.read_write_closer, length) - if len(data) < length: - self.next_length = length - len(data) - else: - self.next_length = None - return data - + @abstractmethod async def next_msg_len(self) -> int: - if self.next_length is None: - self.next_length = await read_length( - self.read_write_closer, self.size_len_bytes - ) - return self.next_length + ... + + @abstractmethod + def encode_msg(self, msg: bytes) -> bytes: + ... async def close(self) -> None: await self.read_write_closer.close() async def write_msg(self, msg: bytes) -> None: - data = encode_msg_with_length(msg, self.size_len_bytes) - await self.read_write_closer.write(data) + encoded_msg = self.encode_msg(msg) + await self.read_write_closer.write(encoded_msg) + + +class FixedSizeLenMsgReadWriter(BaseMsgReadWriter): + size_len_bytes: int + + async def next_msg_len(self) -> int: + return await read_length(self.read_write_closer, self.size_len_bytes) + + def encode_msg(self, msg: bytes) -> bytes: + return encode_msg_with_length(msg, self.size_len_bytes) + + +class VarIntLengthMsgReadWriter(BaseMsgReadWriter): + max_msg_size: int + + async def next_msg_len(self) -> int: + msg_len = await decode_uvarint_from_stream(self.read_write_closer) + if msg_len > self.max_msg_size: + raise MessageTooLarge( + f"msg_len={msg_len} > max_msg_size={self.max_msg_size}" + ) + return msg_len + + def encode_msg(self, msg: bytes) -> bytes: + msg_len = len(msg) + if msg_len > self.max_msg_size: + raise MessageTooLarge( + f"msg_len={msg_len} > max_msg_size={self.max_msg_size}" + ) + return encode_varint_prefixed(msg) diff --git a/libp2p/security/insecure/transport.py b/libp2p/security/insecure/transport.py index 861ca716..42b7bf83 100644 --- a/libp2p/security/insecure/transport.py +++ b/libp2p/security/insecure/transport.py @@ -3,7 +3,7 @@ from libp2p.crypto.keys import PrivateKey, PublicKey from libp2p.crypto.pb import crypto_pb2 from libp2p.crypto.serialization import deserialize_public_key from libp2p.io.abc import ReadWriteCloser -from libp2p.io.msgio import BaseMsgReadWriter +from libp2p.io.msgio import FixedSizeLenMsgReadWriter from libp2p.network.connection.exceptions import RawConnError from libp2p.network.connection.raw_connection_interface import IRawConnection from libp2p.peer.id import ID @@ -23,7 +23,7 @@ PLAINTEXT_PROTOCOL_ID = TProtocol("/plaintext/2.0.0") SIZE_PLAINTEXT_LEN_BYTES = 4 -class PlaintextHandshakeReadWriter(BaseMsgReadWriter): +class PlaintextHandshakeReadWriter(FixedSizeLenMsgReadWriter): size_len_bytes = SIZE_PLAINTEXT_LEN_BYTES diff --git a/libp2p/security/noise/io.py b/libp2p/security/noise/io.py index 5ffeafe1..499bbeba 100644 --- a/libp2p/security/noise/io.py +++ b/libp2p/security/noise/io.py @@ -3,7 +3,7 @@ from typing import cast from noise.connection import NoiseConnection as NoiseState from libp2p.io.abc import EncryptedMsgReadWriter, MsgReadWriteCloser, ReadWriteCloser -from libp2p.io.msgio import BaseMsgReadWriter, encode_msg_with_length +from libp2p.io.msgio import FixedSizeLenMsgReadWriter, encode_msg_with_length from libp2p.network.connection.raw_connection_interface import IRawConnection SIZE_NOISE_MESSAGE_LEN = 2 @@ -19,7 +19,7 @@ BYTE_ORDER = "big" # <-2 bytes-><- max=65533 bytes -> -class NoisePacketReadWriter(BaseMsgReadWriter): +class NoisePacketReadWriter(FixedSizeLenMsgReadWriter): size_len_bytes = SIZE_NOISE_MESSAGE_LEN diff --git a/libp2p/security/secio/transport.py b/libp2p/security/secio/transport.py index 4759ccc3..46f5c2d7 100644 --- a/libp2p/security/secio/transport.py +++ b/libp2p/security/secio/transport.py @@ -19,7 +19,7 @@ from libp2p.crypto.keys import PrivateKey, PublicKey from libp2p.crypto.serialization import deserialize_public_key from libp2p.io.abc import EncryptedMsgReadWriter from libp2p.io.exceptions import DecryptionFailedException, IOException -from libp2p.io.msgio import BaseMsgReadWriter +from libp2p.io.msgio import FixedSizeLenMsgReadWriter from libp2p.network.connection.raw_connection_interface import IRawConnection from libp2p.peer.id import ID as PeerID from libp2p.security.base_transport import BaseSecureTransport @@ -50,18 +50,18 @@ DEFAULT_SUPPORTED_CIPHERS = "AES-128" DEFAULT_SUPPORTED_HASHES = "SHA256" -class MsgIOReadWriter(BaseMsgReadWriter): +class SecioPacketReadWriter(FixedSizeLenMsgReadWriter): size_len_bytes = SIZE_SECIO_LEN_BYTES class SecioMsgReadWriter(EncryptedMsgReadWriter): - read_writer: MsgIOReadWriter + read_writer: SecioPacketReadWriter def __init__( self, local_encryption_parameters: AuthenticatedEncryptionParameters, remote_encryption_parameters: AuthenticatedEncryptionParameters, - read_writer: MsgIOReadWriter, + read_writer: SecioPacketReadWriter, ) -> None: self.local_encryption_parameters = local_encryption_parameters self.remote_encryption_parameters = remote_encryption_parameters @@ -170,7 +170,7 @@ class SessionParameters: pass -async def _response_to_msg(read_writer: MsgIOReadWriter, msg: bytes) -> bytes: +async def _response_to_msg(read_writer: SecioPacketReadWriter, msg: bytes) -> bytes: await read_writer.write_msg(msg) return await read_writer.read_msg() @@ -234,7 +234,7 @@ async def _establish_session_parameters( local_peer: PeerID, local_private_key: PrivateKey, remote_peer: Optional[PeerID], - conn: MsgIOReadWriter, + conn: SecioPacketReadWriter, nonce: bytes, ) -> Tuple[SessionParameters, bytes]: # establish shared encryption parameters @@ -326,7 +326,7 @@ async def _establish_session_parameters( def _mk_session_from( local_private_key: PrivateKey, session_parameters: SessionParameters, - conn: MsgIOReadWriter, + conn: SecioPacketReadWriter, is_initiator: bool, ) -> SecureSession: key_set1, key_set2 = initialize_pair_for_encryption( @@ -371,7 +371,7 @@ async def create_secure_session( to the ``remote_peer``. Raise `SecioException` when `conn` closed. Raise `InconsistentNonce` when handshake failed """ - msg_io = MsgIOReadWriter(conn) + msg_io = SecioPacketReadWriter(conn) try: session_parameters, remote_nonce = await _establish_session_parameters( local_peer, local_private_key, remote_peer, msg_io, local_nonce From f8240bd2cba930c97faa5a2c3ad5f0253b22ba39 Mon Sep 17 00:00:00 2001 From: mhchia Date: Thu, 20 Feb 2020 22:55:11 +0800 Subject: [PATCH 17/22] plaintext: use varint msg read/writer --- libp2p/security/insecure/transport.py | 8 +++----- libp2p/tools/interop/daemon.py | 7 ++----- tests_interop/test_echo.py | 1 - 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/libp2p/security/insecure/transport.py b/libp2p/security/insecure/transport.py index 42b7bf83..8211944c 100644 --- a/libp2p/security/insecure/transport.py +++ b/libp2p/security/insecure/transport.py @@ -3,7 +3,7 @@ from libp2p.crypto.keys import PrivateKey, PublicKey from libp2p.crypto.pb import crypto_pb2 from libp2p.crypto.serialization import deserialize_public_key from libp2p.io.abc import ReadWriteCloser -from libp2p.io.msgio import FixedSizeLenMsgReadWriter +from libp2p.io.msgio import VarIntLengthMsgReadWriter from libp2p.network.connection.exceptions import RawConnError from libp2p.network.connection.raw_connection_interface import IRawConnection from libp2p.peer.id import ID @@ -20,11 +20,9 @@ from .pb import plaintext_pb2 PLAINTEXT_PROTOCOL_ID = TProtocol("/plaintext/2.0.0") -SIZE_PLAINTEXT_LEN_BYTES = 4 - -class PlaintextHandshakeReadWriter(FixedSizeLenMsgReadWriter): - size_len_bytes = SIZE_PLAINTEXT_LEN_BYTES +class PlaintextHandshakeReadWriter(VarIntLengthMsgReadWriter): + max_msg_size = 1 << 16 class InsecureSession(BaseSession): diff --git a/libp2p/tools/interop/daemon.py b/libp2p/tools/interop/daemon.py index 4dfcfb63..95d19ef0 100644 --- a/libp2p/tools/interop/daemon.py +++ b/libp2p/tools/interop/daemon.py @@ -8,7 +8,6 @@ import trio from libp2p.peer.id import ID from libp2p.peer.peerinfo import PeerInfo, info_from_p2p_addr -from libp2p.security.insecure.transport import PLAINTEXT_PROTOCOL_ID from libp2p.typing import TProtocol from .constants import LOCALHOST_IP @@ -28,10 +27,8 @@ class P2PDProcess(BaseInteractiveProcess): is_pubsub_signing: bool = False, is_pubsub_signing_strict: bool = False, ) -> None: - args = [f"-listen={control_maddr!s}"] - # NOTE: To support `-insecure`, we need to hack `go-libp2p-daemon`. - if security_protocol == PLAINTEXT_PROTOCOL_ID: - args.append("-insecure=true") + # NOTE: To support `-security`, we need to hack `go-libp2p-daemon`. + args = [f"-listen={control_maddr!s}", f"-security={security_protocol}"] if is_pubsub_enabled: args.append("-pubsub") if is_gossipsub: diff --git a/tests_interop/test_echo.py b/tests_interop/test_echo.py index 6a96ac76..e19d2673 100644 --- a/tests_interop/test_echo.py +++ b/tests_interop/test_echo.py @@ -6,7 +6,6 @@ import pytest import trio from libp2p.peer.peerinfo import PeerInfo, info_from_p2p_addr -from libp2p.security.insecure.transport import PLAINTEXT_PROTOCOL_ID from libp2p.tools.factories import HostFactory from libp2p.tools.interop.envs import GO_BIN_PATH from libp2p.tools.interop.process import BaseInteractiveProcess From 728d9e5c9cb207c145a24756a4d7b10faaa80b39 Mon Sep 17 00:00:00 2001 From: mhchia Date: Wed, 26 Feb 2020 15:04:43 +0800 Subject: [PATCH 18/22] Noise: update pb --- libp2p/security/noise/pb/noise.proto | 9 ++++++--- libp2p/security/noise/pb/noise_pb2.py | 22 +++++++++++----------- libp2p/security/noise/pb/noise_pb2.pyi | 2 -- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/libp2p/security/noise/pb/noise.proto b/libp2p/security/noise/pb/noise.proto index 54cef926..05a78c6f 100644 --- a/libp2p/security/noise/pb/noise.proto +++ b/libp2p/security/noise/pb/noise.proto @@ -1,5 +1,8 @@ +syntax = "proto3"; +package pb; + message NoiseHandshakePayload { - optional bytes identity_key = 1; - optional bytes identity_sig = 2; - optional bytes data = 3; + bytes identity_key = 1; + bytes identity_sig = 2; + bytes data = 3; } diff --git a/libp2p/security/noise/pb/noise_pb2.py b/libp2p/security/noise/pb/noise_pb2.py index d7b92ae0..63320daf 100644 --- a/libp2p/security/noise/pb/noise_pb2.py +++ b/libp2p/security/noise/pb/noise_pb2.py @@ -17,9 +17,9 @@ _sym_db = _symbol_database.Default() DESCRIPTOR = _descriptor.FileDescriptor( name='libp2p/security/noise/pb/noise.proto', - package='', - syntax='proto2', - serialized_pb=_b('\n$libp2p/security/noise/pb/noise.proto\"Q\n\x15NoiseHandshakePayload\x12\x14\n\x0cidentity_key\x18\x01 \x01(\x0c\x12\x14\n\x0cidentity_sig\x18\x02 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c') + package='pb', + syntax='proto3', + serialized_pb=_b('\n$libp2p/security/noise/pb/noise.proto\x12\x02pb\"Q\n\x15NoiseHandshakePayload\x12\x14\n\x0cidentity_key\x18\x01 \x01(\x0c\x12\x14\n\x0cidentity_sig\x18\x02 \x01(\x0c\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x62\x06proto3') ) @@ -27,27 +27,27 @@ DESCRIPTOR = _descriptor.FileDescriptor( _NOISEHANDSHAKEPAYLOAD = _descriptor.Descriptor( name='NoiseHandshakePayload', - full_name='NoiseHandshakePayload', + full_name='pb.NoiseHandshakePayload', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='identity_key', full_name='NoiseHandshakePayload.identity_key', index=0, + name='identity_key', full_name='pb.NoiseHandshakePayload.identity_key', index=0, number=1, type=12, cpp_type=9, label=1, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='identity_sig', full_name='NoiseHandshakePayload.identity_sig', index=1, + name='identity_sig', full_name='pb.NoiseHandshakePayload.identity_sig', index=1, number=2, type=12, cpp_type=9, label=1, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='data', full_name='NoiseHandshakePayload.data', index=2, + name='data', full_name='pb.NoiseHandshakePayload.data', index=2, number=3, type=12, cpp_type=9, label=1, has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, @@ -61,12 +61,12 @@ _NOISEHANDSHAKEPAYLOAD = _descriptor.Descriptor( ], options=None, is_extendable=False, - syntax='proto2', + syntax='proto3', extension_ranges=[], oneofs=[ ], - serialized_start=40, - serialized_end=121, + serialized_start=44, + serialized_end=125, ) DESCRIPTOR.message_types_by_name['NoiseHandshakePayload'] = _NOISEHANDSHAKEPAYLOAD @@ -75,7 +75,7 @@ _sym_db.RegisterFileDescriptor(DESCRIPTOR) NoiseHandshakePayload = _reflection.GeneratedProtocolMessageType('NoiseHandshakePayload', (_message.Message,), dict( DESCRIPTOR = _NOISEHANDSHAKEPAYLOAD, __module__ = 'libp2p.security.noise.pb.noise_pb2' - # @@protoc_insertion_point(class_scope:NoiseHandshakePayload) + # @@protoc_insertion_point(class_scope:pb.NoiseHandshakePayload) )) _sym_db.RegisterMessage(NoiseHandshakePayload) diff --git a/libp2p/security/noise/pb/noise_pb2.pyi b/libp2p/security/noise/pb/noise_pb2.pyi index eea8fda1..a4bcf5b1 100644 --- a/libp2p/security/noise/pb/noise_pb2.pyi +++ b/libp2p/security/noise/pb/noise_pb2.pyi @@ -34,8 +34,6 @@ class NoiseHandshakePayload(google___protobuf___message___Message): def MergeFrom(self, other_msg: google___protobuf___message___Message) -> None: ... def CopyFrom(self, other_msg: google___protobuf___message___Message) -> None: ... if sys.version_info >= (3,): - def HasField(self, field_name: typing_extensions___Literal[u"data",u"identity_key",u"identity_sig"]) -> bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"data",u"identity_key",u"identity_sig"]) -> None: ... else: - def HasField(self, field_name: typing_extensions___Literal[u"data",b"data",u"identity_key",b"identity_key",u"identity_sig",b"identity_sig"]) -> bool: ... def ClearField(self, field_name: typing_extensions___Literal[u"data",b"data",u"identity_key",b"identity_key",u"identity_sig",b"identity_sig"]) -> None: ... From c1f6054b3cec8e1b642242625d7e9546b094621d Mon Sep 17 00:00:00 2001 From: mhchia Date: Wed, 26 Feb 2020 15:05:05 +0800 Subject: [PATCH 19/22] plaintext: update pb --- libp2p/security/insecure/pb/plaintext_pb2.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/libp2p/security/insecure/pb/plaintext_pb2.py b/libp2p/security/insecure/pb/plaintext_pb2.py index 4ee92ab3..72b27400 100644 --- a/libp2p/security/insecure/pb/plaintext_pb2.py +++ b/libp2p/security/insecure/pb/plaintext_pb2.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: libp2p/security/insecure/pb/plaintext.proto @@ -8,6 +7,7 @@ from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -20,7 +20,6 @@ DESCRIPTOR = _descriptor.FileDescriptor( name='libp2p/security/insecure/pb/plaintext.proto', package='plaintext.pb', syntax='proto2', - serialized_options=None, serialized_pb=_b('\n+libp2p/security/insecure/pb/plaintext.proto\x12\x0cplaintext.pb\x1a\x1dlibp2p/crypto/pb/crypto.proto\"<\n\x08\x45xchange\x12\n\n\x02id\x18\x01 \x01(\x0c\x12$\n\x06pubkey\x18\x02 \x01(\x0b\x32\x14.crypto.pb.PublicKey') , dependencies=[libp2p_dot_crypto_dot_pb_dot_crypto__pb2.DESCRIPTOR,]) @@ -41,21 +40,21 @@ _EXCHANGE = _descriptor.Descriptor( has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='pubkey', full_name='plaintext.pb.Exchange.pubkey', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - serialized_options=None, + options=None, is_extendable=False, syntax='proto2', extension_ranges=[], @@ -69,11 +68,11 @@ _EXCHANGE.fields_by_name['pubkey'].message_type = libp2p_dot_crypto_dot_pb_dot_c DESCRIPTOR.message_types_by_name['Exchange'] = _EXCHANGE _sym_db.RegisterFileDescriptor(DESCRIPTOR) -Exchange = _reflection.GeneratedProtocolMessageType('Exchange', (_message.Message,), { - 'DESCRIPTOR' : _EXCHANGE, - '__module__' : 'libp2p.security.insecure.pb.plaintext_pb2' +Exchange = _reflection.GeneratedProtocolMessageType('Exchange', (_message.Message,), dict( + DESCRIPTOR = _EXCHANGE, + __module__ = 'libp2p.security.insecure.pb.plaintext_pb2' # @@protoc_insertion_point(class_scope:plaintext.pb.Exchange) - }) + )) _sym_db.RegisterMessage(Exchange) From cb4e1115c6c57233df68a1f9ba86007a517756a4 Mon Sep 17 00:00:00 2001 From: mhchia Date: Wed, 26 Feb 2020 15:33:22 +0800 Subject: [PATCH 20/22] Remove wrong encoding --- libp2p/security/noise/io.py | 39 +++++++++---------------------------- 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/libp2p/security/noise/io.py b/libp2p/security/noise/io.py index 499bbeba..3dd024d3 100644 --- a/libp2p/security/noise/io.py +++ b/libp2p/security/noise/io.py @@ -3,7 +3,7 @@ from typing import cast from noise.connection import NoiseConnection as NoiseState from libp2p.io.abc import EncryptedMsgReadWriter, MsgReadWriteCloser, ReadWriteCloser -from libp2p.io.msgio import FixedSizeLenMsgReadWriter, encode_msg_with_length +from libp2p.io.msgio import FixedSizeLenMsgReadWriter from libp2p.network.connection.raw_connection_interface import IRawConnection SIZE_NOISE_MESSAGE_LEN = 2 @@ -15,37 +15,12 @@ BYTE_ORDER = "big" # | Noise packet | # < 2 bytes -><- 65535 -> # | noise msg len | noise msg | -# | body len | body | padding | -# <-2 bytes-><- max=65533 bytes -> class NoisePacketReadWriter(FixedSizeLenMsgReadWriter): size_len_bytes = SIZE_NOISE_MESSAGE_LEN -def encode_msg_body(msg_body: bytes) -> bytes: - encoded_msg_body = encode_msg_with_length(msg_body, SIZE_NOISE_MESSAGE_BODY_LEN) - if len(encoded_msg_body) > MAX_NOISE_MESSAGE_BODY_LEN: - raise ValueError( - f"msg_body is too long: {len(msg_body)}, " - f"maximum={MAX_NOISE_MESSAGE_BODY_LEN}" - ) - # NOTE: Improvements: - # 1. Senders *SHOULD* use a source of random data to populate the padding field. - # 2. and *may* use any length of padding that does not cause the total length of - # the Noise message to exceed 65535 bytes. - # Ref: https://github.com/libp2p/specs/tree/master/noise#encrypted-payloads - return encoded_msg_body # + padding - - -def decode_msg_body(noise_msg: bytes) -> bytes: - len_body = int.from_bytes(noise_msg[:SIZE_NOISE_MESSAGE_BODY_LEN], BYTE_ORDER) - # Just ignore the padding - return noise_msg[ - SIZE_NOISE_MESSAGE_BODY_LEN : (SIZE_NOISE_MESSAGE_BODY_LEN + len_body) - ] - - class BaseNoiseMsgReadWriter(EncryptedMsgReadWriter): """ The base implementation of noise message reader/writer. @@ -62,14 +37,18 @@ class BaseNoiseMsgReadWriter(EncryptedMsgReadWriter): self.noise_state = noise_state async def write_msg(self, data: bytes) -> None: - noise_msg = encode_msg_body(data) - data_encrypted = self.encrypt(noise_msg) + data_encrypted = self.encrypt(data) + # FIXME: Decide whether this prefix should be added or not. + # if not first: + # data_encrypted = b"\x00" * 32 + data_encrypted await self.read_writer.write_msg(data_encrypted) async def read_msg(self) -> bytes: noise_msg_encrypted = await self.read_writer.read_msg() - noise_msg = self.decrypt(noise_msg_encrypted) - return decode_msg_body(noise_msg) + # FIXME: Decide whether this prefix should be added or not. + # if not first: + # noise_msg_encrypted = noise_msg_encrypted[32:] + return self.decrypt(noise_msg_encrypted) async def close(self) -> None: await self.read_writer.close() From 1af12ddf43bed02fbe82a8f7933ede82cbe88ced Mon Sep 17 00:00:00 2001 From: mhchia Date: Wed, 26 Feb 2020 16:07:01 +0800 Subject: [PATCH 21/22] interop test: support both noise and plaintext --- libp2p/security/noise/io.py | 23 +++++++++++++---------- libp2p/security/noise/patterns.py | 4 ++-- tests_interop/conftest.py | 7 ++++--- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/libp2p/security/noise/io.py b/libp2p/security/noise/io.py index 3dd024d3..c6d48e48 100644 --- a/libp2p/security/noise/io.py +++ b/libp2p/security/noise/io.py @@ -32,23 +32,26 @@ class BaseNoiseMsgReadWriter(EncryptedMsgReadWriter): read_writer: MsgReadWriteCloser noise_state: NoiseState + # FIXME: This prefix is added in msg#3 in Go. Check whether it's a desired behavior. + prefix: bytes = b"\x00" * 32 + def __init__(self, conn: IRawConnection, noise_state: NoiseState) -> None: self.read_writer = NoisePacketReadWriter(cast(ReadWriteCloser, conn)) self.noise_state = noise_state - async def write_msg(self, data: bytes) -> None: + async def write_msg(self, data: bytes, prefix_encoded: bool = False) -> None: data_encrypted = self.encrypt(data) - # FIXME: Decide whether this prefix should be added or not. - # if not first: - # data_encrypted = b"\x00" * 32 + data_encrypted - await self.read_writer.write_msg(data_encrypted) + if prefix_encoded: + await self.read_writer.write_msg(self.prefix + data_encrypted) + else: + await self.read_writer.write_msg(data_encrypted) - async def read_msg(self) -> bytes: + async def read_msg(self, prefix_encoded: bool = False) -> bytes: noise_msg_encrypted = await self.read_writer.read_msg() - # FIXME: Decide whether this prefix should be added or not. - # if not first: - # noise_msg_encrypted = noise_msg_encrypted[32:] - return self.decrypt(noise_msg_encrypted) + if prefix_encoded: + return self.decrypt(noise_msg_encrypted[len(self.prefix) :]) + else: + return self.decrypt(noise_msg_encrypted) async def close(self) -> None: await self.read_writer.close() diff --git a/libp2p/security/noise/patterns.py b/libp2p/security/noise/patterns.py index fd6efc56..1e537d05 100644 --- a/libp2p/security/noise/patterns.py +++ b/libp2p/security/noise/patterns.py @@ -88,7 +88,7 @@ class PatternXX(BasePattern): await read_writer.write_msg(msg_2) # Receive and consume msg#3. - msg_3 = await read_writer.read_msg() + msg_3 = await read_writer.read_msg(prefix_encoded=True) peer_handshake_payload = NoiseHandshakePayload.deserialize(msg_3) if handshake_state.rs is None: @@ -156,7 +156,7 @@ class PatternXX(BasePattern): # Send msg#3, which includes our encrypted payload and our noise static key. our_payload = self.make_handshake_payload() msg_3 = our_payload.serialize() - await read_writer.write_msg(msg_3) + await read_writer.write_msg(msg_3, prefix_encoded=True) if not noise_state.handshake_finished: raise HandshakeHasNotFinished( diff --git a/tests_interop/conftest.py b/tests_interop/conftest.py index b14f91cc..a59af28f 100644 --- a/tests_interop/conftest.py +++ b/tests_interop/conftest.py @@ -7,14 +7,15 @@ import trio from libp2p.io.abc import ReadWriteCloser from libp2p.security.insecure.transport import PLAINTEXT_PROTOCOL_ID +from libp2p.security.noise.transport import PROTOCOL_ID as NOISE_PROTOCOL_ID from libp2p.tools.factories import HostFactory, PubsubFactory from libp2p.tools.interop.daemon import make_p2pd from libp2p.tools.interop.utils import connect -@pytest.fixture -def security_protocol(): - return PLAINTEXT_PROTOCOL_ID +@pytest.fixture(params=[PLAINTEXT_PROTOCOL_ID, NOISE_PROTOCOL_ID]) +def security_protocol(request): + return request.param @pytest.fixture From ec8c10965da727b043e4b7cd6664e09d497ac636 Mon Sep 17 00:00:00 2001 From: mhchia Date: Wed, 26 Feb 2020 16:21:05 +0800 Subject: [PATCH 22/22] travis CI: use the latest go version --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1c7856d6..a412fa1c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,11 +16,11 @@ matrix: env: TOXENV=docs - python: 3.7 dist: xenial - env: TOXENV=py37-interop + env: TOXENV=py37-interop GOBINPKG=go1.13.8.linux-amd64.tar.gz sudo: true before_install: - - wget https://dl.google.com/go/go1.12.6.linux-amd64.tar.gz - - sudo tar -C /usr/local -xzf go1.12.6.linux-amd64.tar.gz + - wget https://dl.google.com/go/$GOBINPKG + - sudo tar -C /usr/local -xzf $GOBINPKG - export GOPATH=$HOME/go - export GOROOT=/usr/local/go - export PATH=$GOROOT/bin:$GOPATH/bin:$PATH