From a7955a560ea8f8c10694ecdd57a6b189086f1e9c Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 24 Jan 2018 16:00:28 -0800 Subject: [PATCH 001/178] init --- .bumpversion.cfg | 23 ++++ .gitignore | 107 ++++++++++++++++ .travis.yml | 31 +++++ LICENSE | 21 ++++ Makefile | 54 ++++++++ README.md | 84 +++++++++++++ docs/conf.py | 274 +++++++++++++++++++++++++++++++++++++++++ docs/releases.rst | 7 ++ fill_template_vars.sh | 35 ++++++ pytest.ini | 7 ++ setup.py | 64 ++++++++++ tests/core/conftest.py | 0 tox.ini | 35 ++++++ 13 files changed, 742 insertions(+) create mode 100644 .bumpversion.cfg create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 docs/conf.py create mode 100644 docs/releases.rst create mode 100755 fill_template_vars.sh create mode 100644 pytest.ini create mode 100644 setup.py create mode 100644 tests/core/conftest.py create mode 100644 tox.ini diff --git a/.bumpversion.cfg b/.bumpversion.cfg new file mode 100644 index 00000000..eefe4ad9 --- /dev/null +++ b/.bumpversion.cfg @@ -0,0 +1,23 @@ +[bumpversion] +current_version = 0.1.0-alpha.0 +commit = True +tag = True +parse = (?P\d+)\.(?P\d+)\.(?P\d+)(-(?P[^.]*)\.(?P\d+))? +serialize = + {major}.{minor}.{patch}-{stage}.{devnum} + {major}.{minor}.{patch} + +[bumpversion:part:stage] +optional_value = stable +first_value = stable +values = + alpha + beta + stable + +[bumpversion:part:devnum] + +[bumpversion:file:setup.py] +search = version='{current_version}', +replace = version='{new_version}', + diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..c3802038 --- /dev/null +++ b/.gitignore @@ -0,0 +1,107 @@ +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg +*.egg-info +dist +build +eggs +.eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 +venv* + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Complexity +output/*.html +output/*/index.html + +# Sphinx +docs/_build +docs/modules.rst +docs/web3.* + +# Blockchain +chains + +# Hypothese Property base testing +.hypothesis + +# tox/pytest cache +.cache + +# Test output logs +logs +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/workspace.xml +.idea/tasks.xml +.idea/dictionaries +.idea/vcs.xml +.idea/jsLibraryMappings.xml + +# Sensitive or high-churn files: +.idea/dataSources.ids +.idea/dataSources.xml +.idea/dataSources.local.xml +.idea/sqlDataSources.xml +.idea/dynamic.xml +.idea/uiDesigner.xml + +# Gradle: +.idea/gradle.xml +.idea/libraries + +# Mongo Explorer plugin: +.idea/mongoSettings.xml + +# VIM temp files +*.swp + +## File-based project format: +*.iws + +## Plugin-specific files: + +# IntelliJ +/out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..72be81d4 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,31 @@ +sudo: false +language: python +dist: trusty +matrix: + include: + # + # Python 3.5 testing + # + # lint + - python: "3.5" + env: TOX_POSARGS="-e lint" + # core + - python: "3.5" + env: TOX_POSARGS="-e py35-core" + # + # Python 3.6 testing + # + # core + - python: "3.6" + env: TOX_POSARGS="-e py36-core" +cache: + - pip: true +install: + - travis_retry pip install pip setuptools --upgrade + - travis_retry pip install tox +before_script: + - python --version + - pip --version + - pip freeze +script: + - tox $TOX_POSARGS diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..1b7a2b56 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2018 Jason Carver + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..b8424652 --- /dev/null +++ b/Makefile @@ -0,0 +1,54 @@ +.PHONY: clean-pyc clean-build docs + +help: + @echo "clean-build - remove build artifacts" + @echo "clean-pyc - remove Python file artifacts" + @echo "lint - check style with flake8" + @echo "test - run tests quickly with the default Python" + @echo "testall - run tests on every Python version with tox" + @echo "release - package and upload a release" + @echo "dist - package" + +clean: clean-build clean-pyc + +clean-build: + rm -fr build/ + rm -fr dist/ + rm -fr *.egg-info + +clean-pyc: + find . -name '*.pyc' -exec rm -f {} + + find . -name '*.pyo' -exec rm -f {} + + find . -name '*~' -exec rm -f {} + + +lint: + tox -elint + +test: + py.test tests + +test-all: + tox + +build-docs: + sphinx-apidoc -o docs/ . setup.py "*conftest*" + $(MAKE) -C docs clean + $(MAKE) -C docs html + +docs: build-docs + open docs/_build/html/index.html + +linux-docs: build-docs + xdg-open docs/_build/html/index.html + +release: clean + CURRENT_SIGN_SETTING=$(git config commit.gpgSign) + git config commit.gpgSign true + bumpversion $(bump) + git push && git push --tags + python setup.py sdist bdist_wheel upload + git config commit.gpgSign "$(CURRENT_SIGN_SETTING)" + +dist: clean + python setup.py sdist bdist_wheel + ls -l dist diff --git a/README.md b/README.md new file mode 100644 index 00000000..e29c4d9b --- /dev/null +++ b/README.md @@ -0,0 +1,84 @@ +# + +[![Join the chat at https://gitter.im/ethereum/](https://badges.gitter.im/ethereum/.svg)](https://gitter.im/ethereum/?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + +[![Build Status](https://travis-ci.org/ethereum/.png)](https://travis-ci.org/ethereum/) + + + + +* Python 3.5+ support + +Read more in the [documentation on ReadTheDocs](http://.readthedocs.io/). [View the change log on Github](docs/releases.rst). + +## Quickstart + +```sh +pip install +``` + +## Developer setup + +If you would like to hack on , set up your dev environment with: + +```sh + +git clone git@github.com:ethereum/.git +cd +virtualenv -p python3 venv +. venv/bin/activate +pip install -e .[dev] +``` + +### Testing Setup + +During development, you might like to have tests run on every file save. + +Show flake8 errors on file change: + +```sh +# Test flake8 +when-changed -v -s -r -1 / tests/ -c "clear; flake8 tests && echo 'flake8 success' || echo 'error'" +``` + +Run multi-process tests in one command, but without color: + +```sh +# in the project root: +pytest --numprocesses=4 --looponfail --maxfail=1 +# the same thing, succinctly: +pytest -n 4 -f --maxfail=1 +``` + +Run in one thread, with color and desktop notifications: + +```sh +cd venv +ptw --onfail "notify-send -t 5000 'Test failure ⚠⚠⚠⚠⚠' 'python 3 test on failed'" ../tests ../ +``` + +### Release setup + +For Debian-like systems: +``` +apt install pandoc +``` + +To release a new version: + +```sh +make release bump=$$VERSION_PART_TO_BUMP$$ +``` + +#### How to bumpversion + +The version format for this repo is `{major}.{minor}.{patch}` for stable, and +`{major}.{minor}.{patch}-{stage}.{devnum}` for unstable (`stage` can be alpha or beta). + +To issue the next version in line, specify which part to bump, +like `make release bump=minor` or `make release bump=devnum`. + +If you are in a beta version, `make release bump=stage` will switch to a stable. + +To issue an unstable version when the current version is stable, specify the +new version explicitly, like `make release bump="--new-version 4.0.0-alpha.1 devnum"` diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..77bcb7eb --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,274 @@ +# -*- coding: utf-8 -*- +# +# documentation build configuration file, created by +# sphinx-quickstart on Thu Oct 16 20:43:24 2014. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +import os + +DIR = os.path.dirname('__file__') +with open (os.path.join(DIR, '../setup.py'), 'r') as f: + for line in f: + if 'version=' in line: + setup_version = line.split('\'')[1] + break + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = '' +copyright = '2018, Jason Carver, Piper Merriam' + +__version__ = setup_version +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '.'.join(__version__.split('.')[:2]) +# The full version, including alpha/beta/rc tags. +release = __version__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [ + '_build', + '.rst', + 'modules.rst', +] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'doc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ('index', '.tex', ' Documentation', + 'Jason Carver', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', '', ' Documentation', + ['Jason Carver'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', '', ' Documentation', + 'Jason Carver', '', '', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False + +# -- Intersphinx configuration ------------------------------------------------ + +intersphinx_mapping = { + 'python': ('https://docs.python.org/3.5', None), +} diff --git a/docs/releases.rst b/docs/releases.rst new file mode 100644 index 00000000..ec4804d1 --- /dev/null +++ b/docs/releases.rst @@ -0,0 +1,7 @@ +Release Notes +============= + +v0.1.0-alpha.1 +------------- + +- Launched repository, claimed names for pip, RTD, github, etc diff --git a/fill_template_vars.sh b/fill_template_vars.sh new file mode 100755 index 00000000..ecbdcb08 --- /dev/null +++ b/fill_template_vars.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +# List of all non-executable files +TEMPLATE_FILES=$(find . ! -perm -u=x -type f) + +echo "What is your python module name?" +read MODULE_NAME + +echo "What is your pypi package name? (default: $MODULE_NAME)" +read PYPI_INPUT +PYPI_NAME=${PYPI_INPUT:-$MODULE_NAME} + +echo "What is your github project name? (default: $PYPI_NAME)" +read REPO_INPUT +REPO_NAME=${REPO_INPUT:-$PYPI_NAME} + +echo "What is your readthedocs.org project name? (default: $PYPI_NAME)" +read RTD_INPUT +RTD_NAME=${RTD_INPUT:-$PYPI_NAME} + +echo "What is your project name (ex: at the top of the README)? (default: $REPO_NAME)" +read PROJECT_INPUT +PROJECT_NAME=${PROJECT_INPUT:-$REPO_NAME} + +echo "What is a one-liner describing the project?" +read SHORT_DESCRIPTION + +sed -i "s//$MODULE_NAME/g" $TEMPLATE_FILES +sed -i "s//$PYPI_NAME/g" $TEMPLATE_FILES +sed -i "s//$REPO_NAME/g" $TEMPLATE_FILES +sed -i "s//$SHORT_DESCRIPTION/g" $TEMPLATE_FILES diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..d53de3aa --- /dev/null +++ b/pytest.ini @@ -0,0 +1,7 @@ +[pytest] +addopts= -v --showlocals --durations 10 +python_paths= . +xfail_strict=true + +[pytest-watch] +runner= py.test --failed-first --maxfail=1 --no-success-flaky-report diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..4882a641 --- /dev/null +++ b/setup.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from setuptools import ( + setup, + find_packages, +) + +extras_require={ + 'test': [ + "pytest==3.3.2", + "tox>=2.9.1,<3", + ], + 'lint': [ + "flake8==3.4.1", + "isort>=4.2.15,<5", + ], + 'document': [ + "Sphinx>=1.6.5,<2", + "sphinx_rtd_theme>=0.1.9", + ], + 'dev': [ + "bumpversion>=0.5.3,<1", + "pytest-xdist", + "wheel", + ], +} + +extras_require['dev'] = ( + extras_require['dev'] + + extras_require['test'] + + extras_require['lint'] + + extras_require['document'] +) + +setup( + name='', + # *IMPORTANT*: Don't manually change the version here. Use `make bump`, as described in readme + version='0.1.0-alpha.1', + description=""": """, + long_description_markdown_filename='README.md', + author='Jason Carver', + author_email='ethcalibur+pip@gmail.com', + url='https://github.com/ethereum/', + include_package_data=True, + install_requires=[ + "eth-utils>=0.7.4,<1.0.0", + ], + setup_requires=['setuptools-markdown'], + extras_require=extras_require, + py_modules=[''], + license="MIT", + zip_safe=False, + keywords='ethereum', + packages=find_packages(exclude=["tests", "tests.*"]), + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Natural Language :: English', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + ], +) diff --git a/tests/core/conftest.py b/tests/core/conftest.py new file mode 100644 index 00000000..e69de29b diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..02772752 --- /dev/null +++ b/tox.ini @@ -0,0 +1,35 @@ +[tox] +envlist= + py{35,36}-core + lint + +[isort] +combine_as_imports=True +force_sort_within_sections=True +include_trailing_comma=True +known_standard_library=pytest +known_first_party= +line_length=21 +multi_line_output=3 +use_parentheses=True + +[flake8] +max-line-length= 100 +exclude= venv*,.tox,docs,build +ignore= + +[testenv] +usedevelop=True +commands= + core: py.test {posargs:tests/core} +basepython = + py35: python3.5 + py36: python3.6 +extras=test + +[testenv:lint] +basepython=python +extras=lint +commands= + flake8 {toxinidir}/ {toxinidir}/tests + isort --recursive --check-only --diff {toxinidir}/ {toxinidir}/tests From 3d87ff715396b8a9cdfc9d8f13b71c4e0dd2dace Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 24 Jan 2018 16:24:08 -0800 Subject: [PATCH 002/178] Replace all template vars --- fill_template_vars.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fill_template_vars.sh b/fill_template_vars.sh index ecbdcb08..236733a4 100755 --- a/fill_template_vars.sh +++ b/fill_template_vars.sh @@ -32,4 +32,6 @@ read SHORT_DESCRIPTION sed -i "s//$MODULE_NAME/g" $TEMPLATE_FILES sed -i "s//$PYPI_NAME/g" $TEMPLATE_FILES sed -i "s//$REPO_NAME/g" $TEMPLATE_FILES +sed -i "s//$RTD_NAME/g" $TEMPLATE_FILES +sed -i "s//$PROJECT_NAME/g" $TEMPLATE_FILES sed -i "s//$SHORT_DESCRIPTION/g" $TEMPLATE_FILES From cdba56a29525ba9c4ba5e82dcb7901cdc3bccd9a Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 24 Jan 2018 16:24:31 -0800 Subject: [PATCH 003/178] skip replacing variables inside .git/* files --- fill_template_vars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fill_template_vars.sh b/fill_template_vars.sh index 236733a4..d4dba88e 100755 --- a/fill_template_vars.sh +++ b/fill_template_vars.sh @@ -5,7 +5,7 @@ set -o nounset set -o pipefail # List of all non-executable files -TEMPLATE_FILES=$(find . ! -perm -u=x -type f) +TEMPLATE_FILES=$(find . ! -perm -u=x -type f | grep -v "\.git") echo "What is your python module name?" read MODULE_NAME From 9cdfd649d581c20939f90cb549c28c6b7e03dae8 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 24 Jan 2018 16:31:02 -0800 Subject: [PATCH 004/178] create empty module with supplied name --- fill_template_vars.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fill_template_vars.sh b/fill_template_vars.sh index d4dba88e..a0f9b903 100755 --- a/fill_template_vars.sh +++ b/fill_template_vars.sh @@ -35,3 +35,6 @@ sed -i "s//$REPO_NAME/g" $TEMPLATE_FILES sed -i "s//$RTD_NAME/g" $TEMPLATE_FILES sed -i "s//$PROJECT_NAME/g" $TEMPLATE_FILES sed -i "s//$SHORT_DESCRIPTION/g" $TEMPLATE_FILES + +mkdir $MODULE_NAME +touch $MODULE_NAME/__init__.py From 4a8b7b06afca9a9d0d533e76ba1cca24788244b7 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 24 Jan 2018 16:33:03 -0800 Subject: [PATCH 005/178] shorten extra name for documentation install --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 4882a641..181033eb 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ extras_require={ "flake8==3.4.1", "isort>=4.2.15,<5", ], - 'document': [ + 'doc': [ "Sphinx>=1.6.5,<2", "sphinx_rtd_theme>=0.1.9", ], @@ -29,7 +29,7 @@ extras_require['dev'] = ( extras_require['dev'] + extras_require['test'] + extras_require['lint'] - + extras_require['document'] + + extras_require['doc'] ) setup( From ca670f43a4be6b2d6f326e07419c6b07c2167db1 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 24 Jan 2018 16:35:36 -0800 Subject: [PATCH 006/178] start at devnum 0 so the first bump goes to 1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 181033eb..fbef3442 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ extras_require['dev'] = ( setup( name='', # *IMPORTANT*: Don't manually change the version here. Use `make bump`, as described in readme - version='0.1.0-alpha.1', + version='0.1.0-alpha.0', description=""": """, long_description_markdown_filename='README.md', author='Jason Carver', From 1d4e19d929e6e2845a4da76d3bb2114d9abe932e Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 24 Jan 2018 17:04:52 -0800 Subject: [PATCH 007/178] add docs index --- docs/index.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 docs/index.rst diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..bddbdbeb --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,20 @@ + +======= + + + +Contents +-------- + +.. toctree:: + :maxdepth: 1 + + releases + + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` From 3290919ff1cd31aa1c6c491f347335ffc7d93f73 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Thu, 25 Jan 2018 15:51:14 -0800 Subject: [PATCH 008/178] add github templates for issues and pull requests --- .github/ISSUE_TEMPLATE.md | 38 ++++++++++++++++++++++++++++++++ .github/PULL_REQUEST_TEMPLATE.md | 11 +++++++++ 2 files changed, 49 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 00000000..5ff4880e --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,38 @@ + _If this is a bug report, please fill in the following sections. +If this is a feature request, delete and describe what you would like with examples._ + +## What was wrong? + +### Code that produced the error + +```py +CODE_TO_REPRODUCE +``` + +### Full error output + +```sh +ERROR_HERE +``` + +### Expected Result + +_This section may be deleted if the expectation is "don't crash"._ + +```sh +EXPECTED_RESULT +``` + +### Environment + +```sh +# run this: +$ python -m eth_utils + +# then copy the output here: +OUTPUT_HERE +``` + +## How can it be fixed? + +Fill this section in if you know how this could or should be fixed. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..76ef5ac6 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,11 @@ +## What was wrong? + +Issue # + +## How was it fixed? + +Summary of approach. + +#### Cute Animal Picture + +![put a cute animal picture link inside the parentheses]() From 2938497f022a90e44d7589f19f5d960c5d1fa4c9 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Thu, 25 Jan 2018 17:21:42 -0800 Subject: [PATCH 009/178] Link to dev tactical manual --- README.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e29c4d9b..709130c2 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,18 @@ pip install ## Developer setup -If you would like to hack on , set up your dev environment with: +If you would like to hack on , please check out the +[Ethereum Development Tactical Manual](https://github.com/pipermerriam/ethereum-dev-tactical-manual) +for information on how we do: + +- Testing +- Pull Requests +- Code Style +- Documentation + +### Development Environment Setup + +You can set up your dev environment with: ```sh From 76bfefa66c987ffb8004c234190aebda1d145065 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Fri, 26 Jan 2018 19:01:36 -0800 Subject: [PATCH 010/178] add `make lint-roll` to locally auto de-lint --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index b8424652..1a7f8c6c 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,9 @@ clean-pyc: lint: tox -elint +lint-roll: lint + isort --recursive tests + test: py.test tests From 195249cdabb2f097baed392c0ed37e0aca549768 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 30 Jan 2018 10:46:34 -0800 Subject: [PATCH 011/178] docs cleanup: Makefile, conf, warnings --- docs/Makefile | 177 ++++++++++++++++++++ docs/_static/.suppress-sphinx-build-warning | 0 docs/conf.py | 1 - docs/index.rst | 6 +- docs/releases.rst | 2 +- 5 files changed, 181 insertions(+), 5 deletions(-) create mode 100644 docs/Makefile create mode 100644 docs/_static/.suppress-sphinx-build-warning diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..32800935 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,177 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/web3.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/web3.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/web3" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/web3" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/docs/_static/.suppress-sphinx-build-warning b/docs/_static/.suppress-sphinx-build-warning new file mode 100644 index 00000000..e69de29b diff --git a/docs/conf.py b/docs/conf.py index 77bcb7eb..d2294f77 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -76,7 +76,6 @@ release = __version__ # directories to ignore when looking for source files. exclude_patterns = [ '_build', - '.rst', 'modules.rst', ] diff --git a/docs/index.rst b/docs/index.rst index bddbdbeb..b3598a41 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,5 +1,5 @@ -======= +============================== @@ -7,8 +7,9 @@ Contents -------- .. toctree:: - :maxdepth: 1 + :maxdepth: 3 + releases @@ -17,4 +18,3 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` -* :ref:`search` diff --git a/docs/releases.rst b/docs/releases.rst index ec4804d1..6fdd6c97 100644 --- a/docs/releases.rst +++ b/docs/releases.rst @@ -2,6 +2,6 @@ Release Notes ============= v0.1.0-alpha.1 -------------- +-------------- - Launched repository, claimed names for pip, RTD, github, etc From 109a0866ed06f97e2f7b34d1090e36dd6cbf499a Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 30 Jan 2018 15:17:03 -0800 Subject: [PATCH 012/178] requirements.txt: best way to build on readthedocs --- requirements-docs.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 requirements-docs.txt diff --git a/requirements-docs.txt b/requirements-docs.txt new file mode 100644 index 00000000..1b49b744 --- /dev/null +++ b/requirements-docs.txt @@ -0,0 +1 @@ +[doc] From 96a371705b810bd036817ac33c1aed8c045644d1 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 30 Jan 2018 15:17:50 -0800 Subject: [PATCH 013/178] bugfix: run tox -elint *after* isort otherwise tox fails, and the auto-lint never runs --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1a7f8c6c..97eab7cb 100644 --- a/Makefile +++ b/Makefile @@ -24,8 +24,9 @@ clean-pyc: lint: tox -elint -lint-roll: lint +lint-roll: isort --recursive tests + $(make) lint test: py.test tests From 0e57d8e262a1c46d5191e7fd31fd17026bdf3e2c Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 30 Jan 2018 15:18:59 -0800 Subject: [PATCH 014/178] when do you not want ipython in local dev? never --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index fbef3442..518ee265 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ extras_require={ "bumpversion>=0.5.3,<1", "pytest-xdist", "wheel", + "ipython", ], } From 9a7d72a816dc5352ace5916a4b3ea3991e146b1c Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 30 Jan 2018 15:37:40 -0800 Subject: [PATCH 015/178] pytest default settings, plus pytest-watch --- pytest.ini | 1 - setup.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index d53de3aa..9cc5a18d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,7 +1,6 @@ [pytest] addopts= -v --showlocals --durations 10 python_paths= . -xfail_strict=true [pytest-watch] runner= py.test --failed-first --maxfail=1 --no-success-flaky-report diff --git a/setup.py b/setup.py index 518ee265..a89f4aca 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,7 @@ extras_require={ 'dev': [ "bumpversion>=0.5.3,<1", "pytest-xdist", + "pytest-watch>=4.1.0,<5", "wheel", "ipython", ], From cb71fb443028771e35370a4dd0bfbe848e694dda Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Thu, 1 Feb 2018 09:42:48 -0800 Subject: [PATCH 016/178] add circle ci config --- .circleci/config.yml | 51 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..7384eab5 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,51 @@ +version: 2.0 + +# heavily inspired by https://raw.githubusercontent.com/pinax/pinax-wiki/6bd2a99ab6f702e300d708532a6d1d9aa638b9f8/.circleci/config.yml + +common: &common + working_directory: ~/repo + steps: + - checkout + - restore_cache: + keys: + - cache-{{ .Environment.CIRCLE_JOB }}-{{ checksum "setup.py" }}-{{ checksum "tox.ini" }} + - run: + name: install dependencies + command: pip install --user tox + - run: + name: run tox + command: ~/.local/bin/tox + - save_cache: + paths: + - .tox + - ~/.cache/pip + - ~/.local + - ./eggs + key: cache-{{ .Environment.CIRCLE_JOB }}-{{ checksum "setup.py" }}-{{ checksum "tox.ini" }} + +jobs: + lint: + <<: *common + docker: + - image: circleci/python:3.5 + environment: + TOXENV: lint + py35-core: + <<: *common + docker: + - image: circleci/python:3.5 + environment: + TOXENV: py35-core + py36-core: + <<: *common + docker: + - image: circleci/python:3.6 + environment: + TOXENV: py36-core +workflows: + version: 2 + test: + jobs: + - lint + - py35-core + - py36-core From f7b0f07b45798548550a3dd90444b350b9b95ff7 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Thu, 1 Feb 2018 13:07:33 -0800 Subject: [PATCH 017/178] fixups: rm template filler, xfail strict, make bug --- Makefile | 2 +- fill_template_vars.sh | 3 +++ pytest.ini | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 97eab7cb..3bf24030 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ lint: lint-roll: isort --recursive tests - $(make) lint + $(MAKE) lint test: py.test tests diff --git a/fill_template_vars.sh b/fill_template_vars.sh index a0f9b903..17f25035 100755 --- a/fill_template_vars.sh +++ b/fill_template_vars.sh @@ -38,3 +38,6 @@ sed -i "s//$SHORT_DESCRIPTION/g" $TEMPLATE_FILES mkdir $MODULE_NAME touch $MODULE_NAME/__init__.py + +# template filler is no longer needed, delete it +rm "$0" diff --git a/pytest.ini b/pytest.ini index 9cc5a18d..d53de3aa 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,7 @@ [pytest] addopts= -v --showlocals --durations 10 python_paths= . +xfail_strict=true [pytest-watch] runner= py.test --failed-first --maxfail=1 --no-success-flaky-report From 4bb4f720db893c4e3f44c22d4693869ed934ef6d Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 6 Feb 2018 15:33:04 -0800 Subject: [PATCH 018/178] add badges: pypi version and pythons supported --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 709130c2..228615f5 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,9 @@ # [![Join the chat at https://gitter.im/ethereum/](https://badges.gitter.im/ethereum/.svg)](https://gitter.im/ethereum/?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) - [![Build Status](https://travis-ci.org/ethereum/.png)](https://travis-ci.org/ethereum/) +[![PyPI version](https://badge.fury.io/py/.svg)](https://badge.fury.io/py/) +[![Python versions](https://img.shields.io/pypi/pyversions/.svg)](https://pypi.python.org/pypi/) From b825c36defd58715d02a0dd8b5eb47f52c2795d9 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 6 Feb 2018 15:42:21 -0800 Subject: [PATCH 019/178] docs: show badge, link to release notes --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 228615f5..d478ded1 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,14 @@ [![Build Status](https://travis-ci.org/ethereum/.png)](https://travis-ci.org/ethereum/) [![PyPI version](https://badge.fury.io/py/.svg)](https://badge.fury.io/py/) [![Python versions](https://img.shields.io/pypi/pyversions/.svg)](https://pypi.python.org/pypi/) +[![Docs build](https://readthedocs.org/projects//badge/?version=latest)](http://.readthedocs.io/en/latest/?badge=latest) * Python 3.5+ support -Read more in the [documentation on ReadTheDocs](http://.readthedocs.io/). [View the change log on Github](docs/releases.rst). +Read more in the [documentation on ReadTheDocs](http://.readthedocs.io/). [View the change log](http://.readthedocs.io/en/latest/releases.html). ## Quickstart From 7bfa2ac5d45d274ac7e4a50415cf02eee140cdd3 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 6 Feb 2018 15:52:41 -0800 Subject: [PATCH 020/178] add pypy3 as a supported environment, by default --- .travis.yml | 6 ++++++ setup.py | 1 + tox.ini | 3 ++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 72be81d4..42b63c99 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,12 @@ matrix: # core - python: "3.6" env: TOX_POSARGS="-e py36-core" + # + # pypy3 testing + # + # core + - python: "pypy3" + env: TOX_POSARGS="-e pypy3-core" cache: - pip: true install: diff --git a/setup.py b/setup.py index a89f4aca..a59d9eec 100644 --- a/setup.py +++ b/setup.py @@ -62,5 +62,6 @@ setup( 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: Implementation :: PyPy3', ], ) diff --git a/tox.ini b/tox.ini index 02772752..9b1727e7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist= - py{35,36}-core + py{35,36,py3}-core lint [isort] @@ -25,6 +25,7 @@ commands= basepython = py35: python3.5 py36: python3.6 + pypy3: pypy3 extras=test [testenv:lint] From 6abf0667793022b1b077f6eea69c4e706cc3513d Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 6 Feb 2018 15:56:11 -0800 Subject: [PATCH 021/178] add module import test --- tests/core/test_import.py | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 tests/core/test_import.py diff --git a/tests/core/test_import.py b/tests/core/test_import.py new file mode 100644 index 00000000..2243d057 --- /dev/null +++ b/tests/core/test_import.py @@ -0,0 +1,4 @@ + + +def test_import(): + import From 1695a3b3262f183dbee81229a7cbd326edc5c210 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 6 Feb 2018 16:04:41 -0800 Subject: [PATCH 022/178] bugfix: setuptools reference to pypy --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a59d9eec..4eeb8d77 100644 --- a/setup.py +++ b/setup.py @@ -62,6 +62,6 @@ setup( 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: Implementation :: PyPy3', + 'Programming Language :: Python :: Implementation :: PyPy', ], ) From beb1b10ee893ee68b188048dba8210ebcbd0506c Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 6 Feb 2018 16:10:25 -0800 Subject: [PATCH 023/178] silence flake8 error during module import test --- tests/core/test_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/test_import.py b/tests/core/test_import.py index 2243d057..f146ae79 100644 --- a/tests/core/test_import.py +++ b/tests/core/test_import.py @@ -1,4 +1,4 @@ def test_import(): - import + import # noqa: F401 From a9d9fec258284bc01524cd59d6d18629cca523cd Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 7 Feb 2018 11:14:11 -0800 Subject: [PATCH 024/178] add doctest by default --- Makefile | 1 + docs/conf.py | 17 ++++++++++++++++- tox.ini | 6 +++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 3bf24030..bb5c6a07 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,7 @@ build-docs: sphinx-apidoc -o docs/ . setup.py "*conftest*" $(MAKE) -C docs clean $(MAKE) -C docs html + $(MAKE) -C docs doctest docs: build-docs open docs/_build/html/index.html diff --git a/docs/conf.py b/docs/conf.py index d2294f77..a74a9775 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -34,7 +34,11 @@ with open (os.path.join(DIR, '../setup.py'), 'r') as f: # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx'] +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', +] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -271,3 +275,14 @@ texinfo_documents = [ intersphinx_mapping = { 'python': ('https://docs.python.org/3.5', None), } + +# -- Doctest configuration ---------------------------------------- + +import doctest + +doctest_default_flags = (0 + | doctest.DONT_ACCEPT_TRUE_FOR_1 + | doctest.ELLIPSIS + | doctest.IGNORE_EXCEPTION_DETAIL + | doctest.NORMALIZE_WHITESPACE +) diff --git a/tox.ini b/tox.ini index 9b1727e7..162657f5 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,7 @@ envlist= py{35,36,py3}-core lint + doctest [isort] combine_as_imports=True @@ -22,11 +23,14 @@ ignore= usedevelop=True commands= core: py.test {posargs:tests/core} + doctest: make -C {toxinidir}/docs doctest basepython = py35: python3.5 py36: python3.6 pypy3: pypy3 -extras=test +extras= + test + doctest: doc [testenv:lint] basepython=python From a1ba89ed413b1f6ea2691f14948d0ef3cc03fbf0 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 7 Feb 2018 11:18:47 -0800 Subject: [PATCH 025/178] every tox environment should have a basic python --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 162657f5..f28e277b 100644 --- a/tox.ini +++ b/tox.ini @@ -25,6 +25,7 @@ commands= core: py.test {posargs:tests/core} doctest: make -C {toxinidir}/docs doctest basepython = + python py35: python3.5 py36: python3.6 pypy3: pypy3 @@ -33,7 +34,6 @@ extras= doctest: doc [testenv:lint] -basepython=python extras=lint commands= flake8 {toxinidir}/ {toxinidir}/tests From 37d978c5b3c1de029e624b1335a7d0a3df96c045 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 7 Feb 2018 11:24:37 -0800 Subject: [PATCH 026/178] squash warning about using make inside tox --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index f28e277b..8328bc50 100644 --- a/tox.ini +++ b/tox.ini @@ -32,6 +32,7 @@ basepython = extras= test doctest: doc +whitelest_externals=make [testenv:lint] extras=lint From ecb3731cbe45acfc762443ff9e702e040b642b08 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 7 Feb 2018 11:26:47 -0800 Subject: [PATCH 027/178] can't override python that way in tox --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8328bc50..e7e9713e 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,7 @@ commands= core: py.test {posargs:tests/core} doctest: make -C {toxinidir}/docs doctest basepython = - python + doctest: python py35: python3.5 py36: python3.6 pypy3: pypy3 @@ -35,6 +35,7 @@ extras= whitelest_externals=make [testenv:lint] +basepython=python extras=lint commands= flake8 {toxinidir}/ {toxinidir}/tests From 60b026e3edfc75d696ffb896312f9d35f4e8ab17 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 7 Feb 2018 11:40:35 -0800 Subject: [PATCH 028/178] add doctest run to travis --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 42b63c99..5600ec0f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,9 @@ matrix: # lint - python: "3.5" env: TOX_POSARGS="-e lint" + # doctest + - python: "3.5" + env: TOX_POSARGS="-e doctest" # core - python: "3.5" env: TOX_POSARGS="-e py35-core" From 58902032c755c9a61ec1a32f574a9a655db68490 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 7 Feb 2018 14:37:30 -0800 Subject: [PATCH 029/178] add doctest run to circle-ci --- .circleci/config.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7384eab5..2ba8898d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,6 +24,12 @@ common: &common key: cache-{{ .Environment.CIRCLE_JOB }}-{{ checksum "setup.py" }}-{{ checksum "tox.ini" }} jobs: + doctest: + <<: *common + docker: + - image: circleci/python:3.5 + environment: + TOXENV: doctest lint: <<: *common docker: @@ -46,6 +52,7 @@ workflows: version: 2 test: jobs: + - doctest - lint - py35-core - py36-core From b3461e9c934bcf017c90744192f218d2ab0a504c Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 7 Feb 2018 14:40:01 -0800 Subject: [PATCH 030/178] tox.ini bugfix: whitelist make --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index e7e9713e..34a66b4d 100644 --- a/tox.ini +++ b/tox.ini @@ -32,7 +32,7 @@ basepython = extras= test doctest: doc -whitelest_externals=make +whitelist_externals=make [testenv:lint] basepython=python From f0e0b10cc5f9a6086c249074d91c0ca18f63a575 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 20 Feb 2018 18:09:58 -0800 Subject: [PATCH 031/178] add pypy3 as a default test environment in circle --- .circleci/config.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2ba8898d..821d1b41 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,6 +48,12 @@ jobs: - image: circleci/python:3.6 environment: TOXENV: py36-core + pypy3-core: + <<: *common + docker: + - image: pypy + environment: + TOXENV: pypy3-core workflows: version: 2 test: @@ -56,3 +62,4 @@ workflows: - lint - py35-core - py36-core + - pypy3-core From 974fae3a2b6a414dab8eafc4aea2f968a6d5c8d7 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 20 Feb 2018 18:10:30 -0800 Subject: [PATCH 032/178] travis config: use latest pypy3 container --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5600ec0f..78c56a6e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ matrix: # pypy3 testing # # core - - python: "pypy3" + - python: "pypy3.5" env: TOX_POSARGS="-e pypy3-core" cache: - pip: true From 588b1af6ee556cfd39b5681383c84b326b153a60 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 28 Feb 2018 10:29:45 -0800 Subject: [PATCH 033/178] Makefile: deploy to upstream --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index bb5c6a07..5b0f7997 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,7 @@ release: clean CURRENT_SIGN_SETTING=$(git config commit.gpgSign) git config commit.gpgSign true bumpversion $(bump) - git push && git push --tags + git push upstream && git push upstream --tags python setup.py sdist bdist_wheel upload git config commit.gpgSign "$(CURRENT_SIGN_SETTING)" From 031b1175f4c03ead623b54067fbc95485de17966 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 28 Feb 2018 14:20:39 -0800 Subject: [PATCH 034/178] force tox to rebuild in circle runs --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 821d1b41..2393a065 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ common: &common command: pip install --user tox - run: name: run tox - command: ~/.local/bin/tox + command: ~/.local/bin/tox -r - save_cache: paths: - .tox From 84ecb7effc827b9f070b5eb6d50e4e7a341d078d Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 10 Apr 2018 10:16:17 -0700 Subject: [PATCH 035/178] require python 3.5 during setup (and reject py4) --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 4eeb8d77..43082949 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ setup( "eth-utils>=0.7.4,<1.0.0", ], setup_requires=['setuptools-markdown'], + python_requires='>=3.5, <4', extras_require=extras_require, py_modules=[''], license="MIT", From 30473b3ff54da0b6f06e7cd761fd9ec40f6651f1 Mon Sep 17 00:00:00 2001 From: David Sanders Date: Thu, 19 Apr 2018 16:31:48 -0600 Subject: [PATCH 036/178] Make Circle CI test against merge with PR base --- .circleci/config.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2393a065..7c636af8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,6 +6,22 @@ common: &common working_directory: ~/repo steps: - checkout + - run: + name: merge pull request base + command: | + if [[ -n "${CIRCLE_PR_NUMBER}" ]]; then + PR_INFO_URL=https://api.github.com/repos/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/pulls/$CIRCLE_PR_NUMBER + + PR_BASE_BRANCH=$(curl -L "$PR_INFO_URL" | python -c 'import json, sys; obj = json.load(sys.stdin); sys.stdout.write(obj["base"]["ref"])') + git fetch origin +"$PR_BASE_BRANCH":circleci/pr-base + + # We need these config values or git complains when creating the + # merge commit + git config --global user.name "Circle CI" + git config --global user.email "circleci@example.com" + + git merge --no-edit circleci/pr-base + fi - restore_cache: keys: - cache-{{ .Environment.CIRCLE_JOB }}-{{ checksum "setup.py" }}-{{ checksum "tox.ini" }} From f7535eb99204e2c691aa6b330e5672caa29a29c0 Mon Sep 17 00:00:00 2001 From: David Sanders Date: Sun, 22 Apr 2018 01:34:42 -0600 Subject: [PATCH 037/178] Add circle ci status badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d478ded1..ac1ee8b1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # [![Join the chat at https://gitter.im/ethereum/](https://badges.gitter.im/ethereum/.svg)](https://gitter.im/ethereum/?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -[![Build Status](https://travis-ci.org/ethereum/.png)](https://travis-ci.org/ethereum/) +[![Build Status](https://circleci.com/gh/ethereum/.svg?style=shield)](https://circleci.com/gh/ethereum/) [![PyPI version](https://badge.fury.io/py/.svg)](https://badge.fury.io/py/) [![Python versions](https://img.shields.io/pypi/pyversions/.svg)](https://pypi.python.org/pypi/) [![Docs build](https://readthedocs.org/projects//badge/?version=latest)](http://.readthedocs.io/en/latest/?badge=latest) From cdc83bd498d3e145109a760314d41ea3504f821f Mon Sep 17 00:00:00 2001 From: David Sanders Date: Sun, 22 Apr 2018 01:35:08 -0600 Subject: [PATCH 038/178] Make find replace script more macOS friendly Also, make handling of arbitrary filenames a bit more robust. --- fill_template_vars.sh | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/fill_template_vars.sh b/fill_template_vars.sh index 17f25035..531efcff 100755 --- a/fill_template_vars.sh +++ b/fill_template_vars.sh @@ -4,9 +4,6 @@ set -o errexit set -o nounset set -o pipefail -# List of all non-executable files -TEMPLATE_FILES=$(find . ! -perm -u=x -type f | grep -v "\.git") - echo "What is your python module name?" read MODULE_NAME @@ -29,12 +26,20 @@ PROJECT_NAME=${PROJECT_INPUT:-$REPO_NAME} echo "What is a one-liner describing the project?" read SHORT_DESCRIPTION -sed -i "s//$MODULE_NAME/g" $TEMPLATE_FILES -sed -i "s//$PYPI_NAME/g" $TEMPLATE_FILES -sed -i "s//$REPO_NAME/g" $TEMPLATE_FILES -sed -i "s//$RTD_NAME/g" $TEMPLATE_FILES -sed -i "s//$PROJECT_NAME/g" $TEMPLATE_FILES -sed -i "s//$SHORT_DESCRIPTION/g" $TEMPLATE_FILES +_replace() { + local find_cmd=(find . ! -perm -u=x ! -path '*/.git/*' -type f) + + if [[ $(uname) == Darwin ]]; then + "${find_cmd[@]}" -exec sed -i '' "$1" {} + + else + "${find_cmd[@]}" -exec sed -i "$1" {} + + fi +} +_replace "s//$PYPI_NAME/g" +_replace "s//$REPO_NAME/g" +_replace "s//$RTD_NAME/g" +_replace "s//$PROJECT_NAME/g" +_replace "s//$SHORT_DESCRIPTION/g" mkdir $MODULE_NAME touch $MODULE_NAME/__init__.py From d7ce1100ed8c80b108d98f78dc080c6e64d74b9f Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 25 Apr 2018 15:05:30 -0700 Subject: [PATCH 039/178] gitignore utils and internals docs; fill-in fixup --- .gitignore | 3 ++- fill_template_vars.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index c3802038..e5902e4f 100644 --- a/.gitignore +++ b/.gitignore @@ -43,7 +43,8 @@ output/*/index.html # Sphinx docs/_build docs/modules.rst -docs/web3.* +docs/*.internals.rst +docs/*.utils.rst # Blockchain chains diff --git a/fill_template_vars.sh b/fill_template_vars.sh index 531efcff..197f490b 100755 --- a/fill_template_vars.sh +++ b/fill_template_vars.sh @@ -41,7 +41,7 @@ _replace "s//$RTD_NAME/g" _replace "s//$PROJECT_NAME/g" _replace "s//$SHORT_DESCRIPTION/g" -mkdir $MODULE_NAME +mkdir -p $MODULE_NAME touch $MODULE_NAME/__init__.py # template filler is no longer needed, delete it From 0dd03f0a3fc3a895a4276789f0043c150b6a7705 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 25 Apr 2018 15:19:05 -0700 Subject: [PATCH 040/178] re-add MODULE_NAME in new template filler --- fill_template_vars.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/fill_template_vars.sh b/fill_template_vars.sh index 197f490b..286daad3 100755 --- a/fill_template_vars.sh +++ b/fill_template_vars.sh @@ -35,6 +35,7 @@ _replace() { "${find_cmd[@]}" -exec sed -i "$1" {} + fi } +_replace "s//$MODULE_NAME/g" _replace "s//$PYPI_NAME/g" _replace "s//$REPO_NAME/g" _replace "s//$RTD_NAME/g" From b98147efcabaca4dfbc61c53fa55eaf81b880ee8 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 25 Apr 2018 15:20:03 -0700 Subject: [PATCH 041/178] keep template filler around for future merges --- fill_template_vars.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/fill_template_vars.sh b/fill_template_vars.sh index 286daad3..5360f31d 100755 --- a/fill_template_vars.sh +++ b/fill_template_vars.sh @@ -44,6 +44,3 @@ _replace "s//$SHORT_DESCRIPTION/g" mkdir -p $MODULE_NAME touch $MODULE_NAME/__init__.py - -# template filler is no longer needed, delete it -rm "$0" From 285f5c6c70b6b5ac1204ecd81a31cf1a5d4c591e Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 25 Apr 2018 15:39:52 -0700 Subject: [PATCH 042/178] Remove duplicate python version mention in readme Rely on the badge instead --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index ac1ee8b1..041ad819 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,6 @@ -* Python 3.5+ support - Read more in the [documentation on ReadTheDocs](http://.readthedocs.io/). [View the change log](http://.readthedocs.io/en/latest/releases.html). ## Quickstart From 1f89c6385ab4c9b2cdf6ce18e52c1f40cd25f075 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 25 Apr 2018 18:50:01 -0700 Subject: [PATCH 043/178] fix gitignore typo about internals module in docs/ --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e5902e4f..58654f84 100644 --- a/.gitignore +++ b/.gitignore @@ -43,7 +43,7 @@ output/*/index.html # Sphinx docs/_build docs/modules.rst -docs/*.internals.rst +docs/*.internal.rst docs/*.utils.rst # Blockchain From 8815766abf57e21498c7e8422c7dece38f503ec9 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 15 May 2018 11:52:40 -0700 Subject: [PATCH 044/178] circleci: merge (and reattempt) PR before testing --- .circleci/config.yml | 12 ++++++++++++ .circleci/merge_pr.sh | 12 ++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 .circleci/merge_pr.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 7c636af8..af939637 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,6 +6,17 @@ common: &common working_directory: ~/repo steps: - checkout + - run: + name: merge pull request base + command: ./.circleci/merge_pr.sh + - run: + name: merge pull request base (2nd try) + command: ./.circleci/merge_pr.sh + when: on_fail + - run: + name: merge pull request base (3nd try) + command: ./.circleci/merge_pr.sh + when: on_fail - run: name: merge pull request base command: | @@ -33,6 +44,7 @@ common: &common command: ~/.local/bin/tox -r - save_cache: paths: + - .hypothesis - .tox - ~/.cache/pip - ~/.local diff --git a/.circleci/merge_pr.sh b/.circleci/merge_pr.sh new file mode 100644 index 00000000..91eb47ca --- /dev/null +++ b/.circleci/merge_pr.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +if [[ -n "${CIRCLE_PR_NUMBER}" ]]; then + PR_INFO_URL=https://api.github.com/repos/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/pulls/$CIRCLE_PR_NUMBER + PR_BASE_BRANCH=$(curl -L "$PR_INFO_URL" | python -c 'import json, sys; obj = json.load(sys.stdin); sys.stdout.write(obj["base"]["ref"])') + git fetch origin +"$PR_BASE_BRANCH":circleci/pr-base + # We need these config values or git complains when creating the + # merge commit + git config --global user.name "Circle CI" + git config --global user.email "circleci@example.com" + git merge --no-edit circleci/pr-base +fi From 1091971637ff0afc3531cc011c4d00db1a2fc620 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 15 May 2018 12:09:51 -0700 Subject: [PATCH 045/178] remove old merge code from circle template --- .circleci/config.yml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index af939637..7f542a12 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,22 +17,6 @@ common: &common name: merge pull request base (3nd try) command: ./.circleci/merge_pr.sh when: on_fail - - run: - name: merge pull request base - command: | - if [[ -n "${CIRCLE_PR_NUMBER}" ]]; then - PR_INFO_URL=https://api.github.com/repos/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME/pulls/$CIRCLE_PR_NUMBER - - PR_BASE_BRANCH=$(curl -L "$PR_INFO_URL" | python -c 'import json, sys; obj = json.load(sys.stdin); sys.stdout.write(obj["base"]["ref"])') - git fetch origin +"$PR_BASE_BRANCH":circleci/pr-base - - # We need these config values or git complains when creating the - # merge commit - git config --global user.name "Circle CI" - git config --global user.email "circleci@example.com" - - git merge --no-edit circleci/pr-base - fi - restore_cache: keys: - cache-{{ .Environment.CIRCLE_JOB }}-{{ checksum "setup.py" }}-{{ checksum "tox.ini" }} From 85e33cf3801b19e9d46b237c7837baab0eb1e468 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 15 May 2018 12:17:36 -0700 Subject: [PATCH 046/178] move template filler to own directory --- .../fill_template_vars.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) rename fill_template_vars.sh => .project-template/fill_template_vars.sh (89%) diff --git a/fill_template_vars.sh b/.project-template/fill_template_vars.sh similarity index 89% rename from fill_template_vars.sh rename to .project-template/fill_template_vars.sh index 5360f31d..7ae7ebba 100755 --- a/fill_template_vars.sh +++ b/.project-template/fill_template_vars.sh @@ -4,6 +4,8 @@ set -o errexit set -o nounset set -o pipefail +PROJECT_ROOT=$(dirname $(dirname $(readlink -f $0))) + echo "What is your python module name?" read MODULE_NAME @@ -27,7 +29,7 @@ echo "What is a one-liner describing the project?" read SHORT_DESCRIPTION _replace() { - local find_cmd=(find . ! -perm -u=x ! -path '*/.git/*' -type f) + local find_cmd=(find "$PROJECT_ROOT" ! -perm -u=x ! -path '*/.git/*' -type f) if [[ $(uname) == Darwin ]]; then "${find_cmd[@]}" -exec sed -i '' "$1" {} + From 71ba221c493d70b04046fc624bf39ea05b8b38d5 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 15 May 2018 12:25:21 -0700 Subject: [PATCH 047/178] Enable template refilling --- .project-template/fill_template_vars.sh | 4 ++-- .project-template/refill_template_vars.sh | 1 + .project-template/template_vars.txt | 6 ++++++ 3 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 .project-template/refill_template_vars.sh create mode 100644 .project-template/template_vars.txt diff --git a/.project-template/fill_template_vars.sh b/.project-template/fill_template_vars.sh index 7ae7ebba..1e76d8ad 100755 --- a/.project-template/fill_template_vars.sh +++ b/.project-template/fill_template_vars.sh @@ -44,5 +44,5 @@ _replace "s//$RTD_NAME/g" _replace "s//$PROJECT_NAME/g" _replace "s//$SHORT_DESCRIPTION/g" -mkdir -p $MODULE_NAME -touch $MODULE_NAME/__init__.py +mkdir -p "$PROJECT_ROOT/$MODULE_NAME" +touch "$PROJECT_ROOT/$MODULE_NAME/__init__.py" diff --git a/.project-template/refill_template_vars.sh b/.project-template/refill_template_vars.sh new file mode 100644 index 00000000..bbb39e99 --- /dev/null +++ b/.project-template/refill_template_vars.sh @@ -0,0 +1 @@ + + + + + + From b357fda082fa2d560ea7fb8e07d2560c1404dbb3 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 15 May 2018 12:33:36 -0700 Subject: [PATCH 048/178] refill from any directory --- .project-template/refill_template_vars.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) mode change 100644 => 100755 .project-template/refill_template_vars.sh diff --git a/.project-template/refill_template_vars.sh b/.project-template/refill_template_vars.sh old mode 100644 new mode 100755 index bbb39e99..6e7943fb --- a/.project-template/refill_template_vars.sh +++ b/.project-template/refill_template_vars.sh @@ -1 +1,2 @@ - Date: Tue, 15 May 2018 13:02:56 -0700 Subject: [PATCH 049/178] make merge script executable --- .circleci/merge_pr.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .circleci/merge_pr.sh diff --git a/.circleci/merge_pr.sh b/.circleci/merge_pr.sh old mode 100644 new mode 100755 From 5e509100e7174eef6169f1d152197c0db22163d9 Mon Sep 17 00:00:00 2001 From: David Sanders Date: Fri, 1 Jun 2018 14:05:57 -0600 Subject: [PATCH 050/178] Some readme updates --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 041ad819..937344d2 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ -Read more in the [documentation on ReadTheDocs](http://.readthedocs.io/). [View the change log](http://.readthedocs.io/en/latest/releases.html). +Read more in the [documentation on ReadTheDocs](https://.readthedocs.io/). [View the change log](https://.readthedocs.io/en/latest/releases.html). ## Quickstart @@ -17,7 +17,7 @@ Read more in the [documentation on ReadTheDocs](http://.readthedocs.io pip install ``` -## Developer setup +## Developer Setup If you would like to hack on , please check out the [Ethereum Development Tactical Manual](https://github.com/pipermerriam/ethereum-dev-tactical-manual) @@ -33,7 +33,6 @@ for information on how we do: You can set up your dev environment with: ```sh - git clone git@github.com:ethereum/.git cd virtualenv -p python3 venv From 32c8c981f262248b20730d18e2d4fcad074488dd Mon Sep 17 00:00:00 2001 From: David Sanders Date: Mon, 4 Jun 2018 16:36:02 -0600 Subject: [PATCH 051/178] Remove references to deprecated "py.test" --- Makefile | 2 +- pytest.ini | 2 +- tox.ini | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 5b0f7997..7b93655c 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ lint-roll: $(MAKE) lint test: - py.test tests + pytest tests test-all: tox diff --git a/pytest.ini b/pytest.ini index d53de3aa..f5fdc0ec 100644 --- a/pytest.ini +++ b/pytest.ini @@ -4,4 +4,4 @@ python_paths= . xfail_strict=true [pytest-watch] -runner= py.test --failed-first --maxfail=1 --no-success-flaky-report +runner= pytest --failed-first --maxfail=1 --no-success-flaky-report diff --git a/tox.ini b/tox.ini index 34a66b4d..066b3774 100644 --- a/tox.ini +++ b/tox.ini @@ -22,7 +22,7 @@ ignore= [testenv] usedevelop=True commands= - core: py.test {posargs:tests/core} + core: pytest {posargs:tests/core} doctest: make -C {toxinidir}/docs doctest basepython = doctest: python From 00819dd924174c78829f97679871fd181dd12509 Mon Sep 17 00:00:00 2001 From: David Sanders Date: Mon, 4 Jun 2018 16:36:35 -0600 Subject: [PATCH 052/178] Whitespace --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 066b3774..74ea1d4e 100644 --- a/tox.ini +++ b/tox.ini @@ -38,5 +38,5 @@ whitelist_externals=make basepython=python extras=lint commands= - flake8 {toxinidir}/ {toxinidir}/tests - isort --recursive --check-only --diff {toxinidir}/ {toxinidir}/tests + flake8 {toxinidir}/ {toxinidir}/tests + isort --recursive --check-only --diff {toxinidir}/ {toxinidir}/tests From e7d0a8577b8f6f5cac11e6f86199f7bca24b3ecd Mon Sep 17 00:00:00 2001 From: David Sanders Date: Mon, 4 Jun 2018 16:37:38 -0600 Subject: [PATCH 053/178] Fix testing dependencies --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 43082949..92ee090c 100644 --- a/setup.py +++ b/setup.py @@ -8,6 +8,7 @@ from setuptools import ( extras_require={ 'test': [ "pytest==3.3.2", + "pytest-xdist", "tox>=2.9.1,<3", ], 'lint': [ @@ -20,7 +21,6 @@ extras_require={ ], 'dev': [ "bumpversion>=0.5.3,<1", - "pytest-xdist", "pytest-watch>=4.1.0,<5", "wheel", "ipython", From 0a83658b3ee41fdc8473e3156a5b7591ef992c9b Mon Sep 17 00:00:00 2001 From: David Sanders Date: Mon, 4 Jun 2018 16:37:52 -0600 Subject: [PATCH 054/178] Lint in setup.py --- setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 92ee090c..55b3c90c 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from setuptools import ( find_packages, ) -extras_require={ +extras_require = { 'test': [ "pytest==3.3.2", "pytest-xdist", @@ -28,10 +28,10 @@ extras_require={ } extras_require['dev'] = ( - extras_require['dev'] - + extras_require['test'] - + extras_require['lint'] - + extras_require['doc'] + extras_require['dev'] + + extras_require['test'] + + extras_require['lint'] + + extras_require['doc'] ) setup( From 0088da3781742674729aa54eb423f35e4a3826ae Mon Sep 17 00:00:00 2001 From: David Sanders Date: Mon, 4 Jun 2018 18:40:58 -0600 Subject: [PATCH 055/178] Fix possibly incorrect isort config --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 74ea1d4e..6ae707b1 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,7 @@ envlist= combine_as_imports=True force_sort_within_sections=True include_trailing_comma=True -known_standard_library=pytest +known_third_party=hypothesis,pytest known_first_party= line_length=21 multi_line_output=3 From 6ed7bb011f7c7bd77c0e3a1802e3381d6bf3d8a5 Mon Sep 17 00:00:00 2001 From: David Sanders Date: Wed, 6 Jun 2018 18:30:40 -0600 Subject: [PATCH 056/178] Make template filler more BSD friendly --- .project-template/fill_template_vars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.project-template/fill_template_vars.sh b/.project-template/fill_template_vars.sh index 1e76d8ad..f09e8ffe 100755 --- a/.project-template/fill_template_vars.sh +++ b/.project-template/fill_template_vars.sh @@ -4,7 +4,7 @@ set -o errexit set -o nounset set -o pipefail -PROJECT_ROOT=$(dirname $(dirname $(readlink -f $0))) +PROJECT_ROOT=$(dirname $(dirname $(python -c 'import os, sys; sys.stdout.write(os.path.realpath(sys.argv[1]))' "$0"))) echo "What is your python module name?" read MODULE_NAME From 016f48d17ddf549955f2baea0c6dd32fefdbd5a8 Mon Sep 17 00:00:00 2001 From: David Sanders Date: Wed, 6 Jun 2018 19:35:31 -0600 Subject: [PATCH 057/178] Use twine for pypi uploading per packaging docs See here: https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives --- Makefile | 3 ++- setup.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7b93655c..db660371 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,8 @@ release: clean git config commit.gpgSign true bumpversion $(bump) git push upstream && git push upstream --tags - python setup.py sdist bdist_wheel upload + python setup.py sdist bdist_wheel + twine upload dist/* git config commit.gpgSign "$(CURRENT_SIGN_SETTING)" dist: clean diff --git a/setup.py b/setup.py index 55b3c90c..511490f1 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ extras_require = { "bumpversion>=0.5.3,<1", "pytest-watch>=4.1.0,<5", "wheel", + "twine", "ipython", ], } From a9d531615da67410bc42fe7285e0e049dc42a78d Mon Sep 17 00:00:00 2001 From: David Sanders Date: Thu, 7 Jun 2018 12:23:32 -0600 Subject: [PATCH 058/178] Fix ineffectual commands in Makefile --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7b93655c..8339d7d7 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +CURRENT_SIGN_SETTING := $(shell git config commit.gpgSign) + .PHONY: clean-pyc clean-build docs help: @@ -47,7 +49,6 @@ linux-docs: build-docs xdg-open docs/_build/html/index.html release: clean - CURRENT_SIGN_SETTING=$(git config commit.gpgSign) git config commit.gpgSign true bumpversion $(bump) git push upstream && git push upstream --tags From d3537cf1ab18cf813c5bfd957e51ca6699e3e0a3 Mon Sep 17 00:00:00 2001 From: David Sanders Date: Thu, 7 Jun 2018 12:27:07 -0600 Subject: [PATCH 059/178] Update eth-utils version requirement --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 55b3c90c..4bfc7a7e 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ setup( url='https://github.com/ethereum/', include_package_data=True, install_requires=[ - "eth-utils>=0.7.4,<1.0.0", + "eth-utils>=1,<2", ], setup_requires=['setuptools-markdown'], python_requires='>=3.5, <4', From 5ad0973c3d16c0a43a705db6ce33b1510989d8cb Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 29 Aug 2018 09:47:22 -0700 Subject: [PATCH 060/178] Ignore any internal `_utils` module --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 58654f84..9a48c01a 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ docs/_build docs/modules.rst docs/*.internal.rst docs/*.utils.rst +docs/*._utils.* # Blockchain chains From 0b004a8597fcf27cf259d9002d9bc30c2e2a0179 Mon Sep 17 00:00:00 2001 From: Nick Gheorghita Date: Mon, 27 Aug 2018 11:15:43 -0600 Subject: [PATCH 061/178] Add changelog instructions --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 937344d2..88439600 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,11 @@ The version format for this repo is `{major}.{minor}.{patch}` for stable, and `{major}.{minor}.{patch}-{stage}.{devnum}` for unstable (`stage` can be alpha or beta). To issue the next version in line, specify which part to bump, -like `make release bump=minor` or `make release bump=devnum`. +like `make release bump=minor` or `make release bump=devnum`. This is typically done from the +master branch, except when releasing a beta (in which case the beta is released from master, +and the previous stable branch is released from said branch). To include changes made with each +release, update "docs/releases.rst" with the changes, and apply commit directly to master +before release. If you are in a beta version, `make release bump=stage` will switch to a stable. From caf9050a19a889c90afd23a05b6d5f9aefadd665 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 15 Jan 2019 16:06:18 -0800 Subject: [PATCH 062/178] Drop py3.5, add py3.7 --- .circleci/config.yml | 18 +++++++++--------- .travis.yml | 24 ++++++++++++------------ docs/conf.py | 2 +- setup.py | 4 ++-- tox.ini | 4 ++-- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7f542a12..bbc56d0a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -39,27 +39,27 @@ jobs: doctest: <<: *common docker: - - image: circleci/python:3.5 + - image: circleci/python:3.6 environment: TOXENV: doctest lint: <<: *common docker: - - image: circleci/python:3.5 + - image: circleci/python:3.6 environment: TOXENV: lint - py35-core: - <<: *common - docker: - - image: circleci/python:3.5 - environment: - TOXENV: py35-core py36-core: <<: *common docker: - image: circleci/python:3.6 environment: TOXENV: py36-core + py37-core: + <<: *common + docker: + - image: circleci/python:3.7 + environment: + TOXENV: py37-core pypy3-core: <<: *common docker: @@ -72,6 +72,6 @@ workflows: jobs: - doctest - lint - - py35-core - py36-core + - py37-core - pypy3-core diff --git a/.travis.yml b/.travis.yml index 78c56a6e..c5238746 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,23 +4,23 @@ dist: trusty matrix: include: # - # Python 3.5 testing - # - # lint - - python: "3.5" - env: TOX_POSARGS="-e lint" - # doctest - - python: "3.5" - env: TOX_POSARGS="-e doctest" - # core - - python: "3.5" - env: TOX_POSARGS="-e py35-core" - # # Python 3.6 testing # # core - python: "3.6" env: TOX_POSARGS="-e py36-core" + # lint + - python: "3.6" + env: TOX_POSARGS="-e lint" + # doctest + - python: "3.6" + env: TOX_POSARGS="-e doctest" + # + # Python 3.7 testing + # + # core + - python: "3.7" + env: TOX_POSARGS="-e py37-core" # # pypy3 testing # diff --git a/docs/conf.py b/docs/conf.py index a74a9775..0c63ed46 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -273,7 +273,7 @@ texinfo_documents = [ # -- Intersphinx configuration ------------------------------------------------ intersphinx_mapping = { - 'python': ('https://docs.python.org/3.5', None), + 'python': ('https://docs.python.org/3.6', None), } # -- Doctest configuration ---------------------------------------- diff --git a/setup.py b/setup.py index 99618aae..047d6ff3 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ setup( "eth-utils>=1,<2", ], setup_requires=['setuptools-markdown'], - python_requires='>=3.5, <4', + python_requires='>=3.6, <4', extras_require=extras_require, py_modules=[''], license="MIT", @@ -62,8 +62,8 @@ setup( 'License :: OSI Approved :: MIT License', 'Natural Language :: English', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: Implementation :: PyPy', ], ) diff --git a/tox.ini b/tox.ini index 6ae707b1..71e19f2d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist= - py{35,36,py3}-core + py{36,37,py3}-core lint doctest @@ -26,8 +26,8 @@ commands= doctest: make -C {toxinidir}/docs doctest basepython = doctest: python - py35: python3.5 py36: python3.6 + py37: python3.7 pypy3: pypy3 extras= test From dd74824840c0b5899dcbc4b5a7b61b4398218fd1 Mon Sep 17 00:00:00 2001 From: David Sanders Date: Thu, 24 Jan 2019 13:57:17 -0700 Subject: [PATCH 063/178] Add docstring checking with pydocstyle --- .pydocstyle.ini | 30 ++++++++++++++++++++++++++++++ setup.py | 1 + tox.ini | 1 + 3 files changed, 32 insertions(+) create mode 100644 .pydocstyle.ini diff --git a/.pydocstyle.ini b/.pydocstyle.ini new file mode 100644 index 00000000..0d40aa88 --- /dev/null +++ b/.pydocstyle.ini @@ -0,0 +1,30 @@ +[pydocstyle] +; All error codes found here: +; http://www.pydocstyle.org/en/3.0.0/error_codes.html +; +; Ignored: +; D1 - Missing docstring error codes +; +; Selected: +; D2 - Whitespace error codes +; D3 - Quote error codes +; D4 - Content related error codes +select=D2,D3,D4 + +; Extra ignores: +; D200 - One-line docstring should fit on one line with quotes +; D203 - 1 blank line required before class docstring +; D204 - 1 blank line required after class docstring +; D205 - 1 blank line required between summary line and description +; D212 - Multi-line docstring summary should start at the first line +; D302 - Use u""" for Unicode docstrings +; D400 - First line should end with a period +; D401 - First line should be in imperative mood +; D412 - No blank lines allowed between a section header and its content +add-ignore=D200,D203,D204,D205,D212,D302,D400,D401,D412 + +; Explanation: +; D400 - Enabling this error code seems to make it a requirement that the first +; sentence in a docstring is not split across two lines. It also makes it a +; requirement that no docstring can have a multi-sentence description without a +; summary line. Neither one of those requirements seem appropriate. diff --git a/setup.py b/setup.py index 047d6ff3..6c02fb21 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ extras_require = { 'lint': [ "flake8==3.4.1", "isort>=4.2.15,<5", + "pydocstyle>=3.0.0,<4", ], 'doc': [ "Sphinx>=1.6.5,<2", diff --git a/tox.ini b/tox.ini index 71e19f2d..63721258 100644 --- a/tox.ini +++ b/tox.ini @@ -40,3 +40,4 @@ extras=lint commands= flake8 {toxinidir}/ {toxinidir}/tests isort --recursive --check-only --diff {toxinidir}/ {toxinidir}/tests + pydocstyle {toxinidir}/ {toxinidir}/tests From de4bdf9e308e7eba871e7399a67dcdbf6c5a8bbc Mon Sep 17 00:00:00 2001 From: David Sanders Date: Mon, 25 Mar 2019 15:14:04 -0600 Subject: [PATCH 064/178] Remove attribution to specific person --- LICENSE | 2 +- README.md | 4 ++-- docs/conf.py | 8 ++++---- setup.py | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/LICENSE b/LICENSE index 1b7a2b56..d93175ab 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2018 Jason Carver +Copyright (c) 2019 The Ethereum Foundation Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 88439600..ad7eb27f 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,8 @@ pip install ## Developer Setup -If you would like to hack on , please check out the -[Ethereum Development Tactical Manual](https://github.com/pipermerriam/ethereum-dev-tactical-manual) +If you would like to hack on , please check out the [Snake Charmers +Tactical Manual](https://github.com/ethereum/snake-charmers-tactical-manual) for information on how we do: - Testing diff --git a/docs/conf.py b/docs/conf.py index 0c63ed46..aaf6fb6e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -54,7 +54,7 @@ master_doc = 'index' # General information about the project. project = '' -copyright = '2018, Jason Carver, Piper Merriam' +copyright = '2019, The Ethereum Foundation' __version__ = setup_version # The version info for the project you're documenting, acts as replacement for @@ -210,7 +210,7 @@ latex_elements = { # author, documentclass [howto, manual, or own class]). latex_documents = [ ('index', '.tex', ' Documentation', - 'Jason Carver', 'manual'), + 'The Ethereum Foundation', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -240,7 +240,7 @@ latex_documents = [ # (source start file, name, description, authors, manual section). man_pages = [ ('index', '', ' Documentation', - ['Jason Carver'], 1) + ['The Ethereum Foundation'], 1) ] # If true, show URL addresses after external links. @@ -254,7 +254,7 @@ man_pages = [ # dir menu entry, description, category) texinfo_documents = [ ('index', '', ' Documentation', - 'Jason Carver', '', '', + 'The Ethereum Foundation', '', '', 'Miscellaneous'), ] diff --git a/setup.py b/setup.py index 6c02fb21..904429c9 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ setup( version='0.1.0-alpha.0', description=""": """, long_description_markdown_filename='README.md', - author='Jason Carver', + author='The Ethereum Foundation', author_email='ethcalibur+pip@gmail.com', url='https://github.com/ethereum/', include_package_data=True, From 5f3e6b37bf6f596e23b05bac12069ddc1c34fd83 Mon Sep 17 00:00:00 2001 From: David Sanders Date: Mon, 25 Mar 2019 20:04:26 -0600 Subject: [PATCH 065/178] Remove travis config --- .travis.yml | 40 ---------------------------------------- 1 file changed, 40 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c5238746..00000000 --- a/.travis.yml +++ /dev/null @@ -1,40 +0,0 @@ -sudo: false -language: python -dist: trusty -matrix: - include: - # - # Python 3.6 testing - # - # core - - python: "3.6" - env: TOX_POSARGS="-e py36-core" - # lint - - python: "3.6" - env: TOX_POSARGS="-e lint" - # doctest - - python: "3.6" - env: TOX_POSARGS="-e doctest" - # - # Python 3.7 testing - # - # core - - python: "3.7" - env: TOX_POSARGS="-e py37-core" - # - # pypy3 testing - # - # core - - python: "pypy3.5" - env: TOX_POSARGS="-e pypy3-core" -cache: - - pip: true -install: - - travis_retry pip install pip setuptools --upgrade - - travis_retry pip install tox -before_script: - - python --version - - pip --version - - pip freeze -script: - - tox $TOX_POSARGS From 5e6dc2999162c19f87bf123d839f693399babbb5 Mon Sep 17 00:00:00 2001 From: David Sanders Date: Tue, 26 Mar 2019 11:51:15 -0600 Subject: [PATCH 066/178] Update default author email --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 904429c9..0203cbf7 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ setup( description=""": """, long_description_markdown_filename='README.md', author='The Ethereum Foundation', - author_email='ethcalibur+pip@gmail.com', + author_email='snakecharmers@ethereum.org', url='https://github.com/ethereum/', include_package_data=True, install_requires=[ From 720b2cf3d2cbef46d73cff6d0e237a60d59a6b62 Mon Sep 17 00:00:00 2001 From: Christoph Burgdorf Date: Fri, 26 Apr 2019 15:59:33 +0200 Subject: [PATCH 067/178] Add mypy support --- mypy.ini | 16 ++++++++++++++++ setup.py | 1 + tox.ini | 1 + 3 files changed, 18 insertions(+) create mode 100644 mypy.ini diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 00000000..1fcbd14c --- /dev/null +++ b/mypy.ini @@ -0,0 +1,16 @@ +[mypy] + +check_untyped_defs = True +disallow_incomplete_defs = True +disallow_untyped_defs = True +disallow_any_generics = True +disallow_untyped_calls = True +disallow_untyped_decorators = True +disallow_subclassing_any = True +ignore_missing_imports = True +strict_optional = True +strict_equality = True +warn_redundant_casts = True +warn_return_any = True +warn_unused_configs = True +warn_unused_ignores = True diff --git a/setup.py b/setup.py index 0203cbf7..97ac1c39 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ extras_require = { 'lint': [ "flake8==3.4.1", "isort>=4.2.15,<5", + "mypy==0.701", "pydocstyle>=3.0.0,<4", ], 'doc': [ diff --git a/tox.ini b/tox.ini index 63721258..e7e9920e 100644 --- a/tox.ini +++ b/tox.ini @@ -38,6 +38,7 @@ whitelist_externals=make basepython=python extras=lint commands= + mypy -p {toxinidir}/ --config-file {toxinidir}/mypy.ini flake8 {toxinidir}/ {toxinidir}/tests isort --recursive --check-only --diff {toxinidir}/ {toxinidir}/tests pydocstyle {toxinidir}/ {toxinidir}/tests From 84d2b22a7b9214b3ff40463325b2ffb338594628 Mon Sep 17 00:00:00 2001 From: Piper Merriam Date: Wed, 1 May 2019 14:31:32 -0600 Subject: [PATCH 068/178] switch to native pypy markdown support --- setup.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 97ac1c39..edd66855 100644 --- a/setup.py +++ b/setup.py @@ -31,18 +31,24 @@ extras_require = { } extras_require['dev'] = ( - extras_require['dev'] + - extras_require['test'] + - extras_require['lint'] + + extras_require['dev'] + # noqa: W504 + extras_require['test'] + # noqa: W504 + extras_require['lint'] + # noqa: W504 extras_require['doc'] ) + +with open('./README.md') as readme: + long_description = readme.read() + + setup( name='', # *IMPORTANT*: Don't manually change the version here. Use `make bump`, as described in readme version='0.1.0-alpha.0', description=""": """, - long_description_markdown_filename='README.md', + long_description=long_description, + long_description_content_type='text/markdown', author='The Ethereum Foundation', author_email='snakecharmers@ethereum.org', url='https://github.com/ethereum/', @@ -50,7 +56,6 @@ setup( install_requires=[ "eth-utils>=1,<2", ], - setup_requires=['setuptools-markdown'], python_requires='>=3.6, <4', extras_require=extras_require, py_modules=[''], From a6f607881443a06c3bbf3be3aad0651651b84c8a Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Mon, 3 Jun 2019 12:22:46 -0700 Subject: [PATCH 069/178] Add vim .swo files and .mypy_cache to .gitignore --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9a48c01a..0e057278 100644 --- a/.gitignore +++ b/.gitignore @@ -85,7 +85,10 @@ logs .idea/mongoSettings.xml # VIM temp files -*.swp +*.sw[op] + +# mypy +.mypy_cache ## File-based project format: *.iws From 19907e18ec3c8e9edada23fa85446f06e8ce78c3 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Wed, 20 Nov 2019 23:06:37 +0800 Subject: [PATCH 070/178] Replace (check and) del pattern with pop method --- libp2p/network/swarm.py | 6 ++---- libp2p/pubsub/gossipsub.py | 12 +++++------- libp2p/pubsub/mcache.py | 3 +-- libp2p/pubsub/pubsub.py | 11 +++++------ libp2p/security/security_multistream.py | 3 +-- libp2p/stream_muxer/mplex/mplex.py | 6 ++---- libp2p/stream_muxer/mplex/mplex_stream.py | 10 +++------- libp2p/stream_muxer/muxer_multistream.py | 3 +-- 8 files changed, 20 insertions(+), 34 deletions(-) diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index 7bb40cee..18676b3d 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -248,7 +248,7 @@ class Swarm(INetwork): # TODO: Should be changed to close multisple connections, # if we have several connections per peer in the future. connection = self.connections[peer_id] - # NOTE: `connection.close` will perform `del self.connections[peer_id]` + # NOTE: `connection.close` will delete `peer_id` from `self.connections` # and `notify_disconnected` for us. await connection.close() @@ -270,11 +270,9 @@ class Swarm(INetwork): """Simply remove the connection from Swarm's records, without closing the connection.""" peer_id = swarm_conn.muxed_conn.peer_id - if peer_id not in self.connections: - return # TODO: Should be changed to remove the exact connection, # if we have several connections per peer in the future. - del self.connections[peer_id] + self.connections.pop(peer_id, None) # Notifee diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 93faebdd..e9d0f355 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -144,8 +144,7 @@ class GossipSub(IPubsubRouter): elif peer_id in self.peers_floodsub: self.peers_floodsub.remove(peer_id) - if peer_id in self.peers_to_protocol: - del self.peers_to_protocol[peer_id] + self.peers_to_protocol.pop(peer_id, None) async def handle_rpc(self, rpc: rpc_pb2.RPC, sender_peer_id: ID) -> None: """ @@ -274,8 +273,7 @@ class GossipSub(IPubsubRouter): self.mesh[topic].append(peer) await self.emit_graft(topic, peer) - if topic_in_fanout: - del self.fanout[topic] + self.fanout.pop(topic, None) async def leave(self, topic: str) -> None: # Note: the comments here are the near-exact algorithm description from the spec @@ -294,7 +292,7 @@ class GossipSub(IPubsubRouter): await self.emit_prune(topic, peer) # Forget mesh[topic] - del self.mesh[topic] + self.mesh.pop(topic, None) # Heartbeat async def heartbeat(self) -> None: @@ -355,8 +353,8 @@ class GossipSub(IPubsubRouter): # TODO: there's no way time_since_last_publish gets set anywhere yet if self.time_since_last_publish[topic] > self.time_to_live: # Remove topic from fanout - del self.fanout[topic] - del self.time_since_last_publish[topic] + self.fanout.pop(topic, None) + self.time_since_last_publish.pop(topic, None) else: num_fanout_peers_in_topic = len(self.fanout[topic]) diff --git a/libp2p/pubsub/mcache.py b/libp2p/pubsub/mcache.py index b17f8679..c8489123 100644 --- a/libp2p/pubsub/mcache.py +++ b/libp2p/pubsub/mcache.py @@ -96,8 +96,7 @@ class MessageCache: last_entries: List[CacheEntry] = self.history[len(self.history) - 1] for entry in last_entries: - if entry.mid in self.msgs: - del self.msgs[entry.mid] + self.msgs.pop(entry.mid) i: int = len(self.history) - 2 diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 3834eb4b..ba1994ea 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -232,8 +232,7 @@ class Pubsub: :param topic: the topic to remove validator from """ - if topic in self.topic_validators: - del self.topic_validators[topic] + self.topic_validators.pop(topic, None) def get_msg_validators(self, msg: rpc_pb2.Message) -> Tuple[TopicValidator, ...]: """ @@ -283,7 +282,7 @@ class Pubsub: await stream.write(encode_varint_prefixed(hello.SerializeToString())) except StreamClosed: logger.debug("Fail to add new peer %s: stream closed", peer_id) - del self.peers[peer_id] + self.peers.pop(peer_id, None) return # TODO: Check EOF of this stream. # TODO: Check if the peer in black list. @@ -291,7 +290,7 @@ class Pubsub: self.router.add_peer(peer_id, stream.get_protocol()) except Exception as error: logger.debug("fail to add new peer %s, error %s", peer_id, error) - del self.peers[peer_id] + self.peers.pop(peer_id, None) return logger.debug("added new peer %s", peer_id) @@ -299,7 +298,7 @@ class Pubsub: def _handle_dead_peer(self, peer_id: ID) -> None: if peer_id not in self.peers: return - del self.peers[peer_id] + self.peers.pop(peer_id, None) for topic in self.peer_topics: if peer_id in self.peer_topics[topic]: @@ -411,7 +410,7 @@ class Pubsub: if topic_id not in self.my_topics: return # Remove topic_id from map if present - del self.my_topics[topic_id] + self.my_topics.pop(topic_id, None) # Create unsubscribe message packet: rpc_pb2.RPC = rpc_pb2.RPC() diff --git a/libp2p/security/security_multistream.py b/libp2p/security/security_multistream.py index 52c957c1..0507a524 100644 --- a/libp2p/security/security_multistream.py +++ b/libp2p/security/security_multistream.py @@ -50,8 +50,7 @@ class SecurityMultistream(ABC): :param transport: the corresponding transportation to the ``protocol``. """ # If protocol is already added before, remove it and add it again. - if protocol in self.transports: - del self.transports[protocol] + self.transports.pop(protocol, None) self.transports[protocol] = transport # Note: None is added as the handler for the given protocol since # we only care about selecting the protocol, not any handler function diff --git a/libp2p/stream_muxer/mplex/mplex.py b/libp2p/stream_muxer/mplex/mplex.py index 1a43c7cb..f70cae20 100644 --- a/libp2p/stream_muxer/mplex/mplex.py +++ b/libp2p/stream_muxer/mplex/mplex.py @@ -297,8 +297,7 @@ class Mplex(IMuxedConn): # the entry of this stream, to avoid others from accessing it. if is_local_closed: async with self.streams_lock: - if stream_id in self.streams: - del self.streams[stream_id] + self.streams.pop(stream_id, None) async def _handle_reset(self, stream_id: StreamID) -> None: async with self.streams_lock: @@ -316,8 +315,7 @@ class Mplex(IMuxedConn): if not stream.event_local_closed.is_set(): stream.event_local_closed.set() async with self.streams_lock: - if stream_id in self.streams: - del self.streams[stream_id] + self.streams.pop(stream_id, None) async def _cleanup(self) -> None: if not self.event_shutting_down.is_set(): diff --git a/libp2p/stream_muxer/mplex/mplex_stream.py b/libp2p/stream_muxer/mplex/mplex_stream.py index f080d3cf..7630c964 100644 --- a/libp2p/stream_muxer/mplex/mplex_stream.py +++ b/libp2p/stream_muxer/mplex/mplex_stream.py @@ -180,8 +180,7 @@ class MplexStream(IMuxedStream): if _is_remote_closed: # Both sides are closed, we can safely remove the buffer from the dict. async with self.muxed_conn.streams_lock: - if self.stream_id in self.muxed_conn.streams: - del self.muxed_conn.streams[self.stream_id] + self.muxed_conn.streams.pop(self.stream_id, None) async def reset(self) -> None: """closes both ends of the stream tells this remote side to hang up.""" @@ -208,11 +207,8 @@ class MplexStream(IMuxedStream): self.event_remote_closed.set() async with self.muxed_conn.streams_lock: - if ( - self.muxed_conn.streams is not None - and self.stream_id in self.muxed_conn.streams - ): - del self.muxed_conn.streams[self.stream_id] + if self.muxed_conn.streams is not None: + self.muxed_conn.streams.pop(self.stream_id, None) # TODO deadline not in use def set_deadline(self, ttl: int) -> bool: diff --git a/libp2p/stream_muxer/muxer_multistream.py b/libp2p/stream_muxer/muxer_multistream.py index f82cd19d..d83869f0 100644 --- a/libp2p/stream_muxer/muxer_multistream.py +++ b/libp2p/stream_muxer/muxer_multistream.py @@ -44,8 +44,7 @@ class MuxerMultistream: :param transport: the corresponding transportation to the ``protocol``. """ # If protocol is already added before, remove it and add it again. - if protocol in self.transports: - del self.transports[protocol] + self.transports.pop(protocol, None) self.transports[protocol] = transport self.multiselect.add_handler(protocol, None) From 501eef59deabaa155622648b41c994861ace8439 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Thu, 21 Nov 2019 14:48:03 +0800 Subject: [PATCH 071/178] Apply PR feedback: Only use pop method if graceful failure handling is desired --- libp2p/network/swarm.py | 4 +++- libp2p/pubsub/gossipsub.py | 4 ++-- libp2p/pubsub/pubsub.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index 18676b3d..de80dd82 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -270,9 +270,11 @@ class Swarm(INetwork): """Simply remove the connection from Swarm's records, without closing the connection.""" peer_id = swarm_conn.muxed_conn.peer_id + if peer_id not in self.connections: + return # TODO: Should be changed to remove the exact connection, # if we have several connections per peer in the future. - self.connections.pop(peer_id, None) + del self.connections[peer_id] # Notifee diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index e9d0f355..3bf5fa11 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -353,8 +353,8 @@ class GossipSub(IPubsubRouter): # TODO: there's no way time_since_last_publish gets set anywhere yet if self.time_since_last_publish[topic] > self.time_to_live: # Remove topic from fanout - self.fanout.pop(topic, None) - self.time_since_last_publish.pop(topic, None) + del self.fanout[topic] + del self.time_since_last_publish[topic] else: num_fanout_peers_in_topic = len(self.fanout[topic]) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index ba1994ea..869c9688 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -298,7 +298,7 @@ class Pubsub: def _handle_dead_peer(self, peer_id: ID) -> None: if peer_id not in self.peers: return - self.peers.pop(peer_id, None) + del self.peers[peer_id] for topic in self.peer_topics: if peer_id in self.peer_topics[topic]: From e355cb2600050abf9b8f8c587c0ca80d18834b03 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sat, 23 Nov 2019 16:04:22 +0800 Subject: [PATCH 072/178] Apply PR feedback: Only use pop method if error handling is in place --- libp2p/pubsub/gossipsub.py | 2 +- libp2p/pubsub/pubsub.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 3bf5fa11..d09db76e 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -292,7 +292,7 @@ class GossipSub(IPubsubRouter): await self.emit_prune(topic, peer) # Forget mesh[topic] - self.mesh.pop(topic, None) + del self.mesh[topic] # Heartbeat async def heartbeat(self) -> None: diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 869c9688..493a303c 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -282,7 +282,7 @@ class Pubsub: await stream.write(encode_varint_prefixed(hello.SerializeToString())) except StreamClosed: logger.debug("Fail to add new peer %s: stream closed", peer_id) - self.peers.pop(peer_id, None) + del self.peers[peer_id] return # TODO: Check EOF of this stream. # TODO: Check if the peer in black list. @@ -290,7 +290,7 @@ class Pubsub: self.router.add_peer(peer_id, stream.get_protocol()) except Exception as error: logger.debug("fail to add new peer %s, error %s", peer_id, error) - self.peers.pop(peer_id, None) + del self.peers[peer_id] return logger.debug("added new peer %s", peer_id) @@ -410,7 +410,7 @@ class Pubsub: if topic_id not in self.my_topics: return # Remove topic_id from map if present - self.my_topics.pop(topic_id, None) + del self.my_topics[topic_id] # Create unsubscribe message packet: rpc_pb2.RPC = rpc_pb2.RPC() From 9837f306981037a9824e850235ead5d0c1bbcc71 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sun, 24 Nov 2019 16:52:46 +0800 Subject: [PATCH 073/178] Rename `peer_map` to `peer_data_map` --- libp2p/peer/peerstore.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/libp2p/peer/peerstore.py b/libp2p/peer/peerstore.py index 65f6eb7c..6d6b1f11 100644 --- a/libp2p/peer/peerstore.py +++ b/libp2p/peer/peerstore.py @@ -10,35 +10,35 @@ from .peerstore_interface import IPeerStore class PeerStore(IPeerStore): - peer_map: Dict[ID, PeerData] + peer_data_map: Dict[ID, PeerData] def __init__(self) -> None: IPeerStore.__init__(self) - self.peer_map = {} + self.peer_data_map = {} def __create_or_get_peer(self, peer_id: ID) -> PeerData: """ Returns the peer data for peer_id or creates a new peer data (and - stores it in peer_map) if peer data for peer_id does not yet exist. + stores it in peer_data_map) if peer data for peer_id does not yet exist. :param peer_id: peer ID :return: peer data """ - if peer_id in self.peer_map: - return self.peer_map[peer_id] + if peer_id in self.peer_data_map: + return self.peer_data_map[peer_id] data = PeerData() - self.peer_map[peer_id] = data - return self.peer_map[peer_id] + self.peer_data_map[peer_id] = data + return self.peer_data_map[peer_id] def peer_info(self, peer_id: ID) -> Optional[PeerInfo]: - if peer_id in self.peer_map: - peer_data = self.peer_map[peer_id] + if peer_id in self.peer_data_map: + peer_data = self.peer_data_map[peer_id] return PeerInfo(peer_id, peer_data.addrs) return None def get_protocols(self, peer_id: ID) -> List[str]: - if peer_id in self.peer_map: - return self.peer_map[peer_id].get_protocols() + if peer_id in self.peer_data_map: + return self.peer_data_map[peer_id].get_protocols() raise PeerStoreError("peer ID not found") def add_protocols(self, peer_id: ID, protocols: Sequence[str]) -> None: @@ -50,12 +50,12 @@ class PeerStore(IPeerStore): peer.set_protocols(list(protocols)) def peer_ids(self) -> List[ID]: - return list(self.peer_map.keys()) + return list(self.peer_data_map.keys()) def get(self, peer_id: ID, key: str) -> Any: - if peer_id in self.peer_map: + if peer_id in self.peer_data_map: try: - val = self.peer_map[peer_id].get_metadata(key) + val = self.peer_data_map[peer_id].get_metadata(key) except PeerDataError as error: raise PeerStoreError(error) return val @@ -76,21 +76,21 @@ class PeerStore(IPeerStore): peer.add_addrs(list(addrs)) def addrs(self, peer_id: ID) -> List[Multiaddr]: - if peer_id in self.peer_map: - return self.peer_map[peer_id].get_addrs() + if peer_id in self.peer_data_map: + return self.peer_data_map[peer_id].get_addrs() raise PeerStoreError("peer ID not found") def clear_addrs(self, peer_id: ID) -> None: # Only clear addresses if the peer is in peer map - if peer_id in self.peer_map: - self.peer_map[peer_id].clear_addrs() + if peer_id in self.peer_data_map: + self.peer_data_map[peer_id].clear_addrs() def peers_with_addrs(self) -> List[ID]: # Add all peers with addrs at least 1 to output output: List[ID] = [] - for peer_id in self.peer_map: - if len(self.peer_map[peer_id].get_addrs()) >= 1: + for peer_id in self.peer_data_map: + if len(self.peer_data_map[peer_id].get_addrs()) >= 1: output.append(peer_id) return output From 144d93a02370679b44d20c2624a842bb4d96592b Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sun, 24 Nov 2019 18:04:00 +0800 Subject: [PATCH 074/178] Add pubkey/privkey info to `PeerStore` --- libp2p/peer/peerstore.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/libp2p/peer/peerstore.py b/libp2p/peer/peerstore.py index 6d6b1f11..8a19a3c3 100644 --- a/libp2p/peer/peerstore.py +++ b/libp2p/peer/peerstore.py @@ -1,7 +1,9 @@ -from typing import Any, Dict, List, Optional, Sequence +from typing import Any, Dict, List, Optional, Sequence, Tuple from multiaddr import Multiaddr +from libp2p.crypto.keys import PrivateKey, PublicKey + from .id import ID from .peerdata import PeerData, PeerDataError from .peerinfo import PeerInfo @@ -10,10 +12,14 @@ from .peerstore_interface import IPeerStore class PeerStore(IPeerStore): + peer_pubkey_map: Dict[ID, PublicKey] + peer_privkey_map: Dict[ID, PrivateKey] peer_data_map: Dict[ID, PeerData] def __init__(self) -> None: IPeerStore.__init__(self) + self.peer_pubkey_map = {} + self.peer_privkey_map = {} self.peer_data_map = {} def __create_or_get_peer(self, peer_id: ID) -> PeerData: @@ -94,6 +100,29 @@ class PeerStore(IPeerStore): output.append(peer_id) return output + def add_pubkey(self, peer_id: ID, pubkey: PublicKey) -> None: + if peer_id in self.peer_pubkey_map: + raise PeerStoreError(f"peer ID already has pubkey: {self.peer_pubkey_map[peer_id]}") + self.peer_pubkey_map[peer_id] = pubkey + + def pubkey(self, peer_id: ID) -> PublicKey: + if peer_id in self.peer_pubkey_map: + return self.peer_pubkey_map[peer_id] + raise PeerStoreError("peer ID not found") + + def add_privkey(self, peer_id: ID, privkey: PrivateKey) -> None: + if peer_id in self.peer_privkey_map: + raise PeerStoreError(f"peer ID already has privkey: {self.peer_privkey_map[peer_id]}") + self.peer_privkey_map[peer_id] = privkey + + def privkey(self, peer_id: ID) -> PrivateKey: + if peer_id in self.peer_pubkey_map: + return self.peer_privkey_map[peer_id] + raise PeerStoreError("peer ID not found") + + def peers_with_keys(self) -> Tuple[ID]: + return set(self.peer_pubkey_map.keys()).union(self.peer_privkey_map.keys()) + class PeerStoreError(KeyError): """Raised when peer ID is not found in peer store.""" From a63f00d8f8287d34c82c1d39c998d265be827e61 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Mon, 25 Nov 2019 16:55:55 +0800 Subject: [PATCH 075/178] Store our pubkey/privkey info during `initialize_default_swarm` --- libp2p/__init__.py | 8 ++++++-- libp2p/host/basic_host.py | 5 +---- libp2p/host/routed_host.py | 5 ++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/libp2p/__init__.py b/libp2p/__init__.py index a1dca535..75b67644 100644 --- a/libp2p/__init__.py +++ b/libp2p/__init__.py @@ -105,6 +105,10 @@ def initialize_default_swarm( ) peerstore = peerstore_opt or PeerStore() + # Store our key pair in peerstore + peerstore.add_pubkey(id_opt, key_pair.public_key) + peerstore.add_privkey(id_opt, key_pair.private_key) + # TODO: Initialize discovery if not presented return Swarm(id_opt, peerstore, upgrader, transport) @@ -151,9 +155,9 @@ async def new_node( # TODO routing unimplemented host: IHost # If not explicitly typed, MyPy raises error if disc_opt: - host = RoutedHost(key_pair.public_key, swarm_opt, disc_opt) + host = RoutedHost(swarm_opt, disc_opt) else: - host = BasicHost(key_pair.public_key, swarm_opt) + host = BasicHost(swarm_opt) # Kick off cleanup job asyncio.ensure_future(cleanup_done_tasks()) diff --git a/libp2p/host/basic_host.py b/libp2p/host/basic_host.py index 7469d33c..cd0d2896 100644 --- a/libp2p/host/basic_host.py +++ b/libp2p/host/basic_host.py @@ -39,7 +39,6 @@ class BasicHost(IHost): right after a stream is initialized. """ - _public_key: PublicKey _network: INetwork peerstore: IPeerStore @@ -48,11 +47,9 @@ class BasicHost(IHost): def __init__( self, - public_key: PublicKey, network: INetwork, default_protocols: "OrderedDict[TProtocol, StreamHandlerFn]" = None, ) -> None: - self._public_key = public_key self._network = network self._network.set_stream_handler(self._swarm_stream_handler) self.peerstore = self._network.peerstore @@ -68,7 +65,7 @@ class BasicHost(IHost): return self._network.get_peer_id() def get_public_key(self) -> PublicKey: - return self._public_key + return self.peerstore.pubkey(self.get_id()) def get_network(self) -> INetwork: """ diff --git a/libp2p/host/routed_host.py b/libp2p/host/routed_host.py index e253ce1c..78b6fa54 100644 --- a/libp2p/host/routed_host.py +++ b/libp2p/host/routed_host.py @@ -1,4 +1,3 @@ -from libp2p.crypto.keys import PublicKey from libp2p.host.basic_host import BasicHost from libp2p.host.exceptions import ConnectionFailure from libp2p.network.network_interface import INetwork @@ -11,8 +10,8 @@ from libp2p.routing.interfaces import IPeerRouting class RoutedHost(BasicHost): _router: IPeerRouting - def __init__(self, public_key: PublicKey, network: INetwork, router: IPeerRouting): - super().__init__(public_key, network) + def __init__(self, network: INetwork, router: IPeerRouting): + super().__init__(network) self._router = router async def connect(self, peer_info: PeerInfo) -> None: From e49de15227c1b39678227129e260d5c61267c339 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Mon, 25 Nov 2019 16:57:00 +0800 Subject: [PATCH 076/178] Add `get_private_key` to `Host` --- libp2p/host/basic_host.py | 3 +++ libp2p/host/host_interface.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/libp2p/host/basic_host.py b/libp2p/host/basic_host.py index cd0d2896..48c322bd 100644 --- a/libp2p/host/basic_host.py +++ b/libp2p/host/basic_host.py @@ -67,6 +67,9 @@ class BasicHost(IHost): def get_public_key(self) -> PublicKey: return self.peerstore.pubkey(self.get_id()) + def get_private_key(self) -> PublicKey: + return self.peerstore.privkey(self.get_id()) + def get_network(self) -> INetwork: """ :return: network instance of host diff --git a/libp2p/host/host_interface.py b/libp2p/host/host_interface.py index cfba82c8..5e28fdaf 100644 --- a/libp2p/host/host_interface.py +++ b/libp2p/host/host_interface.py @@ -24,6 +24,12 @@ class IHost(ABC): :return: the public key belonging to the peer """ + @abstractmethod + def get_private_key(self) -> PublicKey: + """ + :return: the private key belonging to the peer + """ + @abstractmethod def get_network(self) -> INetwork: """ From 8b4022328d5530531f70536bfa4cbea54fd6c793 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Mon, 25 Nov 2019 17:16:47 +0800 Subject: [PATCH 077/178] Update peer store interface --- libp2p/peer/addrbook_interface.py | 3 - libp2p/peer/peermetadata_interface.py | 3 - libp2p/peer/peerstore.py | 76 +++++++++++++++++- libp2p/peer/peerstore_interface.py | 106 +++++++++++++++++++++++--- 4 files changed, 172 insertions(+), 16 deletions(-) diff --git a/libp2p/peer/addrbook_interface.py b/libp2p/peer/addrbook_interface.py index a4045a76..849600e4 100644 --- a/libp2p/peer/addrbook_interface.py +++ b/libp2p/peer/addrbook_interface.py @@ -7,9 +7,6 @@ from .id import ID class IAddrBook(ABC): - def __init__(self) -> None: - pass - @abstractmethod def add_addr(self, peer_id: ID, addr: Multiaddr, ttl: int) -> None: """ diff --git a/libp2p/peer/peermetadata_interface.py b/libp2p/peer/peermetadata_interface.py index 22a690e1..6273a714 100644 --- a/libp2p/peer/peermetadata_interface.py +++ b/libp2p/peer/peermetadata_interface.py @@ -5,9 +5,6 @@ from .id import ID class IPeerMetadata(ABC): - def __init__(self) -> None: - pass - @abstractmethod def get(self, peer_id: ID, key: str) -> Any: """ diff --git a/libp2p/peer/peerstore.py b/libp2p/peer/peerstore.py index 8a19a3c3..7dc367ff 100644 --- a/libp2p/peer/peerstore.py +++ b/libp2p/peer/peerstore.py @@ -17,7 +17,6 @@ class PeerStore(IPeerStore): peer_data_map: Dict[ID, PeerData] def __init__(self) -> None: - IPeerStore.__init__(self) self.peer_pubkey_map = {} self.peer_privkey_map = {} self.peer_data_map = {} @@ -37,28 +36,54 @@ class PeerStore(IPeerStore): return self.peer_data_map[peer_id] def peer_info(self, peer_id: ID) -> Optional[PeerInfo]: + """ + :param peer_id: peer ID to get info for + :return: peer info object + """ if peer_id in self.peer_data_map: peer_data = self.peer_data_map[peer_id] return PeerInfo(peer_id, peer_data.addrs) return None def get_protocols(self, peer_id: ID) -> List[str]: + """ + :param peer_id: peer ID to get protocols for + :return: protocols (as list of strings) + :raise PeerStoreError: if peer ID not found + """ if peer_id in self.peer_data_map: return self.peer_data_map[peer_id].get_protocols() raise PeerStoreError("peer ID not found") def add_protocols(self, peer_id: ID, protocols: Sequence[str]) -> None: + """ + :param peer_id: peer ID to add protocols for + :param protocols: protocols to add + """ peer = self.__create_or_get_peer(peer_id) peer.add_protocols(list(protocols)) def set_protocols(self, peer_id: ID, protocols: Sequence[str]) -> None: + """ + :param peer_id: peer ID to set protocols for + :param protocols: protocols to set + """ peer = self.__create_or_get_peer(peer_id) peer.set_protocols(list(protocols)) def peer_ids(self) -> List[ID]: + """ + :return: all of the peer IDs stored in peer store + """ return list(self.peer_data_map.keys()) def get(self, peer_id: ID, key: str) -> Any: + """ + :param peer_id: peer ID to get peer data for + :param key: the key to search value for + :return: value corresponding to the key + :raise PeerStoreError: if peer ID or value not found + """ if peer_id in self.peer_data_map: try: val = self.peer_data_map[peer_id].get_metadata(key) @@ -68,30 +93,56 @@ class PeerStore(IPeerStore): raise PeerStoreError("peer ID not found") def put(self, peer_id: ID, key: str, val: Any) -> None: + """ + :param peer_id: peer ID to put peer data for + :param key: + :param value: + """ # <> # This can output an error, not sure what the possible errors are peer = self.__create_or_get_peer(peer_id) peer.put_metadata(key, val) def add_addr(self, peer_id: ID, addr: Multiaddr, ttl: int) -> None: + """ + :param peer_id: peer ID to add address for + :param addr: + :param ttl: time-to-live for the this record + """ self.add_addrs(peer_id, [addr], ttl) def add_addrs(self, peer_id: ID, addrs: Sequence[Multiaddr], ttl: int) -> None: + """ + :param peer_id: peer ID to add address for + :param addrs: + :param ttl: time-to-live for the this record + """ # Ignore ttl for now peer = self.__create_or_get_peer(peer_id) peer.add_addrs(list(addrs)) def addrs(self, peer_id: ID) -> List[Multiaddr]: + """ + :param peer_id: peer ID to get addrs for + :return: list of addrs + :raise PeerStoreError: if peer ID not found + """ if peer_id in self.peer_data_map: return self.peer_data_map[peer_id].get_addrs() raise PeerStoreError("peer ID not found") def clear_addrs(self, peer_id: ID) -> None: + """ + :param peer_id: peer ID to clear addrs for + """ # Only clear addresses if the peer is in peer map if peer_id in self.peer_data_map: self.peer_data_map[peer_id].clear_addrs() def peers_with_addrs(self) -> List[ID]: + """ + :return: all of the peer IDs which has addrs stored in peer store + """ # Add all peers with addrs at least 1 to output output: List[ID] = [] @@ -101,26 +152,49 @@ class PeerStore(IPeerStore): return output def add_pubkey(self, peer_id: ID, pubkey: PublicKey) -> None: + """ + :param peer_id: peer ID to add public key for + :param pubkey: + :raise PeerStoreError: if peer ID already has pubkey set + """ if peer_id in self.peer_pubkey_map: raise PeerStoreError(f"peer ID already has pubkey: {self.peer_pubkey_map[peer_id]}") self.peer_pubkey_map[peer_id] = pubkey def pubkey(self, peer_id: ID) -> PublicKey: + """ + :param peer_id: peer ID to get public key for + :return: public key of the peer + :raise PeerStoreError: if peer ID not found + """ if peer_id in self.peer_pubkey_map: return self.peer_pubkey_map[peer_id] raise PeerStoreError("peer ID not found") def add_privkey(self, peer_id: ID, privkey: PrivateKey) -> None: + """ + :param peer_id: peer ID to add private key for + :param privkey: + :raise PeerStoreError: if peer ID already has privkey set + """ if peer_id in self.peer_privkey_map: raise PeerStoreError(f"peer ID already has privkey: {self.peer_privkey_map[peer_id]}") self.peer_privkey_map[peer_id] = privkey def privkey(self, peer_id: ID) -> PrivateKey: + """ + :param peer_id: peer ID to get private key for + :return: private key of the peer + :raise PeerStoreError: if peer ID not found + """ if peer_id in self.peer_pubkey_map: return self.peer_privkey_map[peer_id] raise PeerStoreError("peer ID not found") def peers_with_keys(self) -> Tuple[ID]: + """ + :return: all of the peer IDs which has pubkey/privkey stored in peer store + """ return set(self.peer_pubkey_map.keys()).union(self.peer_privkey_map.keys()) diff --git a/libp2p/peer/peerstore_interface.py b/libp2p/peer/peerstore_interface.py index 3ffd9a2d..fbd1a2eb 100644 --- a/libp2p/peer/peerstore_interface.py +++ b/libp2p/peer/peerstore_interface.py @@ -1,5 +1,9 @@ from abc import abstractmethod -from typing import List, Sequence +from typing import Any, List, Sequence, Tuple + +from multiaddr import Multiaddr + +from libp2p.crypto.keys import PrivateKey, PublicKey from .addrbook_interface import IAddrBook from .id import ID @@ -8,10 +12,6 @@ from .peermetadata_interface import IPeerMetadata class IPeerStore(IAddrBook, IPeerMetadata): - def __init__(self) -> None: - IPeerMetadata.__init__(self) - IAddrBook.__init__(self) - @abstractmethod def peer_info(self, peer_id: ID) -> PeerInfo: """ @@ -23,8 +23,8 @@ class IPeerStore(IAddrBook, IPeerMetadata): def get_protocols(self, peer_id: ID) -> List[str]: """ :param peer_id: peer ID to get protocols for - :return: protocols (as strings) - :raise Exception: peer ID not found exception + :return: protocols (as list of strings) + :raise PeerStoreError: if peer ID not found """ @abstractmethod @@ -32,7 +32,6 @@ class IPeerStore(IAddrBook, IPeerMetadata): """ :param peer_id: peer ID to add protocols for :param protocols: protocols to add - :raise Exception: peer ID not found """ @abstractmethod @@ -40,7 +39,6 @@ class IPeerStore(IAddrBook, IPeerMetadata): """ :param peer_id: peer ID to set protocols for :param protocols: protocols to set - :raise Exception: peer ID not found """ @abstractmethod @@ -48,3 +46,93 @@ class IPeerStore(IAddrBook, IPeerMetadata): """ :return: all of the peer IDs stored in peer store """ + + @abstractmethod + def get(self, peer_id: ID, key: str) -> Any: + """ + :param peer_id: peer ID to get peer data for + :param key: the key to search value for + :return: value corresponding to the key + :raise PeerStoreError: if peer ID or value not found + """ + + @abstractmethod + def put(self, peer_id: ID, key: str, val: Any) -> None: + """ + :param peer_id: peer ID to put peer data for + :param key: + :param value: + """ + + @abstractmethod + def add_addr(self, peer_id: ID, addr: Multiaddr, ttl: int) -> None: + """ + :param peer_id: peer ID to add address for + :param addr: + :param ttl: time-to-live for the this record + """ + + @abstractmethod + def add_addrs(self, peer_id: ID, addrs: Sequence[Multiaddr], ttl: int) -> None: + """ + :param peer_id: peer ID to add address for + :param addrs: + :param ttl: time-to-live for the this record + """ + + @abstractmethod + def addrs(self, peer_id: ID) -> List[Multiaddr]: + """ + :param peer_id: peer ID to get addrs for + :return: list of addrs + """ + + @abstractmethod + def clear_addrs(self, peer_id: ID) -> None: + """ + :param peer_id: peer ID to clear addrs for + """ + + @abstractmethod + def peers_with_addrs(self) -> List[ID]: + """ + :return: all of the peer IDs which has addrs stored in peer store + """ + + @abstractmethod + def add_pubkey(self, peer_id: ID, pubkey: PublicKey) -> None: + """ + :param peer_id: peer ID to add public key for + :param pubkey: + :raise PeerStoreError: if peer ID already has pubkey set + """ + + @abstractmethod + def pubkey(self, peer_id: ID) -> PublicKey: + """ + :param peer_id: peer ID to get public key for + :return: public key of the peer + :raise PeerStoreError: if peer ID not found + """ + + @abstractmethod + def add_privkey(self, peer_id: ID, privkey: PrivateKey) -> None: + """ + :param peer_id: peer ID to add private key for + :param privkey: + :raise PeerStoreError: if peer ID already has privkey set + """ + + @abstractmethod + def privkey(self, peer_id: ID) -> PrivateKey: + """ + :param peer_id: peer ID to get private key for + :return: private key of the peer + :raise PeerStoreError: if peer ID not found + """ + + @abstractmethod + def peers_with_keys(self) -> Tuple[ID]: + """ + :return: all of the peer IDs which has pubkey/privkey stored in peer store + """ From e28a9744257a23a16f9bc431572c3e7640363d6f Mon Sep 17 00:00:00 2001 From: NIC619 Date: Mon, 25 Nov 2019 17:17:09 +0800 Subject: [PATCH 078/178] Update peer store initialization in host factory --- libp2p/tools/factories.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/libp2p/tools/factories.py b/libp2p/tools/factories.py index 2b63544d..341d6916 100644 --- a/libp2p/tools/factories.py +++ b/libp2p/tools/factories.py @@ -10,6 +10,7 @@ from libp2p.host.basic_host import BasicHost from libp2p.network.connection.swarm_connection import SwarmConn from libp2p.network.stream.net_stream_interface import INetStream from libp2p.network.swarm import Swarm +from libp2p.peer.id import ID from libp2p.peer.peerstore import PeerStore from libp2p.pubsub.floodsub import FloodSub from libp2p.pubsub.gossipsub import GossipSub @@ -33,6 +34,13 @@ from .constants import ( from .utils import connect, connect_swarm +def initialize_peerstore_with_our_keypair(self_id: ID, key_pair: KeyPair): + peer_store = PeerStore() + peer_store.add_pubkey(self_id, key_pair.public_key) + peer_store.add_privkey(self_id, key_pair.private_key) + return peer_store + + def security_transport_factory( is_secure: bool, key_pair: KeyPair ) -> Dict[TProtocol, BaseSecureTransport]: @@ -52,7 +60,9 @@ class SwarmFactory(factory.Factory): muxer_opt = {MPLEX_PROTOCOL_ID: Mplex} peer_id = factory.LazyAttribute(lambda o: generate_peer_id_from(o.key_pair)) - peerstore = factory.LazyFunction(PeerStore) + peerstore = factory.LazyAttribute( + lambda o: initialize_peerstore_with_our_keypair(o.peer_id, o.key_pair) + ) upgrader = factory.LazyAttribute( lambda o: TransportUpgrader( security_transport_factory(o.is_secure, o.key_pair), o.muxer_opt @@ -97,7 +107,6 @@ class HostFactory(factory.Factory): is_secure = False key_pair = factory.LazyFunction(generate_new_rsa_identity) - public_key = factory.LazyAttribute(lambda o: o.key_pair.public_key) network = factory.LazyAttribute( lambda o: SwarmFactory(is_secure=o.is_secure, key_pair=o.key_pair) ) @@ -113,10 +122,7 @@ class HostFactory(factory.Factory): for key_pair in key_pairs ] ) - return tuple( - BasicHost(key_pair.public_key, swarm) - for key_pair, swarm in zip(key_pairs, swarms) - ) + return tuple(BasicHost(swarm) for swarm in swarms) class FloodsubFactory(factory.Factory): From 566e4c080dd50a9d533b83398e35a3fba101c12a Mon Sep 17 00:00:00 2001 From: NIC619 Date: Mon, 25 Nov 2019 17:32:53 +0800 Subject: [PATCH 079/178] Fix lint --- libp2p/host/basic_host.py | 4 ++-- libp2p/host/host_interface.py | 4 ++-- libp2p/peer/peerstore.py | 15 ++++++++++----- libp2p/peer/peerstore_interface.py | 4 ++-- libp2p/tools/factories.py | 2 +- 5 files changed, 17 insertions(+), 12 deletions(-) diff --git a/libp2p/host/basic_host.py b/libp2p/host/basic_host.py index 48c322bd..253394e5 100644 --- a/libp2p/host/basic_host.py +++ b/libp2p/host/basic_host.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, List, Sequence import multiaddr -from libp2p.crypto.keys import PublicKey +from libp2p.crypto.keys import PrivateKey, PublicKey from libp2p.host.defaults import get_default_protocols from libp2p.host.exceptions import StreamFailure from libp2p.network.network_interface import INetwork @@ -67,7 +67,7 @@ class BasicHost(IHost): def get_public_key(self) -> PublicKey: return self.peerstore.pubkey(self.get_id()) - def get_private_key(self) -> PublicKey: + def get_private_key(self) -> PrivateKey: return self.peerstore.privkey(self.get_id()) def get_network(self) -> INetwork: diff --git a/libp2p/host/host_interface.py b/libp2p/host/host_interface.py index 5e28fdaf..43f4ac40 100644 --- a/libp2p/host/host_interface.py +++ b/libp2p/host/host_interface.py @@ -3,7 +3,7 @@ from typing import Any, List, Sequence import multiaddr -from libp2p.crypto.keys import PublicKey +from libp2p.crypto.keys import PrivateKey, PublicKey from libp2p.network.network_interface import INetwork from libp2p.network.stream.net_stream_interface import INetStream from libp2p.peer.id import ID @@ -25,7 +25,7 @@ class IHost(ABC): """ @abstractmethod - def get_private_key(self) -> PublicKey: + def get_private_key(self) -> PrivateKey: """ :return: the private key belonging to the peer """ diff --git a/libp2p/peer/peerstore.py b/libp2p/peer/peerstore.py index 7dc367ff..4cff54e2 100644 --- a/libp2p/peer/peerstore.py +++ b/libp2p/peer/peerstore.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Sequence, Tuple +from typing import Any, Dict, List, Optional, Sequence, Set from multiaddr import Multiaddr @@ -24,7 +24,8 @@ class PeerStore(IPeerStore): def __create_or_get_peer(self, peer_id: ID) -> PeerData: """ Returns the peer data for peer_id or creates a new peer data (and - stores it in peer_data_map) if peer data for peer_id does not yet exist. + stores it in peer_data_map) if peer data for peer_id does not yet + exist. :param peer_id: peer ID :return: peer data @@ -158,7 +159,9 @@ class PeerStore(IPeerStore): :raise PeerStoreError: if peer ID already has pubkey set """ if peer_id in self.peer_pubkey_map: - raise PeerStoreError(f"peer ID already has pubkey: {self.peer_pubkey_map[peer_id]}") + raise PeerStoreError( + f"peer ID already has pubkey: {self.peer_pubkey_map[peer_id]}" + ) self.peer_pubkey_map[peer_id] = pubkey def pubkey(self, peer_id: ID) -> PublicKey: @@ -178,7 +181,9 @@ class PeerStore(IPeerStore): :raise PeerStoreError: if peer ID already has privkey set """ if peer_id in self.peer_privkey_map: - raise PeerStoreError(f"peer ID already has privkey: {self.peer_privkey_map[peer_id]}") + raise PeerStoreError( + f"peer ID already has privkey: {self.peer_privkey_map[peer_id]}" + ) self.peer_privkey_map[peer_id] = privkey def privkey(self, peer_id: ID) -> PrivateKey: @@ -191,7 +196,7 @@ class PeerStore(IPeerStore): return self.peer_privkey_map[peer_id] raise PeerStoreError("peer ID not found") - def peers_with_keys(self) -> Tuple[ID]: + def peers_with_keys(self) -> Set[ID]: """ :return: all of the peer IDs which has pubkey/privkey stored in peer store """ diff --git a/libp2p/peer/peerstore_interface.py b/libp2p/peer/peerstore_interface.py index fbd1a2eb..8ebe1cc6 100644 --- a/libp2p/peer/peerstore_interface.py +++ b/libp2p/peer/peerstore_interface.py @@ -1,5 +1,5 @@ from abc import abstractmethod -from typing import Any, List, Sequence, Tuple +from typing import Any, List, Sequence, Set from multiaddr import Multiaddr @@ -132,7 +132,7 @@ class IPeerStore(IAddrBook, IPeerMetadata): """ @abstractmethod - def peers_with_keys(self) -> Tuple[ID]: + def peers_with_keys(self) -> Set[ID]: """ :return: all of the peer IDs which has pubkey/privkey stored in peer store """ diff --git a/libp2p/tools/factories.py b/libp2p/tools/factories.py index 341d6916..405e6ed6 100644 --- a/libp2p/tools/factories.py +++ b/libp2p/tools/factories.py @@ -34,7 +34,7 @@ from .constants import ( from .utils import connect, connect_swarm -def initialize_peerstore_with_our_keypair(self_id: ID, key_pair: KeyPair): +def initialize_peerstore_with_our_keypair(self_id: ID, key_pair: KeyPair) -> PeerStore: peer_store = PeerStore() peer_store.add_pubkey(self_id, key_pair.public_key) peer_store.add_privkey(self_id, key_pair.private_key) From 076dae50f364844635dfbba7b18c83dd2638c77e Mon Sep 17 00:00:00 2001 From: NIC619 Date: Mon, 25 Nov 2019 22:05:33 +0800 Subject: [PATCH 080/178] Fix missing `BasicHost` interface update --- tests/host/test_basic_host.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/host/test_basic_host.py b/tests/host/test_basic_host.py index f25f4e72..1eec04a8 100644 --- a/tests/host/test_basic_host.py +++ b/tests/host/test_basic_host.py @@ -7,7 +7,7 @@ from libp2p.host.defaults import get_default_protocols def test_default_protocols(): key_pair = create_new_key_pair() swarm = initialize_default_swarm(key_pair) - host = BasicHost(key_pair.public_key, swarm) + host = BasicHost(swarm) mux = host.get_mux() handlers = mux.handlers From 02fe35663c96b81bbd7512480653fbd6140d7ff5 Mon Sep 17 00:00:00 2001 From: Christoph Burgdorf Date: Mon, 29 Jul 2019 12:51:01 +0200 Subject: [PATCH 081/178] Setup towncrier to generate release notes --- .circleci/config.yml | 6 ++-- .github/PULL_REQUEST_TEMPLATE.md | 10 ++++++ Makefile | 14 ++++++++- README.md | 4 +-- docs/index.rst | 2 +- docs/{releases.rst => release_notes.rst} | 2 ++ newsfragments/README.md | 26 ++++++++++++++++ newsfragments/validate_files.py | 32 +++++++++++++++++++ pyproject.toml | 39 ++++++++++++++++++++++++ setup.py | 1 + tox.ini | 8 ++--- 11 files changed, 132 insertions(+), 12 deletions(-) rename docs/{releases.rst => release_notes.rst} (78%) create mode 100644 newsfragments/README.md create mode 100755 newsfragments/validate_files.py create mode 100644 pyproject.toml diff --git a/.circleci/config.yml b/.circleci/config.yml index bbc56d0a..fa7691e6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,12 +36,12 @@ common: &common key: cache-{{ .Environment.CIRCLE_JOB }}-{{ checksum "setup.py" }}-{{ checksum "tox.ini" }} jobs: - doctest: + docs: <<: *common docker: - image: circleci/python:3.6 environment: - TOXENV: doctest + TOXENV: docs lint: <<: *common docker: @@ -70,7 +70,7 @@ workflows: version: 2 test: jobs: - - doctest + - docs - lint - py36-core - py37-core diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 76ef5ac6..21d4db59 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,6 +6,16 @@ Issue # Summary of approach. +### To-Do + +[//]: # (Stay ahead of things, add list items here!) +- [ ] Clean up commit history + +[//]: # (For important changes that should go into the release notes please add a newsfragment file as explained here: https://github.com/ethereum//blob/master/newsfragments/README.md) + +[//]: # (See: https://.readthedocs.io/en/latest/contributing.html#pull-requests) +- [ ] Add entry to the [release notes](https://github.com/ethereum//blob/master/newsfragments/README.md) + #### Cute Animal Picture ![put a cute animal picture link inside the parentheses]() diff --git a/Makefile b/Makefile index 18f7003b..56033104 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,8 @@ build-docs: $(MAKE) -C docs clean $(MAKE) -C docs html $(MAKE) -C docs doctest + ./newsfragments/validate_files.py + towncrier --draft --version preview docs: build-docs open docs/_build/html/index.html @@ -49,13 +51,23 @@ linux-docs: build-docs xdg-open docs/_build/html/index.html release: clean + CURRENT_SIGN_SETTING=$(git config commit.gpgSign) git config commit.gpgSign true - bumpversion $(bump) + # Let UPCOMING_VERSION be the version that is used for the current bump + $(eval UPCOMING_VERSION=$(shell bumpversion $(bump) --dry-run --list | grep new_version= | sed 's/new_version=//g')) + # Now generate the release notes to have them included in the release commit + towncrier --yes --version $(UPCOMING_VERSION) + # Before we bump the version, make sure that the towncrier-generated docs will build + make build-docs + # We need --allow-dirty because of the generated release_notes file that goes into the release + # commit. No other files are added accidentially. The dry-run still runs *without* --allow-dirty + bumpversion --allow-dirty $(bump) git push upstream && git push upstream --tags python setup.py sdist bdist_wheel twine upload dist/* git config commit.gpgSign "$(CURRENT_SIGN_SETTING)" + dist: clean python setup.py sdist bdist_wheel ls -l dist diff --git a/README.md b/README.md index ad7eb27f..3ceb98c9 100644 --- a/README.md +++ b/README.md @@ -88,9 +88,7 @@ The version format for this repo is `{major}.{minor}.{patch}` for stable, and To issue the next version in line, specify which part to bump, like `make release bump=minor` or `make release bump=devnum`. This is typically done from the master branch, except when releasing a beta (in which case the beta is released from master, -and the previous stable branch is released from said branch). To include changes made with each -release, update "docs/releases.rst" with the changes, and apply commit directly to master -before release. +and the previous stable branch is released from said branch). If you are in a beta version, `make release bump=stage` will switch to a stable. diff --git a/docs/index.rst b/docs/index.rst index b3598a41..89815efa 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -10,7 +10,7 @@ Contents :maxdepth: 3 - releases + release_notes Indices and tables diff --git a/docs/releases.rst b/docs/release_notes.rst similarity index 78% rename from docs/releases.rst rename to docs/release_notes.rst index 6fdd6c97..63786aca 100644 --- a/docs/releases.rst +++ b/docs/release_notes.rst @@ -1,6 +1,8 @@ Release Notes ============= +.. towncrier release notes start + v0.1.0-alpha.1 -------------- diff --git a/newsfragments/README.md b/newsfragments/README.md new file mode 100644 index 00000000..09c1cc8d --- /dev/null +++ b/newsfragments/README.md @@ -0,0 +1,26 @@ +This directory collects "newsfragments": short files that each contain +a snippet of ReST-formatted text that will be added to the next +release notes. This should be a description of aspects of the change +(if any) that are relevant to users. (This contrasts with the +commit message and PR description, which are a description of the change as +relevant to people working on the code itself.) + +Each file should be named like `..rst`, where +`` is an issue numbers, and `` is one of: + +* `feature` +* `bugfix` +* `performance` +* `doc` +* `removal` +* `misc` + +So for example: `123.feature.rst`, `456.bugfix.rst` + +If the PR fixes an issue, use that number here. If there is no issue, +then open up the PR first and use the PR number for the newsfragment. + +Note that the `towncrier` tool will automatically +reflow your text, so don't try to do any fancy formatting. Run + `towncrier --draft` to get a preview of what the release notes entry + will look like in the final release notes. \ No newline at end of file diff --git a/newsfragments/validate_files.py b/newsfragments/validate_files.py new file mode 100755 index 00000000..c6695bc7 --- /dev/null +++ b/newsfragments/validate_files.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +# Towncrier silently ignores files that do not match the expected ending. +# We use this script to ensure we catch these as errors in CI. + +import os +import pathlib + +ALLOWED_EXTENSIONS = { + '.bugfix.rst', + '.doc.rst', + '.feature.rst', + '.misc.rst', + '.performance.rst', + '.removal.rst', +} + +ALLOWED_FILES = { + 'validate_files.py', + 'README.md', +} + +THIS_DIR = pathlib.Path(__file__).parent + +for fragment_file in THIS_DIR.iterdir(): + + if fragment_file.name in ALLOWED_FILES: + continue + + full_extension = "".join(fragment_file.suffixes) + if full_extension not in ALLOWED_EXTENSIONS: + raise Exception(f"Unexpected file: {fragment_file}") diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..42a4130e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,39 @@ +[tool.towncrier] +# Read https://github.com/ethereum//newsfragments/README.md for instructions +package = "" +filename = "docs/release_notes.rst" +directory = "newsfragments" +underlines = ["-", "~", "^"] +issue_format = "`#{issue} /issues/{issue}>`__" + +# Configure all default sections plus an extra one for performance improvements. + +[[tool.towncrier.type]] +directory = "feature" +name = "Features" +showcontent = true + +[[tool.towncrier.type]] +directory = "bugfix" +name = "Bugfixes" +showcontent = true + +[[tool.towncrier.type]] +directory = "performance" +name = "Performance improvements" +showcontent = true + +[[tool.towncrier.type]] +directory = "doc" +name = "Improved Documentation" +showcontent = true + +[[tool.towncrier.type]] +directory = "removal" +name = "Deprecations and Removals" +showcontent = true + +[[tool.towncrier.type]] +directory = "misc" +name = "Miscellaneous internal changes" +showcontent = false \ No newline at end of file diff --git a/setup.py b/setup.py index edd66855..54dbd6ca 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ extras_require = { 'doc': [ "Sphinx>=1.6.5,<2", "sphinx_rtd_theme>=0.1.9", + "towncrier>=19.2.0, <20", ], 'dev': [ "bumpversion>=0.5.3,<1", diff --git a/tox.ini b/tox.ini index e7e9920e..953e83ab 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ envlist= py{36,37,py3}-core lint - doctest + docs [isort] combine_as_imports=True @@ -23,15 +23,15 @@ ignore= usedevelop=True commands= core: pytest {posargs:tests/core} - doctest: make -C {toxinidir}/docs doctest + docs: make build-docs basepython = - doctest: python + docs: python py36: python3.6 py37: python3.7 pypy3: pypy3 extras= test - doctest: doc + docs: doc whitelist_externals=make [testenv:lint] From 07eecb5c69b9c92e5e5699fafded51bbfaddca54 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Mon, 25 Nov 2019 12:22:29 -0800 Subject: [PATCH 082/178] Separate release-note build from release --- Makefile | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 56033104..5e087e37 100644 --- a/Makefile +++ b/Makefile @@ -50,18 +50,24 @@ docs: build-docs linux-docs: build-docs xdg-open docs/_build/html/index.html -release: clean - CURRENT_SIGN_SETTING=$(git config commit.gpgSign) - git config commit.gpgSign true +notes: # Let UPCOMING_VERSION be the version that is used for the current bump $(eval UPCOMING_VERSION=$(shell bumpversion $(bump) --dry-run --list | grep new_version= | sed 's/new_version=//g')) # Now generate the release notes to have them included in the release commit towncrier --yes --version $(UPCOMING_VERSION) # Before we bump the version, make sure that the towncrier-generated docs will build make build-docs - # We need --allow-dirty because of the generated release_notes file that goes into the release - # commit. No other files are added accidentially. The dry-run still runs *without* --allow-dirty - bumpversion --allow-dirty $(bump) + git commit -m "Compile release notes" + +release: clean + # require that you be on a branch that's linked to upstream/master + git status -s -b | head -1 | grep "\.\.upstream/master" + # verify that docs build correctly + ./newsfragments/validate_files.py is-empty + make build-docs + CURRENT_SIGN_SETTING=$(git config commit.gpgSign) + git config commit.gpgSign true + bumpversion $(bump) git push upstream && git push upstream --tags python setup.py sdist bdist_wheel twine upload dist/* From 89363b2d9b67c30a04dc9aa3ef2ad6e1ad1b1709 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Mon, 25 Nov 2019 12:28:05 -0800 Subject: [PATCH 083/178] Add internal type for release notes --- newsfragments/README.md | 3 ++- newsfragments/validate_files.py | 17 ++++++++++++++--- pyproject.toml | 11 +++++++---- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/newsfragments/README.md b/newsfragments/README.md index 09c1cc8d..09a10ddc 100644 --- a/newsfragments/README.md +++ b/newsfragments/README.md @@ -12,6 +12,7 @@ Each file should be named like `..rst`, where * `bugfix` * `performance` * `doc` +* `internal` * `removal` * `misc` @@ -23,4 +24,4 @@ then open up the PR first and use the PR number for the newsfragment. Note that the `towncrier` tool will automatically reflow your text, so don't try to do any fancy formatting. Run `towncrier --draft` to get a preview of what the release notes entry - will look like in the final release notes. \ No newline at end of file + will look like in the final release notes. diff --git a/newsfragments/validate_files.py b/newsfragments/validate_files.py index c6695bc7..c0e9b289 100755 --- a/newsfragments/validate_files.py +++ b/newsfragments/validate_files.py @@ -5,11 +5,13 @@ import os import pathlib +import sys ALLOWED_EXTENSIONS = { '.bugfix.rst', '.doc.rst', '.feature.rst', + '.internal.rst', '.misc.rst', '.performance.rst', '.removal.rst', @@ -22,11 +24,20 @@ ALLOWED_FILES = { THIS_DIR = pathlib.Path(__file__).parent +num_args = len(sys.argv) - 1 +assert num_args in {0, 1} +if num_args == 1: + assert sys.argv[1] in ('is-empty', ) + for fragment_file in THIS_DIR.iterdir(): if fragment_file.name in ALLOWED_FILES: continue - - full_extension = "".join(fragment_file.suffixes) - if full_extension not in ALLOWED_EXTENSIONS: + elif num_args == 0: + full_extension = "".join(fragment_file.suffixes) + if full_extension not in ALLOWED_EXTENSIONS: + raise Exception(f"Unexpected file: {fragment_file}") + elif sys.argv[1] == 'is-empty': raise Exception(f"Unexpected file: {fragment_file}") + else: + raise RuntimeError("Strange: arguments {sys.argv} were validated, but not found") diff --git a/pyproject.toml b/pyproject.toml index 42a4130e..a9724ba6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,8 +6,6 @@ directory = "newsfragments" underlines = ["-", "~", "^"] issue_format = "`#{issue} /issues/{issue}>`__" -# Configure all default sections plus an extra one for performance improvements. - [[tool.towncrier.type]] directory = "feature" name = "Features" @@ -33,7 +31,12 @@ directory = "removal" name = "Deprecations and Removals" showcontent = true +[[tool.towncrier.type]] +directory = "internal" +name = "Internal Changes - for Contributors" +showcontent = true + [[tool.towncrier.type]] directory = "misc" -name = "Miscellaneous internal changes" -showcontent = false \ No newline at end of file +name = "Miscellaneous changes" +showcontent = false From 828ae69c66c8c6fe6964c98f1bfb5624a4ff0501 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 26 Nov 2019 11:52:31 +0800 Subject: [PATCH 084/178] Apply PR feedback: `add_key_pair` --- libp2p/__init__.py | 3 +-- libp2p/peer/peerstore.py | 13 +++++++++++-- libp2p/peer/peerstore_interface.py | 10 +++++++++- libp2p/tools/factories.py | 3 +-- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/libp2p/__init__.py b/libp2p/__init__.py index 75b67644..291b3a52 100644 --- a/libp2p/__init__.py +++ b/libp2p/__init__.py @@ -106,8 +106,7 @@ def initialize_default_swarm( peerstore = peerstore_opt or PeerStore() # Store our key pair in peerstore - peerstore.add_pubkey(id_opt, key_pair.public_key) - peerstore.add_privkey(id_opt, key_pair.private_key) + peerstore.add_key_pair(id_opt, key_pair) # TODO: Initialize discovery if not presented return Swarm(id_opt, peerstore, upgrader, transport) diff --git a/libp2p/peer/peerstore.py b/libp2p/peer/peerstore.py index 4cff54e2..c011e3ef 100644 --- a/libp2p/peer/peerstore.py +++ b/libp2p/peer/peerstore.py @@ -2,7 +2,7 @@ from typing import Any, Dict, List, Optional, Sequence, Set from multiaddr import Multiaddr -from libp2p.crypto.keys import PrivateKey, PublicKey +from libp2p.crypto.keys import KeyPair, PrivateKey, PublicKey from .id import ID from .peerdata import PeerData, PeerDataError @@ -34,7 +34,7 @@ class PeerStore(IPeerStore): return self.peer_data_map[peer_id] data = PeerData() self.peer_data_map[peer_id] = data - return self.peer_data_map[peer_id] + return data def peer_info(self, peer_id: ID) -> Optional[PeerInfo]: """ @@ -196,6 +196,15 @@ class PeerStore(IPeerStore): return self.peer_privkey_map[peer_id] raise PeerStoreError("peer ID not found") + def add_key_pair(self, peer_id: ID, key_pair: KeyPair) -> None: + """ + :param peer_id: peer ID to add private key for + :param key_pair: + :raise PeerStoreError: if peer ID already has pubkey or privkey set + """ + self.add_pubkey(peer_id, key_pair.public_keypubkey) + self.add_privkey(peer_id, key_pair.private_key) + def peers_with_keys(self) -> Set[ID]: """ :return: all of the peer IDs which has pubkey/privkey stored in peer store diff --git a/libp2p/peer/peerstore_interface.py b/libp2p/peer/peerstore_interface.py index 8ebe1cc6..618bbf5e 100644 --- a/libp2p/peer/peerstore_interface.py +++ b/libp2p/peer/peerstore_interface.py @@ -3,7 +3,7 @@ from typing import Any, List, Sequence, Set from multiaddr import Multiaddr -from libp2p.crypto.keys import PrivateKey, PublicKey +from libp2p.crypto.keys import KeyPair, PrivateKey, PublicKey from .addrbook_interface import IAddrBook from .id import ID @@ -131,6 +131,14 @@ class IPeerStore(IAddrBook, IPeerMetadata): :raise PeerStoreError: if peer ID not found """ + @abstractmethod + def add_key_pair(self, peer_id: ID, key_pair: KeyPair) -> None: + """ + :param peer_id: peer ID to add private key for + :param key_pair: + :raise PeerStoreError: if peer ID already has pubkey or privkey set + """ + @abstractmethod def peers_with_keys(self) -> Set[ID]: """ diff --git a/libp2p/tools/factories.py b/libp2p/tools/factories.py index 405e6ed6..b5c16b85 100644 --- a/libp2p/tools/factories.py +++ b/libp2p/tools/factories.py @@ -36,8 +36,7 @@ from .utils import connect, connect_swarm def initialize_peerstore_with_our_keypair(self_id: ID, key_pair: KeyPair) -> PeerStore: peer_store = PeerStore() - peer_store.add_pubkey(self_id, key_pair.public_key) - peer_store.add_privkey(self_id, key_pair.private_key) + peer_store.add_key_pair(self_id, key_pair) return peer_store From 0bfbdf7fab77a15562b32bf892cd23d5d9a5c16d Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 26 Nov 2019 12:27:59 +0800 Subject: [PATCH 085/178] Move keypair into `PeerData` --- libp2p/peer/peerdata.py | 59 ++++++++++++++++++++++ libp2p/peer/peerdata_interface.py | 33 +++++++++++-- libp2p/peer/peerstore.py | 81 ++++++++++++++----------------- 3 files changed, 126 insertions(+), 47 deletions(-) diff --git a/libp2p/peer/peerdata.py b/libp2p/peer/peerdata.py index 9273079d..f26ddd6b 100644 --- a/libp2p/peer/peerdata.py +++ b/libp2p/peer/peerdata.py @@ -2,11 +2,15 @@ from typing import Any, Dict, List, Sequence from multiaddr import Multiaddr +from libp2p.crypto.keys import PrivateKey, PublicKey + from .peerdata_interface import IPeerData class PeerData(IPeerData): + pubkey: PublicKey + privkey: PrivateKey metadata: Dict[Any, Any] protocols: List[str] addrs: List[Multiaddr] @@ -17,31 +21,86 @@ class PeerData(IPeerData): self.addrs = [] def get_protocols(self) -> List[str]: + """ + :return: all protocols associated with given peer + """ return self.protocols def add_protocols(self, protocols: Sequence[str]) -> None: + """ + :param protocols: protocols to add + """ self.protocols.extend(list(protocols)) def set_protocols(self, protocols: Sequence[str]) -> None: + """ + :param protocols: protocols to set + """ self.protocols = list(protocols) def add_addrs(self, addrs: Sequence[Multiaddr]) -> None: + """ + :param addrs: multiaddresses to add + """ self.addrs.extend(addrs) def get_addrs(self) -> List[Multiaddr]: + """ + :return: all multiaddresses + """ return self.addrs def clear_addrs(self) -> None: + """Clear all addresses.""" self.addrs = [] def put_metadata(self, key: str, val: Any) -> None: + """ + :param key: key in KV pair + :param val: val to associate with key + """ self.metadata[key] = val def get_metadata(self, key: str) -> Any: + """ + :param key: key in KV pair + :return: val for key + :raise PeerDataError: key not found + """ if key in self.metadata: return self.metadata[key] raise PeerDataError("key not found") + def add_pubkey(self, pubkey: PublicKey) -> None: + """ + :param pubkey: + """ + self.pubkey = pubkey + + def get_pubkey(self) -> PublicKey: + """ + :return: public key of the peer + :raise PeerDataError: if public key not found + """ + if self.pubkey is None: + raise PeerDataError("public key not found") + return self.pubkey + + def add_privkey(self, privkey: PrivateKey) -> None: + """ + :param privkey: + """ + self.privkey = privkey + + def get_privkey(self) -> PrivateKey: + """ + :return: private key of the peer + :raise PeerDataError: if private key not found + """ + if self.privkey is None: + raise PeerDataError("private key not found") + return self.privkey + class PeerDataError(KeyError): """Raised when a key is not found in peer metadata.""" diff --git a/libp2p/peer/peerdata_interface.py b/libp2p/peer/peerdata_interface.py index e842acb6..ab9016ed 100644 --- a/libp2p/peer/peerdata_interface.py +++ b/libp2p/peer/peerdata_interface.py @@ -3,6 +3,8 @@ from typing import Any, List, Sequence from multiaddr import Multiaddr +from libp2p.crypto.keys import PrivateKey, PublicKey + from .peermetadata_interface import IPeerMetadata @@ -22,7 +24,7 @@ class IPeerData(ABC): @abstractmethod def set_protocols(self, protocols: Sequence[str]) -> None: """ - :param protocols: protocols to add + :param protocols: protocols to set """ @abstractmethod @@ -46,7 +48,6 @@ class IPeerData(ABC): """ :param key: key in KV pair :param val: val to associate with key - :raise Exception: unsuccesful put """ @abstractmethod @@ -54,5 +55,31 @@ class IPeerData(ABC): """ :param key: key in KV pair :return: val for key - :raise Exception: key not found + :raise PeerDataError: key not found + """ + + @abstractmethod + def add_pubkey(self, pubkey: PublicKey) -> None: + """ + :param pubkey: + """ + + @abstractmethod + def get_pubkey(self) -> PublicKey: + """ + :return: public key of the peer + :raise PeerDataError: if public key not found + """ + + @abstractmethod + def add_privkey(self, privkey: PrivateKey) -> None: + """ + :param privkey: + """ + + @abstractmethod + def get_privkey(self) -> PrivateKey: + """ + :return: private key of the peer + :raise PeerDataError: if private key not found """ diff --git a/libp2p/peer/peerstore.py b/libp2p/peer/peerstore.py index c011e3ef..572a7afe 100644 --- a/libp2p/peer/peerstore.py +++ b/libp2p/peer/peerstore.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Sequence, Set +from typing import Any, Dict, List, Sequence from multiaddr import Multiaddr @@ -12,16 +12,12 @@ from .peerstore_interface import IPeerStore class PeerStore(IPeerStore): - peer_pubkey_map: Dict[ID, PublicKey] - peer_privkey_map: Dict[ID, PrivateKey] peer_data_map: Dict[ID, PeerData] def __init__(self) -> None: - self.peer_pubkey_map = {} - self.peer_privkey_map = {} self.peer_data_map = {} - def __create_or_get_peer(self, peer_id: ID) -> PeerData: + def __create_or_get_peer_data(self, peer_id: ID) -> PeerData: """ Returns the peer data for peer_id or creates a new peer data (and stores it in peer_data_map) if peer data for peer_id does not yet @@ -36,15 +32,15 @@ class PeerStore(IPeerStore): self.peer_data_map[peer_id] = data return data - def peer_info(self, peer_id: ID) -> Optional[PeerInfo]: + def peer_info(self, peer_id: ID) -> PeerInfo: """ :param peer_id: peer ID to get info for :return: peer info object """ if peer_id in self.peer_data_map: peer_data = self.peer_data_map[peer_id] - return PeerInfo(peer_id, peer_data.addrs) - return None + return PeerInfo(peer_id, peer_data.get_addrs()) + raise PeerStoreError("peer ID not found") def get_protocols(self, peer_id: ID) -> List[str]: """ @@ -61,16 +57,16 @@ class PeerStore(IPeerStore): :param peer_id: peer ID to add protocols for :param protocols: protocols to add """ - peer = self.__create_or_get_peer(peer_id) - peer.add_protocols(list(protocols)) + peer_data = self.__create_or_get_peer_data(peer_id) + peer_data.add_protocols(list(protocols)) def set_protocols(self, peer_id: ID, protocols: Sequence[str]) -> None: """ :param peer_id: peer ID to set protocols for :param protocols: protocols to set """ - peer = self.__create_or_get_peer(peer_id) - peer.set_protocols(list(protocols)) + peer_data = self.__create_or_get_peer_data(peer_id) + peer_data.set_protocols(list(protocols)) def peer_ids(self) -> List[ID]: """ @@ -101,8 +97,8 @@ class PeerStore(IPeerStore): """ # <> # This can output an error, not sure what the possible errors are - peer = self.__create_or_get_peer(peer_id) - peer.put_metadata(key, val) + peer_data = self.__create_or_get_peer_data(peer_id) + peer_data.put_metadata(key, val) def add_addr(self, peer_id: ID, addr: Multiaddr, ttl: int) -> None: """ @@ -119,8 +115,8 @@ class PeerStore(IPeerStore): :param ttl: time-to-live for the this record """ # Ignore ttl for now - peer = self.__create_or_get_peer(peer_id) - peer.add_addrs(list(addrs)) + peer_data = self.__create_or_get_peer_data(peer_id) + peer_data.add_addrs(list(addrs)) def addrs(self, peer_id: ID) -> List[Multiaddr]: """ @@ -156,60 +152,57 @@ class PeerStore(IPeerStore): """ :param peer_id: peer ID to add public key for :param pubkey: - :raise PeerStoreError: if peer ID already has pubkey set """ - if peer_id in self.peer_pubkey_map: - raise PeerStoreError( - f"peer ID already has pubkey: {self.peer_pubkey_map[peer_id]}" - ) - self.peer_pubkey_map[peer_id] = pubkey + peer_data = self.__create_or_get_peer_data(peer_id) + # TODO: Check if pubkey matches peer ID + peer_data.add_pubkey(pubkey) def pubkey(self, peer_id: ID) -> PublicKey: """ :param peer_id: peer ID to get public key for :return: public key of the peer - :raise PeerStoreError: if peer ID not found + :raise PeerStoreError: if peer ID or peer pubkey not found """ - if peer_id in self.peer_pubkey_map: - return self.peer_pubkey_map[peer_id] + if peer_id in self.peer_data_map: + peer_data = self.peer_data_map[peer_id] + try: + pubkey = peer_data.get_pubkey() + except PeerDataError: + raise PeerStoreError("peer pubkey not found") + return pubkey raise PeerStoreError("peer ID not found") def add_privkey(self, peer_id: ID, privkey: PrivateKey) -> None: """ :param peer_id: peer ID to add private key for :param privkey: - :raise PeerStoreError: if peer ID already has privkey set """ - if peer_id in self.peer_privkey_map: - raise PeerStoreError( - f"peer ID already has privkey: {self.peer_privkey_map[peer_id]}" - ) - self.peer_privkey_map[peer_id] = privkey + peer_data = self.__create_or_get_peer_data(peer_id) + # TODO: Check if privkey matches peer ID + peer_data.add_privkey(privkey) def privkey(self, peer_id: ID) -> PrivateKey: """ :param peer_id: peer ID to get private key for :return: private key of the peer - :raise PeerStoreError: if peer ID not found + :raise PeerStoreError: if peer ID or peer privkey not found """ - if peer_id in self.peer_pubkey_map: - return self.peer_privkey_map[peer_id] + if peer_id in self.peer_data_map: + peer_data = self.peer_data_map[peer_id] + try: + privkey = peer_data.get_privkey() + except PeerDataError: + raise PeerStoreError("peer privkey not found") + return privkey raise PeerStoreError("peer ID not found") def add_key_pair(self, peer_id: ID, key_pair: KeyPair) -> None: """ :param peer_id: peer ID to add private key for :param key_pair: - :raise PeerStoreError: if peer ID already has pubkey or privkey set """ - self.add_pubkey(peer_id, key_pair.public_keypubkey) - self.add_privkey(peer_id, key_pair.private_key) - - def peers_with_keys(self) -> Set[ID]: - """ - :return: all of the peer IDs which has pubkey/privkey stored in peer store - """ - return set(self.peer_pubkey_map.keys()).union(self.peer_privkey_map.keys()) + self.add_pubkey(key_pair.public_key) + self.add_privkey(key_pair.private_key) class PeerStoreError(KeyError): From ffa73f56493dea5fd4f8805ed3aaa9aba9527824 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 26 Nov 2019 12:33:55 +0800 Subject: [PATCH 086/178] Check if pubkey matches peer ID before add --- libp2p/peer/peerstore.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/libp2p/peer/peerstore.py b/libp2p/peer/peerstore.py index 572a7afe..8111f08d 100644 --- a/libp2p/peer/peerstore.py +++ b/libp2p/peer/peerstore.py @@ -95,8 +95,6 @@ class PeerStore(IPeerStore): :param key: :param value: """ - # <> - # This can output an error, not sure what the possible errors are peer_data = self.__create_or_get_peer_data(peer_id) peer_data.put_metadata(key, val) @@ -152,9 +150,11 @@ class PeerStore(IPeerStore): """ :param peer_id: peer ID to add public key for :param pubkey: + :raise PeerStoreError: if peer ID and pubkey does not match """ peer_data = self.__create_or_get_peer_data(peer_id) - # TODO: Check if pubkey matches peer ID + if ID.from_pubkey(pubkey) != peer_id: + raise PeerStoreError("peer ID and pubkey does not match") peer_data.add_pubkey(pubkey) def pubkey(self, peer_id: ID) -> PublicKey: @@ -176,9 +176,11 @@ class PeerStore(IPeerStore): """ :param peer_id: peer ID to add private key for :param privkey: + :raise PeerStoreError: if peer ID or peer privkey not found """ peer_data = self.__create_or_get_peer_data(peer_id) - # TODO: Check if privkey matches peer ID + if ID.from_pubkey(privkey.get_public_key()) != peer_id: + raise PeerStoreError("peer ID and privkey does not match") peer_data.add_privkey(privkey) def privkey(self, peer_id: ID) -> PrivateKey: From fab27b03579635faea047dd5606abbf7418f7aca Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 26 Nov 2019 12:35:50 +0800 Subject: [PATCH 087/178] Fix lint --- libp2p/peer/peerstore.py | 4 ++-- libp2p/peer/peerstore_interface.py | 8 +------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/libp2p/peer/peerstore.py b/libp2p/peer/peerstore.py index 8111f08d..03edffb4 100644 --- a/libp2p/peer/peerstore.py +++ b/libp2p/peer/peerstore.py @@ -203,8 +203,8 @@ class PeerStore(IPeerStore): :param peer_id: peer ID to add private key for :param key_pair: """ - self.add_pubkey(key_pair.public_key) - self.add_privkey(key_pair.private_key) + self.add_pubkey(peer_id, key_pair.public_key) + self.add_privkey(peer_id, key_pair.private_key) class PeerStoreError(KeyError): diff --git a/libp2p/peer/peerstore_interface.py b/libp2p/peer/peerstore_interface.py index 618bbf5e..a9790b07 100644 --- a/libp2p/peer/peerstore_interface.py +++ b/libp2p/peer/peerstore_interface.py @@ -1,5 +1,5 @@ from abc import abstractmethod -from typing import Any, List, Sequence, Set +from typing import Any, List, Sequence from multiaddr import Multiaddr @@ -138,9 +138,3 @@ class IPeerStore(IAddrBook, IPeerMetadata): :param key_pair: :raise PeerStoreError: if peer ID already has pubkey or privkey set """ - - @abstractmethod - def peers_with_keys(self) -> Set[ID]: - """ - :return: all of the peer IDs which has pubkey/privkey stored in peer store - """ From 94c7a0bca472b7c0756565ba755a68a4ba44240c Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 26 Nov 2019 14:04:28 +0800 Subject: [PATCH 088/178] Apply PR feedback and fix --- libp2p/peer/peerdata.py | 2 ++ tests/libp2p/test_libp2p.py | 7 ++++--- tests/peer/test_peerstore.py | 9 +++++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/libp2p/peer/peerdata.py b/libp2p/peer/peerdata.py index f26ddd6b..02df07ec 100644 --- a/libp2p/peer/peerdata.py +++ b/libp2p/peer/peerdata.py @@ -16,6 +16,8 @@ class PeerData(IPeerData): addrs: List[Multiaddr] def __init__(self) -> None: + self.pubkey = None + self.privkey = None self.metadata = {} self.protocols = [] self.addrs = [] diff --git a/tests/libp2p/test_libp2p.py b/tests/libp2p/test_libp2p.py index 330250c4..541c1733 100644 --- a/tests/libp2p/test_libp2p.py +++ b/tests/libp2p/test_libp2p.py @@ -328,18 +328,19 @@ async def test_host_connect(): transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list) - assert not node_a.get_peerstore().peer_ids() + # Only our peer ID is stored in peer store + assert len(node_a.get_peerstore().peer_ids()) == 1 addr = node_b.get_addrs()[0] info = info_from_p2p_addr(addr) await node_a.connect(info) - assert len(node_a.get_peerstore().peer_ids()) == 1 + assert len(node_a.get_peerstore().peer_ids()) == 2 await node_a.connect(info) # make sure we don't do double connection - assert len(node_a.get_peerstore().peer_ids()) == 1 + assert len(node_a.get_peerstore().peer_ids()) == 2 assert node_b.get_id() in node_a.get_peerstore().peer_ids() ma_node_b = multiaddr.Multiaddr("/p2p/%s" % node_b.get_id().pretty()) diff --git a/tests/peer/test_peerstore.py b/tests/peer/test_peerstore.py index ffb7b2dc..4a45651e 100644 --- a/tests/peer/test_peerstore.py +++ b/tests/peer/test_peerstore.py @@ -1,13 +1,14 @@ -from libp2p.peer.peerstore import PeerStore +import pytest + +from libp2p.peer.peerstore import PeerStore, PeerStoreError # Testing methods from IPeerStore base class. def test_peer_info_empty(): store = PeerStore() - info = store.peer_info("peer") - - assert not info + with pytest.raises(PeerStoreError): + store.peer_info("peer") def test_peer_info_basic(): From 9f3f2c53da9ebd96114370e4fa2d97456b71676f Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 26 Nov 2019 16:18:15 +0800 Subject: [PATCH 089/178] Apply PR feedback: use defaultdict for peer data map --- libp2p/peer/peerstore.py | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/libp2p/peer/peerstore.py b/libp2p/peer/peerstore.py index 03edffb4..f9a9fbdb 100644 --- a/libp2p/peer/peerstore.py +++ b/libp2p/peer/peerstore.py @@ -1,3 +1,4 @@ +from collections import defaultdict from typing import Any, Dict, List, Sequence from multiaddr import Multiaddr @@ -15,22 +16,7 @@ class PeerStore(IPeerStore): peer_data_map: Dict[ID, PeerData] def __init__(self) -> None: - self.peer_data_map = {} - - def __create_or_get_peer_data(self, peer_id: ID) -> PeerData: - """ - Returns the peer data for peer_id or creates a new peer data (and - stores it in peer_data_map) if peer data for peer_id does not yet - exist. - - :param peer_id: peer ID - :return: peer data - """ - if peer_id in self.peer_data_map: - return self.peer_data_map[peer_id] - data = PeerData() - self.peer_data_map[peer_id] = data - return data + self.peer_data_map = defaultdict(PeerData) def peer_info(self, peer_id: ID) -> PeerInfo: """ @@ -57,7 +43,7 @@ class PeerStore(IPeerStore): :param peer_id: peer ID to add protocols for :param protocols: protocols to add """ - peer_data = self.__create_or_get_peer_data(peer_id) + peer_data = self.peer_data_map[peer_id] peer_data.add_protocols(list(protocols)) def set_protocols(self, peer_id: ID, protocols: Sequence[str]) -> None: @@ -65,7 +51,7 @@ class PeerStore(IPeerStore): :param peer_id: peer ID to set protocols for :param protocols: protocols to set """ - peer_data = self.__create_or_get_peer_data(peer_id) + peer_data = self.peer_data_map[peer_id] peer_data.set_protocols(list(protocols)) def peer_ids(self) -> List[ID]: @@ -95,7 +81,7 @@ class PeerStore(IPeerStore): :param key: :param value: """ - peer_data = self.__create_or_get_peer_data(peer_id) + peer_data = self.peer_data_map[peer_id] peer_data.put_metadata(key, val) def add_addr(self, peer_id: ID, addr: Multiaddr, ttl: int) -> None: @@ -113,7 +99,7 @@ class PeerStore(IPeerStore): :param ttl: time-to-live for the this record """ # Ignore ttl for now - peer_data = self.__create_or_get_peer_data(peer_id) + peer_data = self.peer_data_map[peer_id] peer_data.add_addrs(list(addrs)) def addrs(self, peer_id: ID) -> List[Multiaddr]: @@ -152,7 +138,7 @@ class PeerStore(IPeerStore): :param pubkey: :raise PeerStoreError: if peer ID and pubkey does not match """ - peer_data = self.__create_or_get_peer_data(peer_id) + peer_data = self.peer_data_map[peer_id] if ID.from_pubkey(pubkey) != peer_id: raise PeerStoreError("peer ID and pubkey does not match") peer_data.add_pubkey(pubkey) @@ -178,7 +164,7 @@ class PeerStore(IPeerStore): :param privkey: :raise PeerStoreError: if peer ID or peer privkey not found """ - peer_data = self.__create_or_get_peer_data(peer_id) + peer_data = self.peer_data_map[peer_id] if ID.from_pubkey(privkey.get_public_key()) != peer_id: raise PeerStoreError("peer ID and privkey does not match") peer_data.add_privkey(privkey) From d1b5a56ccfab2e785bdf0a3b0f009e54957a23f4 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 26 Nov 2019 12:55:59 -0800 Subject: [PATCH 090/178] Match linting rules after merging in template --- mypy.ini | 4 ++-- setup.py | 17 +++++++---------- tests/core/test_import.py | 2 -- tox.ini | 4 ++-- 4 files changed, 11 insertions(+), 16 deletions(-) diff --git a/mypy.ini b/mypy.ini index 1dd91ee7..fffd2aa8 100644 --- a/mypy.ini +++ b/mypy.ini @@ -6,13 +6,13 @@ disallow_untyped_defs = True disallow_any_generics = True disallow_untyped_calls = True disallow_untyped_decorators = True -disallow_subclassing_any = True +disallow_subclassing_any = False ignore_missing_imports = True strict_optional = False warn_unused_ignores = True strict_equality = True warn_redundant_casts = True -warn_return_any = True +warn_return_any = False warn_unused_configs = True warn_unreachable = True diff --git a/setup.py b/setup.py index f823a15c..c0b6f353 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from setuptools import ( - setup, - find_packages, -) +from setuptools import find_packages, setup extras_require = { "test": [ @@ -21,7 +18,7 @@ extras_require = { "flake8-bugbear>=19.8.0,<20", "docformatter>=1.3.1,<2", ], - 'doc': [ + "doc": [ "Sphinx>=1.6.5,<2", "sphinx_rtd_theme>=0.4.3,<=1", "towncrier>=19.2.0, <20", @@ -38,10 +35,10 @@ extras_require = { } extras_require["dev"] = ( - extras_require['dev'] + # noqa: W504 - extras_require['test'] + # noqa: W504 - extras_require['lint'] + # noqa: W504 - extras_require['doc'] + extras_require["dev"] + + extras_require["test"] + + extras_require["lint"] + + extras_require["doc"] ) @@ -74,7 +71,7 @@ setup( ], python_requires=">=3.7,<4", extras_require=extras_require, - py_modules=[''], + py_modules=[""], license="MIT/APACHE2.0", zip_safe=False, keywords="libp2p p2p", diff --git a/tests/core/test_import.py b/tests/core/test_import.py index f146ae79..07b2a18e 100644 --- a/tests/core/test_import.py +++ b/tests/core/test_import.py @@ -1,4 +1,2 @@ - - def test_import(): import # noqa: F401 diff --git a/tox.ini b/tox.ini index 52d621cf..ce0fbe01 100644 --- a/tox.ini +++ b/tox.ini @@ -9,10 +9,10 @@ envlist = docs [isort] -combine_as_imports=True +combine_as_imports=False force_sort_within_sections=True include_trailing_comma=True -known_third_party=hypothesis,pytest,p2pclient,pexpect +known_third_party=hypothesis,pytest,p2pclient,pexpect,factory known_first_party= line_length=88 multi_line_output=3 From d589daf00d1931018f74b4842a5eecbcce33230d Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 26 Nov 2019 12:44:17 -0800 Subject: [PATCH 091/178] Fill in template variables --- .github/PULL_REQUEST_TEMPLATE.md | 6 +++--- .project-template/template_vars.txt | 12 ++++++------ README.md | 18 +++++++++--------- docs/conf.py | 14 +++++++------- docs/index.rst | 6 +++--- pyproject.toml | 8 ++++---- requirements-docs.txt | 2 +- setup.py | 4 ++-- tests/core/test_import.py | 2 +- tox.ini | 2 +- 10 files changed, 37 insertions(+), 37 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 21d4db59..4ef27e52 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -11,10 +11,10 @@ Summary of approach. [//]: # (Stay ahead of things, add list items here!) - [ ] Clean up commit history -[//]: # (For important changes that should go into the release notes please add a newsfragment file as explained here: https://github.com/ethereum//blob/master/newsfragments/README.md) +[//]: # (For important changes that should go into the release notes please add a newsfragment file as explained here: https://github.com/libp2p/py-libp2p/blob/master/newsfragments/README.md) -[//]: # (See: https://.readthedocs.io/en/latest/contributing.html#pull-requests) -- [ ] Add entry to the [release notes](https://github.com/ethereum//blob/master/newsfragments/README.md) +[//]: # (See: https://py-libp2p.readthedocs.io/en/latest/contributing.html#pull-requests) +- [ ] Add entry to the [release notes](https://github.com/libp2p/py-libp2p/blob/master/newsfragments/README.md) #### Cute Animal Picture diff --git a/.project-template/template_vars.txt b/.project-template/template_vars.txt index 5beeccbb..ce0a492e 100644 --- a/.project-template/template_vars.txt +++ b/.project-template/template_vars.txt @@ -1,6 +1,6 @@ - - - - - - +libp2p +libp2p +py-libp2p +py-libp2p +py-libp2p +The Python implementation of the libp2p networking stack diff --git a/README.md b/README.md index f1943603..a375890e 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,9 @@ [![Join the chat at https://gitter.im/py-libp2p/Lobby](https://badges.gitter.im/py-libp2p/Lobby.png)](https://gitter.im/py-libp2p/Lobby) [![Build Status](https://travis-ci.com/libp2p/py-libp2p.svg?branch=master)](https://travis-ci.com/libp2p/py-libp2p) -[![PyPI version](https://badge.fury.io/py/.svg)](https://badge.fury.io/py/) -[![Python versions](https://img.shields.io/pypi/pyversions/.svg)](https://pypi.python.org/pypi/) -[![Docs build](https://readthedocs.org/projects//badge/?version=latest)](http://.readthedocs.io/en/latest/?badge=latest) +[![PyPI version](https://badge.fury.io/py/libp2p.svg)](https://badge.fury.io/py/libp2p) +[![Python versions](https://img.shields.io/pypi/pyversions/libp2p.svg)](https://pypi.python.org/pypi/libp2p) +[![Docs build](https://readthedocs.org/projects/py-libp2p/badge/?version=latest)](http://py-libp2p.readthedocs.io/en/latest/?badge=latest) [![Freenode](https://img.shields.io/badge/freenode-%23libp2p-yellow.svg)](https://webchat.freenode.net/?channels=%23libp2p) [![Matrix](https://img.shields.io/badge/matrix-%23libp2p%3Apermaweb.io-blue.svg)](https://riot.permaweb.io/#/room/#libp2p:permaweb.io) [![Discord](https://img.shields.io/discord/475789330380488707?color=blueviolet&label=discord)](https://discord.gg/66KBrm2) @@ -18,9 +18,9 @@ ## WARNING py-libp2p is an experimental and work-in-progress repo under heavy development. We do not yet recommend using py-libp2p in production environments. - +The Python implementation of the libp2p networking stack -Read more in the [documentation on ReadTheDocs](https://.readthedocs.io/). [View the change log](https://.readthedocs.io/en/latest/releases.html). +Read more in the [documentation on ReadTheDocs](https://py-libp2p.readthedocs.io/). [View the change log](https://py-libp2p.readthedocs.io/en/latest/releases.html). ## Sponsorship This project is graciously sponsored by the Ethereum Foundation through [Wave 5 of their Grants Program](https://blog.ethereum.org/2019/02/21/ethereum-foundation-grants-program-wave-5/). @@ -35,8 +35,8 @@ The py-libp2p team consists of: py-libp2p requires Python 3.7 and the best way to guarantee a clean Python 3.7 environment is with [`virtualenv`](https://virtualenv.pypa.io/en/stable/) ```sh -git clone git@github.com:ethereum/.git -cd +git clone git@github.com:libp2p/py-libp2p.git +cd py-libp2p virtualenv -p python3.7 venv . venv/bin/activate pip install -e .[dev] @@ -50,7 +50,7 @@ Show flake8 errors on file change: ```sh # Test flake8 -when-changed -v -s -r -1 / tests/ -c "clear; flake8 tests && echo 'flake8 success' || echo 'error'" +when-changed -v -s -r -1 libp2p/ tests/ -c "clear; flake8 libp2p tests && echo 'flake8 success' || echo 'error'" ``` Run multi-process tests in one command, but without color: @@ -66,7 +66,7 @@ Run in one thread, with color and desktop notifications: ```sh cd venv -ptw --onfail "notify-send -t 5000 'Test failure ⚠⚠⚠⚠⚠' 'python 3 test on failed'" ../tests ../ +ptw --onfail "notify-send -t 5000 'Test failure ⚠⚠⚠⚠⚠' 'python 3 test on py-libp2p failed'" ../tests ../libp2p ``` Note that tests/libp2p/test_libp2p.py contains an end-to-end messaging test between two libp2p hosts, which is the bulk of our proof of concept. diff --git a/docs/conf.py b/docs/conf.py index aaf6fb6e..54b210a3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# documentation build configuration file, created by +# py-libp2p documentation build configuration file, created by # sphinx-quickstart on Thu Oct 16 20:43:24 2014. # # This file is execfile()d with the current directory set to its @@ -53,7 +53,7 @@ source_suffix = '.rst' master_doc = 'index' # General information about the project. -project = '' +project = 'py-libp2p' copyright = '2019, The Ethereum Foundation' __version__ = setup_version @@ -189,7 +189,7 @@ html_static_path = ['_static'] #html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'doc' +htmlhelp_basename = 'libp2pdoc' # -- Options for LaTeX output --------------------------------------------- @@ -209,7 +209,7 @@ latex_elements = { # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', '.tex', ' Documentation', + ('index', 'libp2p.tex', 'py-libp2p Documentation', 'The Ethereum Foundation', 'manual'), ] @@ -239,7 +239,7 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', '', ' Documentation', + ('index', 'libp2p', 'py-libp2p Documentation', ['The Ethereum Foundation'], 1) ] @@ -253,8 +253,8 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', '', ' Documentation', - 'The Ethereum Foundation', '', '', + ('index', 'py-libp2p', 'py-libp2p Documentation', + 'The Ethereum Foundation', 'py-libp2p', 'The Python implementation of the libp2p networking stack', 'Miscellaneous'), ] diff --git a/docs/index.rst b/docs/index.rst index 89815efa..8fa151f1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,7 +1,7 @@ - +py-libp2p ============================== - +The Python implementation of the libp2p networking stack Contents -------- @@ -9,7 +9,7 @@ Contents .. toctree:: :maxdepth: 3 - + libp2p release_notes diff --git a/pyproject.toml b/pyproject.toml index aaf9dd96..ba94a6cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,10 @@ [tool.towncrier] -# Read https://github.com/ethereum//newsfragments/README.md for instructions -package = "" +# Read https://github.com/libp2p/py-libp2p/newsfragments/README.md for instructions +package = "libp2p" filename = "docs/release_notes.rst" directory = "newsfragments" underlines = ["-", "~", "^"] -issue_format = "`#{issue} /issues/{issue}>`__" +issue_format = "`#{issue} `__" [[tool.towncrier.type]] directory = "feature" @@ -33,7 +33,7 @@ showcontent = true [[tool.towncrier.type]] directory = "internal" -name = "Internal Changes - for Contributors" +name = "Internal Changes - for py-libp2p Contributors" showcontent = true [[tool.towncrier.type]] diff --git a/requirements-docs.txt b/requirements-docs.txt index 1b49b744..ba1ee49e 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1 +1 @@ -[doc] +libp2p[doc] diff --git a/setup.py b/setup.py index c0b6f353..37822312 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ setup( long_description_content_type="text/markdown", maintainer="The Ethereum Foundation", maintainer_email="snakecharmers@ethereum.org", - url="https://github.com/ethereum/py-libp2p", + url="https://github.com/libp2p/py-libp2p", include_package_data=True, install_requires=[ "pycryptodome>=3.9.2,<4.0.0", @@ -71,7 +71,7 @@ setup( ], python_requires=">=3.7,<4", extras_require=extras_require, - py_modules=[""], + py_modules=["libp2p"], license="MIT/APACHE2.0", zip_safe=False, keywords="libp2p p2p", diff --git a/tests/core/test_import.py b/tests/core/test_import.py index 07b2a18e..325444f9 100644 --- a/tests/core/test_import.py +++ b/tests/core/test_import.py @@ -1,2 +1,2 @@ def test_import(): - import # noqa: F401 + import libp2p # noqa: F401 diff --git a/tox.ini b/tox.ini index ce0fbe01..de4f8f1b 100644 --- a/tox.ini +++ b/tox.ini @@ -13,7 +13,7 @@ combine_as_imports=False force_sort_within_sections=True include_trailing_comma=True known_third_party=hypothesis,pytest,p2pclient,pexpect,factory -known_first_party= +known_first_party=libp2p line_length=88 multi_line_output=3 use_parentheses=True From f84edfc13377a3d9840e9fa9e333c0803dd32c39 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 26 Nov 2019 12:57:02 -0800 Subject: [PATCH 092/178] Add missing package test script used by Makefile --- scripts/release/test_package.py | 52 +++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 scripts/release/test_package.py diff --git a/scripts/release/test_package.py b/scripts/release/test_package.py new file mode 100644 index 00000000..89ef36a4 --- /dev/null +++ b/scripts/release/test_package.py @@ -0,0 +1,52 @@ +from pathlib import Path +import subprocess +from tempfile import TemporaryDirectory +import venv + + +def create_venv(parent_path): + venv_path = parent_path / 'package-smoke-test' + venv.create(venv_path, with_pip=True) + subprocess.run([venv_path / 'bin' / 'pip', 'install', '-U', 'pip', 'setuptools'], check=True) + return venv_path + + +def find_wheel(project_path): + wheels = list(project_path.glob('dist/*.whl')) + + if len(wheels) != 1: + raise Exception( + f"Expected one wheel. Instead found: {wheels} in project {project_path.absolute()}" + ) + + return wheels[0] + + +def install_wheel(venv_path, wheel_path, extras=()): + if extras: + extra_suffix = f"[{','.join(extras)}]" + else: + extra_suffix = "" + + subprocess.run( + [ + venv_path / 'bin' / 'pip', + 'install', + f"{wheel_path}{extra_suffix}" + ], + check=True, + ) + + +def test_install_local_wheel(): + with TemporaryDirectory() as tmpdir: + venv_path = create_venv(Path(tmpdir)) + wheel_path = find_wheel(Path('.')) + install_wheel(venv_path, wheel_path) + print("Installed", wheel_path.absolute(), "to", venv_path) + print(f"Activate with `source {venv_path}/bin/activate`") + input("Press enter when the test has completed. The directory will be deleted.") + + +if __name__ == '__main__': + test_install_local_wheel() From 1a283d0b1a40079d66d4974faf54048ccf7c3270 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 26 Nov 2019 15:23:10 -0800 Subject: [PATCH 093/178] Fix merge bug in the readme --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index a375890e..9b799548 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ -<<<<<<< HEAD # py-libp2p [![Join the chat at https://gitter.im/py-libp2p/Lobby](https://badges.gitter.im/py-libp2p/Lobby.png)](https://gitter.im/py-libp2p/Lobby) From 5d3be4f890f7092fe594801f3884a922348869f0 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 26 Nov 2019 15:38:32 -0800 Subject: [PATCH 094/178] Fix comment about mypy being not semver --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 37822312..4ef43fb4 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ extras_require = { "lint": [ "flake8==3.7.9", # flake8 is not semver: it has added new warnings at minor releases "isort==4.3.21", - "mypy==0.740", # flake8 is not semver: it has added new warnings at minor releases + "mypy==0.740", # mypy is not semver: it has added new warnings at minor releases "mypy-protobuf==1.15", "black==19.3b0", "flake8-bugbear>=19.8.0,<20", From 69742da4a3e690e9d43e5679293c3c2425d05740 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 26 Nov 2019 16:50:33 -0800 Subject: [PATCH 095/178] Add docs test to Travis CI --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 824fa247..ce9b061e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,9 @@ matrix: - python: 3.7-dev dist: xenial env: TOXENV=lint + - python: 3.7-dev + dist: xenial + env: TOXENV=docs - python: 3.7-dev dist: xenial env: TOXENV=py37-interop From b672da82ec70e5bf2ed164ca2d1dc1f3a03364a9 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 26 Nov 2019 17:19:19 -0800 Subject: [PATCH 096/178] Fix version parse failure because of double-quote --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 54b210a3..9038ad0a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -23,7 +23,7 @@ DIR = os.path.dirname('__file__') with open (os.path.join(DIR, '../setup.py'), 'r') as f: for line in f: if 'version=' in line: - setup_version = line.split('\'')[1] + setup_version = line.split('"')[1] break # -- General configuration ------------------------------------------------ From 1564ad659b1b8f7d0572fd9971343744f853e53b Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 26 Nov 2019 17:20:01 -0800 Subject: [PATCH 097/178] Convert doc warnings into errors for CI --- docs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Makefile b/docs/Makefile index 32800935..3ecaf132 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -2,7 +2,7 @@ # # You can set these variables from the command line. -SPHINXOPTS = +SPHINXOPTS = -W SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build From 6668e8d33968283cd945079f9381bd76d84f937c Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 26 Nov 2019 17:29:13 -0800 Subject: [PATCH 098/178] sphinx to v2, because starting from blank slate --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4ef43fb4..252252fe 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ extras_require = { "docformatter>=1.3.1,<2", ], "doc": [ - "Sphinx>=1.6.5,<2", + "Sphinx>=2.2.1,<3", "sphinx_rtd_theme>=0.4.3,<=1", "towncrier>=19.2.0, <20", ], From d503950179c810a881a31f3b273021de137d83d8 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 26 Nov 2019 17:33:50 -0800 Subject: [PATCH 099/178] Fix all doc build warnings --- Makefile | 2 +- docs/index.rst | 1 + libp2p/__init__.py | 2 +- libp2p/kademlia/network.py | 7 ++- libp2p/network/swarm.py | 16 +++--- .../floodsub_integration_test_settings.py | 49 ++++++++++--------- 6 files changed, 41 insertions(+), 36 deletions(-) diff --git a/Makefile b/Makefile index aec24219..b82dbca0 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,7 @@ test-all: tox build-docs: - sphinx-apidoc -o docs/ . setup.py "*conftest*" + sphinx-apidoc -o docs/ . setup.py "*conftest*" "libp2p/tools/interop*" $(MAKE) -C docs clean $(MAKE) -C docs html $(MAKE) -C docs doctest diff --git a/docs/index.rst b/docs/index.rst index 8fa151f1..3757abbe 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,6 +11,7 @@ Contents libp2p release_notes + examples Indices and tables diff --git a/libp2p/__init__.py b/libp2p/__init__.py index a1dca535..34a67f13 100644 --- a/libp2p/__init__.py +++ b/libp2p/__init__.py @@ -55,7 +55,7 @@ def initialize_default_kademlia_router( :param alpha: The alpha parameter from the paper :param id_opt: optional id for host :param storage: An instance that implements - :interface:`~kademlia.storage.IStorage` + :class:`~kademlia.storage.IStorage` :return: return a default kademlia instance """ if not id_opt: diff --git a/libp2p/kademlia/network.py b/libp2p/kademlia/network.py index f93ca09e..1077bb5a 100644 --- a/libp2p/kademlia/network.py +++ b/libp2p/kademlia/network.py @@ -232,10 +232,9 @@ class KademliaServer: """ Save the state of node with a given regularity to the given filename. - Args: - fname: File name to save retularly to - frequency: Frequency in seconds that the state should be saved. - By default, 10 minutes. + :param fname: File name to save regularly to + :param frequency: Frequency in seconds that the state should be saved. + By default, 10 minutes. """ self.save_state(fname) loop = asyncio.get_event_loop() diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index de80dd82..0c405106 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -155,13 +155,15 @@ class Swarm(INetwork): :return: true if at least one success For each multiaddr - Check if a listener for multiaddr exists already - If listener already exists, continue - Otherwise: - Capture multiaddr in conn handler - Have conn handler delegate to stream handler - Call listener listen with the multiaddr - Map multiaddr to listener + + - Check if a listener for multiaddr exists already + - If listener already exists, continue + - Otherwise: + + - Capture multiaddr in conn handler + - Have conn handler delegate to stream handler + - Call listener listen with the multiaddr + - Map multiaddr to listener """ for maddr in multiaddrs: if str(maddr) in self.listeners: diff --git a/libp2p/tools/pubsub/floodsub_integration_test_settings.py b/libp2p/tools/pubsub/floodsub_integration_test_settings.py index 90939dec..b7b66244 100644 --- a/libp2p/tools/pubsub/floodsub_integration_test_settings.py +++ b/libp2p/tools/pubsub/floodsub_integration_test_settings.py @@ -145,31 +145,34 @@ floodsub_protocol_pytest_params = [ async def perform_test_from_obj(obj, router_factory) -> None: """ - Perform pubsub tests from a test obj. - test obj are composed as follows: + Perform pubsub tests from a test object, which is composed as follows: - { - "supported_protocols": ["supported/protocol/1.0.0",...], - "adj_list": { - "node1": ["neighbor1_of_node1", "neighbor2_of_node1", ...], - "node2": ["neighbor1_of_node2", "neighbor2_of_node2", ...], - ... - }, - "topic_map": { - "topic1": ["node1_subscribed_to_topic1", "node2_subscribed_to_topic1", ...] - }, - "messages": [ - { - "topics": ["topic1_for_message", "topic2_for_message", ...], - "data": b"some contents of the message (newlines are not supported)", - "node_id": "message sender node id" + .. code-block:: python + + { + "supported_protocols": ["supported/protocol/1.0.0",...], + "adj_list": { + "node1": ["neighbor1_of_node1", "neighbor2_of_node1", ...], + "node2": ["neighbor1_of_node2", "neighbor2_of_node2", ...], + ... }, - ... - ] - } - NOTE: In adj_list, for any neighbors A and B, only list B as a neighbor of A - or B as a neighbor of A once. Do NOT list both A: ["B"] and B:["A"] as the behavior - is undefined (even if it may work) + "topic_map": { + "topic1": ["node1_subscribed_to_topic1", "node2_subscribed_to_topic1", ...] + }, + "messages": [ + { + "topics": ["topic1_for_message", "topic2_for_message", ...], + "data": b"some contents of the message (newlines are not supported)", + "node_id": "message sender node id" + }, + ... + ] + } + + .. note:: + In adj_list, for any neighbors A and B, only list B as a neighbor of A + or B as a neighbor of A once. Do NOT list both A: ["B"] and B:["A"] as the behavior + is undefined (even if it may work) """ # Step 1) Create graph From abf0da925c6d28592237eee247d2b7b42bfb9780 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 26 Nov 2019 17:34:36 -0800 Subject: [PATCH 100/178] Built docs for modules --- docs/examples.chat.rst | 22 +++++++ docs/examples.rst | 17 +++++ docs/libp2p.crypto.pb.rst | 22 +++++++ docs/libp2p.crypto.rst | 93 ++++++++++++++++++++++++++++ docs/libp2p.host.rst | 62 +++++++++++++++++++ docs/libp2p.identity.identify.pb.rst | 22 +++++++ docs/libp2p.identity.identify.rst | 29 +++++++++ docs/libp2p.identity.rst | 17 +++++ docs/libp2p.io.rst | 46 ++++++++++++++ docs/libp2p.kademlia.rst | 70 +++++++++++++++++++++ docs/libp2p.network.connection.rst | 54 ++++++++++++++++ docs/libp2p.network.rst | 54 ++++++++++++++++ docs/libp2p.network.stream.rst | 38 ++++++++++++ docs/libp2p.peer.rst | 78 +++++++++++++++++++++++ docs/libp2p.protocol_muxer.rst | 70 +++++++++++++++++++++ docs/libp2p.pubsub.pb.rst | 22 +++++++ docs/libp2p.pubsub.rst | 77 +++++++++++++++++++++++ docs/libp2p.routing.kademlia.rst | 30 +++++++++ docs/libp2p.routing.rst | 29 +++++++++ docs/libp2p.rst | 58 +++++++++++++++++ docs/libp2p.security.insecure.pb.rst | 22 +++++++ docs/libp2p.security.insecure.rst | 29 +++++++++ docs/libp2p.security.rst | 70 +++++++++++++++++++++ docs/libp2p.security.secio.pb.rst | 22 +++++++ docs/libp2p.security.secio.rst | 37 +++++++++++ docs/libp2p.stream_muxer.mplex.rst | 54 ++++++++++++++++ docs/libp2p.stream_muxer.rst | 45 ++++++++++++++ docs/libp2p.tools.pubsub.rst | 38 ++++++++++++ docs/libp2p.tools.rst | 47 ++++++++++++++ docs/libp2p.transport.rst | 61 ++++++++++++++++++ docs/libp2p.transport.tcp.rst | 22 +++++++ 31 files changed, 1357 insertions(+) create mode 100644 docs/examples.chat.rst create mode 100644 docs/examples.rst create mode 100644 docs/libp2p.crypto.pb.rst create mode 100644 docs/libp2p.crypto.rst create mode 100644 docs/libp2p.host.rst create mode 100644 docs/libp2p.identity.identify.pb.rst create mode 100644 docs/libp2p.identity.identify.rst create mode 100644 docs/libp2p.identity.rst create mode 100644 docs/libp2p.io.rst create mode 100644 docs/libp2p.kademlia.rst create mode 100644 docs/libp2p.network.connection.rst create mode 100644 docs/libp2p.network.rst create mode 100644 docs/libp2p.network.stream.rst create mode 100644 docs/libp2p.peer.rst create mode 100644 docs/libp2p.protocol_muxer.rst create mode 100644 docs/libp2p.pubsub.pb.rst create mode 100644 docs/libp2p.pubsub.rst create mode 100644 docs/libp2p.routing.kademlia.rst create mode 100644 docs/libp2p.routing.rst create mode 100644 docs/libp2p.rst create mode 100644 docs/libp2p.security.insecure.pb.rst create mode 100644 docs/libp2p.security.insecure.rst create mode 100644 docs/libp2p.security.rst create mode 100644 docs/libp2p.security.secio.pb.rst create mode 100644 docs/libp2p.security.secio.rst create mode 100644 docs/libp2p.stream_muxer.mplex.rst create mode 100644 docs/libp2p.stream_muxer.rst create mode 100644 docs/libp2p.tools.pubsub.rst create mode 100644 docs/libp2p.tools.rst create mode 100644 docs/libp2p.transport.rst create mode 100644 docs/libp2p.transport.tcp.rst diff --git a/docs/examples.chat.rst b/docs/examples.chat.rst new file mode 100644 index 00000000..9242803b --- /dev/null +++ b/docs/examples.chat.rst @@ -0,0 +1,22 @@ +examples.chat package +===================== + +Submodules +---------- + +examples.chat.chat module +------------------------- + +.. automodule:: examples.chat.chat + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: examples.chat + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/examples.rst b/docs/examples.rst new file mode 100644 index 00000000..4dd05984 --- /dev/null +++ b/docs/examples.rst @@ -0,0 +1,17 @@ +examples package +================ + +Subpackages +----------- + +.. toctree:: + + examples.chat + +Module contents +--------------- + +.. automodule:: examples + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.crypto.pb.rst b/docs/libp2p.crypto.pb.rst new file mode 100644 index 00000000..2641266b --- /dev/null +++ b/docs/libp2p.crypto.pb.rst @@ -0,0 +1,22 @@ +libp2p.crypto.pb package +======================== + +Submodules +---------- + +libp2p.crypto.pb.crypto\_pb2 module +----------------------------------- + +.. automodule:: libp2p.crypto.pb.crypto_pb2 + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.crypto.pb + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.crypto.rst b/docs/libp2p.crypto.rst new file mode 100644 index 00000000..8acc707a --- /dev/null +++ b/docs/libp2p.crypto.rst @@ -0,0 +1,93 @@ +libp2p.crypto package +===================== + +Subpackages +----------- + +.. toctree:: + + libp2p.crypto.pb + +Submodules +---------- + +libp2p.crypto.authenticated\_encryption module +---------------------------------------------- + +.. automodule:: libp2p.crypto.authenticated_encryption + :members: + :undoc-members: + :show-inheritance: + +libp2p.crypto.ecc module +------------------------ + +.. automodule:: libp2p.crypto.ecc + :members: + :undoc-members: + :show-inheritance: + +libp2p.crypto.ed25519 module +---------------------------- + +.. automodule:: libp2p.crypto.ed25519 + :members: + :undoc-members: + :show-inheritance: + +libp2p.crypto.exceptions module +------------------------------- + +.. automodule:: libp2p.crypto.exceptions + :members: + :undoc-members: + :show-inheritance: + +libp2p.crypto.key\_exchange module +---------------------------------- + +.. automodule:: libp2p.crypto.key_exchange + :members: + :undoc-members: + :show-inheritance: + +libp2p.crypto.keys module +------------------------- + +.. automodule:: libp2p.crypto.keys + :members: + :undoc-members: + :show-inheritance: + +libp2p.crypto.rsa module +------------------------ + +.. automodule:: libp2p.crypto.rsa + :members: + :undoc-members: + :show-inheritance: + +libp2p.crypto.secp256k1 module +------------------------------ + +.. automodule:: libp2p.crypto.secp256k1 + :members: + :undoc-members: + :show-inheritance: + +libp2p.crypto.serialization module +---------------------------------- + +.. automodule:: libp2p.crypto.serialization + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.crypto + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.host.rst b/docs/libp2p.host.rst new file mode 100644 index 00000000..f2e4b075 --- /dev/null +++ b/docs/libp2p.host.rst @@ -0,0 +1,62 @@ +libp2p.host package +=================== + +Submodules +---------- + +libp2p.host.basic\_host module +------------------------------ + +.. automodule:: libp2p.host.basic_host + :members: + :undoc-members: + :show-inheritance: + +libp2p.host.defaults module +--------------------------- + +.. automodule:: libp2p.host.defaults + :members: + :undoc-members: + :show-inheritance: + +libp2p.host.exceptions module +----------------------------- + +.. automodule:: libp2p.host.exceptions + :members: + :undoc-members: + :show-inheritance: + +libp2p.host.host\_interface module +---------------------------------- + +.. automodule:: libp2p.host.host_interface + :members: + :undoc-members: + :show-inheritance: + +libp2p.host.ping module +----------------------- + +.. automodule:: libp2p.host.ping + :members: + :undoc-members: + :show-inheritance: + +libp2p.host.routed\_host module +------------------------------- + +.. automodule:: libp2p.host.routed_host + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.host + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.identity.identify.pb.rst b/docs/libp2p.identity.identify.pb.rst new file mode 100644 index 00000000..a707a4a5 --- /dev/null +++ b/docs/libp2p.identity.identify.pb.rst @@ -0,0 +1,22 @@ +libp2p.identity.identify.pb package +=================================== + +Submodules +---------- + +libp2p.identity.identify.pb.identify\_pb2 module +------------------------------------------------ + +.. automodule:: libp2p.identity.identify.pb.identify_pb2 + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.identity.identify.pb + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.identity.identify.rst b/docs/libp2p.identity.identify.rst new file mode 100644 index 00000000..cd9f94d0 --- /dev/null +++ b/docs/libp2p.identity.identify.rst @@ -0,0 +1,29 @@ +libp2p.identity.identify package +================================ + +Subpackages +----------- + +.. toctree:: + + libp2p.identity.identify.pb + +Submodules +---------- + +libp2p.identity.identify.protocol module +---------------------------------------- + +.. automodule:: libp2p.identity.identify.protocol + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.identity.identify + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.identity.rst b/docs/libp2p.identity.rst new file mode 100644 index 00000000..4535ce6a --- /dev/null +++ b/docs/libp2p.identity.rst @@ -0,0 +1,17 @@ +libp2p.identity package +======================= + +Subpackages +----------- + +.. toctree:: + + libp2p.identity.identify + +Module contents +--------------- + +.. automodule:: libp2p.identity + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.io.rst b/docs/libp2p.io.rst new file mode 100644 index 00000000..87c17bde --- /dev/null +++ b/docs/libp2p.io.rst @@ -0,0 +1,46 @@ +libp2p.io package +================= + +Submodules +---------- + +libp2p.io.abc module +-------------------- + +.. automodule:: libp2p.io.abc + :members: + :undoc-members: + :show-inheritance: + +libp2p.io.exceptions module +--------------------------- + +.. automodule:: libp2p.io.exceptions + :members: + :undoc-members: + :show-inheritance: + +libp2p.io.msgio module +---------------------- + +.. automodule:: libp2p.io.msgio + :members: + :undoc-members: + :show-inheritance: + +libp2p.io.utils module +---------------------- + +.. automodule:: libp2p.io.utils + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.io + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.kademlia.rst b/docs/libp2p.kademlia.rst new file mode 100644 index 00000000..aa0417f0 --- /dev/null +++ b/docs/libp2p.kademlia.rst @@ -0,0 +1,70 @@ +libp2p.kademlia package +======================= + +Submodules +---------- + +libp2p.kademlia.crawling module +------------------------------- + +.. automodule:: libp2p.kademlia.crawling + :members: + :undoc-members: + :show-inheritance: + +libp2p.kademlia.kad\_peerinfo module +------------------------------------ + +.. automodule:: libp2p.kademlia.kad_peerinfo + :members: + :undoc-members: + :show-inheritance: + +libp2p.kademlia.network module +------------------------------ + +.. automodule:: libp2p.kademlia.network + :members: + :undoc-members: + :show-inheritance: + +libp2p.kademlia.protocol module +------------------------------- + +.. automodule:: libp2p.kademlia.protocol + :members: + :undoc-members: + :show-inheritance: + +libp2p.kademlia.routing module +------------------------------ + +.. automodule:: libp2p.kademlia.routing + :members: + :undoc-members: + :show-inheritance: + +libp2p.kademlia.storage module +------------------------------ + +.. automodule:: libp2p.kademlia.storage + :members: + :undoc-members: + :show-inheritance: + +libp2p.kademlia.utils module +---------------------------- + +.. automodule:: libp2p.kademlia.utils + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.kademlia + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.network.connection.rst b/docs/libp2p.network.connection.rst new file mode 100644 index 00000000..f50dde21 --- /dev/null +++ b/docs/libp2p.network.connection.rst @@ -0,0 +1,54 @@ +libp2p.network.connection package +================================= + +Submodules +---------- + +libp2p.network.connection.exceptions module +------------------------------------------- + +.. automodule:: libp2p.network.connection.exceptions + :members: + :undoc-members: + :show-inheritance: + +libp2p.network.connection.net\_connection\_interface module +----------------------------------------------------------- + +.. automodule:: libp2p.network.connection.net_connection_interface + :members: + :undoc-members: + :show-inheritance: + +libp2p.network.connection.raw\_connection module +------------------------------------------------ + +.. automodule:: libp2p.network.connection.raw_connection + :members: + :undoc-members: + :show-inheritance: + +libp2p.network.connection.raw\_connection\_interface module +----------------------------------------------------------- + +.. automodule:: libp2p.network.connection.raw_connection_interface + :members: + :undoc-members: + :show-inheritance: + +libp2p.network.connection.swarm\_connection module +-------------------------------------------------- + +.. automodule:: libp2p.network.connection.swarm_connection + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.network.connection + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.network.rst b/docs/libp2p.network.rst new file mode 100644 index 00000000..2ef65e9c --- /dev/null +++ b/docs/libp2p.network.rst @@ -0,0 +1,54 @@ +libp2p.network package +====================== + +Subpackages +----------- + +.. toctree:: + + libp2p.network.connection + libp2p.network.stream + +Submodules +---------- + +libp2p.network.exceptions module +-------------------------------- + +.. automodule:: libp2p.network.exceptions + :members: + :undoc-members: + :show-inheritance: + +libp2p.network.network\_interface module +---------------------------------------- + +.. automodule:: libp2p.network.network_interface + :members: + :undoc-members: + :show-inheritance: + +libp2p.network.notifee\_interface module +---------------------------------------- + +.. automodule:: libp2p.network.notifee_interface + :members: + :undoc-members: + :show-inheritance: + +libp2p.network.swarm module +--------------------------- + +.. automodule:: libp2p.network.swarm + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.network + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.network.stream.rst b/docs/libp2p.network.stream.rst new file mode 100644 index 00000000..73538216 --- /dev/null +++ b/docs/libp2p.network.stream.rst @@ -0,0 +1,38 @@ +libp2p.network.stream package +============================= + +Submodules +---------- + +libp2p.network.stream.exceptions module +--------------------------------------- + +.. automodule:: libp2p.network.stream.exceptions + :members: + :undoc-members: + :show-inheritance: + +libp2p.network.stream.net\_stream module +---------------------------------------- + +.. automodule:: libp2p.network.stream.net_stream + :members: + :undoc-members: + :show-inheritance: + +libp2p.network.stream.net\_stream\_interface module +--------------------------------------------------- + +.. automodule:: libp2p.network.stream.net_stream_interface + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.network.stream + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.peer.rst b/docs/libp2p.peer.rst new file mode 100644 index 00000000..668daa53 --- /dev/null +++ b/docs/libp2p.peer.rst @@ -0,0 +1,78 @@ +libp2p.peer package +=================== + +Submodules +---------- + +libp2p.peer.addrbook\_interface module +-------------------------------------- + +.. automodule:: libp2p.peer.addrbook_interface + :members: + :undoc-members: + :show-inheritance: + +libp2p.peer.id module +--------------------- + +.. automodule:: libp2p.peer.id + :members: + :undoc-members: + :show-inheritance: + +libp2p.peer.peerdata module +--------------------------- + +.. automodule:: libp2p.peer.peerdata + :members: + :undoc-members: + :show-inheritance: + +libp2p.peer.peerdata\_interface module +-------------------------------------- + +.. automodule:: libp2p.peer.peerdata_interface + :members: + :undoc-members: + :show-inheritance: + +libp2p.peer.peerinfo module +--------------------------- + +.. automodule:: libp2p.peer.peerinfo + :members: + :undoc-members: + :show-inheritance: + +libp2p.peer.peermetadata\_interface module +------------------------------------------ + +.. automodule:: libp2p.peer.peermetadata_interface + :members: + :undoc-members: + :show-inheritance: + +libp2p.peer.peerstore module +---------------------------- + +.. automodule:: libp2p.peer.peerstore + :members: + :undoc-members: + :show-inheritance: + +libp2p.peer.peerstore\_interface module +--------------------------------------- + +.. automodule:: libp2p.peer.peerstore_interface + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.peer + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.protocol_muxer.rst b/docs/libp2p.protocol_muxer.rst new file mode 100644 index 00000000..69044d61 --- /dev/null +++ b/docs/libp2p.protocol_muxer.rst @@ -0,0 +1,70 @@ +libp2p.protocol\_muxer package +============================== + +Submodules +---------- + +libp2p.protocol\_muxer.exceptions module +---------------------------------------- + +.. automodule:: libp2p.protocol_muxer.exceptions + :members: + :undoc-members: + :show-inheritance: + +libp2p.protocol\_muxer.multiselect module +----------------------------------------- + +.. automodule:: libp2p.protocol_muxer.multiselect + :members: + :undoc-members: + :show-inheritance: + +libp2p.protocol\_muxer.multiselect\_client module +------------------------------------------------- + +.. automodule:: libp2p.protocol_muxer.multiselect_client + :members: + :undoc-members: + :show-inheritance: + +libp2p.protocol\_muxer.multiselect\_client\_interface module +------------------------------------------------------------ + +.. automodule:: libp2p.protocol_muxer.multiselect_client_interface + :members: + :undoc-members: + :show-inheritance: + +libp2p.protocol\_muxer.multiselect\_communicator module +------------------------------------------------------- + +.. automodule:: libp2p.protocol_muxer.multiselect_communicator + :members: + :undoc-members: + :show-inheritance: + +libp2p.protocol\_muxer.multiselect\_communicator\_interface module +------------------------------------------------------------------ + +.. automodule:: libp2p.protocol_muxer.multiselect_communicator_interface + :members: + :undoc-members: + :show-inheritance: + +libp2p.protocol\_muxer.multiselect\_muxer\_interface module +----------------------------------------------------------- + +.. automodule:: libp2p.protocol_muxer.multiselect_muxer_interface + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.protocol_muxer + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.pubsub.pb.rst b/docs/libp2p.pubsub.pb.rst new file mode 100644 index 00000000..2a8a8e9a --- /dev/null +++ b/docs/libp2p.pubsub.pb.rst @@ -0,0 +1,22 @@ +libp2p.pubsub.pb package +======================== + +Submodules +---------- + +libp2p.pubsub.pb.rpc\_pb2 module +-------------------------------- + +.. automodule:: libp2p.pubsub.pb.rpc_pb2 + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.pubsub.pb + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.pubsub.rst b/docs/libp2p.pubsub.rst new file mode 100644 index 00000000..d217772b --- /dev/null +++ b/docs/libp2p.pubsub.rst @@ -0,0 +1,77 @@ +libp2p.pubsub package +===================== + +Subpackages +----------- + +.. toctree:: + + libp2p.pubsub.pb + +Submodules +---------- + +libp2p.pubsub.floodsub module +----------------------------- + +.. automodule:: libp2p.pubsub.floodsub + :members: + :undoc-members: + :show-inheritance: + +libp2p.pubsub.gossipsub module +------------------------------ + +.. automodule:: libp2p.pubsub.gossipsub + :members: + :undoc-members: + :show-inheritance: + +libp2p.pubsub.mcache module +--------------------------- + +.. automodule:: libp2p.pubsub.mcache + :members: + :undoc-members: + :show-inheritance: + +libp2p.pubsub.pubsub module +--------------------------- + +.. automodule:: libp2p.pubsub.pubsub + :members: + :undoc-members: + :show-inheritance: + +libp2p.pubsub.pubsub\_notifee module +------------------------------------ + +.. automodule:: libp2p.pubsub.pubsub_notifee + :members: + :undoc-members: + :show-inheritance: + +libp2p.pubsub.pubsub\_router\_interface module +---------------------------------------------- + +.. automodule:: libp2p.pubsub.pubsub_router_interface + :members: + :undoc-members: + :show-inheritance: + +libp2p.pubsub.validators module +------------------------------- + +.. automodule:: libp2p.pubsub.validators + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.pubsub + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.routing.kademlia.rst b/docs/libp2p.routing.kademlia.rst new file mode 100644 index 00000000..c52dcdce --- /dev/null +++ b/docs/libp2p.routing.kademlia.rst @@ -0,0 +1,30 @@ +libp2p.routing.kademlia package +=============================== + +Submodules +---------- + +libp2p.routing.kademlia.kademlia\_content\_router module +-------------------------------------------------------- + +.. automodule:: libp2p.routing.kademlia.kademlia_content_router + :members: + :undoc-members: + :show-inheritance: + +libp2p.routing.kademlia.kademlia\_peer\_router module +----------------------------------------------------- + +.. automodule:: libp2p.routing.kademlia.kademlia_peer_router + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.routing.kademlia + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.routing.rst b/docs/libp2p.routing.rst new file mode 100644 index 00000000..1595e4ef --- /dev/null +++ b/docs/libp2p.routing.rst @@ -0,0 +1,29 @@ +libp2p.routing package +====================== + +Subpackages +----------- + +.. toctree:: + + libp2p.routing.kademlia + +Submodules +---------- + +libp2p.routing.interfaces module +-------------------------------- + +.. automodule:: libp2p.routing.interfaces + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.routing + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.rst b/docs/libp2p.rst new file mode 100644 index 00000000..8aa6bc5e --- /dev/null +++ b/docs/libp2p.rst @@ -0,0 +1,58 @@ +libp2p package +============== + +Subpackages +----------- + +.. toctree:: + + libp2p.crypto + libp2p.host + libp2p.identity + libp2p.io + libp2p.kademlia + libp2p.network + libp2p.peer + libp2p.protocol_muxer + libp2p.pubsub + libp2p.routing + libp2p.security + libp2p.stream_muxer + libp2p.tools + libp2p.transport + +Submodules +---------- + +libp2p.exceptions module +------------------------ + +.. automodule:: libp2p.exceptions + :members: + :undoc-members: + :show-inheritance: + +libp2p.typing module +-------------------- + +.. automodule:: libp2p.typing + :members: + :undoc-members: + :show-inheritance: + +libp2p.utils module +------------------- + +.. automodule:: libp2p.utils + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.security.insecure.pb.rst b/docs/libp2p.security.insecure.pb.rst new file mode 100644 index 00000000..dc24e858 --- /dev/null +++ b/docs/libp2p.security.insecure.pb.rst @@ -0,0 +1,22 @@ +libp2p.security.insecure.pb package +=================================== + +Submodules +---------- + +libp2p.security.insecure.pb.plaintext\_pb2 module +------------------------------------------------- + +.. automodule:: libp2p.security.insecure.pb.plaintext_pb2 + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.security.insecure.pb + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.security.insecure.rst b/docs/libp2p.security.insecure.rst new file mode 100644 index 00000000..0f25b5b3 --- /dev/null +++ b/docs/libp2p.security.insecure.rst @@ -0,0 +1,29 @@ +libp2p.security.insecure package +================================ + +Subpackages +----------- + +.. toctree:: + + libp2p.security.insecure.pb + +Submodules +---------- + +libp2p.security.insecure.transport module +----------------------------------------- + +.. automodule:: libp2p.security.insecure.transport + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.security.insecure + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.security.rst b/docs/libp2p.security.rst new file mode 100644 index 00000000..de60977a --- /dev/null +++ b/docs/libp2p.security.rst @@ -0,0 +1,70 @@ +libp2p.security package +======================= + +Subpackages +----------- + +.. toctree:: + + libp2p.security.insecure + libp2p.security.secio + +Submodules +---------- + +libp2p.security.base\_session module +------------------------------------ + +.. automodule:: libp2p.security.base_session + :members: + :undoc-members: + :show-inheritance: + +libp2p.security.base\_transport module +-------------------------------------- + +.. automodule:: libp2p.security.base_transport + :members: + :undoc-members: + :show-inheritance: + +libp2p.security.exceptions module +--------------------------------- + +.. automodule:: libp2p.security.exceptions + :members: + :undoc-members: + :show-inheritance: + +libp2p.security.secure\_conn\_interface module +---------------------------------------------- + +.. automodule:: libp2p.security.secure_conn_interface + :members: + :undoc-members: + :show-inheritance: + +libp2p.security.secure\_transport\_interface module +--------------------------------------------------- + +.. automodule:: libp2p.security.secure_transport_interface + :members: + :undoc-members: + :show-inheritance: + +libp2p.security.security\_multistream module +-------------------------------------------- + +.. automodule:: libp2p.security.security_multistream + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.security + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.security.secio.pb.rst b/docs/libp2p.security.secio.pb.rst new file mode 100644 index 00000000..c0cf61b9 --- /dev/null +++ b/docs/libp2p.security.secio.pb.rst @@ -0,0 +1,22 @@ +libp2p.security.secio.pb package +================================ + +Submodules +---------- + +libp2p.security.secio.pb.spipe\_pb2 module +------------------------------------------ + +.. automodule:: libp2p.security.secio.pb.spipe_pb2 + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.security.secio.pb + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.security.secio.rst b/docs/libp2p.security.secio.rst new file mode 100644 index 00000000..da20b8b1 --- /dev/null +++ b/docs/libp2p.security.secio.rst @@ -0,0 +1,37 @@ +libp2p.security.secio package +============================= + +Subpackages +----------- + +.. toctree:: + + libp2p.security.secio.pb + +Submodules +---------- + +libp2p.security.secio.exceptions module +--------------------------------------- + +.. automodule:: libp2p.security.secio.exceptions + :members: + :undoc-members: + :show-inheritance: + +libp2p.security.secio.transport module +-------------------------------------- + +.. automodule:: libp2p.security.secio.transport + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.security.secio + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.stream_muxer.mplex.rst b/docs/libp2p.stream_muxer.mplex.rst new file mode 100644 index 00000000..37e92e53 --- /dev/null +++ b/docs/libp2p.stream_muxer.mplex.rst @@ -0,0 +1,54 @@ +libp2p.stream\_muxer.mplex package +================================== + +Submodules +---------- + +libp2p.stream\_muxer.mplex.constants module +------------------------------------------- + +.. automodule:: libp2p.stream_muxer.mplex.constants + :members: + :undoc-members: + :show-inheritance: + +libp2p.stream\_muxer.mplex.datastructures module +------------------------------------------------ + +.. automodule:: libp2p.stream_muxer.mplex.datastructures + :members: + :undoc-members: + :show-inheritance: + +libp2p.stream\_muxer.mplex.exceptions module +-------------------------------------------- + +.. automodule:: libp2p.stream_muxer.mplex.exceptions + :members: + :undoc-members: + :show-inheritance: + +libp2p.stream\_muxer.mplex.mplex module +--------------------------------------- + +.. automodule:: libp2p.stream_muxer.mplex.mplex + :members: + :undoc-members: + :show-inheritance: + +libp2p.stream\_muxer.mplex.mplex\_stream module +----------------------------------------------- + +.. automodule:: libp2p.stream_muxer.mplex.mplex_stream + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.stream_muxer.mplex + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.stream_muxer.rst b/docs/libp2p.stream_muxer.rst new file mode 100644 index 00000000..12114ece --- /dev/null +++ b/docs/libp2p.stream_muxer.rst @@ -0,0 +1,45 @@ +libp2p.stream\_muxer package +============================ + +Subpackages +----------- + +.. toctree:: + + libp2p.stream_muxer.mplex + +Submodules +---------- + +libp2p.stream\_muxer.abc module +------------------------------- + +.. automodule:: libp2p.stream_muxer.abc + :members: + :undoc-members: + :show-inheritance: + +libp2p.stream\_muxer.exceptions module +-------------------------------------- + +.. automodule:: libp2p.stream_muxer.exceptions + :members: + :undoc-members: + :show-inheritance: + +libp2p.stream\_muxer.muxer\_multistream module +---------------------------------------------- + +.. automodule:: libp2p.stream_muxer.muxer_multistream + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.stream_muxer + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.tools.pubsub.rst b/docs/libp2p.tools.pubsub.rst new file mode 100644 index 00000000..91b130c5 --- /dev/null +++ b/docs/libp2p.tools.pubsub.rst @@ -0,0 +1,38 @@ +libp2p.tools.pubsub package +=========================== + +Submodules +---------- + +libp2p.tools.pubsub.dummy\_account\_node module +----------------------------------------------- + +.. automodule:: libp2p.tools.pubsub.dummy_account_node + :members: + :undoc-members: + :show-inheritance: + +libp2p.tools.pubsub.floodsub\_integration\_test\_settings module +---------------------------------------------------------------- + +.. automodule:: libp2p.tools.pubsub.floodsub_integration_test_settings + :members: + :undoc-members: + :show-inheritance: + +libp2p.tools.pubsub.utils module +-------------------------------- + +.. automodule:: libp2p.tools.pubsub.utils + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.tools.pubsub + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.tools.rst b/docs/libp2p.tools.rst new file mode 100644 index 00000000..2d7846fc --- /dev/null +++ b/docs/libp2p.tools.rst @@ -0,0 +1,47 @@ +libp2p.tools package +==================== + +Subpackages +----------- + +.. toctree:: + + libp2p.tools.pubsub + +The interop module is left out for now, because of the extra dependencies it requires. + +Submodules +---------- + +libp2p.tools.constants module +----------------------------- + +.. automodule:: libp2p.tools.constants + :members: + :undoc-members: + :show-inheritance: + +libp2p.tools.factories module +----------------------------- + +.. automodule:: libp2p.tools.factories + :members: + :undoc-members: + :show-inheritance: + +libp2p.tools.utils module +------------------------- + +.. automodule:: libp2p.tools.utils + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.tools + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.transport.rst b/docs/libp2p.transport.rst new file mode 100644 index 00000000..2fe53335 --- /dev/null +++ b/docs/libp2p.transport.rst @@ -0,0 +1,61 @@ +libp2p.transport package +======================== + +Subpackages +----------- + +.. toctree:: + + libp2p.transport.tcp + +Submodules +---------- + +libp2p.transport.exceptions module +---------------------------------- + +.. automodule:: libp2p.transport.exceptions + :members: + :undoc-members: + :show-inheritance: + +libp2p.transport.listener\_interface module +------------------------------------------- + +.. automodule:: libp2p.transport.listener_interface + :members: + :undoc-members: + :show-inheritance: + +libp2p.transport.transport\_interface module +-------------------------------------------- + +.. automodule:: libp2p.transport.transport_interface + :members: + :undoc-members: + :show-inheritance: + +libp2p.transport.typing module +------------------------------ + +.. automodule:: libp2p.transport.typing + :members: + :undoc-members: + :show-inheritance: + +libp2p.transport.upgrader module +-------------------------------- + +.. automodule:: libp2p.transport.upgrader + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.transport + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/libp2p.transport.tcp.rst b/docs/libp2p.transport.tcp.rst new file mode 100644 index 00000000..374e1e5a --- /dev/null +++ b/docs/libp2p.transport.tcp.rst @@ -0,0 +1,22 @@ +libp2p.transport.tcp package +============================ + +Submodules +---------- + +libp2p.transport.tcp.tcp module +------------------------------- + +.. automodule:: libp2p.transport.tcp.tcp + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: libp2p.transport.tcp + :members: + :undoc-members: + :show-inheritance: From 683710573e0216c073648c20d226ab871e3279e8 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 26 Nov 2019 16:09:46 +0800 Subject: [PATCH 101/178] Add `strict_signing: bool and sign_key` to Pubsub --- libp2p/pubsub/pubsub.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 493a303c..076e60cc 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -16,6 +16,7 @@ from typing import ( import base58 from lru import LRU +from libp2p.crypto.keys import PrivateKey from libp2p.exceptions import ParseError, ValidationError from libp2p.host.host_interface import IHost from libp2p.io.exceptions import IncompleteReadError @@ -82,8 +83,17 @@ class Pubsub: _tasks: List["asyncio.Future[Any]"] + # Indicate if we should enforce signature verification + strict_signing: bool + sign_key: PrivateKey + def __init__( - self, host: IHost, router: "IPubsubRouter", my_id: ID, cache_size: int = None + self, + host: IHost, + router: "IPubsubRouter", + my_id: ID, + cache_size: int = None, + strict_signing: bool = True, ) -> None: """ Construct a new Pubsub object, which is responsible for handling all @@ -147,6 +157,12 @@ class Pubsub: self._tasks.append(asyncio.ensure_future(self.handle_peer_queue())) self._tasks.append(asyncio.ensure_future(self.handle_dead_peer_queue())) + self.strict_signing = strict_signing + if strict_signing: + self.sign_key = self.host.get_private_key() + else: + self.sign_key = None + def get_hello_packet(self) -> rpc_pb2.RPC: """Generate subscription message with all topics we are subscribed to only send hello packet if we have subscribed topics.""" @@ -456,8 +472,6 @@ class Pubsub: seqno=self._next_seqno(), ) - # TODO: Sign with our signing key - await self.push_msg(self.host.get_id(), msg) logger.debug("successfully published message %s", msg) From f3c997215919c512cade5ea427e80b8207e3365c Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 26 Nov 2019 16:10:20 +0800 Subject: [PATCH 102/178] Implement Pubsub signature validator --- libp2p/pubsub/validators.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/libp2p/pubsub/validators.py b/libp2p/pubsub/validators.py index d1972075..0f6e55ee 100644 --- a/libp2p/pubsub/validators.py +++ b/libp2p/pubsub/validators.py @@ -1,10 +1,17 @@ -# FIXME: Replace the type of `pubkey` with a custom type `Pubkey` -def signature_validator(pubkey: bytes, msg: bytes) -> bool: +from libp2p.crypto.keys import PublicKey + +from .pb import rpc_pb2 + + +def signature_validator(pubkey: PublicKey, msg: rpc_pb2.Message) -> bool: """ Verify the message against the given public key. :param pubkey: the public key which signs the message. :param msg: the message signed. """ - # TODO: Implement the signature validation + try: + pubkey.verify(msg.SerializeToString(), msg.signature) + except Exception: + return False return True From 0fd400fdf837654836818d904468c6a5e667ce36 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 26 Nov 2019 16:10:58 +0800 Subject: [PATCH 103/178] Sign and verify in Pubsub --- libp2p/pubsub/pubsub.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 076e60cc..e49d0e8f 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -17,6 +17,7 @@ import base58 from lru import LRU from libp2p.crypto.keys import PrivateKey +from libp2p.crypto.serialization import deserialize_public_key from libp2p.exceptions import ParseError, ValidationError from libp2p.host.host_interface import IHost from libp2p.io.exceptions import IncompleteReadError @@ -472,6 +473,12 @@ class Pubsub: seqno=self._next_seqno(), ) + if self.strict_signing: + priv_key = self.sign_key + signature = priv_key.sign(msg.SerializeToString()) + msg.key = self.host.get_public_key().serialize() + msg.signature = signature + await self.push_msg(self.host.get_id(), msg) logger.debug("successfully published message %s", msg) @@ -519,18 +526,20 @@ class Pubsub: # TODO: Check if the `from` is in the blacklist. If yes, reject. - # TODO: Check if signing is required and if so signature should be attached. - # If the message is processed before, return(i.e., don't further process the message). if self._is_msg_seen(msg): return - # TODO: - Validate the message. If failed, reject it. - # Validate the signature of the message - # FIXME: `signature_validator` is currently a stub. - if not signature_validator(msg.key, msg.SerializeToString()): - logger.debug("Signature validation failed for msg: %s", msg) - return + # Check if signing is required and if so signature should be attached. + if self.strict_signing: + if msg.signature == b'': + logger.debug("Reject because no signature attached for msg: %s", msg) + return + # Validate the signature of the message + if not signature_validator(deserialize_public_key(msg.key), msg): + logger.debug("Signature validation failed for msg: %s", msg) + return + # Validate the message with registered topic validators. # If the validation failed, return(i.e., don't further process the message). try: From d5d6962dce9b8e72b0309771ab451c3ef8b79c7b Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 26 Nov 2019 16:12:50 +0800 Subject: [PATCH 104/178] Update Pubsub fixture and test --- libp2p/pubsub/pubsub.py | 2 +- libp2p/tools/factories.py | 1 + tests/pubsub/conftest.py | 28 ++++++++++++++++++++++------ tests/pubsub/test_pubsub.py | 15 +++++++++++++++ 4 files changed, 39 insertions(+), 7 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index e49d0e8f..0fea734f 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -532,7 +532,7 @@ class Pubsub: # Check if signing is required and if so signature should be attached. if self.strict_signing: - if msg.signature == b'': + if msg.signature == b"": logger.debug("Reject because no signature attached for msg: %s", msg) return # Validate the signature of the message diff --git a/libp2p/tools/factories.py b/libp2p/tools/factories.py index b5c16b85..b189cfa1 100644 --- a/libp2p/tools/factories.py +++ b/libp2p/tools/factories.py @@ -153,6 +153,7 @@ class PubsubFactory(factory.Factory): router = None my_id = factory.LazyAttribute(lambda obj: obj.host.get_id()) cache_size = None + strict_signing = False async def swarm_pair_factory( diff --git a/tests/pubsub/conftest.py b/tests/pubsub/conftest.py index 9dbe90b9..520fdf4b 100644 --- a/tests/pubsub/conftest.py +++ b/tests/pubsub/conftest.py @@ -4,14 +4,24 @@ from libp2p.tools.constants import GOSSIPSUB_PARAMS from libp2p.tools.factories import FloodsubFactory, GossipsubFactory, PubsubFactory -def _make_pubsubs(hosts, pubsub_routers, cache_size): +@pytest.fixture +def is_strict_signing(): + return False + + +def _make_pubsubs(hosts, pubsub_routers, cache_size, is_strict_signing): if len(pubsub_routers) != len(hosts): raise ValueError( f"lenght of pubsub_routers={pubsub_routers} should be equaled to the " f"length of hosts={len(hosts)}" ) return tuple( - PubsubFactory(host=host, router=router, cache_size=cache_size) + PubsubFactory( + host=host, + router=router, + cache_size=cache_size, + strict_signing=is_strict_signing, + ) for host, router in zip(hosts, pubsub_routers) ) @@ -27,16 +37,22 @@ def gossipsub_params(): @pytest.fixture -def pubsubs_fsub(num_hosts, hosts, pubsub_cache_size): +def pubsubs_fsub(num_hosts, hosts, pubsub_cache_size, is_strict_signing): floodsubs = FloodsubFactory.create_batch(num_hosts) - _pubsubs_fsub = _make_pubsubs(hosts, floodsubs, pubsub_cache_size) + _pubsubs_fsub = _make_pubsubs( + hosts, floodsubs, pubsub_cache_size, is_strict_signing + ) yield _pubsubs_fsub # TODO: Clean up @pytest.fixture -def pubsubs_gsub(num_hosts, hosts, pubsub_cache_size, gossipsub_params): +def pubsubs_gsub( + num_hosts, hosts, pubsub_cache_size, gossipsub_params, is_strict_signing +): gossipsubs = GossipsubFactory.create_batch(num_hosts, **gossipsub_params._asdict()) - _pubsubs_gsub = _make_pubsubs(hosts, gossipsubs, pubsub_cache_size) + _pubsubs_gsub = _make_pubsubs( + hosts, gossipsubs, pubsub_cache_size, is_strict_signing + ) yield _pubsubs_gsub # TODO: Clean up diff --git a/tests/pubsub/test_pubsub.py b/tests/pubsub/test_pubsub.py index ebe20037..48ef52bb 100644 --- a/tests/pubsub/test_pubsub.py +++ b/tests/pubsub/test_pubsub.py @@ -510,3 +510,18 @@ async def test_push_msg(pubsubs_fsub, monkeypatch): await pubsubs_fsub[0].push_msg(pubsubs_fsub[0].my_id, msg_2) await asyncio.sleep(0.01) assert not event.is_set() + + +@pytest.mark.parametrize("num_hosts, is_strict_signing", ((2, True),)) +@pytest.mark.asyncio +async def test_strict_signing(pubsubs_fsub, hosts, monkeypatch): + await connect(hosts[0], hosts[1]) + await pubsubs_fsub[0].subscribe(TESTING_TOPIC) + await pubsubs_fsub[1].subscribe(TESTING_TOPIC) + await asyncio.sleep(1) + + await pubsubs_fsub[0].publish(TESTING_TOPIC, TESTING_DATA) + await asyncio.sleep(1) + + assert len(pubsubs_fsub[0].seen_messages) == 1 + assert len(pubsubs_fsub[1].seen_messages) == 1 From 84b548beae04d495d57e26e65eb626a9c893aab6 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 26 Nov 2019 15:45:23 -0800 Subject: [PATCH 105/178] Back-generate release notes for v0.1.3 release --- newsfragments/350.bugfix.rst | 2 ++ newsfragments/353.internal.rst | 1 + newsfragments/354.bugfix.rst | 2 ++ newsfragments/355.bugfix.rst | 2 ++ newsfragments/356.internal.rst | 1 + newsfragments/357.misc.rst | 0 6 files changed, 8 insertions(+) create mode 100644 newsfragments/350.bugfix.rst create mode 100644 newsfragments/353.internal.rst create mode 100644 newsfragments/354.bugfix.rst create mode 100644 newsfragments/355.bugfix.rst create mode 100644 newsfragments/356.internal.rst create mode 100644 newsfragments/357.misc.rst diff --git a/newsfragments/350.bugfix.rst b/newsfragments/350.bugfix.rst new file mode 100644 index 00000000..5f49ad6e --- /dev/null +++ b/newsfragments/350.bugfix.rst @@ -0,0 +1,2 @@ +Handle Stream* errors (like ``StreamClosed``) during calls to ``stream.write()`` and +``stream.read()`` diff --git a/newsfragments/353.internal.rst b/newsfragments/353.internal.rst new file mode 100644 index 00000000..b0d2a9e5 --- /dev/null +++ b/newsfragments/353.internal.rst @@ -0,0 +1 @@ +Added Makefile target to test a packaged version of libp2p before release. diff --git a/newsfragments/354.bugfix.rst b/newsfragments/354.bugfix.rst new file mode 100644 index 00000000..c3e55a46 --- /dev/null +++ b/newsfragments/354.bugfix.rst @@ -0,0 +1,2 @@ +Relax the protobuf dependency to play nicely with other libraries. It was pinned to 3.9.0, and now +permits v3.10 up to (but not including) v4. diff --git a/newsfragments/355.bugfix.rst b/newsfragments/355.bugfix.rst new file mode 100644 index 00000000..3844b680 --- /dev/null +++ b/newsfragments/355.bugfix.rst @@ -0,0 +1,2 @@ +Fixes KeyError when peer in a stream accidentally closes and resets the stream, because handlers +for both will try to ``del streams[stream_id]`` without checking if the entry still exists. diff --git a/newsfragments/356.internal.rst b/newsfragments/356.internal.rst new file mode 100644 index 00000000..6c247c41 --- /dev/null +++ b/newsfragments/356.internal.rst @@ -0,0 +1 @@ +Move helper tools from `tests/` to `libp2p/tools/`, and some mildly-related cleanups. diff --git a/newsfragments/357.misc.rst b/newsfragments/357.misc.rst new file mode 100644 index 00000000..e69de29b From 57f1f49a0fa336e97787fab7b99d0a6e1d96e961 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Tue, 26 Nov 2019 16:38:36 -0800 Subject: [PATCH 106/178] Set the latest version in release notes --- docs/release_notes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 63786aca..512ce240 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -3,7 +3,7 @@ Release Notes .. towncrier release notes start -v0.1.0-alpha.1 +v0.1.2 -------------- -- Launched repository, claimed names for pip, RTD, github, etc +Welcome to the great beyond, where changes were not tracked by release... From 3e5f883c50c92235e5152dbd58e589a90ee55748 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 27 Nov 2019 11:21:16 -0800 Subject: [PATCH 107/178] Strip out fastecdsa in setup.py, during doc build --- requirements-docs.txt | 2 +- setup.py | 35 +++++++++++++++++++++++------------ 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/requirements-docs.txt b/requirements-docs.txt index ba1ee49e..3ef37020 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1 +1 @@ -libp2p[doc] +.[doc] diff --git a/setup.py b/setup.py index 252252fe..e4d826ef 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import os + from setuptools import find_packages, setup extras_require = { @@ -46,6 +48,26 @@ with open("./README.md") as readme: long_description = readme.read() +install_requires = [ + "pycryptodome>=3.9.2,<4.0.0", + "base58>=1.0.3,<2.0.0", + "pymultihash>=0.8.2", + "multiaddr>=0.0.8,<0.1.0", + "rpcudp>=3.0.0,<4.0.0", + "lru-dict>=1.1.6", + "protobuf>=3.10.0,<4.0.0", + "coincurve>=10.0.0,<11.0.0", + "pynacl==1.3.0", +] + + +# NOTE: Some dependencies break RTD builds. We can not install system dependencies on the +# RTD system so we have to exclude these dependencies when we are in an RTD environment. +readthedocs_is_building = os.environ.get("READTHEDOCS", False) +if not readthedocs_is_building: + install_requires.append("fastecdsa==1.7.4") + + setup( name="libp2p", # *IMPORTANT*: Don't manually change the version here. Use `make bump`, as described in readme @@ -57,18 +79,7 @@ setup( maintainer_email="snakecharmers@ethereum.org", url="https://github.com/libp2p/py-libp2p", include_package_data=True, - install_requires=[ - "pycryptodome>=3.9.2,<4.0.0", - "base58>=1.0.3,<2.0.0", - "pymultihash>=0.8.2", - "multiaddr>=0.0.8,<0.1.0", - "rpcudp>=3.0.0,<4.0.0", - "lru-dict>=1.1.6", - "protobuf>=3.10.0,<4.0.0", - "coincurve>=10.0.0,<11.0.0", - "fastecdsa==1.7.4", - "pynacl==1.3.0", - ], + install_requires=install_requires, python_requires=">=3.7,<4", extras_require=extras_require, py_modules=["libp2p"], From 27ecd4b0ed2b065683c89a6fa5cba73859326914 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 27 Nov 2019 12:25:11 -0800 Subject: [PATCH 108/178] Mock dependencies that are excluded in readthedocs It seems preferable to import just fastecdsa. But if you do that, then some kind of side-effect doesn't happen, which means that `sec1` is not available as an attribute on `fastecdsa.encoding`. So we specifically mock the sub-modules as well. --- docs/conf.py | 16 ++++++++++++++++ libp2p/crypto/key_exchange.py | 4 +++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 9038ad0a..ebff89ba 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -286,3 +286,19 @@ doctest_default_flags = (0 | doctest.IGNORE_EXCEPTION_DETAIL | doctest.NORMALIZE_WHITESPACE ) + +# -- Mocked dependencies ---------------------------------------- + +# Mock out dependencies that are unbuildable on readthedocs, as recommended here: +# https://docs.readthedocs.io/en/rel/faq.html#i-get-import-errors-on-libraries-that-depend-on-c-modules + +import sys +from unittest.mock import MagicMock + +# Add new modules to mock here (it should be the same list as those excluded in setup.py) +MOCK_MODULES = [ + "fastecdsa", + "fastecdsa.encoding", + "fastecdsa.encoding.sec1", +] +sys.modules.update((mod_name, MagicMock()) for mod_name in MOCK_MODULES) diff --git a/libp2p/crypto/key_exchange.py b/libp2p/crypto/key_exchange.py index 17ec75ab..2af4030d 100644 --- a/libp2p/crypto/key_exchange.py +++ b/libp2p/crypto/key_exchange.py @@ -1,12 +1,14 @@ from typing import Callable, Tuple, cast -from fastecdsa.encoding.util import int_bytelen +from fastecdsa.encoding import util from libp2p.crypto.ecc import ECCPrivateKey, ECCPublicKey, create_new_key_pair from libp2p.crypto.keys import PublicKey SharedKeyGenerator = Callable[[bytes], bytes] +int_bytelen = util.int_bytelen + def create_ephemeral_key_pair(curve_type: str) -> Tuple[PublicKey, SharedKeyGenerator]: """Facilitates ECDH key exchange.""" From ce37082c44cb9e126094f391c622be5765b4726f Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 27 Nov 2019 14:16:55 -0800 Subject: [PATCH 109/178] Add release note for #318 Note that these commits are really just a bugfix, but from a release perspective, it's the first time that docs will be publicly available. --- newsfragments/318.doc.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/318.doc.rst diff --git a/newsfragments/318.doc.rst b/newsfragments/318.doc.rst new file mode 100644 index 00000000..0e49888d --- /dev/null +++ b/newsfragments/318.doc.rst @@ -0,0 +1 @@ +Use Sphinx & autodoc to generate docs, now available on py-libp2p.readthedocs.io From 40973ea106c3c123a59d49c88c4bda6d300e9588 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 27 Nov 2019 16:04:51 -0800 Subject: [PATCH 110/178] Fix "previous" version in bumpversion to: v0.1.2 --- .bumpversion.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index eefe4ad9..72f6def4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.1.0-alpha.0 +current_version = 0.1.2 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(-(?P[^.]*)\.(?P\d+))? From f28227729c9338c70d536cad903c034a1f57593e Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 27 Nov 2019 16:09:08 -0800 Subject: [PATCH 111/178] Customize release notes title towncrier likes to capitalize things, so libp2p was showing up as Libp2P. Uuuugly. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index ba94a6cc..f92e5664 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,7 @@ package = "libp2p" filename = "docs/release_notes.rst" directory = "newsfragments" underlines = ["-", "~", "^"] +title_format = "libp2p v{version} ({project_date})" issue_format = "`#{issue} `__" [[tool.towncrier.type]] From 52ec6a1606947848b5b67d571add976463876c7a Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 27 Nov 2019 16:10:23 -0800 Subject: [PATCH 112/178] Compile release notes --- docs/release_notes.rst | 33 +++++++++++++++++++++++++++++++++ newsfragments/318.doc.rst | 1 - newsfragments/350.bugfix.rst | 2 -- newsfragments/353.internal.rst | 1 - newsfragments/354.bugfix.rst | 2 -- newsfragments/355.bugfix.rst | 2 -- newsfragments/356.internal.rst | 1 - newsfragments/357.misc.rst | 0 8 files changed, 33 insertions(+), 9 deletions(-) delete mode 100644 newsfragments/318.doc.rst delete mode 100644 newsfragments/350.bugfix.rst delete mode 100644 newsfragments/353.internal.rst delete mode 100644 newsfragments/354.bugfix.rst delete mode 100644 newsfragments/355.bugfix.rst delete mode 100644 newsfragments/356.internal.rst delete mode 100644 newsfragments/357.misc.rst diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 512ce240..7537972c 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -3,6 +3,39 @@ Release Notes .. towncrier release notes start +libp2p v0.1.3 (2019-11-27) +-------------------------- + +Bugfixes +~~~~~~~~ + +- Handle Stream* errors (like ``StreamClosed``) during calls to ``stream.write()`` and + ``stream.read()`` (`#350 `__) +- Relax the protobuf dependency to play nicely with other libraries. It was pinned to 3.9.0, and now + permits v3.10 up to (but not including) v4. (`#354 `__) +- Fixes KeyError when peer in a stream accidentally closes and resets the stream, because handlers + for both will try to ``del streams[stream_id]`` without checking if the entry still exists. (`#355 `__) + + +Improved Documentation +~~~~~~~~~~~~~~~~~~~~~~ + +- Use Sphinx & autodoc to generate docs, now available on `py-libp2p.readthedocs.io `_ (`#318 `__) + + +Internal Changes - for py-libp2p Contributors +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Added Makefile target to test a packaged version of libp2p before release. (`#353 `__) +- Move helper tools from ``tests/`` to ``libp2p/tools/``, and some mildly-related cleanups. (`#356 `__) + + +Miscellaneous changes +~~~~~~~~~~~~~~~~~~~~~ + +- `#357 `__ + + v0.1.2 -------------- diff --git a/newsfragments/318.doc.rst b/newsfragments/318.doc.rst deleted file mode 100644 index 0e49888d..00000000 --- a/newsfragments/318.doc.rst +++ /dev/null @@ -1 +0,0 @@ -Use Sphinx & autodoc to generate docs, now available on py-libp2p.readthedocs.io diff --git a/newsfragments/350.bugfix.rst b/newsfragments/350.bugfix.rst deleted file mode 100644 index 5f49ad6e..00000000 --- a/newsfragments/350.bugfix.rst +++ /dev/null @@ -1,2 +0,0 @@ -Handle Stream* errors (like ``StreamClosed``) during calls to ``stream.write()`` and -``stream.read()`` diff --git a/newsfragments/353.internal.rst b/newsfragments/353.internal.rst deleted file mode 100644 index b0d2a9e5..00000000 --- a/newsfragments/353.internal.rst +++ /dev/null @@ -1 +0,0 @@ -Added Makefile target to test a packaged version of libp2p before release. diff --git a/newsfragments/354.bugfix.rst b/newsfragments/354.bugfix.rst deleted file mode 100644 index c3e55a46..00000000 --- a/newsfragments/354.bugfix.rst +++ /dev/null @@ -1,2 +0,0 @@ -Relax the protobuf dependency to play nicely with other libraries. It was pinned to 3.9.0, and now -permits v3.10 up to (but not including) v4. diff --git a/newsfragments/355.bugfix.rst b/newsfragments/355.bugfix.rst deleted file mode 100644 index 3844b680..00000000 --- a/newsfragments/355.bugfix.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixes KeyError when peer in a stream accidentally closes and resets the stream, because handlers -for both will try to ``del streams[stream_id]`` without checking if the entry still exists. diff --git a/newsfragments/356.internal.rst b/newsfragments/356.internal.rst deleted file mode 100644 index 6c247c41..00000000 --- a/newsfragments/356.internal.rst +++ /dev/null @@ -1 +0,0 @@ -Move helper tools from `tests/` to `libp2p/tools/`, and some mildly-related cleanups. diff --git a/newsfragments/357.misc.rst b/newsfragments/357.misc.rst deleted file mode 100644 index e69de29b..00000000 From 3c0675bbba70a864f2df954067011337104c6429 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 27 Nov 2019 16:19:24 -0800 Subject: [PATCH 113/178] Fix bumpversion to look for double-quotes --- .bumpversion.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 72f6def4..a9ddf6d0 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -18,6 +18,6 @@ values = [bumpversion:part:devnum] [bumpversion:file:setup.py] -search = version='{current_version}', -replace = version='{new_version}', +search = version="{current_version}", +replace = version="{new_version}", From 14bcc2a7a7f519af6d3d87c19b1d19bda2f5a3a4 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Wed, 27 Nov 2019 16:20:36 -0800 Subject: [PATCH 114/178] =?UTF-8?q?Bump=20version:=200.1.2=20=E2=86=92=200?= =?UTF-8?q?.1.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index a9ddf6d0..08e5fbfc 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.1.2 +current_version = 0.1.3 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(-(?P[^.]*)\.(?P\d+))? diff --git a/setup.py b/setup.py index e4d826ef..a7b1c367 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ if not readthedocs_is_building: setup( name="libp2p", # *IMPORTANT*: Don't manually change the version here. Use `make bump`, as described in readme - version="0.1.2", + version="0.1.3", description="libp2p implementation written in python", long_description=long_description, long_description_content_type="text/markdown", From 064c109b647b2f1452ad1e96ceb74503c9261793 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Thu, 28 Nov 2019 18:45:00 +0800 Subject: [PATCH 115/178] Fix signature validator: Add prefix and return verify result --- libp2p/pubsub/pubsub.py | 20 ++++++++++++++++++-- libp2p/pubsub/validators.py | 7 ++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 0fea734f..69f873af 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -49,6 +49,8 @@ SyncValidatorFn = Callable[[ID, rpc_pb2.Message], bool] AsyncValidatorFn = Callable[[ID, rpc_pb2.Message], Awaitable[bool]] ValidatorFn = Union[SyncValidatorFn, AsyncValidatorFn] +PUBSUB_SIGNING_PREFIX = "libp2p-pubsub:" + class TopicValidator(NamedTuple): validator: ValidatorFn @@ -475,7 +477,9 @@ class Pubsub: if self.strict_signing: priv_key = self.sign_key - signature = priv_key.sign(msg.SerializeToString()) + signature = priv_key.sign( + PUBSUB_SIGNING_PREFIX.encode() + msg.SerializeToString() + ) msg.key = self.host.get_public_key().serialize() msg.signature = signature @@ -536,7 +540,19 @@ class Pubsub: logger.debug("Reject because no signature attached for msg: %s", msg) return # Validate the signature of the message - if not signature_validator(deserialize_public_key(msg.key), msg): + # First, construct the original payload that's signed by 'msg.key' + msg_without_key_sig = rpc_pb2.Message( + data=msg.data, + topicIDs=msg.topicIDs, + from_id=msg.from_id, + seqno=msg.seqno, + ) + payload = ( + PUBSUB_SIGNING_PREFIX.encode() + msg_without_key_sig.SerializeToString() + ) + if not signature_validator( + deserialize_public_key(msg.key), payload, msg.signature + ): logger.debug("Signature validation failed for msg: %s", msg) return diff --git a/libp2p/pubsub/validators.py b/libp2p/pubsub/validators.py index 0f6e55ee..2683c0e9 100644 --- a/libp2p/pubsub/validators.py +++ b/libp2p/pubsub/validators.py @@ -1,9 +1,7 @@ from libp2p.crypto.keys import PublicKey -from .pb import rpc_pb2 - -def signature_validator(pubkey: PublicKey, msg: rpc_pb2.Message) -> bool: +def signature_validator(pubkey: PublicKey, payload: bytes, signature: bytes) -> bool: """ Verify the message against the given public key. @@ -11,7 +9,6 @@ def signature_validator(pubkey: PublicKey, msg: rpc_pb2.Message) -> bool: :param msg: the message signed. """ try: - pubkey.verify(msg.SerializeToString(), msg.signature) + return pubkey.verify(payload, signature) except Exception: return False - return True From a262b94836c9f5d8f51ecc92239e17d4dae4bbf8 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Fri, 29 Nov 2019 14:12:42 +0800 Subject: [PATCH 116/178] Apply PR feedback: check if signing key and ID match --- libp2p/pubsub/pubsub.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 69f873af..ac08b413 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -539,6 +539,12 @@ class Pubsub: if msg.signature == b"": logger.debug("Reject because no signature attached for msg: %s", msg) return + # Validate if message sender matches message signer, + # i.e., check if `msg.key` matches `msg.from_id` + msg_pubkey = deserialize_public_key(msg.key) + if ID.from_pubkey(msg_pubkey) != msg.from_id: + logger.debug("Reject because signing key does not match sender ID for msg: %s", msg) + return # Validate the signature of the message # First, construct the original payload that's signed by 'msg.key' msg_without_key_sig = rpc_pb2.Message( @@ -551,7 +557,7 @@ class Pubsub: PUBSUB_SIGNING_PREFIX.encode() + msg_without_key_sig.SerializeToString() ) if not signature_validator( - deserialize_public_key(msg.key), payload, msg.signature + msg_pubkey, payload, msg.signature ): logger.debug("Signature validation failed for msg: %s", msg) return From f4e86b117259366e679a81a118d0adb35ca41d68 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Fri, 29 Nov 2019 14:13:07 +0800 Subject: [PATCH 117/178] Add tests for failed signature validation cases --- tests/pubsub/test_pubsub.py | 57 ++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/tests/pubsub/test_pubsub.py b/tests/pubsub/test_pubsub.py index 48ef52bb..80529266 100644 --- a/tests/pubsub/test_pubsub.py +++ b/tests/pubsub/test_pubsub.py @@ -5,6 +5,7 @@ import pytest from libp2p.exceptions import ValidationError from libp2p.peer.id import ID +from libp2p.pubsub.pubsub import PUBSUB_SIGNING_PREFIX from libp2p.pubsub.pb import rpc_pb2 from libp2p.tools.pubsub.utils import make_pubsub_msg from libp2p.tools.utils import connect @@ -514,7 +515,7 @@ async def test_push_msg(pubsubs_fsub, monkeypatch): @pytest.mark.parametrize("num_hosts, is_strict_signing", ((2, True),)) @pytest.mark.asyncio -async def test_strict_signing(pubsubs_fsub, hosts, monkeypatch): +async def test_strict_signing(pubsubs_fsub, hosts): await connect(hosts[0], hosts[1]) await pubsubs_fsub[0].subscribe(TESTING_TOPIC) await pubsubs_fsub[1].subscribe(TESTING_TOPIC) @@ -525,3 +526,57 @@ async def test_strict_signing(pubsubs_fsub, hosts, monkeypatch): assert len(pubsubs_fsub[0].seen_messages) == 1 assert len(pubsubs_fsub[1].seen_messages) == 1 + + +@pytest.mark.parametrize("num_hosts, is_strict_signing", ((2, True),)) +@pytest.mark.asyncio +async def test_strict_signing_failed_validation(pubsubs_fsub, hosts, monkeypatch): + msg = make_pubsub_msg( + origin_id=pubsubs_fsub[0].my_id, + topic_ids=[TESTING_TOPIC], + data=TESTING_DATA, + seqno=b"\x00" * 8, + ) + priv_key = pubsubs_fsub[0].sign_key + signature = priv_key.sign( + PUBSUB_SIGNING_PREFIX.encode() + msg.SerializeToString() + ) + + event = asyncio.Event() + + def _is_msg_seen(msg): + return False + + # Use router publish to check if `push_msg` succeed. + async def router_publish(*args, **kwargs): + # The event will only be set if `push_msg` succeed. + event.set() + + monkeypatch.setattr(pubsubs_fsub[0], "_is_msg_seen", _is_msg_seen) + monkeypatch.setattr(pubsubs_fsub[0].router, "publish", router_publish) + + # Test: no signature attached in `msg` + await pubsubs_fsub[0].push_msg(pubsubs_fsub[0].my_id, msg) + await asyncio.sleep(0.01) + assert not event.is_set() + + # Test: `msg.key` does not match `msg.from_id` + msg.key = hosts[1].get_public_key().serialize() + msg.signature = signature + await pubsubs_fsub[0].push_msg(pubsubs_fsub[0].my_id, msg) + await asyncio.sleep(0.01) + assert not event.is_set() + + # Test: invalid signature + msg.key = hosts[0].get_public_key().serialize() + msg.signature = b"\x12" * 100 + await pubsubs_fsub[0].push_msg(pubsubs_fsub[0].my_id, msg) + await asyncio.sleep(0.01) + assert not event.is_set() + + # Finally, assert the signature indeed will pass validation + msg.key = hosts[0].get_public_key().serialize() + msg.signature = signature + await pubsubs_fsub[0].push_msg(pubsubs_fsub[0].my_id, msg) + await asyncio.sleep(0.01) + assert event.is_set() From 1c54c38ca7a6ceb569687329cd81171f0bad47ee Mon Sep 17 00:00:00 2001 From: NIC619 Date: Fri, 29 Nov 2019 17:24:40 +0800 Subject: [PATCH 118/178] Fix lint and add `signing_strict` to interop tests --- libp2p/pubsub/pubsub.py | 9 +++++---- tests/pubsub/test_pubsub.py | 6 ++---- tests_interop/conftest.py | 26 +++++++++++++++++++++++--- tests_interop/test_pubsub.py | 3 +++ 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index ac08b413..0b4d6059 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -543,7 +543,10 @@ class Pubsub: # i.e., check if `msg.key` matches `msg.from_id` msg_pubkey = deserialize_public_key(msg.key) if ID.from_pubkey(msg_pubkey) != msg.from_id: - logger.debug("Reject because signing key does not match sender ID for msg: %s", msg) + logger.debug( + "Reject because signing key does not match sender ID for msg: %s", + msg, + ) return # Validate the signature of the message # First, construct the original payload that's signed by 'msg.key' @@ -556,9 +559,7 @@ class Pubsub: payload = ( PUBSUB_SIGNING_PREFIX.encode() + msg_without_key_sig.SerializeToString() ) - if not signature_validator( - msg_pubkey, payload, msg.signature - ): + if not signature_validator(msg_pubkey, payload, msg.signature): logger.debug("Signature validation failed for msg: %s", msg) return diff --git a/tests/pubsub/test_pubsub.py b/tests/pubsub/test_pubsub.py index 80529266..01d8ba74 100644 --- a/tests/pubsub/test_pubsub.py +++ b/tests/pubsub/test_pubsub.py @@ -5,8 +5,8 @@ import pytest from libp2p.exceptions import ValidationError from libp2p.peer.id import ID -from libp2p.pubsub.pubsub import PUBSUB_SIGNING_PREFIX from libp2p.pubsub.pb import rpc_pb2 +from libp2p.pubsub.pubsub import PUBSUB_SIGNING_PREFIX from libp2p.tools.pubsub.utils import make_pubsub_msg from libp2p.tools.utils import connect from libp2p.utils import encode_varint_prefixed @@ -538,9 +538,7 @@ async def test_strict_signing_failed_validation(pubsubs_fsub, hosts, monkeypatch seqno=b"\x00" * 8, ) priv_key = pubsubs_fsub[0].sign_key - signature = priv_key.sign( - PUBSUB_SIGNING_PREFIX.encode() + msg.SerializeToString() - ) + signature = priv_key.sign(PUBSUB_SIGNING_PREFIX.encode() + msg.SerializeToString()) event = asyncio.Event() diff --git a/tests_interop/conftest.py b/tests_interop/conftest.py index e75e5190..8ae9769b 100644 --- a/tests_interop/conftest.py +++ b/tests_interop/conftest.py @@ -76,7 +76,24 @@ def is_gossipsub(): @pytest.fixture -async def p2pds(num_p2pds, is_host_secure, is_gossipsub, unused_tcp_port_factory): +def is_pubsub_signing(): + return True + + +@pytest.fixture +def is_pubsub_signing_strict(): + return True + + +@pytest.fixture +async def p2pds( + num_p2pds, + is_host_secure, + is_gossipsub, + unused_tcp_port_factory, + is_pubsub_signing, + is_pubsub_signing_strict, +): p2pds: Union[Daemon, Exception] = await asyncio.gather( *[ make_p2pd( @@ -84,6 +101,8 @@ async def p2pds(num_p2pds, is_host_secure, is_gossipsub, unused_tcp_port_factory unused_tcp_port_factory(), is_host_secure, is_gossipsub=is_gossipsub, + is_pubsub_signing=is_pubsub_signing, + is_pubsub_signing_strict=is_pubsub_signing_strict, ) for _ in range(num_p2pds) ], @@ -102,13 +121,14 @@ async def p2pds(num_p2pds, is_host_secure, is_gossipsub, unused_tcp_port_factory @pytest.fixture -def pubsubs(num_hosts, hosts, is_gossipsub): +def pubsubs(num_hosts, hosts, is_gossipsub, is_pubsub_signing_strict): if is_gossipsub: routers = GossipsubFactory.create_batch(num_hosts, **GOSSIPSUB_PARAMS._asdict()) else: routers = FloodsubFactory.create_batch(num_hosts) _pubsubs = tuple( - PubsubFactory(host=host, router=router) for host, router in zip(hosts, routers) + PubsubFactory(host=host, router=router, strict_signing=is_pubsub_signing_strict) + for host, router in zip(hosts, routers) ) yield _pubsubs # TODO: Clean up diff --git a/tests_interop/test_pubsub.py b/tests_interop/test_pubsub.py index 4e845d7e..f67b47ab 100644 --- a/tests_interop/test_pubsub.py +++ b/tests_interop/test_pubsub.py @@ -55,6 +55,9 @@ def validate_pubsub_msg(msg: rpc_pb2.Message, data: bytes, from_peer_id: ID) -> assert msg.data == data and msg.from_id == from_peer_id +@pytest.mark.parametrize( + "is_pubsub_signing, is_pubsub_signing_strict", ((True, True), (False, False)) +) @pytest.mark.parametrize("is_gossipsub", (True, False)) @pytest.mark.parametrize("num_hosts, num_p2pds", ((1, 2),)) @pytest.mark.asyncio From 658a0ae156454c61f7aff6c98459448c7ce4a639 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Fri, 29 Nov 2019 19:37:48 +0800 Subject: [PATCH 119/178] Apply PR feedback: move signature validation logic into signature validator --- libp2p/pubsub/pubsub.py | 31 +++---------------------------- libp2p/pubsub/validators.py | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 0b4d6059..fab8024b 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -17,7 +17,6 @@ import base58 from lru import LRU from libp2p.crypto.keys import PrivateKey -from libp2p.crypto.serialization import deserialize_public_key from libp2p.exceptions import ParseError, ValidationError from libp2p.host.host_interface import IHost from libp2p.io.exceptions import IncompleteReadError @@ -30,7 +29,7 @@ from libp2p.utils import encode_varint_prefixed, read_varint_prefixed_bytes from .pb import rpc_pb2 from .pubsub_notifee import PubsubNotifee -from .validators import signature_validator +from .validators import PUBSUB_SIGNING_PREFIX, signature_validator if TYPE_CHECKING: from .pubsub_router_interface import IPubsubRouter # noqa: F401 @@ -49,8 +48,6 @@ SyncValidatorFn = Callable[[ID, rpc_pb2.Message], bool] AsyncValidatorFn = Callable[[ID, rpc_pb2.Message], Awaitable[bool]] ValidatorFn = Union[SyncValidatorFn, AsyncValidatorFn] -PUBSUB_SIGNING_PREFIX = "libp2p-pubsub:" - class TopicValidator(NamedTuple): validator: ValidatorFn @@ -534,32 +531,10 @@ class Pubsub: if self._is_msg_seen(msg): return - # Check if signing is required and if so signature should be attached. + # Check if signing is required and if so validate the signature if self.strict_signing: - if msg.signature == b"": - logger.debug("Reject because no signature attached for msg: %s", msg) - return - # Validate if message sender matches message signer, - # i.e., check if `msg.key` matches `msg.from_id` - msg_pubkey = deserialize_public_key(msg.key) - if ID.from_pubkey(msg_pubkey) != msg.from_id: - logger.debug( - "Reject because signing key does not match sender ID for msg: %s", - msg, - ) - return # Validate the signature of the message - # First, construct the original payload that's signed by 'msg.key' - msg_without_key_sig = rpc_pb2.Message( - data=msg.data, - topicIDs=msg.topicIDs, - from_id=msg.from_id, - seqno=msg.seqno, - ) - payload = ( - PUBSUB_SIGNING_PREFIX.encode() + msg_without_key_sig.SerializeToString() - ) - if not signature_validator(msg_pubkey, payload, msg.signature): + if not signature_validator(msg): logger.debug("Signature validation failed for msg: %s", msg) return diff --git a/libp2p/pubsub/validators.py b/libp2p/pubsub/validators.py index 2683c0e9..22f6579a 100644 --- a/libp2p/pubsub/validators.py +++ b/libp2p/pubsub/validators.py @@ -1,14 +1,41 @@ -from libp2p.crypto.keys import PublicKey +import logging + +from libp2p.crypto.serialization import deserialize_public_key +from libp2p.peer.id import ID + +from .pb import rpc_pb2 + +logger = logging.getLogger("libp2p.pubsub") + +PUBSUB_SIGNING_PREFIX = "libp2p-pubsub:" -def signature_validator(pubkey: PublicKey, payload: bytes, signature: bytes) -> bool: +def signature_validator(msg: rpc_pb2.Message) -> bool: """ Verify the message against the given public key. :param pubkey: the public key which signs the message. :param msg: the message signed. """ + # Check if signature is attached + if msg.signature == b"": + logger.debug("Reject because no signature attached for msg: %s", msg) + return False + + # Validate if message sender matches message signer, + # i.e., check if `msg.key` matches `msg.from_id` + msg_pubkey = deserialize_public_key(msg.key) + if ID.from_pubkey(msg_pubkey) != msg.from_id: + logger.debug( + "Reject because signing key does not match sender ID for msg: %s", msg + ) + return False + # First, construct the original payload that's signed by 'msg.key' + msg_without_key_sig = rpc_pb2.Message( + data=msg.data, topicIDs=msg.topicIDs, from_id=msg.from_id, seqno=msg.seqno + ) + payload = PUBSUB_SIGNING_PREFIX.encode() + msg_without_key_sig.SerializeToString() try: - return pubkey.verify(payload, signature) + return msg_pubkey.verify(payload, msg.signature) except Exception: return False From e59ac6a250ee44d24139c693ed1dd897f753d4ad Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sat, 30 Nov 2019 17:12:37 +0800 Subject: [PATCH 120/178] Cleanup TODOs in pubsub --- libp2p/pubsub/pubsub.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index fab8024b..8a85a894 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -78,7 +78,6 @@ class Pubsub: topic_validators: Dict[str, TopicValidator] - # TODO: Be sure it is increased atomically everytime. counter: int # uint64 _tasks: List["asyncio.Future[Any]"] @@ -300,7 +299,6 @@ class Pubsub: logger.debug("Fail to add new peer %s: stream closed", peer_id) del self.peers[peer_id] return - # TODO: Check EOF of this stream. # TODO: Check if the peer in black list. try: self.router.add_peer(peer_id, stream.get_protocol()) @@ -328,7 +326,6 @@ class Pubsub: """ Continuously read from peer queue and each time a new peer is found, open a stream to the peer using a supported pubsub protocol - TODO: Handle failure for when the peer does not support any of the pubsub protocols we support """ while True: From 0a52a053758213a682e0d059289e1a1b6027b046 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sat, 30 Nov 2019 20:02:11 +0800 Subject: [PATCH 121/178] Del entry if no more peers subscribe to the topic --- libp2p/pubsub/pubsub.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 8a85a894..a3827f9f 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -316,7 +316,11 @@ class Pubsub: for topic in self.peer_topics: if peer_id in self.peer_topics[topic]: - self.peer_topics[topic].remove(peer_id) + # Delete the entry if no other peers left + if len(self.peer_topics[topic]) == 1: + del self.peer_topics[topic] + else: + self.peer_topics[topic].remove(peer_id) self.router.remove_peer(peer_id) @@ -361,7 +365,11 @@ class Pubsub: else: if sub_message.topicid in self.peer_topics: if origin_id in self.peer_topics[sub_message.topicid]: - self.peer_topics[sub_message.topicid].remove(origin_id) + # Delete the entry if no other peers left + if len(self.peer_topics[sub_message.topicid]) == 1: + del self.peer_topics[sub_message.topicid] + else: + self.peer_topics[sub_message.topicid].remove(origin_id) # FIXME(mhchia): Change the function name? async def handle_talk(self, publish_message: rpc_pb2.Message) -> None: From 50fd0acf41f929eb50c0f3b2ba672e9ece876227 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Sat, 30 Nov 2019 20:19:17 +0800 Subject: [PATCH 122/178] Cleanup outdated TODOs in gossipsub --- libp2p/pubsub/gossipsub.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index d09db76e..ea2594e0 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -106,7 +106,8 @@ class GossipSub(IPubsubRouter): logger.debug("attached to pusub") # Start heartbeat now that we have a pubsub instance - # TODO: Start after delay + # Start after a delay. Ref: https://github.com/libp2p/go-libp2p-pubsub/blob/01b9825fbee1848751d90a8469e3f5f43bac8466/gossipsub.go#L410 # Noqa: E501 + await asyncio.sleep(0.1) asyncio.ensure_future(self.heartbeat()) def add_peer(self, peer_id: ID, protocol_id: TProtocol) -> None: @@ -127,8 +128,7 @@ class GossipSub(IPubsubRouter): # instance in multistream-select, but it is not the protocol that gossipsub supports. # In this case, probably we registered gossipsub to a wrong `protocol_id` # in multistream-select, or wrong versions. - # TODO: Better handling - raise Exception(f"protocol is not supported: protocol_id={protocol_id}") + raise Exception(f"This should not happen. Protocol={protocol_id} is not supported.") self.peers_to_protocol[peer_id] = protocol_id def remove_peer(self, peer_id: ID) -> None: @@ -187,7 +187,6 @@ class GossipSub(IPubsubRouter): stream = self.pubsub.peers[peer_id] # FIXME: We should add a `WriteMsg` similar to write delimited messages. # Ref: https://github.com/libp2p/go-libp2p-pubsub/blob/master/comm.go#L107 - # TODO: Go use `sendRPC`, which possibly piggybacks gossip/control messages. try: await stream.write(encode_varint_prefixed(rpc_msg.SerializeToString())) except StreamClosed: @@ -218,15 +217,9 @@ class GossipSub(IPubsubRouter): # gossipsub peers in_topic_gossipsub_peers: List[ID] = None - # TODO: Do we need to check `topic in self.pubsub.my_topics`? if topic in self.mesh: in_topic_gossipsub_peers = self.mesh[topic] else: - # TODO(robzajac): Is topic DEFINITELY supposed to be in fanout if we are not - # subscribed? - # I assume there could be short periods between heartbeats where topic may not - # be but we should check that this path gets hit appropriately - if (topic not in self.fanout) or (len(self.fanout[topic]) == 0): # If no peers in fanout, choose some peers from gossipsub peers in topic. self.fanout[topic] = self._get_in_topic_gossipsub_peers_from_minus( @@ -373,7 +366,6 @@ class GossipSub(IPubsubRouter): for topic in self.mesh: msg_ids = self.mcache.window(topic) if msg_ids: - # TODO: Make more efficient, possibly using a generator? # Get all pubsub peers in a topic and only add them if they are gossipsub peers too if topic in self.pubsub.peer_topics: # Select D peers from peers.gossipsub[topic] @@ -397,7 +389,6 @@ class GossipSub(IPubsubRouter): if topic not in self.mesh: msg_ids = self.mcache.window(topic) if msg_ids: - # TODO: Make more efficient, possibly using a generator? # Get all pubsub peers in topic and only add if they are gossipsub peers also if topic in self.pubsub.peer_topics: # Select D peers from peers.gossipsub[topic] @@ -409,7 +400,6 @@ class GossipSub(IPubsubRouter): peer not in self.mesh[topic] and peer not in self.fanout[topic] ): - msg_id_strs = [str(msg) for msg in msg_ids] await self.emit_ihave(topic, msg_id_strs, peer) From 0672f5ae6dca04a07731fb4bedbd423619067bb2 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Mon, 2 Dec 2019 16:38:48 +0800 Subject: [PATCH 123/178] Fix: move heartbeat delay to `heartbeat` --- libp2p/pubsub/gossipsub.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index ea2594e0..6d88f874 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -106,8 +106,6 @@ class GossipSub(IPubsubRouter): logger.debug("attached to pusub") # Start heartbeat now that we have a pubsub instance - # Start after a delay. Ref: https://github.com/libp2p/go-libp2p-pubsub/blob/01b9825fbee1848751d90a8469e3f5f43bac8466/gossipsub.go#L410 # Noqa: E501 - await asyncio.sleep(0.1) asyncio.ensure_future(self.heartbeat()) def add_peer(self, peer_id: ID, protocol_id: TProtocol) -> None: @@ -295,6 +293,8 @@ class GossipSub(IPubsubRouter): Note: the heartbeats are called with awaits because each heartbeat depends on the state changes in the preceding heartbeat """ + # Start after a delay. Ref: https://github.com/libp2p/go-libp2p-pubsub/blob/01b9825fbee1848751d90a8469e3f5f43bac8466/gossipsub.go#L410 # Noqa: E501 + await asyncio.sleep(0.1) while True: await self.mesh_heartbeat() From c2d88962c7751d67b13c4d658395e498239dac03 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Mon, 2 Dec 2019 16:55:16 +0800 Subject: [PATCH 124/178] Add gossipsub `heartbeat_initial_delay` --- libp2p/pubsub/gossipsub.py | 5 ++++- libp2p/tools/constants.py | 1 + libp2p/tools/factories.py | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 6d88f874..7fbcfdfb 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -43,6 +43,7 @@ class GossipSub(IPubsubRouter): mcache: MessageCache + heartbeat_initial_delay: float heartbeat_interval: int def __init__( @@ -54,6 +55,7 @@ class GossipSub(IPubsubRouter): time_to_live: int, gossip_window: int = 3, gossip_history: int = 5, + heartbeat_initial_delay: int = 0.1, heartbeat_interval: int = 120, ) -> None: self.protocols = list(protocols) @@ -84,6 +86,7 @@ class GossipSub(IPubsubRouter): self.mcache = MessageCache(gossip_window, gossip_history) # Create heartbeat timer + self.heartbeat_initial_delay = heartbeat_initial_delay self.heartbeat_interval = heartbeat_interval # Interface functions @@ -294,7 +297,7 @@ class GossipSub(IPubsubRouter): state changes in the preceding heartbeat """ # Start after a delay. Ref: https://github.com/libp2p/go-libp2p-pubsub/blob/01b9825fbee1848751d90a8469e3f5f43bac8466/gossipsub.go#L410 # Noqa: E501 - await asyncio.sleep(0.1) + await asyncio.sleep(self.heartbeat_initial_delay) while True: await self.mesh_heartbeat() diff --git a/libp2p/tools/constants.py b/libp2p/tools/constants.py index 34dade46..2f132bb0 100644 --- a/libp2p/tools/constants.py +++ b/libp2p/tools/constants.py @@ -24,6 +24,7 @@ class GossipsubParams(NamedTuple): time_to_live: int = 30 gossip_window: int = 3 gossip_history: int = 5 + heartbeat_initial_delay: int = 0.1 heartbeat_interval: float = 0.5 diff --git a/libp2p/tools/factories.py b/libp2p/tools/factories.py index b189cfa1..1b98eaa6 100644 --- a/libp2p/tools/factories.py +++ b/libp2p/tools/factories.py @@ -142,6 +142,7 @@ class GossipsubFactory(factory.Factory): time_to_live = GOSSIPSUB_PARAMS.time_to_live gossip_window = GOSSIPSUB_PARAMS.gossip_window gossip_history = GOSSIPSUB_PARAMS.gossip_history + heartbeat_initial_delay = GOSSIPSUB_PARAMS.heartbeat_initial_delay heartbeat_interval = GOSSIPSUB_PARAMS.heartbeat_interval From 357341e0d82b3ab0d3c0e4c3ffe11fe2f758ccd2 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Mon, 2 Dec 2019 22:40:35 +0800 Subject: [PATCH 125/178] Remove unneccessary filter and check in gossipsub --- libp2p/pubsub/gossipsub.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 7fbcfdfb..78e55040 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -320,10 +320,7 @@ class GossipSub(IPubsubRouter): topic, self.degree - num_mesh_peers_in_topic, self.mesh[topic] ) - fanout_peers_not_in_mesh: List[ID] = [ - peer for peer in selected_peers if peer not in self.mesh[topic] - ] - for peer in fanout_peers_not_in_mesh: + for peer in selected_peers: # Add peer to mesh[topic] self.mesh[topic].append(peer) @@ -377,12 +374,7 @@ class GossipSub(IPubsubRouter): ) for peer in peers_to_emit_ihave_to: - # TODO: this line is a monster, can hopefully be simplified - if ( - topic not in self.mesh or (peer not in self.mesh[topic]) - ) and ( - topic not in self.fanout or (peer not in self.fanout[topic]) - ): + if peer not in self.mesh[topic]: msg_id_strs = [str(msg_id) for msg_id in msg_ids] await self.emit_ihave(topic, msg_id_strs, peer) @@ -399,10 +391,7 @@ class GossipSub(IPubsubRouter): topic, self.degree, [] ) for peer in peers_to_emit_ihave_to: - if ( - peer not in self.mesh[topic] - and peer not in self.fanout[topic] - ): + if peer not in self.fanout[topic]: msg_id_strs = [str(msg) for msg in msg_ids] await self.emit_ihave(topic, msg_id_strs, peer) From a7e0c5d7378a9bfd28ac3f1431ab3cb851b31b6d Mon Sep 17 00:00:00 2001 From: NIC619 Date: Mon, 2 Dec 2019 22:41:49 +0800 Subject: [PATCH 126/178] Add missing cleanup in gossipsub `remove_peer` --- libp2p/pubsub/gossipsub.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 78e55040..39397003 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -145,6 +145,21 @@ class GossipSub(IPubsubRouter): elif peer_id in self.peers_floodsub: self.peers_floodsub.remove(peer_id) + for topic in self.mesh: + if peer_id in self.mesh[topic]: + # Delete the entry if no other peers left + if len(self.mesh[topic]) == 1: + del self.mesh[topic] + else: + self.mesh[topic].remove(peer_id) + for topic in self.fanout: + if peer_id in self.fanout[topic]: + # Delete the entry if no other peers left + if len(self.fanout[topic]) == 1: + del self.fanout[topic] + else: + self.fanout[topic].remove(peer_id) + self.peers_to_protocol.pop(peer_id, None) async def handle_rpc(self, rpc: rpc_pb2.RPC, sender_peer_id: ID) -> None: From 920cf646efe8cb979b55c96979008a08f368da4e Mon Sep 17 00:00:00 2001 From: NIC619 Date: Mon, 2 Dec 2019 22:49:27 +0800 Subject: [PATCH 127/178] Fix lint and add check in fanout heartbeat --- libp2p/pubsub/gossipsub.py | 19 +++++++++++++++---- libp2p/pubsub/pubsub.py | 8 +++----- libp2p/tools/constants.py | 2 +- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 39397003..831e6d3e 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -55,7 +55,7 @@ class GossipSub(IPubsubRouter): time_to_live: int, gossip_window: int = 3, gossip_history: int = 5, - heartbeat_initial_delay: int = 0.1, + heartbeat_initial_delay: float = 0.1, heartbeat_interval: int = 120, ) -> None: self.protocols = list(protocols) @@ -129,7 +129,9 @@ class GossipSub(IPubsubRouter): # instance in multistream-select, but it is not the protocol that gossipsub supports. # In this case, probably we registered gossipsub to a wrong `protocol_id` # in multistream-select, or wrong versions. - raise Exception(f"This should not happen. Protocol={protocol_id} is not supported.") + raise Exception( + f"This should not happen. Protocol={protocol_id} is not supported." + ) self.peers_to_protocol[peer_id] = protocol_id def remove_peer(self, peer_id: ID) -> None: @@ -357,13 +359,22 @@ class GossipSub(IPubsubRouter): async def fanout_heartbeat(self) -> None: # Note: the comments here are the exact pseudocode from the spec for topic in self.fanout: - # If time since last published > ttl + # Delete topic entry if it's not in `pubsub.peer_topics` + # or if it's time-since-last-published > ttl # TODO: there's no way time_since_last_publish gets set anywhere yet - if self.time_since_last_publish[topic] > self.time_to_live: + if ( + topic not in self.pubsub.peer_topics + or self.time_since_last_publish[topic] > self.time_to_live + ): # Remove topic from fanout del self.fanout[topic] del self.time_since_last_publish[topic] else: + # Check whether our peers are still in the topic + # ref: https://github.com/libp2p/go-libp2p-pubsub/blob/01b9825fbee1848751d90a8469e3f5f43bac8466/gossipsub.go#L498-L504 # noqa: E501 + for peer in self.fanout[topic]: + if peer not in self.pubsub.peer_topics[topic]: + self.fanout[topic].remove(peer) num_fanout_peers_in_topic = len(self.fanout[topic]) # If |fanout[topic]| < D diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index a3827f9f..45e7ffca 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -327,11 +327,9 @@ class Pubsub: logger.debug("removed dead peer %s", peer_id) async def handle_peer_queue(self) -> None: - """ - Continuously read from peer queue and each time a new peer is found, - open a stream to the peer using a supported pubsub protocol - pubsub protocols we support - """ + """Continuously read from peer queue and each time a new peer is found, + open a stream to the peer using a supported pubsub protocol pubsub + protocols we support.""" while True: peer_id: ID = await self.peer_queue.get() # Add Peer diff --git a/libp2p/tools/constants.py b/libp2p/tools/constants.py index 2f132bb0..8c22d151 100644 --- a/libp2p/tools/constants.py +++ b/libp2p/tools/constants.py @@ -24,7 +24,7 @@ class GossipsubParams(NamedTuple): time_to_live: int = 30 gossip_window: int = 3 gossip_history: int = 5 - heartbeat_initial_delay: int = 0.1 + heartbeat_initial_delay: float = 0.1 heartbeat_interval: float = 0.5 From 63fd531ed031e883f64f15fa94df1f051c06d45d Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Mon, 2 Dec 2019 15:57:22 -0800 Subject: [PATCH 128/178] Fixes to add python 3.6 compatibility --- libp2p/network/connection/raw_connection.py | 4 +- libp2p/pubsub/pubsub.py | 2 +- libp2p/tools/factories.py | 7 +- .../floodsub_integration_test_settings.py | 72 ++++++++----------- libp2p/transport/tcp/tcp.py | 4 +- setup.py | 4 +- tests/pubsub/test_pubsub.py | 4 +- tests/security/test_secio.py | 4 +- tests_interop/conftest.py | 6 +- 9 files changed, 54 insertions(+), 53 deletions(-) diff --git a/libp2p/network/connection/raw_connection.py b/libp2p/network/connection/raw_connection.py index 08d22055..55b47efc 100644 --- a/libp2p/network/connection/raw_connection.py +++ b/libp2p/network/connection/raw_connection.py @@ -1,4 +1,5 @@ import asyncio +import sys from .exceptions import RawConnError from .raw_connection_interface import IRawConnection @@ -52,4 +53,5 @@ class RawConnection(IRawConnection): async def close(self) -> None: self.writer.close() - await self.writer.wait_closed() + if sys.version_info[0:2] > (3, 6): + await self.writer.wait_closed() diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index fab8024b..5233a32d 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -150,7 +150,7 @@ class Pubsub: # Map of topic to topic validator self.topic_validators = {} - self.counter = time.time_ns() + self.counter = int(time.time()) self._tasks = [] # Call handle peer to keep waiting for updates to peer queue diff --git a/libp2p/tools/factories.py b/libp2p/tools/factories.py index b189cfa1..5e223130 100644 --- a/libp2p/tools/factories.py +++ b/libp2p/tools/factories.py @@ -1,5 +1,4 @@ import asyncio -from contextlib import asynccontextmanager from typing import Any, AsyncIterator, Dict, Tuple, cast import factory @@ -33,6 +32,12 @@ from .constants import ( ) from .utils import connect, connect_swarm +try: + from contextlib import asynccontextmanager +except ImportError: + # NOTE: mypy complains about a duplicate import without the following ``# type: ignore`` + from async_generator import asynccontextmanager # type: ignore + def initialize_peerstore_with_our_keypair(self_id: ID, key_pair: KeyPair) -> PeerStore: peer_store = PeerStore() diff --git a/libp2p/tools/pubsub/floodsub_integration_test_settings.py b/libp2p/tools/pubsub/floodsub_integration_test_settings.py index b7b66244..d6b5b678 100644 --- a/libp2p/tools/pubsub/floodsub_integration_test_settings.py +++ b/libp2p/tools/pubsub/floodsub_integration_test_settings.py @@ -143,6 +143,14 @@ floodsub_protocol_pytest_params = [ ] +def _collect_node_ids(adj_list): + node_ids = set() + for node, neighbors in adj_list.items(): + node_ids.add(node) + node_ids.update(set(neighbors)) + return node_ids + + async def perform_test_from_obj(obj, router_factory) -> None: """ Perform pubsub tests from a test object, which is composed as follows: @@ -180,59 +188,43 @@ async def perform_test_from_obj(obj, router_factory) -> None: node_map = {} pubsub_map = {} - async def add_node(node_id_str: str) -> None: + async def add_node(node_id_str: str): pubsub_router = router_factory(protocols=obj["supported_protocols"]) pubsub = PubsubFactory(router=pubsub_router) await pubsub.host.get_network().listen(LISTEN_MADDR) node_map[node_id_str] = pubsub.host pubsub_map[node_id_str] = pubsub - tasks_connect = [] - for start_node_id in adj_list: - # Create node if node does not yet exist - if start_node_id not in node_map: - await add_node(start_node_id) + all_node_ids = _collect_node_ids(adj_list) - # For each neighbor of start_node, create if does not yet exist, - # then connect start_node to neighbor - for neighbor_id in adj_list[start_node_id]: - # Create neighbor if neighbor does not yet exist - if neighbor_id not in node_map: - await add_node(neighbor_id) - tasks_connect.append( - connect(node_map[start_node_id], node_map[neighbor_id]) - ) - # Connect nodes and wait at least for 2 seconds - await asyncio.gather(*tasks_connect, asyncio.sleep(2)) + for node in all_node_ids: + await add_node(node) + + for node, neighbors in adj_list.items(): + for neighbor_id in neighbors: + await connect(node_map[node], node_map[neighbor_id]) + + # NOTE: the test using this routine will fail w/o these sleeps... + await asyncio.sleep(1) # Step 2) Subscribe to topics queues_map = {} topic_map = obj["topic_map"] - tasks_topic = [] - tasks_topic_data = [] for topic, node_ids in topic_map.items(): for node_id in node_ids: - tasks_topic.append(pubsub_map[node_id].subscribe(topic)) - tasks_topic_data.append((node_id, topic)) - tasks_topic.append(asyncio.sleep(2)) + queue = await pubsub_map[node_id].subscribe(topic) + if node_id not in queues_map: + queues_map[node_id] = {} + # Store queue in topic-queue map for node + queues_map[node_id][topic] = queue - # Gather is like Promise.all - responses = await asyncio.gather(*tasks_topic) - for i in range(len(responses) - 1): - node_id, topic = tasks_topic_data[i] - if node_id not in queues_map: - queues_map[node_id] = {} - # Store queue in topic-queue map for node - queues_map[node_id][topic] = responses[i] - - # Allow time for subscribing before continuing - await asyncio.sleep(0.01) + # NOTE: the test using this routine will fail w/o these sleeps... + await asyncio.sleep(1) # Step 3) Publish messages topics_in_msgs_ordered = [] messages = obj["messages"] - tasks_publish = [] for msg in messages: topics = msg["topics"] @@ -242,21 +234,17 @@ async def perform_test_from_obj(obj, router_factory) -> None: # Publish message # TODO: Should be single RPC package with several topics for topic in topics: - tasks_publish.append(pubsub_map[node_id].publish(topic, data)) - - # For each topic in topics, add (topic, node_id, data) tuple to ordered test list - for topic in topics: + await pubsub_map[node_id].publish(topic, data) + # For each topic in topics, add (topic, node_id, data) tuple to ordered test list topics_in_msgs_ordered.append((topic, node_id, data)) - # Allow time for publishing before continuing - await asyncio.gather(*tasks_publish, asyncio.sleep(2)) - # Step 4) Check that all messages were received correctly. for topic, origin_node_id, data in topics_in_msgs_ordered: # Look at each node in each topic for node_id in topic_map[topic]: # Get message from subscription queue - msg = await queues_map[node_id][topic].get() + queue = queues_map[node_id][topic] + msg = await queue.get() assert data == msg.data # Check the message origin assert node_map[origin_node_id].get_id().to_bytes() == msg.from_id diff --git a/libp2p/transport/tcp/tcp.py b/libp2p/transport/tcp/tcp.py index 7470510d..c27de0e1 100644 --- a/libp2p/transport/tcp/tcp.py +++ b/libp2p/transport/tcp/tcp.py @@ -1,5 +1,6 @@ import asyncio from socket import socket +import sys from typing import List from multiaddr import Multiaddr @@ -53,7 +54,8 @@ class TCPListener(IListener): if self.server is None: return self.server.close() - await self.server.wait_closed() + if sys.version_info[0:2] > (3, 6): + await self.server.wait_closed() self.server = None diff --git a/setup.py b/setup.py index a7b1c367..35b17bf8 100644 --- a/setup.py +++ b/setup.py @@ -58,6 +58,8 @@ install_requires = [ "protobuf>=3.10.0,<4.0.0", "coincurve>=10.0.0,<11.0.0", "pynacl==1.3.0", + "dataclasses>=0.7, <1;python_version<'3.7'", + "async_generator==1.10;python_version<'3.7'", ] @@ -80,7 +82,7 @@ setup( url="https://github.com/libp2p/py-libp2p", include_package_data=True, install_requires=install_requires, - python_requires=">=3.7,<4", + python_requires=">=3.6,<4", extras_require=extras_require, py_modules=["libp2p"], license="MIT/APACHE2.0", diff --git a/tests/pubsub/test_pubsub.py b/tests/pubsub/test_pubsub.py index 01d8ba74..6f3c6725 100644 --- a/tests/pubsub/test_pubsub.py +++ b/tests/pubsub/test_pubsub.py @@ -58,11 +58,11 @@ async def test_peers_subscribe(pubsubs_fsub): await connect(pubsubs_fsub[0].host, pubsubs_fsub[1].host) await pubsubs_fsub[0].subscribe(TESTING_TOPIC) # Yield to let 0 notify 1 - await asyncio.sleep(0.1) + await asyncio.sleep(1) assert pubsubs_fsub[0].my_id in pubsubs_fsub[1].peer_topics[TESTING_TOPIC] await pubsubs_fsub[0].unsubscribe(TESTING_TOPIC) # Yield to let 0 notify 1 - await asyncio.sleep(0.1) + await asyncio.sleep(1) assert pubsubs_fsub[0].my_id not in pubsubs_fsub[1].peer_topics[TESTING_TOPIC] diff --git a/tests/security/test_secio.py b/tests/security/test_secio.py index c7808b46..50374809 100644 --- a/tests/security/test_secio.py +++ b/tests/security/test_secio.py @@ -76,8 +76,8 @@ async def test_create_secure_session(): local_conn = InMemoryConnection(local_peer, is_initiator=True) remote_conn = InMemoryConnection(remote_peer) - local_pipe_task = asyncio.create_task(create_pipe(local_conn, remote_conn)) - remote_pipe_task = asyncio.create_task(create_pipe(remote_conn, local_conn)) + local_pipe_task = asyncio.ensure_future(create_pipe(local_conn, remote_conn)) + remote_pipe_task = asyncio.ensure_future(create_pipe(remote_conn, local_conn)) local_session_builder = create_secure_session( local_nonce, local_peer, local_key_pair.private_key, local_conn, remote_peer diff --git a/tests_interop/conftest.py b/tests_interop/conftest.py index 8ae9769b..3fcc74f3 100644 --- a/tests_interop/conftest.py +++ b/tests_interop/conftest.py @@ -151,7 +151,8 @@ class DaemonStream(ReadWriteCloser): async def close(self) -> None: self.writer.close() - await self.writer.wait_closed() + if sys.version_info[0:2] > (3, 6): + await self.writer.wait_closed() async def read(self, n: int = -1) -> bytes: return await self.reader.read(n) @@ -196,7 +197,8 @@ async def py_to_daemon_stream_pair(hosts, p2pds, is_to_fail_daemon_stream): # some day. listener = p2pds[0].control.control.listener listener.close() - await listener.wait_closed() + if sys.version_info[0:2] > (3, 6): + await listener.wait_closed() stream_py = await host.new_stream(p2pd.peer_id, [protocol_id]) if not is_to_fail_daemon_stream: await event_stream_handled.wait() From 4c0f511516632975b706f106cfe5cbb88a4e6061 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Mon, 2 Dec 2019 16:00:56 -0800 Subject: [PATCH 129/178] Add `py36` tox env for testing --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index de4f8f1b..a482e70f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,9 @@ # Reference: https://github.com/ethereum/ethereum-python-project-template/blob/master/tox.ini -# TODO: consider py36 and pypy3 support +# TODO: consider pypy3 support [tox] envlist = + py36-test py37-test py37-interop lint @@ -37,6 +38,7 @@ commands = basepython = docs: python py37: python3.7 + py36: python3.6 extras = test docs: doc From a1fd106bf3e74ce0e6508a8bcb73cb433dab4a50 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Mon, 2 Dec 2019 16:01:06 -0800 Subject: [PATCH 130/178] Ensure correct names of test envs in circle ci --- .circleci/config.yml | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fa7691e6..c3094b9b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,30 +48,24 @@ jobs: - image: circleci/python:3.6 environment: TOXENV: lint - py36-core: + py36-test: <<: *common docker: - image: circleci/python:3.6 environment: - TOXENV: py36-core - py37-core: + TOXENV: py36-test + py37-test: <<: *common docker: - image: circleci/python:3.7 environment: - TOXENV: py37-core - pypy3-core: - <<: *common - docker: - - image: pypy - environment: - TOXENV: pypy3-core + TOXENV: py37-test + workflows: version: 2 test: jobs: - docs - lint - - py36-core - - py37-core - - pypy3-core + - py36-test + - py37-test From d516cf51b814aeb17a303e422b7dc6e35a234e3f Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Mon, 2 Dec 2019 16:05:28 -0800 Subject: [PATCH 131/178] Add py3.6 to travis config --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index ce9b061e..6b6b3ccc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,9 @@ language: python matrix: include: + - python: 3.6-dev + dist: xenial + env: TOXENV=py36-test - python: 3.7-dev dist: xenial env: TOXENV=py37-test From ab1500c7084ba377e76fa4bd97a19ad03a6922ff Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 3 Dec 2019 15:03:06 +0800 Subject: [PATCH 132/178] Remove unneccessary check in gossip heartbeat --- libp2p/pubsub/gossipsub.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 831e6d3e..cf48f53a 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -396,13 +396,12 @@ class GossipSub(IPubsubRouter): if topic in self.pubsub.peer_topics: # Select D peers from peers.gossipsub[topic] peers_to_emit_ihave_to = self._get_in_topic_gossipsub_peers_from_minus( - topic, self.degree, [] + topic, self.degree, self.mesh[topic] ) + msg_id_strs = [str(msg_id) for msg_id in msg_ids] for peer in peers_to_emit_ihave_to: - if peer not in self.mesh[topic]: - msg_id_strs = [str(msg_id) for msg_id in msg_ids] - await self.emit_ihave(topic, msg_id_strs, peer) + await self.emit_ihave(topic, msg_id_strs, peer) # TODO: Refactor and Dedup. This section is the roughly the same as the above. # Do the same for fanout, for all topics not already hit in mesh @@ -414,12 +413,11 @@ class GossipSub(IPubsubRouter): if topic in self.pubsub.peer_topics: # Select D peers from peers.gossipsub[topic] peers_to_emit_ihave_to = self._get_in_topic_gossipsub_peers_from_minus( - topic, self.degree, [] + topic, self.degree, self.fanout[topic] ) + msg_id_strs = [str(msg) for msg in msg_ids] for peer in peers_to_emit_ihave_to: - if peer not in self.fanout[topic]: - msg_id_strs = [str(msg) for msg in msg_ids] - await self.emit_ihave(topic, msg_id_strs, peer) + await self.emit_ihave(topic, msg_id_strs, peer) self.mcache.shift() From 5efdf4c7036cacce8a2c49fbc67534c721801845 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 3 Dec 2019 15:48:23 +0800 Subject: [PATCH 133/178] Group messages for peer in heartbeat --- libp2p/pubsub/gossipsub.py | 134 +++++++++++++++++++++++++++++++++---- 1 file changed, 120 insertions(+), 14 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index cf48f53a..29e23ebb 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -2,7 +2,7 @@ from ast import literal_eval import asyncio import logging import random -from typing import Any, Dict, Iterable, List, Sequence, Set +from typing import Any, Dict, Iterable, List, Sequence, Set, Tuple from libp2p.network.stream.exceptions import StreamClosed from libp2p.peer.id import ID @@ -305,6 +305,75 @@ class GossipSub(IPubsubRouter): # Forget mesh[topic] del self.mesh[topic] + async def _emit_control_msgs( + self, + peers_to_graft: Dict[ID, List[str]], + peers_to_prune: Dict[ID, List[str]], + peers_to_gossip: Dict[ID, Dict[str, List[str]]], + ) -> None: + # Starting with GRAFT messages + for peer, topics in peers_to_graft.items(): + graft_msgs: List[rpc_pb2.ControlGraft] = [] + for topic in topics: + graft_msg: rpc_pb2.ControlGraft = rpc_pb2.ControlGraft() + graft_msg.topicID = topic + graft_msgs.append(graft_msg) + + # If there are also PRUNE messages to send to this peer + if peer in peers_to_prune: + prune_msgs: List[rpc_pb2.ControlPrune] = [] + for topic in peers_to_prune[peer]: + prune_msg: rpc_pb2.ControlPrune = rpc_pb2.ControlPrune() + prune_msg.topicID = topic + prune_msgs.append(prune_msg) + del peers_to_prune[peer] + + # If there are also IHAVE messages to send to this peer + if peer in peers_to_gossip: + ihave_msgs: List[rpc_pb2.ControlIHave] = [] + for topic in peers_to_gossip[peer]: + ihave_msg: rpc_pb2.ControlIHave = rpc_pb2.ControlIHave() + ihave_msg.messageIDs.extend(peers_to_gossip[peer][topic]) + ihave_msg.topicID = topic + ihave_msgs.append(ihave_msg) + del peers_to_gossip[peer] + + control_msg = self.pack_control_msgs(ihave_msgs, graft_msgs, prune_msgs) + await self.emit_control_message(control_msg, peer) + + # Next with PRUNE messages + for peer, topics in peers_to_prune.items(): + prune_msgs = [] + for topic in topics: + prune_msg = rpc_pb2.ControlPrune() + prune_msg.topicID = topic + prune_msgs.append(prune_msg) + + # If there are also IHAVE messages to send to this peer + if peer in peers_to_gossip: + ihave_msgs = [] + for topic in peers_to_gossip[peer]: + ihave_msg = rpc_pb2.ControlIHave() + ihave_msg.messageIDs.extend(peers_to_gossip[peer][topic]) + ihave_msg.topicID = topic + ihave_msgs.append(ihave_msg) + del peers_to_gossip[peer] + + control_msg = self.pack_control_msgs(ihave_msgs, None, prune_msgs) + await self.emit_control_message(control_msg, peer) + + # Fianlly IHAVE messages + for peer in peers_to_gossip: + ihave_msgs = [] + for topic in peers_to_gossip[peer]: + ihave_msg = rpc_pb2.ControlIHave() + ihave_msg.messageIDs.extend(peers_to_gossip[peer][topic]) + ihave_msg.topicID = topic + ihave_msgs.append(ihave_msg) + + control_msg = self.pack_control_msgs(ihave_msgs, None, None) + await self.emit_control_message(control_msg, peer) + # Heartbeat async def heartbeat(self) -> None: """ @@ -316,15 +385,24 @@ class GossipSub(IPubsubRouter): # Start after a delay. Ref: https://github.com/libp2p/go-libp2p-pubsub/blob/01b9825fbee1848751d90a8469e3f5f43bac8466/gossipsub.go#L410 # Noqa: E501 await asyncio.sleep(self.heartbeat_initial_delay) while True: + # Maintain mesh and keep track of which peers to send GRAFT or PRUNE to + peers_to_graft, peers_to_prune = self.mesh_heartbeat() + # Maintain fanout + self.fanout_heartbeat() + # Get the peers to send IHAVE to + peers_to_gossip = self.gossip_heartbeat() + # Pack GRAFT, PRUNE and IHAVE for the same peer into one control message and send it + await self._emit_control_msgs( + peers_to_graft, peers_to_prune, peers_to_gossip + ) - await self.mesh_heartbeat() - await self.fanout_heartbeat() - await self.gossip_heartbeat() + self.mcache.shift() await asyncio.sleep(self.heartbeat_interval) - async def mesh_heartbeat(self) -> None: - # Note: the comments here are the exact pseudocode from the spec + def mesh_heartbeat(self) -> Tuple[Dict[ID, List[str]], Dict[ID, List[str]]]: + peers_to_graft: Dict[ID, List[str]] = {} + peers_to_prune: Dict[ID, List[str]] = {} for topic in self.mesh: # Skip if no peers have subscribed to the topic if topic not in self.pubsub.peer_topics: @@ -342,7 +420,10 @@ class GossipSub(IPubsubRouter): self.mesh[topic].append(peer) # Emit GRAFT(topic) control message to peer - await self.emit_graft(topic, peer) + if peer not in peers_to_graft: + peers_to_graft[peer] = [topic] + else: + peers_to_graft[peer].append(topic) if num_mesh_peers_in_topic > self.degree_high: # Select |mesh[topic]| - D peers from mesh[topic] @@ -354,9 +435,13 @@ class GossipSub(IPubsubRouter): self.mesh[topic].remove(peer) # Emit PRUNE(topic) control message to peer - await self.emit_prune(topic, peer) + if peer not in peers_to_prune: + peers_to_prune[peer] = [topic] + else: + peers_to_prune[peer].append(topic) + return peers_to_graft, peers_to_prune - async def fanout_heartbeat(self) -> None: + def fanout_heartbeat(self) -> None: # Note: the comments here are the exact pseudocode from the spec for topic in self.fanout: # Delete topic entry if it's not in `pubsub.peer_topics` @@ -388,7 +473,8 @@ class GossipSub(IPubsubRouter): # Add the peers to fanout[topic] self.fanout[topic].extend(selected_peers) - async def gossip_heartbeat(self) -> None: + def gossip_heartbeat(self) -> Dict[ID, Dict[str, List[str]]]: + peers_to_gossip: Dict[ID, Dict[str, List[str]]] = {} for topic in self.mesh: msg_ids = self.mcache.window(topic) if msg_ids: @@ -401,7 +487,10 @@ class GossipSub(IPubsubRouter): msg_id_strs = [str(msg_id) for msg_id in msg_ids] for peer in peers_to_emit_ihave_to: - await self.emit_ihave(topic, msg_id_strs, peer) + if peer not in peers_to_gossip: + peers_to_gossip[peer] = {topic: msg_id_strs} + else: + peers_to_gossip[peer][topic] = msg_id_strs # TODO: Refactor and Dedup. This section is the roughly the same as the above. # Do the same for fanout, for all topics not already hit in mesh @@ -417,9 +506,11 @@ class GossipSub(IPubsubRouter): ) msg_id_strs = [str(msg) for msg in msg_ids] for peer in peers_to_emit_ihave_to: - await self.emit_ihave(topic, msg_id_strs, peer) - - self.mcache.shift() + if peer not in peers_to_gossip: + peers_to_gossip[peer] = {topic: msg_id_strs} + else: + peers_to_gossip[peer][topic] = msg_id_strs + return peers_to_gossip @staticmethod def select_from_minus( @@ -554,6 +645,21 @@ class GossipSub(IPubsubRouter): # RPC emitters + def pack_control_msgs( + self, + ihave_msgs: List[rpc_pb2.ControlIHave], + graft_msgs: List[rpc_pb2.ControlGraft], + prune_msgs: List[rpc_pb2.ControlPrune], + ) -> rpc_pb2.ControlMessage: + control_msg: rpc_pb2.ControlMessage = rpc_pb2.ControlMessage() + if ihave_msgs: + control_msg.ihave.extend(ihave_msgs) + if graft_msgs: + control_msg.graft.extend(graft_msgs) + if prune_msgs: + control_msg.prune.extend(prune_msgs) + return control_msg + async def emit_ihave(self, topic: str, msg_ids: Any, to_peer: ID) -> None: """Emit ihave message, sent to to_peer, for topic and msg_ids.""" From 8dec0b111d861cc64202d19fada86eead3dde045 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 3 Dec 2019 15:49:45 +0800 Subject: [PATCH 134/178] Add test for mesh heartbeat --- tests/pubsub/test_gossipsub.py | 42 +++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/pubsub/test_gossipsub.py b/tests/pubsub/test_gossipsub.py index 2121f8fb..3ff602d1 100644 --- a/tests/pubsub/test_gossipsub.py +++ b/tests/pubsub/test_gossipsub.py @@ -3,7 +3,8 @@ import random import pytest -from libp2p.tools.constants import GossipsubParams +from libp2p.peer.id import ID +from libp2p.tools.constants import GOSSIPSUB_PARAMS, GossipsubParams from libp2p.tools.pubsub.utils import dense_connect, one_to_all_connect from libp2p.tools.utils import connect @@ -366,3 +367,42 @@ async def test_gossip_propagation(hosts, pubsubs_gsub): # should be able to read message msg = await queue_1.get() assert msg.data == msg_content + + +@pytest.mark.parametrize( + "num_hosts, gossipsub_params", ((1, GossipsubParams(heartbeat_initial_delay=100)),) +) +@pytest.mark.parametrize("initial_mesh_peer_count", (7, 10, 13)) +@pytest.mark.asyncio +async def test_mesh_heartbeat( + num_hosts, initial_mesh_peer_count, pubsubs_gsub, hosts, monkeypatch +): + total_peer_count = 14 + topic = "TEST_MESH_HEARTBEAT" + + fake_peer_ids = [ + ID((i).to_bytes(2, byteorder="big")) for i in range(total_peer_count) + ] + monkeypatch.setattr(pubsubs_gsub[0].router, "peers_gossipsub", fake_peer_ids) + + peer_topics = {topic: fake_peer_ids} + monkeypatch.setattr(pubsubs_gsub[0], "peer_topics", peer_topics) + + mesh_peer_indices = random.sample(range(total_peer_count), initial_mesh_peer_count) + mesh_peers = [fake_peer_ids[i] for i in mesh_peer_indices] + router_mesh = {topic: list(mesh_peers)} + monkeypatch.setattr(pubsubs_gsub[0].router, "mesh", router_mesh) + + peers_to_graft, peers_to_prune = pubsubs_gsub[0].router.mesh_heartbeat() + if initial_mesh_peer_count > GOSSIPSUB_PARAMS.degree: + assert len(peers_to_graft) == 0 + assert len(peers_to_prune) == initial_mesh_peer_count - GOSSIPSUB_PARAMS.degree + for peer in peers_to_prune: + assert peer in mesh_peers + elif initial_mesh_peer_count < GOSSIPSUB_PARAMS.degree: + assert len(peers_to_prune) == 0 + assert len(peers_to_graft) == GOSSIPSUB_PARAMS.degree - initial_mesh_peer_count + for peer in peers_to_graft: + assert peer not in mesh_peers + else: + assert len(peers_to_prune) == 0 and len(peers_to_graft) == 0 From b405fd76e95dee070c556f3921bb2473fe1b765f Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 3 Dec 2019 15:49:58 +0800 Subject: [PATCH 135/178] Add test for gossip heartbeat --- tests/pubsub/test_gossipsub.py | 60 ++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/pubsub/test_gossipsub.py b/tests/pubsub/test_gossipsub.py index 3ff602d1..a8a89a6c 100644 --- a/tests/pubsub/test_gossipsub.py +++ b/tests/pubsub/test_gossipsub.py @@ -406,3 +406,63 @@ async def test_mesh_heartbeat( assert peer not in mesh_peers else: assert len(peers_to_prune) == 0 and len(peers_to_graft) == 0 + + +@pytest.mark.parametrize( + "num_hosts, gossipsub_params", ((1, GossipsubParams(heartbeat_initial_delay=100)),) +) +@pytest.mark.parametrize("initial_peer_count", (1, 4, 7)) +@pytest.mark.asyncio +async def test_gossip_heartbeat( + num_hosts, initial_peer_count, pubsubs_gsub, hosts, monkeypatch +): + total_peer_count = 28 + topic_mesh = "TEST_GOSSIP_HEARTBEAT_1" + topic_fanout = "TEST_GOSSIP_HEARTBEAT_2" + + fake_peer_ids = [ + ID((i).to_bytes(2, byteorder="big")) for i in range(total_peer_count) + ] + monkeypatch.setattr(pubsubs_gsub[0].router, "peers_gossipsub", fake_peer_ids) + + topic_mesh_peer_count = 14 + peer_topics = { + topic_mesh: fake_peer_ids[:topic_mesh_peer_count], + topic_fanout: fake_peer_ids[topic_mesh_peer_count:], + } + monkeypatch.setattr(pubsubs_gsub[0], "peer_topics", peer_topics) + + mesh_peer_indices = random.sample(range(topic_mesh_peer_count), initial_peer_count) + mesh_peers = [fake_peer_ids[i] for i in mesh_peer_indices] + router_mesh = {topic_mesh: list(mesh_peers)} + monkeypatch.setattr(pubsubs_gsub[0].router, "mesh", router_mesh) + fanout_peer_indices = random.sample( + range(topic_mesh_peer_count, total_peer_count), initial_peer_count + ) + fanout_peers = [fake_peer_ids[i] for i in fanout_peer_indices] + router_fanout = {topic_fanout: list(fanout_peers)} + monkeypatch.setattr(pubsubs_gsub[0].router, "fanout", router_fanout) + + def window(topic): + if topic == topic_mesh: + return [topic_mesh] + elif topic == topic_fanout: + return [topic_fanout] + else: + return [] + + monkeypatch.setattr(pubsubs_gsub[0].router.mcache, "window", window) + + peers_to_gossip = pubsubs_gsub[0].router.gossip_heartbeat() + if topic_mesh_peer_count - initial_peer_count < GOSSIPSUB_PARAMS.degree: + assert len(peers_to_gossip) == 2 * (topic_mesh_peer_count - initial_peer_count) + elif topic_mesh_peer_count - initial_peer_count >= GOSSIPSUB_PARAMS.degree: + assert len(peers_to_gossip) == 2 * (GOSSIPSUB_PARAMS.degree) + + for peer in peers_to_gossip: + if peer in peer_topics[topic_mesh]: + assert peer not in mesh_peers + assert topic_mesh in peers_to_gossip[peer] + elif peer in peer_topics[topic_fanout]: + assert peer not in fanout_peers + assert topic_fanout in peers_to_gossip[peer] From 60bd4694a4cdb12822ca1e2769635b2adea81449 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 3 Dec 2019 18:03:45 +0800 Subject: [PATCH 136/178] Extend wait time for test to pass --- tests/pubsub/test_gossipsub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pubsub/test_gossipsub.py b/tests/pubsub/test_gossipsub.py index a8a89a6c..d05f024d 100644 --- a/tests/pubsub/test_gossipsub.py +++ b/tests/pubsub/test_gossipsub.py @@ -197,7 +197,7 @@ async def test_dense(num_hosts, pubsubs_gsub, hosts): # publish from the randomly chosen host await pubsubs_gsub[origin_idx].publish("foobar", msg_content) - await asyncio.sleep(0.5) + await asyncio.sleep(1) # Assert that all blocking queues receive the message for queue in queues: msg = await queue.get() From ea6cd30a16269537e8f93f96b7e6ce35777c0b00 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 3 Dec 2019 18:45:33 +0800 Subject: [PATCH 137/178] Add back some comment and TODO. Add comment to tests --- libp2p/pubsub/gossipsub.py | 3 +++ tests/pubsub/test_gossipsub.py | 25 ++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 29e23ebb..f0b12195 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -205,6 +205,7 @@ class GossipSub(IPubsubRouter): stream = self.pubsub.peers[peer_id] # FIXME: We should add a `WriteMsg` similar to write delimited messages. # Ref: https://github.com/libp2p/go-libp2p-pubsub/blob/master/comm.go#L107 + # TODO: Go use `sendRPC`, which possibly piggybacks gossip/control messages. try: await stream.write(encode_varint_prefixed(rpc_msg.SerializeToString())) except StreamClosed: @@ -238,6 +239,8 @@ class GossipSub(IPubsubRouter): if topic in self.mesh: in_topic_gossipsub_peers = self.mesh[topic] else: + # It could be the case that we publish to a topic that we have not subscribe + # and the topic is not yet added to our `fanout`. if (topic not in self.fanout) or (len(self.fanout[topic]) == 0): # If no peers in fanout, choose some peers from gossipsub peers in topic. self.fanout[topic] = self._get_in_topic_gossipsub_peers_from_minus( diff --git a/tests/pubsub/test_gossipsub.py b/tests/pubsub/test_gossipsub.py index d05f024d..ca175e53 100644 --- a/tests/pubsub/test_gossipsub.py +++ b/tests/pubsub/test_gossipsub.py @@ -197,7 +197,7 @@ async def test_dense(num_hosts, pubsubs_gsub, hosts): # publish from the randomly chosen host await pubsubs_gsub[origin_idx].publish("foobar", msg_content) - await asyncio.sleep(1) + await asyncio.sleep(2) # Assert that all blocking queues receive the message for queue in queues: msg = await queue.get() @@ -377,6 +377,12 @@ async def test_gossip_propagation(hosts, pubsubs_gsub): async def test_mesh_heartbeat( num_hosts, initial_mesh_peer_count, pubsubs_gsub, hosts, monkeypatch ): + # It's difficult to set up the initial peer subscription condition. + # Ideally I would like to have initial mesh peer count that's below ``GossipSubDegree`` + # so I can test if `mesh_heartbeat` return correct peers to GRAFT. + # The problem is that I can not set it up so that we have peers subscribe to the topic + # but not being part of our mesh peers (as these peers are the peers to GRAFT). + # So I monkeypatch the peer subscriptions and our mesh peers. total_peer_count = 14 topic = "TEST_MESH_HEARTBEAT" @@ -386,20 +392,24 @@ async def test_mesh_heartbeat( monkeypatch.setattr(pubsubs_gsub[0].router, "peers_gossipsub", fake_peer_ids) peer_topics = {topic: fake_peer_ids} + # Monkeypatch the peer subscriptions monkeypatch.setattr(pubsubs_gsub[0], "peer_topics", peer_topics) mesh_peer_indices = random.sample(range(total_peer_count), initial_mesh_peer_count) mesh_peers = [fake_peer_ids[i] for i in mesh_peer_indices] router_mesh = {topic: list(mesh_peers)} + # Monkeypatch our mesh peers monkeypatch.setattr(pubsubs_gsub[0].router, "mesh", router_mesh) peers_to_graft, peers_to_prune = pubsubs_gsub[0].router.mesh_heartbeat() if initial_mesh_peer_count > GOSSIPSUB_PARAMS.degree: + # If number of initial mesh peers is more than `GossipSubDegree`, we should PRUNE mesh peers assert len(peers_to_graft) == 0 assert len(peers_to_prune) == initial_mesh_peer_count - GOSSIPSUB_PARAMS.degree for peer in peers_to_prune: assert peer in mesh_peers elif initial_mesh_peer_count < GOSSIPSUB_PARAMS.degree: + # If number of initial mesh peers is less than `GossipSubDegree`, we should GRAFT more peers assert len(peers_to_prune) == 0 assert len(peers_to_graft) == GOSSIPSUB_PARAMS.degree - initial_mesh_peer_count for peer in peers_to_graft: @@ -416,6 +426,9 @@ async def test_mesh_heartbeat( async def test_gossip_heartbeat( num_hosts, initial_peer_count, pubsubs_gsub, hosts, monkeypatch ): + # The problem is that I can not set it up so that we have peers subscribe to the topic + # but not being part of our mesh peers (as these peers are the peers to GRAFT). + # So I monkeypatch the peer subscriptions and our mesh peers. total_peer_count = 28 topic_mesh = "TEST_GOSSIP_HEARTBEAT_1" topic_fanout = "TEST_GOSSIP_HEARTBEAT_2" @@ -426,21 +439,25 @@ async def test_gossip_heartbeat( monkeypatch.setattr(pubsubs_gsub[0].router, "peers_gossipsub", fake_peer_ids) topic_mesh_peer_count = 14 + # Split into mesh peers and fanout peers peer_topics = { topic_mesh: fake_peer_ids[:topic_mesh_peer_count], topic_fanout: fake_peer_ids[topic_mesh_peer_count:], } + # Monkeypatch the peer subscriptions monkeypatch.setattr(pubsubs_gsub[0], "peer_topics", peer_topics) mesh_peer_indices = random.sample(range(topic_mesh_peer_count), initial_peer_count) mesh_peers = [fake_peer_ids[i] for i in mesh_peer_indices] router_mesh = {topic_mesh: list(mesh_peers)} + # Monkeypatch our mesh peers monkeypatch.setattr(pubsubs_gsub[0].router, "mesh", router_mesh) fanout_peer_indices = random.sample( range(topic_mesh_peer_count, total_peer_count), initial_peer_count ) fanout_peers = [fake_peer_ids[i] for i in fanout_peer_indices] router_fanout = {topic_fanout: list(fanout_peers)} + # Monkeypatch our fanout peers monkeypatch.setattr(pubsubs_gsub[0].router, "fanout", router_fanout) def window(topic): @@ -451,18 +468,24 @@ async def test_gossip_heartbeat( else: return [] + # Monkeypatch the memory cache messages monkeypatch.setattr(pubsubs_gsub[0].router.mcache, "window", window) peers_to_gossip = pubsubs_gsub[0].router.gossip_heartbeat() + # If our mesh peer count is less than `GossipSubDegree`, we should gossip to up to + # `GossipSubDegree` peers (exclude mesh peers). if topic_mesh_peer_count - initial_peer_count < GOSSIPSUB_PARAMS.degree: + # The same goes for fanout so it's two times the number of peers to gossip. assert len(peers_to_gossip) == 2 * (topic_mesh_peer_count - initial_peer_count) elif topic_mesh_peer_count - initial_peer_count >= GOSSIPSUB_PARAMS.degree: assert len(peers_to_gossip) == 2 * (GOSSIPSUB_PARAMS.degree) for peer in peers_to_gossip: if peer in peer_topics[topic_mesh]: + # Check that the peer to gossip to is not in our mesh peers assert peer not in mesh_peers assert topic_mesh in peers_to_gossip[peer] elif peer in peer_topics[topic_fanout]: + # Check that the peer to gossip to is not in our fanout peers assert peer not in fanout_peers assert topic_fanout in peers_to_gossip[peer] From bb15c817b1000eff7d5e23934ab412df523d2b3a Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 3 Dec 2019 22:14:45 +0800 Subject: [PATCH 138/178] Fix var access before assignment --- libp2p/pubsub/gossipsub.py | 6 +++--- tests/pubsub/test_gossipsub.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index f0b12195..3b6e7366 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -314,9 +314,11 @@ class GossipSub(IPubsubRouter): peers_to_prune: Dict[ID, List[str]], peers_to_gossip: Dict[ID, Dict[str, List[str]]], ) -> None: + graft_msgs: List[rpc_pb2.ControlGraft] = [] + prune_msgs: List[rpc_pb2.ControlPrune] = [] + ihave_msgs: List[rpc_pb2.ControlIHave] = [] # Starting with GRAFT messages for peer, topics in peers_to_graft.items(): - graft_msgs: List[rpc_pb2.ControlGraft] = [] for topic in topics: graft_msg: rpc_pb2.ControlGraft = rpc_pb2.ControlGraft() graft_msg.topicID = topic @@ -324,7 +326,6 @@ class GossipSub(IPubsubRouter): # If there are also PRUNE messages to send to this peer if peer in peers_to_prune: - prune_msgs: List[rpc_pb2.ControlPrune] = [] for topic in peers_to_prune[peer]: prune_msg: rpc_pb2.ControlPrune = rpc_pb2.ControlPrune() prune_msg.topicID = topic @@ -333,7 +334,6 @@ class GossipSub(IPubsubRouter): # If there are also IHAVE messages to send to this peer if peer in peers_to_gossip: - ihave_msgs: List[rpc_pb2.ControlIHave] = [] for topic in peers_to_gossip[peer]: ihave_msg: rpc_pb2.ControlIHave = rpc_pb2.ControlIHave() ihave_msg.messageIDs.extend(peers_to_gossip[peer][topic]) diff --git a/tests/pubsub/test_gossipsub.py b/tests/pubsub/test_gossipsub.py index ca175e53..183ff6be 100644 --- a/tests/pubsub/test_gossipsub.py +++ b/tests/pubsub/test_gossipsub.py @@ -197,7 +197,7 @@ async def test_dense(num_hosts, pubsubs_gsub, hosts): # publish from the randomly chosen host await pubsubs_gsub[origin_idx].publish("foobar", msg_content) - await asyncio.sleep(2) + await asyncio.sleep(0.5) # Assert that all blocking queues receive the message for queue in queues: msg = await queue.get() From a9abf1e3dd5540fd59f29822f91c2ec07eb563e8 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 3 Dec 2019 22:37:49 +0800 Subject: [PATCH 139/178] Fix list deletion and add list remove check --- libp2p/pubsub/gossipsub.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 3b6e7366..65253e25 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -460,9 +460,12 @@ class GossipSub(IPubsubRouter): else: # Check whether our peers are still in the topic # ref: https://github.com/libp2p/go-libp2p-pubsub/blob/01b9825fbee1848751d90a8469e3f5f43bac8466/gossipsub.go#L498-L504 # noqa: E501 - for peer in self.fanout[topic]: - if peer not in self.pubsub.peer_topics[topic]: - self.fanout[topic].remove(peer) + in_topic_fanout_peers = [ + peer + for peer in self.fanout[topic] + if peer in self.pubsub.peer_topics[topic] + ] + self.fanout[topic] = in_topic_fanout_peers num_fanout_peers_in_topic = len(self.fanout[topic]) # If |fanout[topic]| < D @@ -644,7 +647,10 @@ class GossipSub(IPubsubRouter): # Remove peer from mesh for topic, if peer is in topic if topic in self.mesh and sender_peer_id in self.mesh[topic]: - self.mesh[topic].remove(sender_peer_id) + if len(self.mesh[topic]) == 1: + del self.mesh[topic] + else: + self.mesh[topic].remove(sender_peer_id) # RPC emitters From c08b2375e133cf5af31bec932ac835791a0618ea Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 3 Dec 2019 23:10:47 +0800 Subject: [PATCH 140/178] Fix: should not remove topic if no peers --- libp2p/pubsub/gossipsub.py | 15 +++------------ libp2p/pubsub/pubsub.py | 10 ++-------- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 65253e25..a6cee2f7 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -150,17 +150,11 @@ class GossipSub(IPubsubRouter): for topic in self.mesh: if peer_id in self.mesh[topic]: # Delete the entry if no other peers left - if len(self.mesh[topic]) == 1: - del self.mesh[topic] - else: - self.mesh[topic].remove(peer_id) + self.mesh[topic].remove(peer_id) for topic in self.fanout: if peer_id in self.fanout[topic]: # Delete the entry if no other peers left - if len(self.fanout[topic]) == 1: - del self.fanout[topic] - else: - self.fanout[topic].remove(peer_id) + self.fanout[topic].remove(peer_id) self.peers_to_protocol.pop(peer_id, None) @@ -647,10 +641,7 @@ class GossipSub(IPubsubRouter): # Remove peer from mesh for topic, if peer is in topic if topic in self.mesh and sender_peer_id in self.mesh[topic]: - if len(self.mesh[topic]) == 1: - del self.mesh[topic] - else: - self.mesh[topic].remove(sender_peer_id) + self.mesh[topic].remove(sender_peer_id) # RPC emitters diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 45e7ffca..25ab81e6 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -317,10 +317,7 @@ class Pubsub: for topic in self.peer_topics: if peer_id in self.peer_topics[topic]: # Delete the entry if no other peers left - if len(self.peer_topics[topic]) == 1: - del self.peer_topics[topic] - else: - self.peer_topics[topic].remove(peer_id) + self.peer_topics[topic].remove(peer_id) self.router.remove_peer(peer_id) @@ -364,10 +361,7 @@ class Pubsub: if sub_message.topicid in self.peer_topics: if origin_id in self.peer_topics[sub_message.topicid]: # Delete the entry if no other peers left - if len(self.peer_topics[sub_message.topicid]) == 1: - del self.peer_topics[sub_message.topicid] - else: - self.peer_topics[sub_message.topicid].remove(origin_id) + self.peer_topics[sub_message.topicid].remove(origin_id) # FIXME(mhchia): Change the function name? async def handle_talk(self, publish_message: rpc_pb2.Message) -> None: From 8e591229fddb3f02989530527877fdd9c02ae8e0 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 3 Dec 2019 23:10:56 +0800 Subject: [PATCH 141/178] Update the sleep time in `test_handle_prune` --- tests/pubsub/test_gossipsub.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/pubsub/test_gossipsub.py b/tests/pubsub/test_gossipsub.py index 183ff6be..5b70e19a 100644 --- a/tests/pubsub/test_gossipsub.py +++ b/tests/pubsub/test_gossipsub.py @@ -149,7 +149,7 @@ async def test_handle_prune(pubsubs_gsub, hosts): await connect(hosts[index_alice], hosts[index_bob]) # Wait 3 seconds for heartbeat to allow mesh to connect - await asyncio.sleep(3) + await asyncio.sleep(1) # Check that they are each other's mesh peer assert id_alice in gossipsubs[index_bob].mesh[topic] @@ -158,15 +158,17 @@ async def test_handle_prune(pubsubs_gsub, hosts): # alice emit prune message to bob, alice should be removed # from bob's mesh peer await gossipsubs[index_alice].emit_prune(topic, id_bob) + # `emit_prune` does not remove bob from alice's mesh peers + assert id_bob in gossipsubs[index_alice].mesh[topic] # FIXME: This test currently works because the heartbeat interval # is increased to 3 seconds, so alice won't get add back into # bob's mesh peer during heartbeat. - await asyncio.sleep(1) + # Wait for bob to `handle_prune` + await asyncio.sleep(0.1) # Check that alice is no longer bob's mesh peer assert id_alice not in gossipsubs[index_bob].mesh[topic] - assert id_bob in gossipsubs[index_alice].mesh[topic] @pytest.mark.parametrize("num_hosts", (10,)) From e6813da5f5a9815288b9ef76eda923deec71f245 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Thu, 5 Dec 2019 14:35:34 +0800 Subject: [PATCH 142/178] Refactor `_get_peers_to_send` --- libp2p/pubsub/gossipsub.py | 51 +++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index a6cee2f7..7f80ddd5 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -2,7 +2,7 @@ from ast import literal_eval import asyncio import logging import random -from typing import Any, Dict, Iterable, List, Sequence, Set, Tuple +from typing import Any, Dict, Iterable, List, Sequence, Tuple from libp2p.network.stream.exceptions import StreamClosed from libp2p.peer.id import ID @@ -216,38 +216,40 @@ class GossipSub(IPubsubRouter): :param origin: peer id of the peer the message originate from. :return: a generator of the peer ids who we send data to. """ - send_to: Set[ID] = set() for topic in topic_ids: if topic not in self.pubsub.peer_topics: continue # floodsub peers - for peer_id in self.pubsub.peer_topics[topic]: - # FIXME: `gossipsub.peers_floodsub` can be changed to `gossipsub.peers` in go. - # This will improve the efficiency when searching for a peer's protocol id. - if peer_id in self.peers_floodsub: - send_to.add(peer_id) + # FIXME: `gossipsub.peers_floodsub` can be changed to `gossipsub.peers` in go. + # This will improve the efficiency when searching for a peer's protocol id. + floodsub_peers: List[ID] = [ + peer_id + for peer_id in self.pubsub.peer_topics[topic] + if peer_id in self.peers_floodsub + ] # gossipsub peers - in_topic_gossipsub_peers: List[ID] = None + gossipsub_peers: List[ID] = [] if topic in self.mesh: - in_topic_gossipsub_peers = self.mesh[topic] + gossipsub_peers = self.mesh[topic] else: - # It could be the case that we publish to a topic that we have not subscribe - # and the topic is not yet added to our `fanout`. - if (topic not in self.fanout) or (len(self.fanout[topic]) == 0): - # If no peers in fanout, choose some peers from gossipsub peers in topic. - self.fanout[topic] = self._get_in_topic_gossipsub_peers_from_minus( - topic, self.degree, [] + # When we publish to a topic that we have not subscribe to, we randomly pick + # `self.degree` number of peers who have subscribe to the topic and add them + # as our `fanout` peers. + topic_in_fanout: bool = topic in self.fanout + fanout_peers: List[ID] = self.fanout[topic] if topic_in_fanout else [] + fanout_size = len(fanout_peers) + if not topic_in_fanout or (topic_in_fanout and fanout_size < self.degree): + # Combine fanout peers with selected peers + self.fanout[topic] += self._get_in_topic_gossipsub_peers_from_minus( + topic, self.degree - fanout_size, fanout_peers ) - in_topic_gossipsub_peers = self.fanout[topic] - for peer_id in in_topic_gossipsub_peers: - send_to.add(peer_id) + gossipsub_peers = fanout_peers # Excludes `msg_forwarder` and `origin` - yield from send_to.difference([msg_forwarder, origin]) + yield from set(floodsub_peers + gossipsub_peers).difference([msg_forwarder, origin]) async def join(self, topic: str) -> None: - # Note: the comments here are the near-exact algorithm description from the spec """ Join notifies the router that we want to receive and forward messages in a topic. It is invoked after the subscription announcement. @@ -277,9 +279,8 @@ class GossipSub(IPubsubRouter): # Add fanout peers to mesh and notifies them with a GRAFT(topic) control message. for peer in fanout_peers: - if peer not in self.mesh[topic]: - self.mesh[topic].append(peer) - await self.emit_graft(topic, peer) + self.mesh[topic].append(peer) + await self.emit_graft(topic, peer) self.fanout.pop(topic, None) @@ -300,7 +301,7 @@ class GossipSub(IPubsubRouter): await self.emit_prune(topic, peer) # Forget mesh[topic] - del self.mesh[topic] + self.mesh.pop(topic, None) async def _emit_control_msgs( self, @@ -452,7 +453,7 @@ class GossipSub(IPubsubRouter): del self.fanout[topic] del self.time_since_last_publish[topic] else: - # Check whether our peers are still in the topic + # Check if fanout peers are still in the topic and remove the ones that are not # ref: https://github.com/libp2p/go-libp2p-pubsub/blob/01b9825fbee1848751d90a8469e3f5f43bac8466/gossipsub.go#L498-L504 # noqa: E501 in_topic_fanout_peers = [ peer From fae3798ca9828a674298c6f7b74eb73c8fa97040 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Thu, 5 Dec 2019 14:40:49 +0800 Subject: [PATCH 143/178] Apply PR feedback: correct the comment in test --- tests/pubsub/test_gossipsub.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/pubsub/test_gossipsub.py b/tests/pubsub/test_gossipsub.py index 5b70e19a..e1d562e8 100644 --- a/tests/pubsub/test_gossipsub.py +++ b/tests/pubsub/test_gossipsub.py @@ -161,9 +161,8 @@ async def test_handle_prune(pubsubs_gsub, hosts): # `emit_prune` does not remove bob from alice's mesh peers assert id_bob in gossipsubs[index_alice].mesh[topic] - # FIXME: This test currently works because the heartbeat interval - # is increased to 3 seconds, so alice won't get add back into - # bob's mesh peer during heartbeat. + # NOTE: We increase `heartbeat_interval` to 3 seconds so that bob will not + # add alice back to his mesh after heartbeat. # Wait for bob to `handle_prune` await asyncio.sleep(0.1) From 67f02c512a81cb2bb55f428d67781ad8a188f1fd Mon Sep 17 00:00:00 2001 From: NIC619 Date: Thu, 5 Dec 2019 15:10:04 +0800 Subject: [PATCH 144/178] Remove unnecessary check and fix test --- libp2p/pubsub/gossipsub.py | 47 +++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 7f80ddd5..7dcac133 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -216,6 +216,7 @@ class GossipSub(IPubsubRouter): :param origin: peer id of the peer the message originate from. :return: a generator of the peer ids who we send data to. """ + send_to: List[ID] = [] for topic in topic_ids: if topic not in self.pubsub.peer_topics: continue @@ -240,14 +241,19 @@ class GossipSub(IPubsubRouter): topic_in_fanout: bool = topic in self.fanout fanout_peers: List[ID] = self.fanout[topic] if topic_in_fanout else [] fanout_size = len(fanout_peers) - if not topic_in_fanout or (topic_in_fanout and fanout_size < self.degree): - # Combine fanout peers with selected peers - self.fanout[topic] += self._get_in_topic_gossipsub_peers_from_minus( - topic, self.degree - fanout_size, fanout_peers - ) + if not topic_in_fanout or ( + topic_in_fanout and fanout_size < self.degree + ): + if topic in self.pubsub.peer_topics: + # Combine fanout peers with selected peers + fanout_peers += self._get_in_topic_gossipsub_peers_from_minus( + topic, self.degree - fanout_size, fanout_peers + ) + self.fanout[topic] = fanout_peers gossipsub_peers = fanout_peers + send_to.extend(floodsub_peers + gossipsub_peers) # Excludes `msg_forwarder` and `origin` - yield from set(floodsub_peers + gossipsub_peers).difference([msg_forwarder, origin]) + yield from set(send_to).difference([msg_forwarder, origin]) async def join(self, topic: str) -> None: """ @@ -496,21 +502,20 @@ class GossipSub(IPubsubRouter): # TODO: Refactor and Dedup. This section is the roughly the same as the above. # Do the same for fanout, for all topics not already hit in mesh for topic in self.fanout: - if topic not in self.mesh: - msg_ids = self.mcache.window(topic) - if msg_ids: - # Get all pubsub peers in topic and only add if they are gossipsub peers also - if topic in self.pubsub.peer_topics: - # Select D peers from peers.gossipsub[topic] - peers_to_emit_ihave_to = self._get_in_topic_gossipsub_peers_from_minus( - topic, self.degree, self.fanout[topic] - ) - msg_id_strs = [str(msg) for msg in msg_ids] - for peer in peers_to_emit_ihave_to: - if peer not in peers_to_gossip: - peers_to_gossip[peer] = {topic: msg_id_strs} - else: - peers_to_gossip[peer][topic] = msg_id_strs + msg_ids = self.mcache.window(topic) + if msg_ids: + # Get all pubsub peers in topic and only add if they are gossipsub peers also + if topic in self.pubsub.peer_topics: + # Select D peers from peers.gossipsub[topic] + peers_to_emit_ihave_to = self._get_in_topic_gossipsub_peers_from_minus( + topic, self.degree, self.fanout[topic] + ) + msg_id_strs = [str(msg) for msg in msg_ids] + for peer in peers_to_emit_ihave_to: + if peer not in peers_to_gossip: + peers_to_gossip[peer] = {topic: msg_id_strs} + else: + peers_to_gossip[peer][topic] = msg_id_strs return peers_to_gossip @staticmethod From b4900d53da39667fb740882af0db145f0231c710 Mon Sep 17 00:00:00 2001 From: NIC Lin Date: Thu, 5 Dec 2019 15:21:09 +0800 Subject: [PATCH 145/178] Apply suggestions from code review Co-Authored-By: Chih Cheng Liang --- libp2p/pubsub/gossipsub.py | 2 +- tests/pubsub/test_gossipsub.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 7dcac133..af5ea771 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -130,7 +130,7 @@ class GossipSub(IPubsubRouter): # In this case, probably we registered gossipsub to a wrong `protocol_id` # in multistream-select, or wrong versions. raise Exception( - f"This should not happen. Protocol={protocol_id} is not supported." + f"Unreachable: Protocol={protocol_id} is not supported." ) self.peers_to_protocol[peer_id] = protocol_id diff --git a/tests/pubsub/test_gossipsub.py b/tests/pubsub/test_gossipsub.py index e1d562e8..030dba43 100644 --- a/tests/pubsub/test_gossipsub.py +++ b/tests/pubsub/test_gossipsub.py @@ -148,7 +148,7 @@ async def test_handle_prune(pubsubs_gsub, hosts): await connect(hosts[index_alice], hosts[index_bob]) - # Wait 3 seconds for heartbeat to allow mesh to connect + # Wait for heartbeat to allow mesh to connect await asyncio.sleep(1) # Check that they are each other's mesh peer From db0017ddbb64c49b20ae3afb220f810f9333c79d Mon Sep 17 00:00:00 2001 From: NIC619 Date: Thu, 5 Dec 2019 17:33:07 +0800 Subject: [PATCH 146/178] Fix lint after applying suggestion --- libp2p/pubsub/gossipsub.py | 4 +--- tests/pubsub/test_gossipsub.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index af5ea771..208d2878 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -129,9 +129,7 @@ class GossipSub(IPubsubRouter): # instance in multistream-select, but it is not the protocol that gossipsub supports. # In this case, probably we registered gossipsub to a wrong `protocol_id` # in multistream-select, or wrong versions. - raise Exception( - f"Unreachable: Protocol={protocol_id} is not supported." - ) + raise Exception(f"Unreachable: Protocol={protocol_id} is not supported.") self.peers_to_protocol[peer_id] = protocol_id def remove_peer(self, peer_id: ID) -> None: diff --git a/tests/pubsub/test_gossipsub.py b/tests/pubsub/test_gossipsub.py index 030dba43..19ec07c5 100644 --- a/tests/pubsub/test_gossipsub.py +++ b/tests/pubsub/test_gossipsub.py @@ -148,7 +148,7 @@ async def test_handle_prune(pubsubs_gsub, hosts): await connect(hosts[index_alice], hosts[index_bob]) - # Wait for heartbeat to allow mesh to connect + # Wait for heartbeat to allow mesh to connect await asyncio.sleep(1) # Check that they are each other's mesh peer From 82dcce214a5a0b21dff38087bc128fe105a20438 Mon Sep 17 00:00:00 2001 From: Chih Cheng Liang Date: Fri, 6 Dec 2019 14:14:33 +0800 Subject: [PATCH 147/178] Remove kademlia module (#377) * Remove kademlia and routing/kademlia * cleanup * Fix routed_host test * lint mypy * fix doc * remove set_up_nodes_by_transport_and_disc_opt and fix typing --- README.md | 6 +- docs/libp2p.kademlia.rst | 70 ------- docs/libp2p.routing.kademlia.rst | 30 --- docs/libp2p.routing.rst | 6 - docs/libp2p.rst | 1 - libp2p/__init__.py | 28 --- libp2p/kademlia/__init__.py | 3 - libp2p/kademlia/crawling.py | 173 ---------------- libp2p/kademlia/kad_peerinfo.py | 153 -------------- libp2p/kademlia/protocol.py | 188 ------------------ libp2p/kademlia/routing.py | 184 ----------------- libp2p/kademlia/rpc.proto | 78 -------- libp2p/kademlia/storage.py | 93 --------- libp2p/kademlia/utils.py | 56 ------ libp2p/routing/kademlia/__init__.py | 0 .../kademlia/kademlia_content_router.py | 21 -- .../routing/kademlia/kademlia_peer_router.py | 43 ---- libp2p/tools/utils.py | 67 ++++--- mypy.ini | 3 - tests/host/test_routed_host.py | 55 +---- tests/kademlia/test_basic.py | 79 -------- tests/kademlia/test_providers.py | 30 --- tests/routing/test_kad_peer_router.py | 75 ------- 23 files changed, 44 insertions(+), 1398 deletions(-) delete mode 100644 docs/libp2p.kademlia.rst delete mode 100644 docs/libp2p.routing.kademlia.rst delete mode 100644 libp2p/kademlia/__init__.py delete mode 100644 libp2p/kademlia/crawling.py delete mode 100644 libp2p/kademlia/kad_peerinfo.py delete mode 100644 libp2p/kademlia/protocol.py delete mode 100644 libp2p/kademlia/routing.py delete mode 100644 libp2p/kademlia/rpc.proto delete mode 100644 libp2p/kademlia/storage.py delete mode 100644 libp2p/kademlia/utils.py delete mode 100644 libp2p/routing/kademlia/__init__.py delete mode 100644 libp2p/routing/kademlia/kademlia_content_router.py delete mode 100644 libp2p/routing/kademlia/kademlia_peer_router.py delete mode 100644 tests/kademlia/test_basic.py delete mode 100644 tests/kademlia/test_providers.py delete mode 100644 tests/routing/test_kad_peer_router.py diff --git a/README.md b/README.md index 9b799548..417c27c2 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ py-libp2p aims for conformity with [the standard libp2p modules](https://github. | Peer Discovery | Status | | -------------------------------------------- | :-----------: | | **`bootstrap list`** | :tomato: | -| **`Kademlia DHT`** | :lemon: | +| **`Kademlia DHT`** | :chestnut: | | **`mDNS`** | :chestnut: | | **`PEX`** | :chestnut: | | **`DNS`** | :chestnut: | @@ -147,7 +147,7 @@ py-libp2p aims for conformity with [the standard libp2p modules](https://github. | Content Routing | Status | | -------------------------------------------- | :-----------: | -| **`Kademlia DHT`** | :lemon: | +| **`Kademlia DHT`** | :chestnut: | | **`floodsub`** | :green_apple: | | **`gossipsub`** | :green_apple: | | **`PHT`** | :chestnut: | @@ -155,7 +155,7 @@ py-libp2p aims for conformity with [the standard libp2p modules](https://github. | Peer Routing | Status | | -------------------------------------------- | :-----------: | -| **`Kademlia DHT`** | :green_apple: | +| **`Kademlia DHT`** | :chestnut: | | **`floodsub`** | :green_apple: | | **`gossipsub`** | :green_apple: | | **`PHT`** | :chestnut: | diff --git a/docs/libp2p.kademlia.rst b/docs/libp2p.kademlia.rst deleted file mode 100644 index aa0417f0..00000000 --- a/docs/libp2p.kademlia.rst +++ /dev/null @@ -1,70 +0,0 @@ -libp2p.kademlia package -======================= - -Submodules ----------- - -libp2p.kademlia.crawling module -------------------------------- - -.. automodule:: libp2p.kademlia.crawling - :members: - :undoc-members: - :show-inheritance: - -libp2p.kademlia.kad\_peerinfo module ------------------------------------- - -.. automodule:: libp2p.kademlia.kad_peerinfo - :members: - :undoc-members: - :show-inheritance: - -libp2p.kademlia.network module ------------------------------- - -.. automodule:: libp2p.kademlia.network - :members: - :undoc-members: - :show-inheritance: - -libp2p.kademlia.protocol module -------------------------------- - -.. automodule:: libp2p.kademlia.protocol - :members: - :undoc-members: - :show-inheritance: - -libp2p.kademlia.routing module ------------------------------- - -.. automodule:: libp2p.kademlia.routing - :members: - :undoc-members: - :show-inheritance: - -libp2p.kademlia.storage module ------------------------------- - -.. automodule:: libp2p.kademlia.storage - :members: - :undoc-members: - :show-inheritance: - -libp2p.kademlia.utils module ----------------------------- - -.. automodule:: libp2p.kademlia.utils - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: libp2p.kademlia - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/libp2p.routing.kademlia.rst b/docs/libp2p.routing.kademlia.rst deleted file mode 100644 index c52dcdce..00000000 --- a/docs/libp2p.routing.kademlia.rst +++ /dev/null @@ -1,30 +0,0 @@ -libp2p.routing.kademlia package -=============================== - -Submodules ----------- - -libp2p.routing.kademlia.kademlia\_content\_router module --------------------------------------------------------- - -.. automodule:: libp2p.routing.kademlia.kademlia_content_router - :members: - :undoc-members: - :show-inheritance: - -libp2p.routing.kademlia.kademlia\_peer\_router module ------------------------------------------------------ - -.. automodule:: libp2p.routing.kademlia.kademlia_peer_router - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: libp2p.routing.kademlia - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/libp2p.routing.rst b/docs/libp2p.routing.rst index 1595e4ef..88b3c5b2 100644 --- a/docs/libp2p.routing.rst +++ b/docs/libp2p.routing.rst @@ -1,12 +1,6 @@ libp2p.routing package ====================== -Subpackages ------------ - -.. toctree:: - - libp2p.routing.kademlia Submodules ---------- diff --git a/docs/libp2p.rst b/docs/libp2p.rst index 8aa6bc5e..a50db7ed 100644 --- a/docs/libp2p.rst +++ b/docs/libp2p.rst @@ -10,7 +10,6 @@ Subpackages libp2p.host libp2p.identity libp2p.io - libp2p.kademlia libp2p.network libp2p.peer libp2p.protocol_muxer diff --git a/libp2p/__init__.py b/libp2p/__init__.py index 27c8239e..8813fd37 100644 --- a/libp2p/__init__.py +++ b/libp2p/__init__.py @@ -6,15 +6,12 @@ from libp2p.crypto.rsa import create_new_key_pair from libp2p.host.basic_host import BasicHost from libp2p.host.host_interface import IHost from libp2p.host.routed_host import RoutedHost -from libp2p.kademlia.network import KademliaServer -from libp2p.kademlia.storage import IStorage from libp2p.network.network_interface import INetwork from libp2p.network.swarm import Swarm from libp2p.peer.id import ID from libp2p.peer.peerstore import PeerStore from libp2p.peer.peerstore_interface import IPeerStore from libp2p.routing.interfaces import IPeerRouting -from libp2p.routing.kademlia.kademlia_peer_router import KadmeliaPeerRouter from libp2p.security.insecure.transport import PLAINTEXT_PROTOCOL_ID, InsecureTransport import libp2p.security.secio.transport as secio from libp2p.stream_muxer.mplex.mplex import MPLEX_PROTOCOL_ID, Mplex @@ -45,31 +42,6 @@ def generate_peer_id_from(key_pair: KeyPair) -> ID: return ID.from_pubkey(public_key) -def initialize_default_kademlia_router( - ksize: int = 20, alpha: int = 3, id_opt: ID = None, storage: IStorage = None -) -> KadmeliaPeerRouter: - """ - initialize kadmelia router when no kademlia router is passed in. - - :param ksize: The k parameter from the paper - :param alpha: The alpha parameter from the paper - :param id_opt: optional id for host - :param storage: An instance that implements - :class:`~kademlia.storage.IStorage` - :return: return a default kademlia instance - """ - if not id_opt: - key_pair = generate_new_rsa_identity() - id_opt = generate_peer_id_from(key_pair) - - node_id = id_opt.to_bytes() - # ignore type for Kademlia module - server = KademliaServer( # type: ignore - ksize=ksize, alpha=alpha, node_id=node_id, storage=storage - ) - return KadmeliaPeerRouter(server) - - def initialize_default_swarm( key_pair: KeyPair, id_opt: ID = None, diff --git a/libp2p/kademlia/__init__.py b/libp2p/kademlia/__init__.py deleted file mode 100644 index 14568595..00000000 --- a/libp2p/kademlia/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Kademlia is a Python implementation of the Kademlia protocol which utilizes -the asyncio library.""" -__version__ = "2.0" diff --git a/libp2p/kademlia/crawling.py b/libp2p/kademlia/crawling.py deleted file mode 100644 index 1a5566fb..00000000 --- a/libp2p/kademlia/crawling.py +++ /dev/null @@ -1,173 +0,0 @@ -from collections import Counter -import logging - -from .kad_peerinfo import KadPeerHeap, create_kad_peerinfo -from .utils import gather_dict - -log = logging.getLogger(__name__) - - -class SpiderCrawl: - """Crawl the network and look for given 160-bit keys.""" - - def __init__(self, protocol, node, peers, ksize, alpha): - """ - Create a new C{SpiderCrawl}er. - - Args: - protocol: A :class:`~kademlia.protocol.KademliaProtocol` instance. - node: A :class:`~kademlia.node.Node` representing the key we're - looking for - peers: A list of :class:`~kademlia.node.Node` instances that - provide the entry point for the network - ksize: The value for k based on the paper - alpha: The value for alpha based on the paper - """ - self.protocol = protocol - self.ksize = ksize - self.alpha = alpha - self.node = node - self.nearest = KadPeerHeap(self.node, self.ksize) - self.last_ids_crawled = [] - log.info("creating spider with peers: %s", peers) - self.nearest.push(peers) - - async def _find(self, rpcmethod): - """ - Get either a value or list of nodes. - - Args: - rpcmethod: The protocol's callfindValue or call_find_node. - - The process: - 1. calls find_* to current ALPHA nearest not already queried nodes, - adding results to current nearest list of k nodes. - 2. current nearest list needs to keep track of who has been queried - already sort by nearest, keep KSIZE - 3. if list is same as last time, next call should be to everyone not - yet queried - 4. repeat, unless nearest list has all been queried, then ur done - """ - log.info("crawling network with nearest: %s", str(tuple(self.nearest))) - count = self.alpha - if self.nearest.get_ids() == self.last_ids_crawled: - count = len(self.nearest) - self.last_ids_crawled = self.nearest.get_ids() - - dicts = {} - for peer in self.nearest.get_uncontacted()[:count]: - dicts[peer.peer_id_bytes] = rpcmethod(peer, self.node) - self.nearest.mark_contacted(peer) - found = await gather_dict(dicts) - return await self._nodes_found(found) - - async def _nodes_found(self, responses): - raise NotImplementedError - - -class ValueSpiderCrawl(SpiderCrawl): - def __init__(self, protocol, node, peers, ksize, alpha): - SpiderCrawl.__init__(self, protocol, node, peers, ksize, alpha) - # keep track of the single nearest node without value - per - # section 2.3 so we can set the key there if found - self.nearest_without_value = KadPeerHeap(self.node, 1) - - async def find(self): - """Find either the closest nodes or the value requested.""" - return await self._find(self.protocol.call_find_value) - - async def _nodes_found(self, responses): - """Handle the result of an iteration in _find.""" - toremove = [] - found_values = [] - for peerid, response in responses.items(): - response = RPCFindResponse(response) - if not response.happened(): - toremove.append(peerid) - elif response.has_value(): - found_values.append(response.get_value()) - else: - peer = self.nearest.get_node(peerid) - self.nearest_without_value.push(peer) - self.nearest.push(response.get_node_list()) - self.nearest.remove(toremove) - - if found_values: - return await self._handle_found_values(found_values) - if self.nearest.have_contacted_all(): - # not found! - return None - return await self.find() - - async def _handle_found_values(self, values): - """ - We got some values! - - Exciting. But let's make sure they're all the same or freak out - a little bit. Also, make sure we tell the nearest node that - *didn't* have the value to store it. - """ - value_counts = Counter(values) - if len(value_counts) != 1: - log.warning( - "Got multiple values for key %i: %s", self.node.xor_id, str(values) - ) - value = value_counts.most_common(1)[0][0] - - peer = self.nearest_without_value.popleft() - if peer: - await self.protocol.call_store(peer, self.node.peer_id_bytes, value) - return value - - -class NodeSpiderCrawl(SpiderCrawl): - async def find(self): - """Find the closest nodes.""" - return await self._find(self.protocol.call_find_node) - - async def _nodes_found(self, responses): - """Handle the result of an iteration in _find.""" - toremove = [] - for peerid, response in responses.items(): - response = RPCFindResponse(response) - if not response.happened(): - toremove.append(peerid) - else: - self.nearest.push(response.get_node_list()) - self.nearest.remove(toremove) - - if self.nearest.have_contacted_all(): - return list(self.nearest) - return await self.find() - - -class RPCFindResponse: - def __init__(self, response): - """ - A wrapper for the result of a RPC find. - - Args: - response: This will be a tuple of (, ) - where will be a list of tuples if not found or - a dictionary of {'value': v} where v is the value desired - """ - self.response = response - - def happened(self): - """Did the other host actually respond?""" - return self.response[0] - - def has_value(self): - return isinstance(self.response[1], dict) - - def get_value(self): - return self.response[1]["value"] - - def get_node_list(self): - """ - Get the node list in the response. - - If there's no value, this should be set. - """ - nodelist = self.response[1] or [] - return [create_kad_peerinfo(*nodeple) for nodeple in nodelist] diff --git a/libp2p/kademlia/kad_peerinfo.py b/libp2p/kademlia/kad_peerinfo.py deleted file mode 100644 index efb20f17..00000000 --- a/libp2p/kademlia/kad_peerinfo.py +++ /dev/null @@ -1,153 +0,0 @@ -import heapq -from operator import itemgetter -import random -from typing import List - -from multiaddr import Multiaddr - -from libp2p.peer.id import ID -from libp2p.peer.peerinfo import PeerInfo - -from .utils import digest - -P_IP = "ip4" -P_UDP = "udp" - - -class KadPeerInfo(PeerInfo): - def __init__(self, peer_id, addrs): - super(KadPeerInfo, self).__init__(peer_id, addrs) - - self.peer_id_bytes = peer_id.to_bytes() - self.xor_id = peer_id.xor_id - - self.addrs = addrs - - self.ip = self.addrs[0].value_for_protocol(P_IP) if addrs else None - self.port = int(self.addrs[0].value_for_protocol(P_UDP)) if addrs else None - - def same_home_as(self, node): - return sorted(self.addrs) == sorted(node.addrs) - - def distance_to(self, node): - """Get the distance between this node and another.""" - return self.xor_id ^ node.xor_id - - def __iter__(self): - """ - Enables use of Node as a tuple - i.e., tuple(node) works. - """ - return iter([self.peer_id_bytes, self.ip, self.port]) - - def __repr__(self): - return repr([self.xor_id, self.ip, self.port, self.peer_id_bytes]) - - def __str__(self): - return "%s:%s" % (self.ip, str(self.port)) - - def encode(self): - return ( - str(self.peer_id_bytes) - + "\n" - + str("/ip4/" + str(self.ip) + "/udp/" + str(self.port)) - ) - - -class KadPeerHeap: - """A heap of peers ordered by distance to a given node.""" - - def __init__(self, node, maxsize): - """ - Constructor. - - @param node: The node to measure all distnaces from. - @param maxsize: The maximum size that this heap can grow to. - """ - self.node = node - self.heap = [] - self.contacted = set() - self.maxsize = maxsize - - def remove(self, peers): - """ - Remove a list of peer ids from this heap. - - Note that while this heap retains a constant visible size (based - on the iterator), it's actual size may be quite a bit larger - than what's exposed. Therefore, removal of nodes may not change - the visible size as previously added nodes suddenly become - visible. - """ - peers = set(peers) - if not peers: - return - nheap = [] - for distance, node in self.heap: - if node.peer_id_bytes not in peers: - heapq.heappush(nheap, (distance, node)) - self.heap = nheap - - def get_node(self, node_id): - for _, node in self.heap: - if node.peer_id_bytes == node_id: - return node - return None - - def have_contacted_all(self): - return len(self.get_uncontacted()) == 0 - - def get_ids(self): - return [n.peer_id_bytes for n in self] - - def mark_contacted(self, node): - self.contacted.add(node.peer_id_bytes) - - def popleft(self): - return heapq.heappop(self.heap)[1] if self else None - - def push(self, nodes): - """ - Push nodes onto heap. - - @param nodes: This can be a single item or a C{list}. - """ - if not isinstance(nodes, list): - nodes = [nodes] - - for node in nodes: - if node not in self: - distance = self.node.distance_to(node) - heapq.heappush(self.heap, (distance, node)) - - def __len__(self): - return min(len(self.heap), self.maxsize) - - def __iter__(self): - nodes = heapq.nsmallest(self.maxsize, self.heap) - return iter(map(itemgetter(1), nodes)) - - def __contains__(self, node): - for _, other in self.heap: - if node.peer_id_bytes == other.peer_id_bytes: - return True - return False - - def get_uncontacted(self): - return [n for n in self if n.peer_id_bytes not in self.contacted] - - -def create_kad_peerinfo(node_id_bytes=None, sender_ip=None, sender_port=None): - node_id = ( - ID(node_id_bytes) if node_id_bytes else ID(digest(random.getrandbits(255))) - ) - addrs: List[Multiaddr] - if sender_ip and sender_port: - addrs = [ - Multiaddr( - "/" + P_IP + "/" + str(sender_ip) + "/" + P_UDP + "/" + str(sender_port) - ) - ] - else: - addrs = [] - - return KadPeerInfo(node_id, addrs) diff --git a/libp2p/kademlia/protocol.py b/libp2p/kademlia/protocol.py deleted file mode 100644 index 59f4206a..00000000 --- a/libp2p/kademlia/protocol.py +++ /dev/null @@ -1,188 +0,0 @@ -import asyncio -import logging -import random - -from rpcudp.protocol import RPCProtocol - -from .kad_peerinfo import create_kad_peerinfo -from .routing import RoutingTable - -log = logging.getLogger(__name__) - - -class KademliaProtocol(RPCProtocol): - """ - There are four main RPCs in the Kademlia protocol PING, STORE, FIND_NODE, - FIND_VALUE. - - - PING probes if a node is still online - - STORE instructs a node to store (key, value) - - FIND_NODE takes a 160-bit ID and gets back - (ip, udp_port, node_id) for k closest nodes to target - - FIND_VALUE behaves like FIND_NODE unless a value is stored. - """ - - def __init__(self, source_node, storage, ksize): - RPCProtocol.__init__(self) - self.router = RoutingTable(self, ksize, source_node) - self.storage = storage - self.source_node = source_node - - def get_refresh_ids(self): - """Get ids to search for to keep old buckets up to date.""" - ids = [] - for bucket in self.router.lonely_buckets(): - rid = random.randint(*bucket.range).to_bytes(20, byteorder="big") - ids.append(rid) - return ids - - def rpc_stun(self, sender): - return sender - - def rpc_ping(self, sender, nodeid): - source = create_kad_peerinfo(nodeid, sender[0], sender[1]) - - self.welcome_if_new(source) - return self.source_node.peer_id_bytes - - def rpc_store(self, sender, nodeid, key, value): - source = create_kad_peerinfo(nodeid, sender[0], sender[1]) - - self.welcome_if_new(source) - log.debug( - "got a store request from %s, storing '%s'='%s'", sender, key.hex(), value - ) - self.storage[key] = value - return True - - def rpc_find_node(self, sender, nodeid, key): - log.info("finding neighbors of %i in local table", int(nodeid.hex(), 16)) - source = create_kad_peerinfo(nodeid, sender[0], sender[1]) - - self.welcome_if_new(source) - node = create_kad_peerinfo(key) - neighbors = self.router.find_neighbors(node, exclude=source) - return list(map(tuple, neighbors)) - - def rpc_find_value(self, sender, nodeid, key): - source = create_kad_peerinfo(nodeid, sender[0], sender[1]) - - self.welcome_if_new(source) - value = self.storage.get(key, None) - if value is None: - return self.rpc_find_node(sender, nodeid, key) - return {"value": value} - - def rpc_add_provider(self, sender, nodeid, key, provider_id): - """rpc when receiving an add_provider call should validate received - PeerInfo matches sender nodeid if it does, receipient must store a - record in its datastore we store a map of content_id to peer_id (non - xor)""" - if nodeid == provider_id: - log.info( - "adding provider %s for key %s in local table", provider_id, str(key) - ) - self.storage[key] = provider_id - return True - return False - - def rpc_get_providers(self, sender, key): - """rpc when receiving a get_providers call should look up key in data - store and respond with records plus a list of closer peers in its - routing table.""" - providers = [] - record = self.storage.get(key, None) - - if record: - providers.append(record) - - keynode = create_kad_peerinfo(key) - neighbors = self.router.find_neighbors(keynode) - for neighbor in neighbors: - if neighbor.peer_id_bytes != record: - providers.append(neighbor.peer_id_bytes) - - return providers - - async def call_find_node(self, node_to_ask, node_to_find): - address = (node_to_ask.ip, node_to_ask.port) - result = await self.find_node( - address, self.source_node.peer_id_bytes, node_to_find.peer_id_bytes - ) - return self.handle_call_response(result, node_to_ask) - - async def call_find_value(self, node_to_ask, node_to_find): - address = (node_to_ask.ip, node_to_ask.port) - result = await self.find_value( - address, self.source_node.peer_id_bytes, node_to_find.peer_id_bytes - ) - return self.handle_call_response(result, node_to_ask) - - async def call_ping(self, node_to_ask): - address = (node_to_ask.ip, node_to_ask.port) - result = await self.ping(address, self.source_node.peer_id_bytes) - return self.handle_call_response(result, node_to_ask) - - async def call_store(self, node_to_ask, key, value): - address = (node_to_ask.ip, node_to_ask.port) - result = await self.store(address, self.source_node.peer_id_bytes, key, value) - return self.handle_call_response(result, node_to_ask) - - async def call_add_provider(self, node_to_ask, key, provider_id): - address = (node_to_ask.ip, node_to_ask.port) - result = await self.add_provider( - address, self.source_node.peer_id_bytes, key, provider_id - ) - - return self.handle_call_response(result, node_to_ask) - - async def call_get_providers(self, node_to_ask, key): - address = (node_to_ask.ip, node_to_ask.port) - result = await self.get_providers(address, key) - return self.handle_call_response(result, node_to_ask) - - def welcome_if_new(self, node): - """ - Given a new node, send it all the keys/values it should be storing, - then add it to the routing table. - - @param node: A new node that just joined (or that we just found out - about). - - Process: - For each key in storage, get k closest nodes. If newnode is closer - than the furtherst in that list, and the node for this server - is closer than the closest in that list, then store the key/value - on the new node (per section 2.5 of the paper) - """ - if not self.router.is_new_node(node): - return - - log.info("never seen %s before, adding to router", node) - for key, value in self.storage: - keynode = create_kad_peerinfo(key) - neighbors = self.router.find_neighbors(keynode) - if neighbors: - last = neighbors[-1].distance_to(keynode) - new_node_close = node.distance_to(keynode) < last - first = neighbors[0].distance_to(keynode) - this_closest = self.source_node.distance_to(keynode) < first - if not neighbors or (new_node_close and this_closest): - asyncio.ensure_future(self.call_store(node, key, value)) - self.router.add_contact(node) - - def handle_call_response(self, result, node): - """ - If we get a response, add the node to the routing table. - - If we get no response, make sure it's removed from the routing - table. - """ - if not result[0]: - log.warning("no response from %s, removing from router", node) - self.router.remove_contact(node) - return result - - log.info("got successful response from %s", node) - self.welcome_if_new(node) - return result diff --git a/libp2p/kademlia/routing.py b/libp2p/kademlia/routing.py deleted file mode 100644 index f65c90a1..00000000 --- a/libp2p/kademlia/routing.py +++ /dev/null @@ -1,184 +0,0 @@ -import asyncio -from collections import OrderedDict -import heapq -import operator -import time - -from .utils import OrderedSet, bytes_to_bit_string, shared_prefix - - -class KBucket: - """each node keeps a list of (ip, udp_port, node_id) for nodes of distance - between 2^i and 2^(i+1) this list that every node keeps is a k-bucket each - k-bucket implements a last seen eviction policy except that live nodes are - never removed.""" - - def __init__(self, rangeLower, rangeUpper, ksize): - self.range = (rangeLower, rangeUpper) - self.nodes = OrderedDict() - self.replacement_nodes = OrderedSet() - self.touch_last_updated() - self.ksize = ksize - - def touch_last_updated(self): - self.last_updated = time.monotonic() - - def get_nodes(self): - return list(self.nodes.values()) - - def split(self): - midpoint = (self.range[0] + self.range[1]) / 2 - one = KBucket(self.range[0], midpoint, self.ksize) - two = KBucket(midpoint + 1, self.range[1], self.ksize) - for node in self.nodes.values(): - bucket = one if node.xor_id <= midpoint else two - bucket.nodes[node.peer_id_bytes] = node - return (one, two) - - def remove_node(self, node): - if node.peer_id_bytes not in self.nodes: - return - - # delete node, and see if we can add a replacement - del self.nodes[node.peer_id_bytes] - if self.replacement_nodes: - newnode = self.replacement_nodes.pop() - self.nodes[newnode.peer_id_bytes] = newnode - - def has_in_range(self, node): - return self.range[0] <= node.xor_id <= self.range[1] - - def is_new_node(self, node): - return node.peer_id_bytes not in self.nodes - - def add_node(self, node): - """ - Add a C{Node} to the C{KBucket}. Return True if successful, False if - the bucket is full. - - If the bucket is full, keep track of node in a replacement list, - per section 4.1 of the paper. - """ - if node.peer_id_bytes in self.nodes: - del self.nodes[node.peer_id_bytes] - self.nodes[node.peer_id_bytes] = node - elif len(self) < self.ksize: - self.nodes[node.peer_id_bytes] = node - else: - self.replacement_nodes.push(node) - return False - return True - - def depth(self): - vals = self.nodes.values() - sprefix = shared_prefix([bytes_to_bit_string(n.peer_id_bytes) for n in vals]) - return len(sprefix) - - def head(self): - return list(self.nodes.values())[0] - - def __getitem__(self, node_id): - return self.nodes.get(node_id, None) - - def __len__(self): - return len(self.nodes) - - -class TableTraverser: - def __init__(self, table, startNode): - index = table.get_bucket_for(startNode) - table.buckets[index].touch_last_updated() - self.current_nodes = table.buckets[index].get_nodes() - self.left_buckets = table.buckets[:index] - self.right_buckets = table.buckets[(index + 1) :] - self.left = True - - def __iter__(self): - return self - - def __next__(self): - """Pop an item from the left subtree, then right, then left, etc.""" - if self.current_nodes: - return self.current_nodes.pop() - - if self.left and self.left_buckets: - self.current_nodes = self.left_buckets.pop().get_nodes() - self.left = False - return next(self) - - if self.right_buckets: - self.current_nodes = self.right_buckets.pop(0).get_nodes() - self.left = True - return next(self) - - raise StopIteration - - -class RoutingTable: - def __init__(self, protocol, ksize, node): - """ - @param node: The node that represents this server. It won't - be added to the routing table, but will be needed later to - determine which buckets to split or not. - """ - self.node = node - self.protocol = protocol - self.ksize = ksize - self.flush() - - def flush(self): - self.buckets = [KBucket(0, 2 ** 160, self.ksize)] - - def split_bucket(self, index): - one, two = self.buckets[index].split() - self.buckets[index] = one - self.buckets.insert(index + 1, two) - - def lonely_buckets(self): - """Get all of the buckets that haven't been updated in over an hour.""" - hrago = time.monotonic() - 3600 - return [b for b in self.buckets if b.last_updated < hrago] - - def remove_contact(self, node): - index = self.get_bucket_for(node) - self.buckets[index].remove_node(node) - - def is_new_node(self, node): - index = self.get_bucket_for(node) - return self.buckets[index].is_new_node(node) - - def add_contact(self, node): - index = self.get_bucket_for(node) - bucket = self.buckets[index] - - # this will succeed unless the bucket is full - if bucket.add_node(node): - return - - # Per section 4.2 of paper, split if the bucket has the node - # in its range or if the depth is not congruent to 0 mod 5 - if bucket.has_in_range(self.node) or bucket.depth() % 5 != 0: - self.split_bucket(index) - self.add_contact(node) - else: - asyncio.ensure_future(self.protocol.call_ping(bucket.head())) - - def get_bucket_for(self, node): - """Get the index of the bucket that the given node would fall into.""" - for index, bucket in enumerate(self.buckets): - if node.xor_id < bucket.range[1]: - return index - # we should never be here, but make linter happy - return None - - def find_neighbors(self, node, k=None, exclude=None): - k = k or self.ksize - nodes = [] - for neighbor in TableTraverser(self, node): - notexcluded = exclude is None or not neighbor.same_home_as(exclude) - if neighbor.peer_id_bytes != node.peer_id_bytes and notexcluded: - heapq.heappush(nodes, (node.distance_to(neighbor), neighbor)) - if len(nodes) == k: - break - - return list(map(operator.itemgetter(1), heapq.nsmallest(k, nodes))) diff --git a/libp2p/kademlia/rpc.proto b/libp2p/kademlia/rpc.proto deleted file mode 100644 index 96c14a81..00000000 --- a/libp2p/kademlia/rpc.proto +++ /dev/null @@ -1,78 +0,0 @@ -// Record represents a dht record that contains a value -// for a key value pair -message Record { - // The key that references this record - bytes key = 1; - - // The actual value this record is storing - bytes value = 2; - - // Note: These fields were removed from the Record message - // hash of the authors public key - //optional string author = 3; - // A PKI signature for the key+value+author - //optional bytes signature = 4; - - // Time the record was received, set by receiver - string timeReceived = 5; -}; - -message Message { - enum MessageType { - PUT_VALUE = 0; - GET_VALUE = 1; - ADD_PROVIDER = 2; - GET_PROVIDERS = 3; - FIND_NODE = 4; - PING = 5; - } - - enum ConnectionType { - // sender does not have a connection to peer, and no extra information (default) - NOT_CONNECTED = 0; - - // sender has a live connection to peer - CONNECTED = 1; - - // sender recently connected to peer - CAN_CONNECT = 2; - - // sender recently tried to connect to peer repeatedly but failed to connect - // ("try" here is loose, but this should signal "made strong effort, failed") - CANNOT_CONNECT = 3; - } - - message Peer { - // ID of a given peer. - bytes id = 1; - - // multiaddrs for a given peer - repeated bytes addrs = 2; - - // used to signal the sender's connection capabilities to the peer - ConnectionType connection = 3; - } - - // defines what type of message it is. - MessageType type = 1; - - // defines what coral cluster level this query/response belongs to. - // in case we want to implement coral's cluster rings in the future. - int32 clusterLevelRaw = 10; // NOT USED - - // Used to specify the key associated with this message. - // PUT_VALUE, GET_VALUE, ADD_PROVIDER, GET_PROVIDERS - bytes key = 2; - - // Used to return a value - // PUT_VALUE, GET_VALUE - Record record = 3; - - // Used to return peers closer to a key in a query - // GET_VALUE, GET_PROVIDERS, FIND_NODE - repeated Peer closerPeers = 8; - - // Used to return Providers - // GET_VALUE, ADD_PROVIDER, GET_PROVIDERS - repeated Peer providerPeers = 9; -} \ No newline at end of file diff --git a/libp2p/kademlia/storage.py b/libp2p/kademlia/storage.py deleted file mode 100644 index 014853c1..00000000 --- a/libp2p/kademlia/storage.py +++ /dev/null @@ -1,93 +0,0 @@ -from abc import ABC, abstractmethod -from collections import OrderedDict -from itertools import takewhile -import operator -import time - - -class IStorage(ABC): - """ - Local storage for this node. - - IStorage implementations of get must return the same type as put in - by set - """ - - @abstractmethod - def __setitem__(self, key, value): - """Set a key to the given value.""" - - @abstractmethod - def __getitem__(self, key): - """ - Get the given key. - - If item doesn't exist, raises C{KeyError} - """ - - @abstractmethod - def get(self, key, default=None): - """ - Get given key. - - If not found, return default. - """ - - @abstractmethod - def iter_older_than(self, seconds_old): - """Return the an iterator over (key, value) tuples for items older than - the given seconds_old.""" - - @abstractmethod - def __iter__(self): - """Get the iterator for this storage, should yield tuple of (key, - value)""" - - -class ForgetfulStorage(IStorage): - def __init__(self, ttl=604800): - """By default, max age is a week.""" - self.data = OrderedDict() - self.ttl = ttl - - def __setitem__(self, key, value): - if key in self.data: - del self.data[key] - self.data[key] = (time.monotonic(), value) - self.cull() - - def cull(self): - for _, _ in self.iter_older_than(self.ttl): - self.data.popitem(last=False) - - def get(self, key, default=None): - self.cull() - if key in self.data: - return self[key] - return default - - def __getitem__(self, key): - self.cull() - return self.data[key][1] - - def __repr__(self): - self.cull() - return repr(self.data) - - def iter_older_than(self, seconds_old): - min_birthday = time.monotonic() - seconds_old - zipped = self._triple_iter() - matches = takewhile(lambda r: min_birthday >= r[1], zipped) - return list(map(operator.itemgetter(0, 2), matches)) - - def _triple_iter(self): - ikeys = self.data.keys() - ibirthday = map(operator.itemgetter(0), self.data.values()) - ivalues = map(operator.itemgetter(1), self.data.values()) - return zip(ikeys, ibirthday, ivalues) - - def __iter__(self): - self.cull() - ikeys = self.data.keys() - ivalues = map(operator.itemgetter(1), self.data.values()) - return zip(ikeys, ivalues) diff --git a/libp2p/kademlia/utils.py b/libp2p/kademlia/utils.py deleted file mode 100644 index f18a6713..00000000 --- a/libp2p/kademlia/utils.py +++ /dev/null @@ -1,56 +0,0 @@ -"""General catchall for functions that don't make sense as methods.""" -import asyncio -import hashlib -import operator - - -async def gather_dict(dic): - cors = list(dic.values()) - results = await asyncio.gather(*cors) - return dict(zip(dic.keys(), results)) - - -def digest(string): - if not isinstance(string, bytes): - string = str(string).encode("utf8") - return hashlib.sha1(string).digest() - - -class OrderedSet(list): - """ - Acts like a list in all ways, except in the behavior of the. - - :meth:`push` method. - """ - - def push(self, thing): - """ - 1. If the item exists in the list, it's removed - 2. The item is pushed to the end of the list - """ - if thing in self: - self.remove(thing) - self.append(thing) - - -def shared_prefix(args): - """ - Find the shared prefix between the strings. - - For instance: - - sharedPrefix(['blahblah', 'blahwhat']) - - returns 'blah'. - """ - i = 0 - while i < min(map(len, args)): - if len(set(map(operator.itemgetter(i), args))) != 1: - break - i += 1 - return args[0][:i] - - -def bytes_to_bit_string(bites): - bits = [bin(bite)[2:].rjust(8, "0") for bite in bites] - return "".join(bits) diff --git a/libp2p/routing/kademlia/__init__.py b/libp2p/routing/kademlia/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/libp2p/routing/kademlia/kademlia_content_router.py b/libp2p/routing/kademlia/kademlia_content_router.py deleted file mode 100644 index b623a252..00000000 --- a/libp2p/routing/kademlia/kademlia_content_router.py +++ /dev/null @@ -1,21 +0,0 @@ -from typing import Iterable - -from libp2p.peer.peerinfo import PeerInfo -from libp2p.routing.interfaces import IContentRouting - - -class KadmeliaContentRouter(IContentRouting): - def provide(self, cid: bytes, announce: bool = True) -> None: - """ - Provide adds the given cid to the content routing system. - - If announce is True, it also announces it, otherwise it is just - kept in the local accounting of which objects are being - provided. - """ - # the DHT finds the closest peers to `key` using the `FIND_NODE` RPC - # then sends a `ADD_PROVIDER` RPC with its own `PeerInfo` to each of these peers. - - def find_provider_iter(self, cid: bytes, count: int) -> Iterable[PeerInfo]: - """Search for peers who are able to provide a given key returns an - iterator of peer.PeerInfo.""" diff --git a/libp2p/routing/kademlia/kademlia_peer_router.py b/libp2p/routing/kademlia/kademlia_peer_router.py deleted file mode 100644 index 32f20757..00000000 --- a/libp2p/routing/kademlia/kademlia_peer_router.py +++ /dev/null @@ -1,43 +0,0 @@ -import json - -import multiaddr - -from libp2p.kademlia.network import KademliaServer -from libp2p.peer.id import ID -from libp2p.peer.peerinfo import PeerInfo -from libp2p.routing.interfaces import IPeerRouting - - -class KadmeliaPeerRouter(IPeerRouting): - server: KademliaServer - - def __init__(self, dht_server: KademliaServer) -> None: - self.server = dht_server - - async def find_peer(self, peer_id: ID) -> PeerInfo: - """ - Find a specific peer. - - :param peer_id: peer to search for - :return: PeerInfo of specified peer - """ - # switching peer_id to xor_id used by kademlia as node_id - xor_id = peer_id.xor_id - # ignore type for kad - value = await self.server.get(xor_id) # type: ignore - return ( - peer_info_from_str(value) if value else None - ) # TODO: should raise error if None? - - -def peer_info_to_str(peer_info: PeerInfo) -> str: - return json.dumps( - [peer_info.peer_id.to_string(), list(map(lambda a: str(a), peer_info.addrs))] - ) - - -def peer_info_from_str(string: str) -> PeerInfo: - peer_id, raw_addrs = json.loads(string) - return PeerInfo( - ID.from_base58(peer_id), list(map(lambda a: multiaddr.Multiaddr(a), raw_addrs)) - ) diff --git a/libp2p/tools/utils.py b/libp2p/tools/utils.py index 84e3edf9..d7792663 100644 --- a/libp2p/tools/utils.py +++ b/libp2p/tools/utils.py @@ -1,16 +1,16 @@ -from typing import List, Sequence, Tuple +from typing import Dict, Sequence, Tuple, cast import multiaddr from libp2p import new_node from libp2p.host.basic_host import BasicHost from libp2p.host.host_interface import IHost -from libp2p.kademlia.network import KademliaServer +from libp2p.host.routed_host import RoutedHost from libp2p.network.stream.net_stream_interface import INetStream from libp2p.network.swarm import Swarm -from libp2p.peer.peerinfo import info_from_p2p_addr +from libp2p.peer.id import ID +from libp2p.peer.peerinfo import PeerInfo, info_from_p2p_addr from libp2p.routing.interfaces import IPeerRouting -from libp2p.routing.kademlia.kademlia_peer_router import KadmeliaPeerRouter from libp2p.typing import StreamHandlerFn, TProtocol from .constants import MAX_READ_LEN @@ -47,35 +47,6 @@ async def set_up_nodes_by_transport_opt( return tuple(nodes_list) -async def set_up_nodes_by_transport_and_disc_opt( - transport_disc_opt_list: Sequence[Tuple[Sequence[str], IPeerRouting]] -) -> Tuple[BasicHost, ...]: - nodes_list = [] - for transport_opt, disc_opt in transport_disc_opt_list: - node = await new_node(transport_opt=transport_opt, disc_opt=disc_opt) - await node.get_network().listen(multiaddr.Multiaddr(transport_opt[0])) - nodes_list.append(node) - return tuple(nodes_list) - - -async def set_up_routers( - router_confs: Tuple[int, int] = (0, 0) -) -> List[KadmeliaPeerRouter]: - """The default ``router_confs`` selects two free ports local to this - machine.""" - bootstrap_node = KademliaServer() # type: ignore - await bootstrap_node.listen(router_confs[0]) - - routers = [KadmeliaPeerRouter(bootstrap_node)] - for port in router_confs[1:]: - node = KademliaServer() # type: ignore - await node.listen(port) - - await node.bootstrap_node(bootstrap_node.address) - routers.append(KadmeliaPeerRouter(node)) - return routers - - async def echo_stream_handler(stream: INetStream) -> None: while True: read_string = (await stream.read(MAX_READ_LEN)).decode() @@ -95,3 +66,33 @@ async def perform_two_host_set_up( # Associate the peer with local ip address (see default parameters of Libp2p()) node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10) return node_a, node_b + + +class DummyRouter(IPeerRouting): + _routing_table: Dict[ID, PeerInfo] + + def __init__(self) -> None: + self._routing_table = dict() + + async def find_peer(self, peer_id: ID) -> PeerInfo: + return self._routing_table.get(peer_id, None) + + +async def set_up_routed_hosts() -> Tuple[RoutedHost, RoutedHost]: + router_a, router_b = DummyRouter(), DummyRouter() + transport = "/ip4/127.0.0.1/tcp/0" + host_a = await new_node(transport_opt=[transport], disc_opt=router_a) + host_b = await new_node(transport_opt=[transport], disc_opt=router_b) + + address = multiaddr.Multiaddr(transport) + await host_a.get_network().listen(address) + await host_b.get_network().listen(address) + + mock_routing_table = { + host_a.get_id(): PeerInfo(host_a.get_id(), host_a.get_addrs()), + host_b.get_id(): PeerInfo(host_b.get_id(), host_b.get_addrs()), + } + + router_a._routing_table = router_b._routing_table = mock_routing_table + + return cast(RoutedHost, host_a), cast(RoutedHost, host_b) diff --git a/mypy.ini b/mypy.ini index fffd2aa8..653e3e2a 100644 --- a/mypy.ini +++ b/mypy.ini @@ -15,6 +15,3 @@ warn_redundant_casts = True warn_return_any = False warn_unused_configs = True warn_unreachable = True - -[mypy-libp2p.kademlia.*] -ignore_errors = True diff --git a/tests/host/test_routed_host.py b/tests/host/test_routed_host.py index 9083d3fc..271246cb 100644 --- a/tests/host/test_routed_host.py +++ b/tests/host/test_routed_host.py @@ -4,71 +4,30 @@ import pytest from libp2p.host.exceptions import ConnectionFailure from libp2p.peer.peerinfo import PeerInfo -from libp2p.routing.kademlia.kademlia_peer_router import peer_info_to_str -from libp2p.tools.utils import ( - set_up_nodes_by_transport_and_disc_opt, - set_up_nodes_by_transport_opt, - set_up_routers, -) +from libp2p.tools.utils import set_up_nodes_by_transport_opt, set_up_routed_hosts @pytest.mark.asyncio async def test_host_routing_success(): - routers = await set_up_routers() - transports = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] - transport_disc_opt_list = zip(transports, routers) - (host_a, host_b) = await set_up_nodes_by_transport_and_disc_opt( - transport_disc_opt_list - ) - - # Set routing info - await routers[0].server.set( - host_a.get_id().xor_id, - peer_info_to_str(PeerInfo(host_a.get_id(), host_a.get_addrs())), - ) - await routers[1].server.set( - host_b.get_id().xor_id, - peer_info_to_str(PeerInfo(host_b.get_id(), host_b.get_addrs())), - ) - + host_a, host_b = await set_up_routed_hosts() # forces to use routing as no addrs are provided await host_a.connect(PeerInfo(host_b.get_id(), [])) await host_b.connect(PeerInfo(host_a.get_id(), [])) # Clean up await asyncio.gather(*[host_a.close(), host_b.close()]) - routers[0].server.stop() - routers[1].server.stop() @pytest.mark.asyncio async def test_host_routing_fail(): - routers = await set_up_routers() - transports = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]] - transport_disc_opt_list = zip(transports, routers) - (host_a, host_b) = await set_up_nodes_by_transport_and_disc_opt( - transport_disc_opt_list - ) - - host_c = (await set_up_nodes_by_transport_opt([["/ip4/127.0.0.1/tcp/0"]]))[0] - - # Set routing info - await routers[0].server.set( - host_a.get_id().xor_id, - peer_info_to_str(PeerInfo(host_a.get_id(), host_a.get_addrs())), - ) - await routers[1].server.set( - host_b.get_id().xor_id, - peer_info_to_str(PeerInfo(host_b.get_id(), host_b.get_addrs())), - ) + host_a, host_b = await set_up_routed_hosts() + basic_host_c = (await set_up_nodes_by_transport_opt([["/ip4/127.0.0.1/tcp/0"]]))[0] # routing fails because host_c does not use routing with pytest.raises(ConnectionFailure): - await host_a.connect(PeerInfo(host_c.get_id(), [])) + await host_a.connect(PeerInfo(basic_host_c.get_id(), [])) with pytest.raises(ConnectionFailure): - await host_b.connect(PeerInfo(host_c.get_id(), [])) + await host_b.connect(PeerInfo(basic_host_c.get_id(), [])) # Clean up - await asyncio.gather(*[host_a.close(), host_b.close(), host_c.close()]) - routers[0].server.stop() - routers[1].server.stop() + await asyncio.gather(*[host_a.close(), host_b.close(), basic_host_c.close()]) diff --git a/tests/kademlia/test_basic.py b/tests/kademlia/test_basic.py deleted file mode 100644 index 655d4717..00000000 --- a/tests/kademlia/test_basic.py +++ /dev/null @@ -1,79 +0,0 @@ -import pytest - -from libp2p.kademlia.network import KademliaServer - - -@pytest.mark.asyncio -async def test_example(): - node_a = KademliaServer() - await node_a.listen() - - node_b = KademliaServer() - await node_b.listen() - - # Bootstrap the node by connecting to other known nodes, in this case - # replace 123.123.123.123 with the IP of another node and optionally - # give as many ip/port combos as you can for other nodes. - await node_b.bootstrap([node_a.address]) - - # set a value for the key "my-key" on the network - value = "my-value" - key = "my-key" - await node_b.set(key, value) - - # get the value associated with "my-key" from the network - assert await node_b.get(key) == value - assert await node_a.get(key) == value - - -@pytest.mark.parametrize("nodes_nr", [(2 ** i) for i in range(2, 5)]) -@pytest.mark.asyncio -async def test_multiple_nodes_bootstrap_set_get(nodes_nr): - - node_bootstrap = KademliaServer() - await node_bootstrap.listen(3000 + nodes_nr * 2) - - nodes = [] - for i in range(nodes_nr): - node = KademliaServer() - addrs = [("127.0.0.1", 3000 + nodes_nr * 2)] - await node.listen(3001 + i + nodes_nr * 2) - await node.bootstrap(addrs) - nodes.append(node) - - for i, node in enumerate(nodes): - # set a value for the key "my-key" on the network - value = "my awesome value %d" % i - key = "set from %d" % i - await node.set(key, value) - - for i in range(nodes_nr): - for node in nodes: - value = "my awesome value %d" % i - key = "set from %d" % i - assert await node.get(key) == value - - -@pytest.mark.parametrize("nodes_nr", [(2 ** i) for i in range(2, 5)]) -@pytest.mark.asyncio -async def test_multiple_nodes_set_bootstrap_get(nodes_nr): - node_bootstrap = KademliaServer() - await node_bootstrap.listen(2000 + nodes_nr * 2) - - nodes = [] - for i in range(nodes_nr): - node = KademliaServer() - addrs = [("127.0.0.1", 2000 + nodes_nr * 2)] - await node.listen(2001 + i + nodes_nr * 2) - await node.bootstrap(addrs) - - value = "my awesome value %d" % i - key = "set from %d" % i - await node.set(key, value) - nodes.append(node) - - for i in range(nodes_nr): - for node in nodes: - value = "my awesome value %d" % i - key = "set from %d" % i - assert await node.get(key) == value diff --git a/tests/kademlia/test_providers.py b/tests/kademlia/test_providers.py deleted file mode 100644 index 45993d92..00000000 --- a/tests/kademlia/test_providers.py +++ /dev/null @@ -1,30 +0,0 @@ -import pytest - -from libp2p.kademlia.network import KademliaServer - - -@pytest.mark.asyncio -async def test_example(): - node_a = KademliaServer() - await node_a.listen() - - node_b = KademliaServer() - await node_b.listen() - await node_b.bootstrap([node_a.address]) - - key = "hello" - value = "world" - await node_b.set(key, value) - await node_b.provide("hello") - - providers = await node_b.get_providers("hello") - - # bmuller's handle_call_response wraps - # every rpc call result in a list of tuples - # [(True, [b'\xf9\xa1\xf5\x10a\xe5\xe0F'])] - first_tuple = providers[0] - # (True, [b'\xf9\xa1\xf5\x10a\xe5\xe0F']) - first_providers = first_tuple[1] - # [b'\xf9\xa1\xf5\x10a\xe5\xe0F'] - first_provider = first_providers[0] - assert node_b.node.peer_id_bytes == first_provider diff --git a/tests/routing/test_kad_peer_router.py b/tests/routing/test_kad_peer_router.py deleted file mode 100644 index d3f38364..00000000 --- a/tests/routing/test_kad_peer_router.py +++ /dev/null @@ -1,75 +0,0 @@ -import pytest - -from libp2p.kademlia.network import KademliaServer -from libp2p.peer.id import ID -from libp2p.routing.kademlia.kademlia_peer_router import ( - KadmeliaPeerRouter, - peer_info_to_str, -) - - -@pytest.mark.asyncio -async def test_simple_two_nodes(): - node_a = KademliaServer() - await node_a.listen(5678) - - node_b = KademliaServer() - await node_b.listen(5679) - - node_a_value = await node_b.bootstrap([("127.0.0.1", 5678)]) - node_a_kad_peerinfo = node_a_value[0] - await node_a.set(node_a_kad_peerinfo.xor_id, peer_info_to_str(node_a_kad_peerinfo)) - - router = KadmeliaPeerRouter(node_b) - returned_info = await router.find_peer(ID(node_a_kad_peerinfo.peer_id_bytes)) - assert returned_info == node_a_kad_peerinfo - - -@pytest.mark.asyncio -async def test_simple_three_nodes(): - node_a = KademliaServer() - await node_a.listen(5701) - - node_b = KademliaServer() - await node_b.listen(5702) - - node_c = KademliaServer() - await node_c.listen(5703) - - node_a_value = await node_b.bootstrap([("127.0.0.1", 5701)]) - node_a_kad_peerinfo = node_a_value[0] - - await node_c.bootstrap([("127.0.0.1", 5702)]) - await node_a.set(node_a_kad_peerinfo.xor_id, peer_info_to_str(node_a_kad_peerinfo)) - - router = KadmeliaPeerRouter(node_c) - returned_info = await router.find_peer(ID(node_a_kad_peerinfo.peer_id_bytes)) - assert returned_info == node_a_kad_peerinfo - - -@pytest.mark.asyncio -async def test_simple_four_nodes(): - node_a = KademliaServer() - await node_a.listen(5801) - - node_b = KademliaServer() - await node_b.listen(5802) - - node_c = KademliaServer() - await node_c.listen(5803) - - node_d = KademliaServer() - await node_d.listen(5804) - - node_a_value = await node_b.bootstrap([("127.0.0.1", 5801)]) - node_a_kad_peerinfo = node_a_value[0] - - await node_c.bootstrap([("127.0.0.1", 5802)]) - - await node_d.bootstrap([("127.0.0.1", 5803)]) - - await node_b.set(node_a_kad_peerinfo.xor_id, peer_info_to_str(node_a_kad_peerinfo)) - - router = KadmeliaPeerRouter(node_d) - returned_info = await router.find_peer(ID(node_a_kad_peerinfo.peer_id_bytes)) - assert returned_info == node_a_kad_peerinfo From 2d3bfc8184e892ef51967c60b73402cac0ccb873 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Fri, 6 Dec 2019 23:42:31 +0800 Subject: [PATCH 148/178] Apply PR feedback: use defaultdict and init control message --- libp2p/pubsub/gossipsub.py | 64 ++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 37 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 208d2878..b8b0e624 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -1,8 +1,9 @@ from ast import literal_eval import asyncio +from collections import defaultdict import logging import random -from typing import Any, Dict, Iterable, List, Sequence, Tuple +from typing import Any, DefaultDict, Dict, Iterable, List, Sequence, Tuple from libp2p.network.stream.exceptions import StreamClosed from libp2p.peer.id import ID @@ -319,24 +320,24 @@ class GossipSub(IPubsubRouter): # Starting with GRAFT messages for peer, topics in peers_to_graft.items(): for topic in topics: - graft_msg: rpc_pb2.ControlGraft = rpc_pb2.ControlGraft() - graft_msg.topicID = topic + graft_msg: rpc_pb2.ControlGraft = rpc_pb2.ControlGraft(topicID=topic) graft_msgs.append(graft_msg) # If there are also PRUNE messages to send to this peer if peer in peers_to_prune: for topic in peers_to_prune[peer]: - prune_msg: rpc_pb2.ControlPrune = rpc_pb2.ControlPrune() - prune_msg.topicID = topic + prune_msg: rpc_pb2.ControlPrune = rpc_pb2.ControlPrune( + topicID=topic + ) prune_msgs.append(prune_msg) del peers_to_prune[peer] # If there are also IHAVE messages to send to this peer if peer in peers_to_gossip: for topic in peers_to_gossip[peer]: - ihave_msg: rpc_pb2.ControlIHave = rpc_pb2.ControlIHave() - ihave_msg.messageIDs.extend(peers_to_gossip[peer][topic]) - ihave_msg.topicID = topic + ihave_msg: rpc_pb2.ControlIHave = rpc_pb2.ControlIHave( + messageIDs=peers_to_gossip[peer][topic], topicID=topic + ) ihave_msgs.append(ihave_msg) del peers_to_gossip[peer] @@ -347,17 +348,16 @@ class GossipSub(IPubsubRouter): for peer, topics in peers_to_prune.items(): prune_msgs = [] for topic in topics: - prune_msg = rpc_pb2.ControlPrune() - prune_msg.topicID = topic + prune_msg = rpc_pb2.ControlPrune(topicID=topic) prune_msgs.append(prune_msg) # If there are also IHAVE messages to send to this peer if peer in peers_to_gossip: ihave_msgs = [] for topic in peers_to_gossip[peer]: - ihave_msg = rpc_pb2.ControlIHave() - ihave_msg.messageIDs.extend(peers_to_gossip[peer][topic]) - ihave_msg.topicID = topic + ihave_msg = rpc_pb2.ControlIHave( + messageIDs=peers_to_gossip[peer][topic], topicID=topic + ) ihave_msgs.append(ihave_msg) del peers_to_gossip[peer] @@ -368,9 +368,9 @@ class GossipSub(IPubsubRouter): for peer in peers_to_gossip: ihave_msgs = [] for topic in peers_to_gossip[peer]: - ihave_msg = rpc_pb2.ControlIHave() - ihave_msg.messageIDs.extend(peers_to_gossip[peer][topic]) - ihave_msg.topicID = topic + ihave_msg = rpc_pb2.ControlIHave( + messageIDs=peers_to_gossip[peer][topic], topicID=topic + ) ihave_msgs.append(ihave_msg) control_msg = self.pack_control_msgs(ihave_msgs, None, None) @@ -402,9 +402,11 @@ class GossipSub(IPubsubRouter): await asyncio.sleep(self.heartbeat_interval) - def mesh_heartbeat(self) -> Tuple[Dict[ID, List[str]], Dict[ID, List[str]]]: - peers_to_graft: Dict[ID, List[str]] = {} - peers_to_prune: Dict[ID, List[str]] = {} + def mesh_heartbeat( + self + ) -> Tuple[DefaultDict[ID, List[str]], DefaultDict[ID, List[str]]]: + peers_to_graft: DefaultDict[ID, List[str]] = defaultdict(list) + peers_to_prune: DefaultDict[ID, List[str]] = defaultdict(list) for topic in self.mesh: # Skip if no peers have subscribed to the topic if topic not in self.pubsub.peer_topics: @@ -422,10 +424,7 @@ class GossipSub(IPubsubRouter): self.mesh[topic].append(peer) # Emit GRAFT(topic) control message to peer - if peer not in peers_to_graft: - peers_to_graft[peer] = [topic] - else: - peers_to_graft[peer].append(topic) + peers_to_graft[peer].append(topic) if num_mesh_peers_in_topic > self.degree_high: # Select |mesh[topic]| - D peers from mesh[topic] @@ -437,10 +436,7 @@ class GossipSub(IPubsubRouter): self.mesh[topic].remove(peer) # Emit PRUNE(topic) control message to peer - if peer not in peers_to_prune: - peers_to_prune[peer] = [topic] - else: - peers_to_prune[peer].append(topic) + peers_to_prune[peer].append(topic) return peers_to_graft, peers_to_prune def fanout_heartbeat(self) -> None: @@ -478,8 +474,8 @@ class GossipSub(IPubsubRouter): # Add the peers to fanout[topic] self.fanout[topic].extend(selected_peers) - def gossip_heartbeat(self) -> Dict[ID, Dict[str, List[str]]]: - peers_to_gossip: Dict[ID, Dict[str, List[str]]] = {} + def gossip_heartbeat(self) -> DefaultDict[ID, Dict[str, List[str]]]: + peers_to_gossip: DefaultDict[ID, Dict[str, List[str]]] = defaultdict(dict) for topic in self.mesh: msg_ids = self.mcache.window(topic) if msg_ids: @@ -492,10 +488,7 @@ class GossipSub(IPubsubRouter): msg_id_strs = [str(msg_id) for msg_id in msg_ids] for peer in peers_to_emit_ihave_to: - if peer not in peers_to_gossip: - peers_to_gossip[peer] = {topic: msg_id_strs} - else: - peers_to_gossip[peer][topic] = msg_id_strs + peers_to_gossip[peer][topic] = msg_id_strs # TODO: Refactor and Dedup. This section is the roughly the same as the above. # Do the same for fanout, for all topics not already hit in mesh @@ -510,10 +503,7 @@ class GossipSub(IPubsubRouter): ) msg_id_strs = [str(msg) for msg in msg_ids] for peer in peers_to_emit_ihave_to: - if peer not in peers_to_gossip: - peers_to_gossip[peer] = {topic: msg_id_strs} - else: - peers_to_gossip[peer][topic] = msg_id_strs + peers_to_gossip[peer][topic] = msg_id_strs return peers_to_gossip @staticmethod From a675da52ee4063dd659edd36d0c3ce665f219625 Mon Sep 17 00:00:00 2001 From: NIC Lin Date: Sat, 7 Dec 2019 15:46:42 +0800 Subject: [PATCH 149/178] Update libp2p/pubsub/gossipsub.py Co-Authored-By: Kevin Mai-Husan Chia --- libp2p/pubsub/gossipsub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index b8b0e624..045ef397 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -235,7 +235,7 @@ class GossipSub(IPubsubRouter): gossipsub_peers = self.mesh[topic] else: # When we publish to a topic that we have not subscribe to, we randomly pick - # `self.degree` number of peers who have subscribe to the topic and add them + # `self.degree` number of peers who have subscribed to the topic and add them # as our `fanout` peers. topic_in_fanout: bool = topic in self.fanout fanout_peers: List[ID] = self.fanout[topic] if topic_in_fanout else [] From 3b9d7c7acd7af955b4e1855effac85aec44ca81f Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 10 Dec 2019 17:07:21 -0800 Subject: [PATCH 150/178] Apply PR feedback --- .circleci/config.yml | 20 +++++++++++++------- libp2p/network/connection/raw_connection.py | 5 +++-- libp2p/tools/factories.py | 10 +++------- libp2p/transport/tcp/tcp.py | 6 ++++-- setup.py | 3 ++- tests_interop/conftest.py | 5 +++-- tox.ini | 3 +-- 7 files changed, 29 insertions(+), 23 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c3094b9b..fa7691e6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -48,24 +48,30 @@ jobs: - image: circleci/python:3.6 environment: TOXENV: lint - py36-test: + py36-core: <<: *common docker: - image: circleci/python:3.6 environment: - TOXENV: py36-test - py37-test: + TOXENV: py36-core + py37-core: <<: *common docker: - image: circleci/python:3.7 environment: - TOXENV: py37-test - + TOXENV: py37-core + pypy3-core: + <<: *common + docker: + - image: pypy + environment: + TOXENV: pypy3-core workflows: version: 2 test: jobs: - docs - lint - - py36-test - - py37-test + - py36-core + - py37-core + - pypy3-core diff --git a/libp2p/network/connection/raw_connection.py b/libp2p/network/connection/raw_connection.py index 55b47efc..4f01b124 100644 --- a/libp2p/network/connection/raw_connection.py +++ b/libp2p/network/connection/raw_connection.py @@ -53,5 +53,6 @@ class RawConnection(IRawConnection): async def close(self) -> None: self.writer.close() - if sys.version_info[0:2] > (3, 6): - await self.writer.wait_closed() + if sys.version_info < (3, 7): + return + await self.writer.wait_closed() diff --git a/libp2p/tools/factories.py b/libp2p/tools/factories.py index 5e223130..5c9c310d 100644 --- a/libp2p/tools/factories.py +++ b/libp2p/tools/factories.py @@ -1,6 +1,8 @@ import asyncio from typing import Any, AsyncIterator, Dict, Tuple, cast +# NOTE: import ``asynccontextmanager`` from ``contextlib`` when support for python 3.6 is dropped. +from async_generator import asynccontextmanager import factory from libp2p import generate_new_rsa_identity, generate_peer_id_from @@ -32,12 +34,6 @@ from .constants import ( ) from .utils import connect, connect_swarm -try: - from contextlib import asynccontextmanager -except ImportError: - # NOTE: mypy complains about a duplicate import without the following ``# type: ignore`` - from async_generator import asynccontextmanager # type: ignore - def initialize_peerstore_with_our_keypair(self_id: ID, key_pair: KeyPair) -> PeerStore: peer_store = PeerStore() @@ -177,7 +173,7 @@ async def host_pair_factory(is_secure: bool) -> Tuple[BasicHost, BasicHost]: return hosts[0], hosts[1] -@asynccontextmanager +@asynccontextmanager # type: ignore async def pair_of_connected_hosts( is_secure: bool = True ) -> AsyncIterator[Tuple[BasicHost, BasicHost]]: diff --git a/libp2p/transport/tcp/tcp.py b/libp2p/transport/tcp/tcp.py index c27de0e1..f5c2aa40 100644 --- a/libp2p/transport/tcp/tcp.py +++ b/libp2p/transport/tcp/tcp.py @@ -54,9 +54,11 @@ class TCPListener(IListener): if self.server is None: return self.server.close() - if sys.version_info[0:2] > (3, 6): - await self.server.wait_closed() + server = self.server self.server = None + if sys.version_info < (3, 7): + return + await server.wait_closed() class TCP(ITransport): diff --git a/setup.py b/setup.py index 35b17bf8..56a6bf20 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ install_requires = [ "coincurve>=10.0.0,<11.0.0", "pynacl==1.3.0", "dataclasses>=0.7, <1;python_version<'3.7'", - "async_generator==1.10;python_version<'3.7'", + "async_generator==1.10", ] @@ -96,6 +96,7 @@ setup( "License :: OSI Approved :: Apache Software License", "Natural Language :: English", "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", ], platforms=["unix", "linux", "osx"], diff --git a/tests_interop/conftest.py b/tests_interop/conftest.py index 3fcc74f3..08df614c 100644 --- a/tests_interop/conftest.py +++ b/tests_interop/conftest.py @@ -151,8 +151,9 @@ class DaemonStream(ReadWriteCloser): async def close(self) -> None: self.writer.close() - if sys.version_info[0:2] > (3, 6): - await self.writer.wait_closed() + if sys.version_info < (3, 7): + return + await self.writer.wait_closed() async def read(self, n: int = -1) -> bytes: return await self.reader.read(n) diff --git a/tox.ini b/tox.ini index a482e70f..21f46435 100644 --- a/tox.ini +++ b/tox.ini @@ -3,8 +3,7 @@ # TODO: consider pypy3 support [tox] envlist = - py36-test - py37-test + py{36,37}-test py37-interop lint docs From 23fa86979db43102e44c300b75008c6394c49b5f Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Thu, 12 Dec 2019 13:53:09 -0800 Subject: [PATCH 151/178] Added release notes --- newsfragments/362.feature.rst | 1 + newsfragments/372.feature.rst | 1 + newsfragments/373.internal.rst | 1 + 3 files changed, 3 insertions(+) create mode 100644 newsfragments/362.feature.rst create mode 100644 newsfragments/372.feature.rst create mode 100644 newsfragments/373.internal.rst diff --git a/newsfragments/362.feature.rst b/newsfragments/362.feature.rst new file mode 100644 index 00000000..c631965c --- /dev/null +++ b/newsfragments/362.feature.rst @@ -0,0 +1 @@ +Add signing and verification to pubsub diff --git a/newsfragments/372.feature.rst b/newsfragments/372.feature.rst new file mode 100644 index 00000000..08df186d --- /dev/null +++ b/newsfragments/372.feature.rst @@ -0,0 +1 @@ +Added support for Python 3.6 diff --git a/newsfragments/373.internal.rst b/newsfragments/373.internal.rst new file mode 100644 index 00000000..0adfaae3 --- /dev/null +++ b/newsfragments/373.internal.rst @@ -0,0 +1 @@ +Refactor and cleanup gossipsub From 368ac7ef7f89ad619f7d73e2e1e89054b39ef2e3 Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Thu, 12 Dec 2019 13:57:27 -0800 Subject: [PATCH 152/178] Compile release notes --- docs/release_notes.rst | 16 ++++++++++++++++ newsfragments/362.feature.rst | 1 - newsfragments/372.feature.rst | 1 - newsfragments/373.internal.rst | 1 - 4 files changed, 16 insertions(+), 3 deletions(-) delete mode 100644 newsfragments/362.feature.rst delete mode 100644 newsfragments/372.feature.rst delete mode 100644 newsfragments/373.internal.rst diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 7537972c..67501e54 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -3,6 +3,22 @@ Release Notes .. towncrier release notes start +libp2p v0.1.4 (2019-12-12) +-------------------------- + +Features +~~~~~~~~ + +- Added support for Python 3.6 (`#372 `__) +- Add signing and verification to pubsub (`#362 `__) + + +Internal Changes - for py-libp2p Contributors +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Refactor and cleanup gossipsub (`#373 `__) + + libp2p v0.1.3 (2019-11-27) -------------------------- diff --git a/newsfragments/362.feature.rst b/newsfragments/362.feature.rst deleted file mode 100644 index c631965c..00000000 --- a/newsfragments/362.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Add signing and verification to pubsub diff --git a/newsfragments/372.feature.rst b/newsfragments/372.feature.rst deleted file mode 100644 index 08df186d..00000000 --- a/newsfragments/372.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Added support for Python 3.6 diff --git a/newsfragments/373.internal.rst b/newsfragments/373.internal.rst deleted file mode 100644 index 0adfaae3..00000000 --- a/newsfragments/373.internal.rst +++ /dev/null @@ -1 +0,0 @@ -Refactor and cleanup gossipsub From ef31f7f6d658c6cee1b0f6a83c769d90ee83275e Mon Sep 17 00:00:00 2001 From: Jason Carver Date: Thu, 12 Dec 2019 13:59:52 -0800 Subject: [PATCH 153/178] =?UTF-8?q?Bump=20version:=200.1.3=20=E2=86=92=200?= =?UTF-8?q?.1.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 08e5fbfc..e96e5c85 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.1.3 +current_version = 0.1.4 commit = True tag = True parse = (?P\d+)\.(?P\d+)\.(?P\d+)(-(?P[^.]*)\.(?P\d+))? diff --git a/setup.py b/setup.py index 56a6bf20..28d29fa2 100644 --- a/setup.py +++ b/setup.py @@ -73,7 +73,7 @@ if not readthedocs_is_building: setup( name="libp2p", # *IMPORTANT*: Don't manually change the version here. Use `make bump`, as described in readme - version="0.1.3", + version="0.1.4", description="libp2p implementation written in python", long_description=long_description, long_description_content_type="text/markdown", From 7d6daa8e10d3c75cd88b80c2f9f8fec2f3dfe37e Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 17 Dec 2019 17:17:03 +0800 Subject: [PATCH 154/178] Minor cleanup: - remove outdated comment - add new peer at the end - turn peers to send from list to set --- libp2p/pubsub/floodsub.py | 10 ++++++---- libp2p/pubsub/gossipsub.py | 2 -- libp2p/pubsub/pubsub.py | 8 ++------ 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index bac0bd77..678e5090 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -77,10 +77,12 @@ class FloodSub(IPubsubRouter): :param pubsub_msg: pubsub message in protobuf. """ - peers_gen = self._get_peers_to_send( - pubsub_msg.topicIDs, - msg_forwarder=msg_forwarder, - origin=ID(pubsub_msg.from_id), + peers_gen = set( + self._get_peers_to_send( + pubsub_msg.topicIDs, + msg_forwarder=msg_forwarder, + origin=ID(pubsub_msg.from_id), + ) ) rpc_msg = rpc_pb2.RPC(publish=[pubsub_msg]) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 045ef397..1969d4f3 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -148,11 +148,9 @@ class GossipSub(IPubsubRouter): for topic in self.mesh: if peer_id in self.mesh[topic]: - # Delete the entry if no other peers left self.mesh[topic].remove(peer_id) for topic in self.fanout: if peer_id in self.fanout[topic]: - # Delete the entry if no other peers left self.fanout[topic].remove(peer_id) self.peers_to_protocol.pop(peer_id, None) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index a44aa052..6e9d9480 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -289,24 +289,22 @@ class Pubsub: logger.debug("fail to add new peer %s, error %s", peer_id, error) return - self.peers[peer_id] = stream - # Send hello packet hello = self.get_hello_packet() try: await stream.write(encode_varint_prefixed(hello.SerializeToString())) except StreamClosed: logger.debug("Fail to add new peer %s: stream closed", peer_id) - del self.peers[peer_id] return # TODO: Check if the peer in black list. try: self.router.add_peer(peer_id, stream.get_protocol()) except Exception as error: logger.debug("fail to add new peer %s, error %s", peer_id, error) - del self.peers[peer_id] return + self.peers[peer_id] = stream + logger.debug("added new peer %s", peer_id) def _handle_dead_peer(self, peer_id: ID) -> None: @@ -316,7 +314,6 @@ class Pubsub: for topic in self.peer_topics: if peer_id in self.peer_topics[topic]: - # Delete the entry if no other peers left self.peer_topics[topic].remove(peer_id) self.router.remove_peer(peer_id) @@ -360,7 +357,6 @@ class Pubsub: else: if sub_message.topicid in self.peer_topics: if origin_id in self.peer_topics[sub_message.topicid]: - # Delete the entry if no other peers left self.peer_topics[sub_message.topicid].remove(origin_id) # FIXME(mhchia): Change the function name? From f1d58ef8ffe85bf3e6bb3140080f7709b9b2b152 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 17 Dec 2019 17:30:24 +0800 Subject: [PATCH 155/178] Change type of peers from list to set: `peers_gossipsub`, `peers_floodsub` and mesh/fanout peers --- libp2p/pubsub/gossipsub.py | 42 +++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 1969d4f3..7ef1372d 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -3,7 +3,7 @@ import asyncio from collections import defaultdict import logging import random -from typing import Any, DefaultDict, Dict, Iterable, List, Sequence, Tuple +from typing import Any, DefaultDict, Dict, Iterable, List, Sequence, Set, Tuple from libp2p.network.stream.exceptions import StreamClosed from libp2p.peer.id import ID @@ -32,15 +32,15 @@ class GossipSub(IPubsubRouter): time_to_live: int - mesh: Dict[str, List[ID]] - fanout: Dict[str, List[ID]] + mesh: Dict[str, Set[ID]] + fanout: Dict[str, Set[ID]] peers_to_protocol: Dict[ID, str] time_since_last_publish: Dict[str, int] - peers_gossipsub: List[ID] - peers_floodsub: List[ID] + peers_gossipsub: Set[ID] + peers_floodsub: Set[ID] mcache: MessageCache @@ -80,8 +80,8 @@ class GossipSub(IPubsubRouter): # Create topic --> time since last publish map self.time_since_last_publish = {} - self.peers_gossipsub = [] - self.peers_floodsub = [] + self.peers_gossipsub = set() + self.peers_floodsub = set() # Create message cache self.mcache = MessageCache(gossip_window, gossip_history) @@ -122,9 +122,9 @@ class GossipSub(IPubsubRouter): logger.debug("adding peer %s with protocol %s", peer_id, protocol_id) if protocol_id == PROTOCOL_ID: - self.peers_gossipsub.append(peer_id) + self.peers_gossipsub.add(peer_id) elif protocol_id == floodsub.PROTOCOL_ID: - self.peers_floodsub.append(peer_id) + self.peers_floodsub.add(peer_id) else: # We should never enter here. Becuase the `protocol_id` is registered by your pubsub # instance in multistream-select, but it is not the protocol that gossipsub supports. @@ -142,16 +142,16 @@ class GossipSub(IPubsubRouter): logger.debug("removing peer %s", peer_id) if peer_id in self.peers_gossipsub: - self.peers_gossipsub.remove(peer_id) + self.peers_gossipsub.discard(peer_id) elif peer_id in self.peers_floodsub: - self.peers_floodsub.remove(peer_id) + self.peers_floodsub.discard(peer_id) for topic in self.mesh: if peer_id in self.mesh[topic]: - self.mesh[topic].remove(peer_id) + self.mesh[topic].discard(peer_id) for topic in self.fanout: if peer_id in self.fanout[topic]: - self.fanout[topic].remove(peer_id) + self.fanout[topic].discard(peer_id) self.peers_to_protocol.pop(peer_id, None) @@ -246,7 +246,7 @@ class GossipSub(IPubsubRouter): fanout_peers += self._get_in_topic_gossipsub_peers_from_minus( topic, self.degree - fanout_size, fanout_peers ) - self.fanout[topic] = fanout_peers + self.fanout[topic] = set(fanout_peers) gossipsub_peers = fanout_peers send_to.extend(floodsub_peers + gossipsub_peers) # Excludes `msg_forwarder` and `origin` @@ -282,7 +282,7 @@ class GossipSub(IPubsubRouter): # Add fanout peers to mesh and notifies them with a GRAFT(topic) control message. for peer in fanout_peers: - self.mesh[topic].append(peer) + self.mesh[topic].add(peer) await self.emit_graft(topic, peer) self.fanout.pop(topic, None) @@ -419,7 +419,7 @@ class GossipSub(IPubsubRouter): for peer in selected_peers: # Add peer to mesh[topic] - self.mesh[topic].append(peer) + self.mesh[topic].add(peer) # Emit GRAFT(topic) control message to peer peers_to_graft[peer].append(topic) @@ -431,7 +431,7 @@ class GossipSub(IPubsubRouter): ) for peer in selected_peers: # Remove peer from mesh[topic] - self.mesh[topic].remove(peer) + self.mesh[topic].discard(peer) # Emit PRUNE(topic) control message to peer peers_to_prune[peer].append(topic) @@ -458,7 +458,7 @@ class GossipSub(IPubsubRouter): for peer in self.fanout[topic] if peer in self.pubsub.peer_topics[topic] ] - self.fanout[topic] = in_topic_fanout_peers + self.fanout[topic] = set(in_topic_fanout_peers) num_fanout_peers_in_topic = len(self.fanout[topic]) # If |fanout[topic]| < D @@ -470,7 +470,7 @@ class GossipSub(IPubsubRouter): self.fanout[topic], ) # Add the peers to fanout[topic] - self.fanout[topic].extend(selected_peers) + self.fanout[topic].Update(selected_peers) def gossip_heartbeat(self) -> DefaultDict[ID, Dict[str, List[str]]]: peers_to_gossip: DefaultDict[ID, Dict[str, List[str]]] = defaultdict(dict) @@ -621,7 +621,7 @@ class GossipSub(IPubsubRouter): # Add peer to mesh for topic if topic in self.mesh: if sender_peer_id not in self.mesh[topic]: - self.mesh[topic].append(sender_peer_id) + self.mesh[topic].add(sender_peer_id) else: # Respond with PRUNE if not subscribed to the topic await self.emit_prune(topic, sender_peer_id) @@ -633,7 +633,7 @@ class GossipSub(IPubsubRouter): # Remove peer from mesh for topic, if peer is in topic if topic in self.mesh and sender_peer_id in self.mesh[topic]: - self.mesh[topic].remove(sender_peer_id) + self.mesh[topic].discard(sender_peer_id) # RPC emitters From 65766ec9aca3a21eac7243ec834fd71f19521747 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 17 Dec 2019 17:36:15 +0800 Subject: [PATCH 156/178] Change type of local peers var from list to set --- libp2p/pubsub/gossipsub.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 7ef1372d..61287940 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -213,7 +213,7 @@ class GossipSub(IPubsubRouter): :param origin: peer id of the peer the message originate from. :return: a generator of the peer ids who we send data to. """ - send_to: List[ID] = [] + send_to: Set[ID] = set() for topic in topic_ids: if topic not in self.pubsub.peer_topics: continue @@ -221,14 +221,15 @@ class GossipSub(IPubsubRouter): # floodsub peers # FIXME: `gossipsub.peers_floodsub` can be changed to `gossipsub.peers` in go. # This will improve the efficiency when searching for a peer's protocol id. - floodsub_peers: List[ID] = [ + floodsub_peers: Set[ID] = set( peer_id for peer_id in self.pubsub.peer_topics[topic] if peer_id in self.peers_floodsub - ] + ) + send_to.update(floodsub_peers) # gossipsub peers - gossipsub_peers: List[ID] = [] + gossipsub_peers: Set[ID] = set() if topic in self.mesh: gossipsub_peers = self.mesh[topic] else: @@ -236,21 +237,23 @@ class GossipSub(IPubsubRouter): # `self.degree` number of peers who have subscribed to the topic and add them # as our `fanout` peers. topic_in_fanout: bool = topic in self.fanout - fanout_peers: List[ID] = self.fanout[topic] if topic_in_fanout else [] + fanout_peers: Set[ID] = self.fanout[topic] if topic_in_fanout else set() fanout_size = len(fanout_peers) if not topic_in_fanout or ( topic_in_fanout and fanout_size < self.degree ): if topic in self.pubsub.peer_topics: # Combine fanout peers with selected peers - fanout_peers += self._get_in_topic_gossipsub_peers_from_minus( - topic, self.degree - fanout_size, fanout_peers + fanout_peers.update( + self._get_in_topic_gossipsub_peers_from_minus( + topic, self.degree - fanout_size, fanout_peers + ) ) - self.fanout[topic] = set(fanout_peers) + self.fanout[topic] = fanout_peers gossipsub_peers = fanout_peers - send_to.extend(floodsub_peers + gossipsub_peers) + send_to.update(gossipsub_peers) # Excludes `msg_forwarder` and `origin` - yield from set(send_to).difference([msg_forwarder, origin]) + yield from send_to.difference([msg_forwarder, origin]) async def join(self, topic: str) -> None: """ @@ -264,10 +267,10 @@ class GossipSub(IPubsubRouter): if topic in self.mesh: return # Create mesh[topic] if it does not yet exist - self.mesh[topic] = [] + self.mesh[topic] = set() topic_in_fanout: bool = topic in self.fanout - fanout_peers: List[ID] = self.fanout[topic] if topic_in_fanout else [] + fanout_peers: Set[ID] = self.fanout[topic] if topic_in_fanout else set() fanout_size = len(fanout_peers) if not topic_in_fanout or (topic_in_fanout and fanout_size < self.degree): # There are less than D peers (let this number be x) @@ -278,7 +281,7 @@ class GossipSub(IPubsubRouter): topic, self.degree - fanout_size, fanout_peers ) # Combine fanout peers with selected peers - fanout_peers += selected_peers + fanout_peers.update(selected_peers) # Add fanout peers to mesh and notifies them with a GRAFT(topic) control message. for peer in fanout_peers: From b4bd997932ff5ee78a7f4f3c63e7e5cf1dddc397 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 17 Dec 2019 17:49:49 +0800 Subject: [PATCH 157/178] Fix mypy --- libp2p/pubsub/gossipsub.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 61287940..bddafa0a 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -246,7 +246,7 @@ class GossipSub(IPubsubRouter): # Combine fanout peers with selected peers fanout_peers.update( self._get_in_topic_gossipsub_peers_from_minus( - topic, self.degree - fanout_size, fanout_peers + topic, self.degree - fanout_size, list(fanout_peers) ) ) self.fanout[topic] = fanout_peers @@ -278,7 +278,7 @@ class GossipSub(IPubsubRouter): # Selects the remaining number of peers (D-x) from peers.gossipsub[topic]. if topic in self.pubsub.peer_topics: selected_peers = self._get_in_topic_gossipsub_peers_from_minus( - topic, self.degree - fanout_size, fanout_peers + topic, self.degree - fanout_size, list(fanout_peers) ) # Combine fanout peers with selected peers fanout_peers.update(selected_peers) @@ -417,7 +417,7 @@ class GossipSub(IPubsubRouter): if num_mesh_peers_in_topic < self.degree_low: # Select D - |mesh[topic]| peers from peers.gossipsub[topic] - mesh[topic] selected_peers = self._get_in_topic_gossipsub_peers_from_minus( - topic, self.degree - num_mesh_peers_in_topic, self.mesh[topic] + topic, self.degree - num_mesh_peers_in_topic, list(self.mesh[topic]) ) for peer in selected_peers: @@ -430,7 +430,7 @@ class GossipSub(IPubsubRouter): if num_mesh_peers_in_topic > self.degree_high: # Select |mesh[topic]| - D peers from mesh[topic] selected_peers = GossipSub.select_from_minus( - num_mesh_peers_in_topic - self.degree, self.mesh[topic], [] + num_mesh_peers_in_topic - self.degree, list(self.mesh[topic]), [] ) for peer in selected_peers: # Remove peer from mesh[topic] @@ -470,10 +470,10 @@ class GossipSub(IPubsubRouter): selected_peers = self._get_in_topic_gossipsub_peers_from_minus( topic, self.degree - num_fanout_peers_in_topic, - self.fanout[topic], + list(self.fanout[topic]), ) # Add the peers to fanout[topic] - self.fanout[topic].Update(selected_peers) + self.fanout[topic].update(selected_peers) def gossip_heartbeat(self) -> DefaultDict[ID, Dict[str, List[str]]]: peers_to_gossip: DefaultDict[ID, Dict[str, List[str]]] = defaultdict(dict) @@ -484,7 +484,7 @@ class GossipSub(IPubsubRouter): if topic in self.pubsub.peer_topics: # Select D peers from peers.gossipsub[topic] peers_to_emit_ihave_to = self._get_in_topic_gossipsub_peers_from_minus( - topic, self.degree, self.mesh[topic] + topic, self.degree, list(self.mesh[topic]) ) msg_id_strs = [str(msg_id) for msg_id in msg_ids] @@ -500,7 +500,7 @@ class GossipSub(IPubsubRouter): if topic in self.pubsub.peer_topics: # Select D peers from peers.gossipsub[topic] peers_to_emit_ihave_to = self._get_in_topic_gossipsub_peers_from_minus( - topic, self.degree, self.fanout[topic] + topic, self.degree, list(self.fanout[topic]) ) msg_id_strs = [str(msg) for msg in msg_ids] for peer in peers_to_emit_ihave_to: @@ -528,7 +528,7 @@ class GossipSub(IPubsubRouter): # If num_to_select > size(selection_pool), then return selection_pool (which has the most # possible elements s.t. the number of elements is less than num_to_select) - if num_to_select > len(selection_pool): + if num_to_select >= len(selection_pool): return selection_pool # Random selection From f10e3099cb04cd2b475759b283d0038b2fb02d7c Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 17 Dec 2019 17:55:13 +0800 Subject: [PATCH 158/178] Change type of peers in pubsub from list to set --- libp2p/pubsub/pubsub.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 6e9d9480..2d5625f7 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -8,6 +8,7 @@ from typing import ( Dict, List, NamedTuple, + Set, Tuple, Union, cast, @@ -73,7 +74,7 @@ class Pubsub: my_topics: Dict[str, "asyncio.Queue[rpc_pb2.Message]"] - peer_topics: Dict[str, List[ID]] + peer_topics: Dict[str, Set[ID]] peers: Dict[ID, INetStream] topic_validators: Dict[str, TopicValidator] @@ -314,7 +315,7 @@ class Pubsub: for topic in self.peer_topics: if peer_id in self.peer_topics[topic]: - self.peer_topics[topic].remove(peer_id) + self.peer_topics[topic].discard(peer_id) self.router.remove_peer(peer_id) @@ -353,11 +354,11 @@ class Pubsub: self.peer_topics[sub_message.topicid] = [origin_id] elif origin_id not in self.peer_topics[sub_message.topicid]: # Add peer to topic - self.peer_topics[sub_message.topicid].append(origin_id) + self.peer_topics[sub_message.topicid].add(origin_id) else: if sub_message.topicid in self.peer_topics: if origin_id in self.peer_topics[sub_message.topicid]: - self.peer_topics[sub_message.topicid].remove(origin_id) + self.peer_topics[sub_message.topicid].discard(origin_id) # FIXME(mhchia): Change the function name? async def handle_talk(self, publish_message: rpc_pb2.Message) -> None: From 794d2101e9b5ce4d1efd3d94b8dacfa508145382 Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Tue, 17 Dec 2019 11:00:45 +0100 Subject: [PATCH 159/178] fixes #197 --- libp2p/peer/id.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libp2p/peer/id.py b/libp2p/peer/id.py index ab20ed04..9870011b 100644 --- a/libp2p/peer/id.py +++ b/libp2p/peer/id.py @@ -44,7 +44,7 @@ class ID: @property def xor_id(self) -> int: if not self._xor_id: - self._xor_id = int(digest(self._bytes).hex(), 16) + self._xor_id = int(sha256_digest(self._bytes).hex(), 16) return self._xor_id def to_bytes(self) -> bytes: @@ -89,7 +89,7 @@ class ID: return cls(mh_digest.encode()) -def digest(data: Union[str, bytes]) -> bytes: +def sha256_digest(data: Union[str, bytes]) -> bytes: if isinstance(data, str): data = data.encode("utf8") - return hashlib.sha1(data).digest() + return hashlib.sha256(data).digest() From 009df257bcd1b5bc15a86ea8f21cd27508db59ea Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 17 Dec 2019 18:47:58 +0800 Subject: [PATCH 160/178] Check peer id exist in dict before access --- libp2p/pubsub/floodsub.py | 2 ++ libp2p/pubsub/gossipsub.py | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index 678e5090..b7b7910b 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -89,6 +89,8 @@ class FloodSub(IPubsubRouter): logger.debug("publishing message %s", pubsub_msg) for peer_id in peers_gen: + if peer_id not in self.pubsub.peers: + continue stream = self.pubsub.peers[peer_id] # FIXME: We should add a `WriteMsg` similar to write delimited messages. # Ref: https://github.com/libp2p/go-libp2p-pubsub/blob/master/comm.go#L107 diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index bddafa0a..a64b9ca8 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -193,6 +193,8 @@ class GossipSub(IPubsubRouter): logger.debug("publishing message %s", pubsub_msg) for peer_id in peers_gen: + if peer_id not in self.pubsub.peers: + continue stream = self.pubsub.peers[peer_id] # FIXME: We should add a `WriteMsg` similar to write delimited messages. # Ref: https://github.com/libp2p/go-libp2p-pubsub/blob/master/comm.go#L107 @@ -604,6 +606,11 @@ class GossipSub(IPubsubRouter): rpc_msg: bytes = packet.SerializeToString() # 3) Get the stream to this peer + if sender_peer_id not in self.pubsub.peers: + logger.debug( + "Fail to responed to iwant request from %s: peer disconnected", + sender_peer_id, + ) peer_stream = self.pubsub.peers[sender_peer_id] # 4) And write the packet to the stream @@ -710,6 +717,8 @@ class GossipSub(IPubsubRouter): rpc_msg: bytes = packet.SerializeToString() # Get stream for peer from pubsub + if to_peer not in self.pubsub.peers: + logger.debug("Fail to emit control message to %s: peer disconnected", to_peer) peer_stream = self.pubsub.peers[to_peer] # Write rpc to stream From 474ed41652fc7a5f3439cec4d35ced98d14fa5b8 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 17 Dec 2019 18:48:25 +0800 Subject: [PATCH 161/178] Remove dead peer if floodsub write stream fail --- libp2p/pubsub/floodsub.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libp2p/pubsub/floodsub.py b/libp2p/pubsub/floodsub.py index b7b7910b..bae2bf27 100644 --- a/libp2p/pubsub/floodsub.py +++ b/libp2p/pubsub/floodsub.py @@ -98,6 +98,7 @@ class FloodSub(IPubsubRouter): await stream.write(encode_varint_prefixed(rpc_msg.SerializeToString())) except StreamClosed: logger.debug("Fail to publish message to %s: stream closed", peer_id) + self.pubsub._handle_dead_peer(peer_id) async def join(self, topic: str) -> None: """ From 3cbe24caab851eb1f1ebdf2691445a34425a53a1 Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Tue, 17 Dec 2019 12:00:11 +0100 Subject: [PATCH 162/178] fixes #384 also adds MultiError to libp2p/exceptions.py and an additional fixme I have noticed --- libp2p/exceptions.py | 6 ++++++ libp2p/network/swarm.py | 44 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/libp2p/exceptions.py b/libp2p/exceptions.py index bdecfad0..5b781145 100644 --- a/libp2p/exceptions.py +++ b/libp2p/exceptions.py @@ -8,3 +8,9 @@ class ValidationError(BaseLibp2pError): class ParseError(BaseLibp2pError): pass + + +class MultiError(BaseLibp2pError): + """Raised with multiple exceptions.""" + + # todo: find some way for this to fancy-print all encapsulated errors diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index 0c405106..5099a5d2 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -19,6 +19,7 @@ from libp2p.transport.transport_interface import ITransport from libp2p.transport.upgrader import TransportUpgrader from libp2p.typing import StreamHandlerFn +from ..exceptions import MultiError from .connection.raw_connection import RawConnection from .connection.swarm_connection import SwarmConn from .exceptions import SwarmException @@ -90,14 +91,49 @@ class Swarm(INetwork): except PeerStoreError: raise SwarmException(f"No known addresses to peer {peer_id}") - if not addrs: + if len(addrs) == 0: raise SwarmException(f"No known addresses to peer {peer_id}") - multiaddr = addrs[0] + exceptions: List[SwarmException] = [] + + # Try all known addresses + for multiaddr in addrs: + try: + return await self.dial_addr(multiaddr, peer_id) + except SwarmException as e: + exceptions.append(e) + logger.debug( + "encountered swarm exception when trying to connect to %s, " + "trying next address...", + multiaddr, + exc_info=e, + ) + + # Tried all addresses, raising exception. + if len(exceptions) > 0: + raise SwarmException( + "unable to connect to %s, all addresses failed to dial (with exceptions)", + peer_id, + ) from MultiError(exceptions) + else: + raise SwarmException( + "unable to connect to %s, all addresses failed to dial", peer_id + ) + + async def dial_addr(self, addr: Multiaddr, peer_id: ID) -> INetConn: + """ + dial_addr try to create a connection to peer_id with addr. + + :param addr: the address we want to connect with + :param peer_id: the peer we want to connect to + :raises SwarmException: raised when an error occurs + :return: muxed connection + """ + # Dial peer (connection to peer does not yet exist) # Transport dials peer (gets back a raw conn) try: - raw_conn = await self.transport.dial(multiaddr) + raw_conn = await self.transport.dial(addr) except OpenConnectionError as error: logger.debug("fail to dial peer %s over base transport", peer_id) raise SwarmException( @@ -137,7 +173,7 @@ class Swarm(INetwork): async def new_stream(self, peer_id: ID) -> INetStream: """ :param peer_id: peer_id of destination - :param protocol_id: protocol id + :param protocol_id: protocol id fixme: protocol_id not in parameters :raises SwarmException: raised when an error occurs :return: net stream instance """ From 04b9d688f8ed55e3d3bb35d912dec9a9702fd399 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 17 Dec 2019 19:09:15 +0800 Subject: [PATCH 163/178] Add newsfragment --- newsfragments/387.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/387.bugfix.rst diff --git a/newsfragments/387.bugfix.rst b/newsfragments/387.bugfix.rst new file mode 100644 index 00000000..0ba3f07c --- /dev/null +++ b/newsfragments/387.bugfix.rst @@ -0,0 +1 @@ +Store peer ids in ``set`` instead of ``list`` and check if peer id exists in ``dict`` before accessing to prevent ``KeyError``. \ No newline at end of file From 19ce8a2140e0c00a1273c36f1bf6282f89f98e32 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Tue, 17 Dec 2019 21:56:02 +0800 Subject: [PATCH 164/178] Fix mypy --- libp2p/pubsub/gossipsub.py | 2 ++ libp2p/pubsub/pubsub.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index a64b9ca8..55029cdb 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -611,6 +611,7 @@ class GossipSub(IPubsubRouter): "Fail to responed to iwant request from %s: peer disconnected", sender_peer_id, ) + return peer_stream = self.pubsub.peers[sender_peer_id] # 4) And write the packet to the stream @@ -719,6 +720,7 @@ class GossipSub(IPubsubRouter): # Get stream for peer from pubsub if to_peer not in self.pubsub.peers: logger.debug("Fail to emit control message to %s: peer disconnected", to_peer) + return peer_stream = self.pubsub.peers[to_peer] # Write rpc to stream diff --git a/libp2p/pubsub/pubsub.py b/libp2p/pubsub/pubsub.py index 2d5625f7..48ed7eba 100644 --- a/libp2p/pubsub/pubsub.py +++ b/libp2p/pubsub/pubsub.py @@ -351,7 +351,7 @@ class Pubsub: """ if sub_message.subscribe: if sub_message.topicid not in self.peer_topics: - self.peer_topics[sub_message.topicid] = [origin_id] + self.peer_topics[sub_message.topicid] = set([origin_id]) elif origin_id not in self.peer_topics[sub_message.topicid]: # Add peer to topic self.peer_topics[sub_message.topicid].add(origin_id) From 445c0f8e65c61c1ded426d18a8d638130e4c2c66 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 17 Dec 2019 10:11:17 -0800 Subject: [PATCH 165/178] Dangling `kademlia` cleanup --- libp2p/kademlia/network.py | 250 ------------------------------------- 1 file changed, 250 deletions(-) delete mode 100644 libp2p/kademlia/network.py diff --git a/libp2p/kademlia/network.py b/libp2p/kademlia/network.py deleted file mode 100644 index 1077bb5a..00000000 --- a/libp2p/kademlia/network.py +++ /dev/null @@ -1,250 +0,0 @@ -"""Package for interacting on the network at a high level.""" -import asyncio -import logging -import pickle - -from .crawling import NodeSpiderCrawl, ValueSpiderCrawl -from .kad_peerinfo import create_kad_peerinfo -from .protocol import KademliaProtocol -from .storage import ForgetfulStorage -from .utils import digest - -log = logging.getLogger(__name__) - - -class KademliaServer: - """ - High level view of a node instance. - - This is the object that should be created to start listening as an - active node on the network. - """ - - protocol_class = KademliaProtocol - - def __init__(self, ksize=20, alpha=3, node_id=None, storage=None): - """ - Create a server instance. This will start listening on the given port. - - Args: - ksize (int): The k parameter from the paper - alpha (int): The alpha parameter from the paper - node_id: The id for this node on the network. - storage: An instance that implements - :interface:`~kademlia.storage.IStorage` - """ - self.ksize = ksize - self.alpha = alpha - self.storage = storage or ForgetfulStorage() - self.node = create_kad_peerinfo(node_id) - self.transport = None - self.protocol = None - self.refresh_loop = None - self.save_state_loop = None - - def stop(self): - if self.transport is not None: - self.transport.close() - - if self.refresh_loop: - self.refresh_loop.cancel() - - if self.save_state_loop: - self.save_state_loop.cancel() - - def _create_protocol(self): - return self.protocol_class(self.node, self.storage, self.ksize) - - async def listen(self, port=0, interface="0.0.0.0"): - """ - Start listening on the given port. - - Provide interface="::" to accept ipv6 address - """ - loop = asyncio.get_event_loop() - listen = loop.create_datagram_endpoint( - self._create_protocol, local_addr=(interface, port) - ) - self.transport, self.protocol = await listen - socket = self.transport.get_extra_info("socket") - self.address = socket.getsockname() - log.info( - "Node %i listening on %s:%i", - self.node.xor_id, - self.address[0], - self.address[1], - ) - # finally, schedule refreshing table - self.refresh_table() - - def refresh_table(self): - log.debug("Refreshing routing table") - asyncio.ensure_future(self._refresh_table()) - loop = asyncio.get_event_loop() - self.refresh_loop = loop.call_later(3600, self.refresh_table) - - async def _refresh_table(self): - """Refresh buckets that haven't had any lookups in the last hour (per - section 2.3 of the paper).""" - results = [] - for node_id in self.protocol.get_refresh_ids(): - node = create_kad_peerinfo(node_id) - nearest = self.protocol.router.find_neighbors(node, self.alpha) - spider = NodeSpiderCrawl( - self.protocol, node, nearest, self.ksize, self.alpha - ) - results.append(spider.find()) - - # do our crawling - await asyncio.gather(*results) - - # now republish keys older than one hour - for dkey, value in self.storage.iter_older_than(3600): - await self.set_digest(dkey, value) - - def bootstrappable_neighbors(self): - """ - Get a :class:`list` of (ip, port) :class:`tuple` pairs suitable for use - as an argument to the bootstrap method. - - The server should have been bootstrapped - already - this is just a utility for getting some neighbors and then - storing them if this server is going down for a while. When it comes - back up, the list of nodes can be used to bootstrap. - """ - neighbors = self.protocol.router.find_neighbors(self.node) - return [tuple(n)[-2:] for n in neighbors] - - async def bootstrap(self, addrs): - """ - Bootstrap the server by connecting to other known nodes in the network. - - Args: - addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP - addresses are acceptable - hostnames will cause an error. - """ - log.debug("Attempting to bootstrap node with %i initial contacts", len(addrs)) - cos = list(map(self.bootstrap_node, addrs)) - gathered = await asyncio.gather(*cos) - nodes = [node for node in gathered if node is not None] - spider = NodeSpiderCrawl( - self.protocol, self.node, nodes, self.ksize, self.alpha - ) - return await spider.find() - - async def bootstrap_node(self, addr): - result = await self.protocol.ping(addr, self.node.peer_id_bytes) - return create_kad_peerinfo(result[1], addr[0], addr[1]) if result[0] else None - - async def get(self, key): - """ - Get a key if the network has it. - - Returns: - :class:`None` if not found, the value otherwise. - """ - log.info("Looking up key %s", key) - dkey = digest(key) - # if this node has it, return it - if self.storage.get(dkey) is not None: - return self.storage.get(dkey) - - node = create_kad_peerinfo(dkey) - nearest = self.protocol.router.find_neighbors(node) - if not nearest: - log.warning("There are no known neighbors to get key %s", key) - return None - spider = ValueSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) - return await spider.find() - - async def set(self, key, value): - """Set the given string key to the given value in the network.""" - if not check_dht_value_type(value): - raise TypeError("Value must be of type int, float, bool, str, or bytes") - log.info("setting '%s' = '%s' on network", key, value) - dkey = digest(key) - return await self.set_digest(dkey, value) - - async def provide(self, key): - """publish to the network that it provides for a particular key.""" - neighbors = self.protocol.router.find_neighbors(self.node) - return [ - await self.protocol.call_add_provider(n, key, self.node.peer_id_bytes) - for n in neighbors - ] - - async def get_providers(self, key): - """get the list of providers for a key.""" - neighbors = self.protocol.router.find_neighbors(self.node) - return [await self.protocol.call_get_providers(n, key) for n in neighbors] - - async def set_digest(self, dkey, value): - """Set the given SHA1 digest key (bytes) to the given value in the - network.""" - node = create_kad_peerinfo(dkey) - - nearest = self.protocol.router.find_neighbors(node) - if not nearest: - log.warning("There are no known neighbors to set key %s", dkey.hex()) - return False - - spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha) - nodes = await spider.find() - log.info("setting '%s' on %s", dkey.hex(), list(map(str, nodes))) - - # if this node is close too, then store here as well - biggest = max([n.distance_to(node) for n in nodes]) - if self.node.distance_to(node) < biggest: - self.storage[dkey] = value - results = [self.protocol.call_store(n, dkey, value) for n in nodes] - # return true only if at least one store call succeeded - return any(await asyncio.gather(*results)) - - def save_state(self, fname): - """Save the state of this node (the alpha/ksize/id/immediate neighbors) - to a cache file with the given fname.""" - log.info("Saving state to %s", fname) - data = { - "ksize": self.ksize, - "alpha": self.alpha, - "id": self.node.peer_id_bytes, - "neighbors": self.bootstrappable_neighbors(), - } - if not data["neighbors"]: - log.warning("No known neighbors, so not writing to cache.") - return - with open(fname, "wb") as file: - pickle.dump(data, file) - - @classmethod - def load_state(cls, fname): - """Load the state of this node (the alpha/ksize/id/immediate neighbors) - from a cache file with the given fname.""" - log.info("Loading state from %s", fname) - with open(fname, "rb") as file: - data = pickle.load(file) - svr = KademliaServer(data["ksize"], data["alpha"], data["id"]) - if data["neighbors"]: - svr.bootstrap(data["neighbors"]) - return svr - - def save_state_regularly(self, fname, frequency=600): - """ - Save the state of node with a given regularity to the given filename. - - :param fname: File name to save regularly to - :param frequency: Frequency in seconds that the state should be saved. - By default, 10 minutes. - """ - self.save_state(fname) - loop = asyncio.get_event_loop() - self.save_state_loop = loop.call_later( - frequency, self.save_state_regularly, fname, frequency - ) - - -def check_dht_value_type(value): - """Checks to see if the type of the value is a valid type for placing in - the dht.""" - typeset = [int, float, bool, str, bytes] - return type(value) in typeset From 6b759012438773067e1642a4a7a7e158d54eeec5 Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Tue, 17 Dec 2019 20:20:09 +0100 Subject: [PATCH 166/178] apply PR feedback (remote len == 0 block, remove redundant fixme comment + docstring line) change wording of final SwarmException to include possible case of no addresses in returned address set add `from error` in except clause --- libp2p/network/swarm.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index 5099a5d2..2e36dddb 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -88,11 +88,8 @@ class Swarm(INetwork): try: # Get peer info from peer store addrs = self.peerstore.addrs(peer_id) - except PeerStoreError: - raise SwarmException(f"No known addresses to peer {peer_id}") - - if len(addrs) == 0: - raise SwarmException(f"No known addresses to peer {peer_id}") + except PeerStoreError as error: + raise SwarmException(f"No known addresses to peer {peer_id}") from error exceptions: List[SwarmException] = [] @@ -112,12 +109,16 @@ class Swarm(INetwork): # Tried all addresses, raising exception. if len(exceptions) > 0: raise SwarmException( - "unable to connect to %s, all addresses failed to dial (with exceptions)", + "unable to connect to %s, no addresses established a successful connection " + "(with exceptions)", peer_id, ) from MultiError(exceptions) else: raise SwarmException( - "unable to connect to %s, all addresses failed to dial", peer_id + "unable to connect to %s, no addresses established a successful connection " + "(tried %d addresses)", + peer_id, + len(addrs), ) async def dial_addr(self, addr: Multiaddr, peer_id: ID) -> INetConn: @@ -173,7 +174,6 @@ class Swarm(INetwork): async def new_stream(self, peer_id: ID) -> INetStream: """ :param peer_id: peer_id of destination - :param protocol_id: protocol id fixme: protocol_id not in parameters :raises SwarmException: raised when an error occurs :return: net stream instance """ From f3732f94809e2ae7314641bcb066be6222370d9d Mon Sep 17 00:00:00 2001 From: NIC619 Date: Wed, 18 Dec 2019 12:37:04 +0800 Subject: [PATCH 167/178] Fix tests --- libp2p/pubsub/gossipsub.py | 4 +++- tests/pubsub/test_gossipsub.py | 16 ++++++++-------- tests_interop/test_pubsub.py | 2 +- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 55029cdb..51e1c92c 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -719,7 +719,9 @@ class GossipSub(IPubsubRouter): # Get stream for peer from pubsub if to_peer not in self.pubsub.peers: - logger.debug("Fail to emit control message to %s: peer disconnected", to_peer) + logger.debug( + "Fail to emit control message to %s: peer disconnected", to_peer + ) return peer_stream = self.pubsub.peers[to_peer] diff --git a/tests/pubsub/test_gossipsub.py b/tests/pubsub/test_gossipsub.py index 19ec07c5..fe6bf3b8 100644 --- a/tests/pubsub/test_gossipsub.py +++ b/tests/pubsub/test_gossipsub.py @@ -390,15 +390,15 @@ async def test_mesh_heartbeat( fake_peer_ids = [ ID((i).to_bytes(2, byteorder="big")) for i in range(total_peer_count) ] - monkeypatch.setattr(pubsubs_gsub[0].router, "peers_gossipsub", fake_peer_ids) + monkeypatch.setattr(pubsubs_gsub[0].router, "peers_gossipsub", set(fake_peer_ids)) - peer_topics = {topic: fake_peer_ids} + peer_topics = {topic: set(fake_peer_ids)} # Monkeypatch the peer subscriptions monkeypatch.setattr(pubsubs_gsub[0], "peer_topics", peer_topics) mesh_peer_indices = random.sample(range(total_peer_count), initial_mesh_peer_count) mesh_peers = [fake_peer_ids[i] for i in mesh_peer_indices] - router_mesh = {topic: list(mesh_peers)} + router_mesh = {topic: set(mesh_peers)} # Monkeypatch our mesh peers monkeypatch.setattr(pubsubs_gsub[0].router, "mesh", router_mesh) @@ -437,27 +437,27 @@ async def test_gossip_heartbeat( fake_peer_ids = [ ID((i).to_bytes(2, byteorder="big")) for i in range(total_peer_count) ] - monkeypatch.setattr(pubsubs_gsub[0].router, "peers_gossipsub", fake_peer_ids) + monkeypatch.setattr(pubsubs_gsub[0].router, "peers_gossipsub", set(fake_peer_ids)) topic_mesh_peer_count = 14 # Split into mesh peers and fanout peers peer_topics = { - topic_mesh: fake_peer_ids[:topic_mesh_peer_count], - topic_fanout: fake_peer_ids[topic_mesh_peer_count:], + topic_mesh: set(fake_peer_ids[:topic_mesh_peer_count]), + topic_fanout: set(fake_peer_ids[topic_mesh_peer_count:]), } # Monkeypatch the peer subscriptions monkeypatch.setattr(pubsubs_gsub[0], "peer_topics", peer_topics) mesh_peer_indices = random.sample(range(topic_mesh_peer_count), initial_peer_count) mesh_peers = [fake_peer_ids[i] for i in mesh_peer_indices] - router_mesh = {topic_mesh: list(mesh_peers)} + router_mesh = {topic_mesh: set(mesh_peers)} # Monkeypatch our mesh peers monkeypatch.setattr(pubsubs_gsub[0].router, "mesh", router_mesh) fanout_peer_indices = random.sample( range(topic_mesh_peer_count, total_peer_count), initial_peer_count ) fanout_peers = [fake_peer_ids[i] for i in fanout_peer_indices] - router_fanout = {topic_fanout: list(fanout_peers)} + router_fanout = {topic_fanout: set(fanout_peers)} # Monkeypatch our fanout peers monkeypatch.setattr(pubsubs_gsub[0].router, "fanout", router_fanout) diff --git a/tests_interop/test_pubsub.py b/tests_interop/test_pubsub.py index f67b47ab..db42c7cd 100644 --- a/tests_interop/test_pubsub.py +++ b/tests_interop/test_pubsub.py @@ -99,7 +99,7 @@ async def test_pubsub(pubsubs, p2pds): go_0_topic_1_peers = await p2pds[0].control.pubsub_list_peers(TOPIC_1) assert len(go_0_topic_1_peers) == 1 and py_peer_id == go_0_topic_1_peers[0] # py - py_topic_0_peers = py_pubsub.peer_topics[TOPIC_0] + py_topic_0_peers = list(py_pubsub.peer_topics[TOPIC_0]) assert len(py_topic_0_peers) == 1 and p2pds[0].peer_id == py_topic_0_peers[0] # go_1 go_1_topic_1_peers = await p2pds[1].control.pubsub_list_peers(TOPIC_1) From 4e4d91b2e20a76f90cd23245a69d3c1e7bf73666 Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Wed, 18 Dec 2019 10:54:52 +0100 Subject: [PATCH 168/178] Apply PR review suggestion (change "muxed" to "network" in docstrings) Co-Authored-By: Kevin Mai-Husan Chia --- libp2p/network/swarm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index 2e36dddb..4c36ab43 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -128,7 +128,7 @@ class Swarm(INetwork): :param addr: the address we want to connect with :param peer_id: the peer we want to connect to :raises SwarmException: raised when an error occurs - :return: muxed connection + :return: network connection """ # Dial peer (connection to peer does not yet exist) From 81fe4049cf038d098f8b3f714e65471a4a998dfb Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Wed, 18 Dec 2019 18:47:03 +0100 Subject: [PATCH 169/178] Apply PR review feedback > add `if not addr` clause back > use f-strings for exceptions instead of %s --- libp2p/network/swarm.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index 4c36ab43..eb7e3069 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -91,6 +91,9 @@ class Swarm(INetwork): except PeerStoreError as error: raise SwarmException(f"No known addresses to peer {peer_id}") from error + if not addrs: + raise SwarmException(f"No known addresses to peer {peer_id}") + exceptions: List[SwarmException] = [] # Try all known addresses @@ -107,19 +110,10 @@ class Swarm(INetwork): ) # Tried all addresses, raising exception. - if len(exceptions) > 0: - raise SwarmException( - "unable to connect to %s, no addresses established a successful connection " - "(with exceptions)", - peer_id, - ) from MultiError(exceptions) - else: - raise SwarmException( - "unable to connect to %s, no addresses established a successful connection " - "(tried %d addresses)", - peer_id, - len(addrs), - ) + raise SwarmException( + f"unable to connect to {peer_id}, no addresses established a successful connection " + "(with exceptions)", + ) from MultiError(exceptions) async def dial_addr(self, addr: Multiaddr, peer_id: ID) -> INetConn: """ @@ -138,7 +132,7 @@ class Swarm(INetwork): except OpenConnectionError as error: logger.debug("fail to dial peer %s over base transport", peer_id) raise SwarmException( - "fail to open connection to peer %s", peer_id + f"fail to open connection to peer {peer_id}" ) from error logger.debug("dialed peer %s over base transport", peer_id) From f54bc9d1afcad7b0ac883f597277ab1db1e3ea78 Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Wed, 18 Dec 2019 19:05:22 +0100 Subject: [PATCH 170/178] Make linter happy --- libp2p/network/swarm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libp2p/network/swarm.py b/libp2p/network/swarm.py index eb7e3069..c8618779 100644 --- a/libp2p/network/swarm.py +++ b/libp2p/network/swarm.py @@ -112,7 +112,7 @@ class Swarm(INetwork): # Tried all addresses, raising exception. raise SwarmException( f"unable to connect to {peer_id}, no addresses established a successful connection " - "(with exceptions)", + "(with exceptions)" ) from MultiError(exceptions) async def dial_addr(self, addr: Multiaddr, peer_id: ID) -> INetConn: From 1124fc8211565b060a9169647d93af199ee49c9c Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Thu, 19 Dec 2019 01:25:57 +0100 Subject: [PATCH 171/178] add eth_utils add fastecdsa-any requirements --- setup.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 28d29fa2..0a9f8a09 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ extras_require = { "ipython", "setuptools>=36.2.0", "tox>=3.13.2,<4.0.0", + "eth_utils", ], } @@ -43,6 +44,20 @@ extras_require["dev"] = ( + extras_require["doc"] ) +fastecdsa = [ + # No official fastecdsa==1.7.4,1.7.5 wheels for Windows, using a pypi package that includes + # the original library, but also windows-built wheels (32+64-bit) on those versions. + # Fixme: Remove section when fastecdsa has released a windows-compatible wheel + # (specifically: both win32 and win_amd64 targets) + # See the following issues for more information; + # https://github.com/libp2p/py-libp2p/issues/363 + # https://github.com/AntonKueltz/fastecdsa/issues/11 + "fastecdsa-any==1.7.5;sys_platform=='win32'", + # Wheels are provided for these platforms, or compiling one is minimally frustrating in a + # default python installation. + "fastecdsa==1.7.5;sys_platform!='win32'", +] + with open("./README.md") as readme: long_description = readme.read() @@ -67,7 +82,7 @@ install_requires = [ # RTD system so we have to exclude these dependencies when we are in an RTD environment. readthedocs_is_building = os.environ.get("READTHEDOCS", False) if not readthedocs_is_building: - install_requires.append("fastecdsa==1.7.4") + install_requires.extend(fastecdsa) setup( From 6cd3eb8faec2e60ca053356a37e58cf51399026d Mon Sep 17 00:00:00 2001 From: NIC619 Date: Thu, 19 Dec 2019 14:15:51 +0800 Subject: [PATCH 172/178] Apply PR feedback: change param type and remove check before `discard` --- libp2p/pubsub/gossipsub.py | 40 +++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 51e1c92c..7b0759cf 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -141,17 +141,13 @@ class GossipSub(IPubsubRouter): """ logger.debug("removing peer %s", peer_id) - if peer_id in self.peers_gossipsub: - self.peers_gossipsub.discard(peer_id) - elif peer_id in self.peers_floodsub: - self.peers_floodsub.discard(peer_id) + self.peers_gossipsub.discard(peer_id) + self.peers_floodsub.discard(peer_id) for topic in self.mesh: - if peer_id in self.mesh[topic]: - self.mesh[topic].discard(peer_id) + self.mesh[topic].discard(peer_id) for topic in self.fanout: - if peer_id in self.fanout[topic]: - self.fanout[topic].discard(peer_id) + self.fanout[topic].discard(peer_id) self.peers_to_protocol.pop(peer_id, None) @@ -248,7 +244,7 @@ class GossipSub(IPubsubRouter): # Combine fanout peers with selected peers fanout_peers.update( self._get_in_topic_gossipsub_peers_from_minus( - topic, self.degree - fanout_size, list(fanout_peers) + topic, self.degree - fanout_size, fanout_peers ) ) self.fanout[topic] = fanout_peers @@ -280,7 +276,7 @@ class GossipSub(IPubsubRouter): # Selects the remaining number of peers (D-x) from peers.gossipsub[topic]. if topic in self.pubsub.peer_topics: selected_peers = self._get_in_topic_gossipsub_peers_from_minus( - topic, self.degree - fanout_size, list(fanout_peers) + topic, self.degree - fanout_size, fanout_peers ) # Combine fanout peers with selected peers fanout_peers.update(selected_peers) @@ -419,7 +415,7 @@ class GossipSub(IPubsubRouter): if num_mesh_peers_in_topic < self.degree_low: # Select D - |mesh[topic]| peers from peers.gossipsub[topic] - mesh[topic] selected_peers = self._get_in_topic_gossipsub_peers_from_minus( - topic, self.degree - num_mesh_peers_in_topic, list(self.mesh[topic]) + topic, self.degree - num_mesh_peers_in_topic, self.mesh[topic] ) for peer in selected_peers: @@ -432,7 +428,7 @@ class GossipSub(IPubsubRouter): if num_mesh_peers_in_topic > self.degree_high: # Select |mesh[topic]| - D peers from mesh[topic] selected_peers = GossipSub.select_from_minus( - num_mesh_peers_in_topic - self.degree, list(self.mesh[topic]), [] + num_mesh_peers_in_topic - self.degree, self.mesh[topic], set() ) for peer in selected_peers: # Remove peer from mesh[topic] @@ -472,7 +468,7 @@ class GossipSub(IPubsubRouter): selected_peers = self._get_in_topic_gossipsub_peers_from_minus( topic, self.degree - num_fanout_peers_in_topic, - list(self.fanout[topic]), + self.fanout[topic], ) # Add the peers to fanout[topic] self.fanout[topic].update(selected_peers) @@ -486,7 +482,7 @@ class GossipSub(IPubsubRouter): if topic in self.pubsub.peer_topics: # Select D peers from peers.gossipsub[topic] peers_to_emit_ihave_to = self._get_in_topic_gossipsub_peers_from_minus( - topic, self.degree, list(self.mesh[topic]) + topic, self.degree, self.mesh[topic] ) msg_id_strs = [str(msg_id) for msg_id in msg_ids] @@ -502,7 +498,7 @@ class GossipSub(IPubsubRouter): if topic in self.pubsub.peer_topics: # Select D peers from peers.gossipsub[topic] peers_to_emit_ihave_to = self._get_in_topic_gossipsub_peers_from_minus( - topic, self.degree, list(self.fanout[topic]) + topic, self.degree, self.fanout[topic] ) msg_id_strs = [str(msg) for msg in msg_ids] for peer in peers_to_emit_ihave_to: @@ -511,7 +507,7 @@ class GossipSub(IPubsubRouter): @staticmethod def select_from_minus( - num_to_select: int, pool: Sequence[Any], minus: Sequence[Any] + num_to_select: int, pool: Iterable[Any], minus: Iterable[Any] ) -> List[Any]: """ Select at most num_to_select subset of elements from the set (pool - minus) randomly. @@ -539,15 +535,15 @@ class GossipSub(IPubsubRouter): return selection def _get_in_topic_gossipsub_peers_from_minus( - self, topic: str, num_to_select: int, minus: Sequence[ID] + self, topic: str, num_to_select: int, minus: Iterable[ID] ) -> List[ID]: - gossipsub_peers_in_topic = [ + gossipsub_peers_in_topic = set( peer_id for peer_id in self.pubsub.peer_topics[topic] if peer_id in self.peers_gossipsub - ] + ) return self.select_from_minus( - num_to_select, gossipsub_peers_in_topic, list(minus) + num_to_select, gossipsub_peers_in_topic, minus ) # RPC handlers @@ -642,8 +638,8 @@ class GossipSub(IPubsubRouter): ) -> None: topic: str = prune_msg.topicID - # Remove peer from mesh for topic, if peer is in topic - if topic in self.mesh and sender_peer_id in self.mesh[topic]: + # Remove peer from mesh for topic + if topic in self.mesh: self.mesh[topic].discard(sender_peer_id) # RPC emitters From e51d376d5eb2568c4f7228e8a1e3c935bc64480b Mon Sep 17 00:00:00 2001 From: NIC619 Date: Thu, 19 Dec 2019 14:44:49 +0800 Subject: [PATCH 173/178] Combine `peers_gossipsub` and `peers_floodsub` --- libp2p/pubsub/gossipsub.py | 34 +++++++++------------------------- tests/pubsub/test_gossipsub.py | 11 +++++++---- 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 7b0759cf..8097f705 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -35,13 +35,11 @@ class GossipSub(IPubsubRouter): mesh: Dict[str, Set[ID]] fanout: Dict[str, Set[ID]] - peers_to_protocol: Dict[ID, str] + # The protocol peer supports + peer_protocol: Dict[ID, TProtocol] time_since_last_publish: Dict[str, int] - peers_gossipsub: Set[ID] - peers_floodsub: Set[ID] - mcache: MessageCache heartbeat_initial_delay: float @@ -75,14 +73,11 @@ class GossipSub(IPubsubRouter): self.fanout = {} # Create peer --> protocol mapping - self.peers_to_protocol = {} + self.peer_protocol = {} # Create topic --> time since last publish map self.time_since_last_publish = {} - self.peers_gossipsub = set() - self.peers_floodsub = set() - # Create message cache self.mcache = MessageCache(gossip_window, gossip_history) @@ -121,17 +116,13 @@ class GossipSub(IPubsubRouter): """ logger.debug("adding peer %s with protocol %s", peer_id, protocol_id) - if protocol_id == PROTOCOL_ID: - self.peers_gossipsub.add(peer_id) - elif protocol_id == floodsub.PROTOCOL_ID: - self.peers_floodsub.add(peer_id) - else: + if protocol_id not in (PROTOCOL_ID, floodsub.PROTOCOL_ID): # We should never enter here. Becuase the `protocol_id` is registered by your pubsub # instance in multistream-select, but it is not the protocol that gossipsub supports. # In this case, probably we registered gossipsub to a wrong `protocol_id` # in multistream-select, or wrong versions. raise Exception(f"Unreachable: Protocol={protocol_id} is not supported.") - self.peers_to_protocol[peer_id] = protocol_id + self.peer_protocol[peer_id] = protocol_id def remove_peer(self, peer_id: ID) -> None: """ @@ -141,15 +132,12 @@ class GossipSub(IPubsubRouter): """ logger.debug("removing peer %s", peer_id) - self.peers_gossipsub.discard(peer_id) - self.peers_floodsub.discard(peer_id) - for topic in self.mesh: self.mesh[topic].discard(peer_id) for topic in self.fanout: self.fanout[topic].discard(peer_id) - self.peers_to_protocol.pop(peer_id, None) + self.peer_protocol.pop(peer_id, None) async def handle_rpc(self, rpc: rpc_pb2.RPC, sender_peer_id: ID) -> None: """ @@ -217,12 +205,10 @@ class GossipSub(IPubsubRouter): continue # floodsub peers - # FIXME: `gossipsub.peers_floodsub` can be changed to `gossipsub.peers` in go. - # This will improve the efficiency when searching for a peer's protocol id. floodsub_peers: Set[ID] = set( peer_id for peer_id in self.pubsub.peer_topics[topic] - if peer_id in self.peers_floodsub + if self.peer_protocol[peer_id] == floodsub.PROTOCOL_ID ) send_to.update(floodsub_peers) @@ -540,11 +526,9 @@ class GossipSub(IPubsubRouter): gossipsub_peers_in_topic = set( peer_id for peer_id in self.pubsub.peer_topics[topic] - if peer_id in self.peers_gossipsub - ) - return self.select_from_minus( - num_to_select, gossipsub_peers_in_topic, minus + if self.peer_protocol[peer_id] == PROTOCOL_ID ) + return self.select_from_minus(num_to_select, gossipsub_peers_in_topic, minus) # RPC handlers diff --git a/tests/pubsub/test_gossipsub.py b/tests/pubsub/test_gossipsub.py index fe6bf3b8..1bc34260 100644 --- a/tests/pubsub/test_gossipsub.py +++ b/tests/pubsub/test_gossipsub.py @@ -4,6 +4,7 @@ import random import pytest from libp2p.peer.id import ID +from libp2p.pubsub.gossipsub import PROTOCOL_ID from libp2p.tools.constants import GOSSIPSUB_PARAMS, GossipsubParams from libp2p.tools.pubsub.utils import dense_connect, one_to_all_connect from libp2p.tools.utils import connect @@ -108,7 +109,7 @@ async def test_handle_graft(pubsubs_gsub, hosts, event_loop, monkeypatch): monkeypatch.setattr(gossipsubs[index_bob], "emit_prune", emit_prune) # Check that alice is bob's peer but not his mesh peer - assert id_alice in gossipsubs[index_bob].peers_gossipsub + assert gossipsubs[index_bob].peer_protocol[id_alice] == PROTOCOL_ID assert topic not in gossipsubs[index_bob].mesh await gossipsubs[index_alice].emit_graft(topic, id_bob) @@ -120,7 +121,7 @@ async def test_handle_graft(pubsubs_gsub, hosts, event_loop, monkeypatch): # Check that bob is alice's peer but not her mesh peer assert topic in gossipsubs[index_alice].mesh assert id_bob not in gossipsubs[index_alice].mesh[topic] - assert id_bob in gossipsubs[index_alice].peers_gossipsub + assert gossipsubs[index_alice].peer_protocol[id_bob] == PROTOCOL_ID await gossipsubs[index_bob].emit_graft(topic, id_alice) @@ -390,7 +391,8 @@ async def test_mesh_heartbeat( fake_peer_ids = [ ID((i).to_bytes(2, byteorder="big")) for i in range(total_peer_count) ] - monkeypatch.setattr(pubsubs_gsub[0].router, "peers_gossipsub", set(fake_peer_ids)) + peer_protocol = {peer_id: PROTOCOL_ID for peer_id in fake_peer_ids} + monkeypatch.setattr(pubsubs_gsub[0].router, "peer_protocol", peer_protocol) peer_topics = {topic: set(fake_peer_ids)} # Monkeypatch the peer subscriptions @@ -437,7 +439,8 @@ async def test_gossip_heartbeat( fake_peer_ids = [ ID((i).to_bytes(2, byteorder="big")) for i in range(total_peer_count) ] - monkeypatch.setattr(pubsubs_gsub[0].router, "peers_gossipsub", set(fake_peer_ids)) + peer_protocol = {peer_id: PROTOCOL_ID for peer_id in fake_peer_ids} + monkeypatch.setattr(pubsubs_gsub[0].router, "peer_protocol", peer_protocol) topic_mesh_peer_count = 14 # Split into mesh peers and fanout peers From 74092c13712e4d09d57d2431d5eebd53dbce0578 Mon Sep 17 00:00:00 2001 From: NIC619 Date: Thu, 19 Dec 2019 16:26:37 +0800 Subject: [PATCH 174/178] Apply PR feedback: update error msg --- libp2p/pubsub/gossipsub.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 8097f705..2c0f7802 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -588,7 +588,7 @@ class GossipSub(IPubsubRouter): # 3) Get the stream to this peer if sender_peer_id not in self.pubsub.peers: logger.debug( - "Fail to responed to iwant request from %s: peer disconnected", + "Fail to responed to iwant request from %s: peer record not exist", sender_peer_id, ) return @@ -700,7 +700,7 @@ class GossipSub(IPubsubRouter): # Get stream for peer from pubsub if to_peer not in self.pubsub.peers: logger.debug( - "Fail to emit control message to %s: peer disconnected", to_peer + "Fail to emit control message to %s: peer record not exist", to_peer ) return peer_stream = self.pubsub.peers[to_peer] From cb80cfc50b886a614c3e02bbf477ae63e7589b1a Mon Sep 17 00:00:00 2001 From: NIC Lin Date: Thu, 19 Dec 2019 16:33:56 +0800 Subject: [PATCH 175/178] Update libp2p/pubsub/gossipsub.py Co-Authored-By: Chih Cheng Liang --- libp2p/pubsub/gossipsub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 2c0f7802..37a1e5dd 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -121,7 +121,7 @@ class GossipSub(IPubsubRouter): # instance in multistream-select, but it is not the protocol that gossipsub supports. # In this case, probably we registered gossipsub to a wrong `protocol_id` # in multistream-select, or wrong versions. - raise Exception(f"Unreachable: Protocol={protocol_id} is not supported.") + raise ValueError(f"Protocol={protocol_id} is not supported.") self.peer_protocol[peer_id] = protocol_id def remove_peer(self, peer_id: ID) -> None: From 3c75c85d7f1ee993b8f2f37d5290e65cb80c40cc Mon Sep 17 00:00:00 2001 From: NIC619 Date: Thu, 19 Dec 2019 23:07:20 +0800 Subject: [PATCH 176/178] Fix extra white space --- libp2p/pubsub/gossipsub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libp2p/pubsub/gossipsub.py b/libp2p/pubsub/gossipsub.py index 37a1e5dd..05efe84e 100644 --- a/libp2p/pubsub/gossipsub.py +++ b/libp2p/pubsub/gossipsub.py @@ -121,7 +121,7 @@ class GossipSub(IPubsubRouter): # instance in multistream-select, but it is not the protocol that gossipsub supports. # In this case, probably we registered gossipsub to a wrong `protocol_id` # in multistream-select, or wrong versions. - raise ValueError(f"Protocol={protocol_id} is not supported.") + raise ValueError(f"Protocol={protocol_id} is not supported.") self.peer_protocol[peer_id] = protocol_id def remove_peer(self, peer_id: ID) -> None: From 0b97f4d57cc988976a7909d75df8e4c7c9228ee0 Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Sat, 21 Dec 2019 08:59:07 +0100 Subject: [PATCH 177/178] remove eth_utils per PR request --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 0a9f8a09..356487a8 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,6 @@ extras_require = { "ipython", "setuptools>=36.2.0", "tox>=3.13.2,<4.0.0", - "eth_utils", ], } From 17074dded0a48b2fb942e2da55ec02c3f4a56292 Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Sat, 21 Dec 2019 10:35:34 +0100 Subject: [PATCH 178/178] add tests to new multiple multiaddr change --- tests/network/test_swarm.py | 57 +++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/network/test_swarm.py b/tests/network/test_swarm.py index 6fe25434..3cc9375b 100644 --- a/tests/network/test_swarm.py +++ b/tests/network/test_swarm.py @@ -1,5 +1,6 @@ import asyncio +from multiaddr import Multiaddr import pytest from libp2p.network.exceptions import SwarmException @@ -91,3 +92,59 @@ async def test_swarm_remove_conn(swarm_pair): # Test: Remove twice. There should not be errors. swarm_0.remove_conn(conn_0) assert swarm_1.get_peer_id() not in swarm_0.connections + + +@pytest.mark.asyncio +async def test_swarm_multiaddr(is_host_secure): + swarms = await SwarmFactory.create_batch_and_listen(is_host_secure, 3) + + def clear(): + swarms[0].peerstore.clear_addrs(swarms[1].get_peer_id()) + + clear() + # No addresses + with pytest.raises(SwarmException): + await swarms[0].dial_peer(swarms[1].get_peer_id()) + + clear() + # Wrong addresses + swarms[0].peerstore.add_addrs( + swarms[1].get_peer_id(), [Multiaddr("/ip4/0.0.0.0/tcp/9999")], 10000 + ) + + with pytest.raises(SwarmException): + await swarms[0].dial_peer(swarms[1].get_peer_id()) + + clear() + # Multiple wrong addresses + swarms[0].peerstore.add_addrs( + swarms[1].get_peer_id(), + [Multiaddr("/ip4/0.0.0.0/tcp/9999"), Multiaddr("/ip4/0.0.0.0/tcp/9998")], + 10000, + ) + + with pytest.raises(SwarmException): + await swarms[0].dial_peer(swarms[1].get_peer_id()) + + # Test one address + addrs = tuple( + addr + for transport in swarms[1].listeners.values() + for addr in transport.get_addrs() + ) + + swarms[0].peerstore.add_addrs(swarms[1].get_peer_id(), addrs[:1], 10000) + await swarms[0].dial_peer(swarms[1].get_peer_id()) + + # Test multiple addresses + addrs = tuple( + addr + for transport in swarms[1].listeners.values() + for addr in transport.get_addrs() + ) + + swarms[0].peerstore.add_addrs(swarms[1].get_peer_id(), addrs + addrs, 10000) + await swarms[0].dial_peer(swarms[1].get_peer_id()) + + for swarm in swarms: + await swarm.close()