Merge pull request 'diff hotfix 2' (#365) from wowario/wownero:fix into master

Reviewed-on: https://git.wownero.com/wownero/wownero/pulls/365

.drone.yml

@@ -7,4 +7,4 @@ steps:
 - name: linux-build
   image: wownero/wow-dependencies:v1.0
   commands:
-  - make -j2 release-static-linux-x86_64
+  - make release-static-linux-x86_64

.gitignore

@@ -23,6 +23,7 @@ cscope.po.out
 external/miniupnpc/Makefile
 miniupnpcstrings.h
 version/
+ClangBuildAnalyzerSession.txt
 # Created by https://www.gitignore.io
 
 ### C++ ###

CMakeLists.txt

@@ -54,6 +54,13 @@ if (USE_CCACHE)
 else()
 	message(STATUS "ccache deselected")
 endif()
+option (USE_COMPILATION_TIME_PROFILER "Use compilation time profiler (for CLang >= 9 only)" OFF)
+if (USE_COMPILATION_TIME_PROFILER)
+	if (NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
+		message(FATAL_ERROR "The flag USE_COMPILATION_TIME_PROFILER is meant to be set only for CLang compiler!")
+	endif()
+	add_compile_options("-ftime-trace")
+endif()
 
 # Job pool feature requires Ninja.
 if (${CMAKE_VERSION} VERSION_GREATER "3.0.0")
@@ -68,6 +75,20 @@ if (${CMAKE_VERSION} VERSION_GREATER "3.0.0")
     set_property(GLOBAL APPEND PROPERTY JOB_POOLS link_job_pool=${WOWNERO_PARALLEL_LINK_JOBS})
     set(CMAKE_JOB_POOL_LINK link_job_pool)
   endif ()
+endif ()
+  
+option (USE_CLANG_TIDY_C   "Lint the code with clang-tidy - variant C"   OFF)
+option (USE_CLANG_TIDY_CXX "Lint the code with clang-tidy - variant C++" OFF)
+if (USE_CLANG_TIDY_C AND USE_CLANG_TIDY_CXX)
+	message(FATAL_ERROR "Enabling both USE_CLANG_TIDY_C and USE_CLANG_TIDY_CXX simultaneously crashes clang-tidy.")
+endif()
+if (USE_CLANG_TIDY_C OR USE_CLANG_TIDY_CXX)
+	include(SetClangTidy)
+endif()
+if (USE_CLANG_TIDY_C)
+	monero_clang_tidy("C")
+elseif (USE_CLANG_TIDY_CXX)
+	monero_clang_tidy("CXX")
 endif()
 
 enable_language(C ASM)
@@ -126,6 +147,24 @@ function (add_definition_if_library_exists library function header var)
   endif()
 endfunction()
 
+option(RELINK_TARGETS "Relink targets, when just a dependant .so changed, but not its header?" OFF)
+function (monero_set_target_no_relink target)
+  if (RELINK_TARGETS MATCHES OFF)
+    # Will not relink the target, when just its dependant .so has changed, but not it's interface
+    set_target_properties("${target}" PROPERTIES LINK_DEPENDS_NO_SHARED true)
+  endif()
+endfunction()
+
+function (monero_add_minimal_executable name)
+  source_group("${name}"
+    FILES
+      ${ARGN})
+
+  add_executable("${name}"
+    ${ARGN})
+    monero_set_target_no_relink( ${name} )
+endfunction()
+
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
   message(STATUS "Setting default build type: ${CMAKE_BUILD_TYPE}")
@@ -943,10 +982,12 @@ list(APPEND EXTRA_LIBRARIES ${CMAKE_DL_LIBS})
 if (HIDAPI_FOUND OR LibUSB_COMPILE_TEST_PASSED)
   if (APPLE)
     if(DEPENDS)
-      list(APPEND EXTRA_LIBRARIES "-framework Foundation -framework IOKit")
+      list(APPEND EXTRA_LIBRARIES "-framework Foundation -framework AppKit -framework IOKit")
     else()
       find_library(COREFOUNDATION CoreFoundation)
+      find_library(APPKIT AppKit)
       find_library(IOKIT IOKit)
+      list(APPEND EXTRA_LIBRARIES ${APPKIT})
       list(APPEND EXTRA_LIBRARIES ${IOKIT})
       list(APPEND EXTRA_LIBRARIES ${COREFOUNDATION})
     endif()

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2014-2020, The Monero Project
+Copyright (c) 2014-2021, The Monero Project
 
 All rights reserved.
 

README.md

@@ -1,43 +1,34 @@
 [![Build Status](https://ci.wownero.com/api/badges/wownero/wownero/status.svg)](https://ci.wownero.com/wownero/wownero)
 # ~~Mo~~Wownero -  Such privacy! Many coins! Wow! 🐕
 
-Copyright (c) 2014-2020 The Monero Project.   
+Copyright (c) 2014-2021 The Monero Project.   
 Portions Copyright (c) 2012-2013 The Cryptonote developers.
 
 ## Resources
 
-- Web: [wownero.org](http://wownero.org)
+- Web: [wownero.org](https://wownero.org)
 - Twitter: [@w0wn3r0](https://twitter.com/w0wn3r0)
 - Reddit: [/r/wownero](https://www.reddit.com/r/wownero)
 - Mail: [wownero@protonmail.com](mailto:wownero@protonmail.com)
-- GitHub: [git.wownero.com/wownero/wownero](https://git.wownero.com/wownero/wownero)
-- IRC: [#wownero on Freenode](https://kiwiirc.com/client/irc.freenode.net/?nick=suchchatter|?#wownero)
-- Bitmessage Chan: wownero (`BM-2cSzWtrj2pzLva9GF1Jp2TYsnLjrnJpvba`)
+- Git: [git.wownero.com/wownero/wownero](https://git.wownero.com/wownero/wownero)
 - Wownero Funding System: [funding.wownero.com](https://funding.wownero.com)
 - Wownero Forum: [forum.wownero.com](https://forum.wownero.com)
-- Discord: [discord.gg/DSC9TWx](https://discord.gg/DSC9TWx)
+- Discord: [discord.gg/ENbgme4bWq](https://discord.com/invite/ENbgme4bWq)
 - Telegram: [t.me/wownero](https://t.me/wownero)
 - Feather-WOW Desktop Wallet: [featherwallet.org/wownero](https://featherwallet.org/wownero)
 - WOW Stash Web Wallet: [wowstash.app](https://wowstash.app)
 - Public Node Status: [monero.fail](https://monero.fail/?crypto=wownero)
+- Map of Nodes: [wownero.fyi](https://wownero.fyi)
+- Wownero Memes: [suchwow.xyz]https://suchwow.xyz/posts/top)
+- XMR/WOW Swap: [nero Swap](https://neroswap.com)
+- Mining Pools: [miningpoolstats.stream](https://miningpoolstats.stream/wownero)
+- Market Info: [coinmarketcap.com](https://coinmarketcap.com/currencies/wownero), [coingecko.com](https://www.coingecko.com/en/coins/wownero/usd)
 
 ### Blockchain Explorers
 - https://explore.wownero.com
 - http://wow5eqtzqvsg5jctqzg5g7uk3u62sfqiacj5x6lo4by7bvnj6jkvubyd.onion
 - https://wownero.club
-
-### Free Public Nodes
-- global.wownodes.com:34568 (Global)
-- node.suchwow.xyz:34568    (US)
-- wow.pwned.systems:34568   (NL)
-- wowbux.org:34568          (CA)
-- super.fast.node.xmr.pm:34568 (DE)
-- wow.pwned.systems          (NL)
-- wowbuxx535x4exuexja2xfezpwcyznxkofui4ndjiectj4yuh2xheiid.onion:34568
-
-### Tor Peers
-- wowp2p5gelm6vhl2d5tvfqills63jilgy6hkvlrqljooov5ktaxgqdad.onion
-- f3moshycuklu3mb3wnlfjwn26nsgzreqtfzfuyjjk46u7jqxlhf7d5id.onion
+- https://explorer.wownero.fyi
 
 ## Introduction
 
@@ -50,6 +41,9 @@ Wownero is a 100% community-sponsored endeavor. Supporting services are also gra
 [<img src="https://git.wownero.com/wownero/meta/raw/branch/master/images/macstadium.png"
       alt="MacStadium"
       height="100">](https://www.macstadium.com)
+[<img src="https://git.wownero.com/wownero/meta/raw/branch/master/images/jetbrains.png"
+      alt="JetBrains"
+      height="100">](https://www.jetbrains.com)
 
 Developers are volunteers doing this mostly for shits and giggles. If you would like to support our shenanigans and stimulant addictions, please consider donating to [WFS proposals](https://funding.wownero.com/proposals) or the dev slush fund.
 
@@ -90,7 +84,7 @@ Dates are provided in the format YYYY-MM-DD.
 | 114,969                          | 2019-06-14 | F For Fappening    | v0.6.1.0               | v0.6.1.2                  | RandomWOW, new block weight algorithm, slightly more efficient RingCT format
 | 160,777                          | 2019-11-20 | Gaping Goatse    | v0.7.0.0               | v0.7.1.0                  | Only allow >= 2 outputs, change to the block median used to calculate penalty, rct sigs in coinbase forbidden, 4 unlock time as protocol rule
 | -                         | 2020-06-28 | Hallucinogenic Hypnotoad  | v0.8.0.0               | v0.8.0.2                  | Dandelion++ support
-| 253,999                         | 2020-10-09 | Illiterate Illuminati  | v0.9.0.0               | v0.9.1.0                  | Dynamic coinbase unlock (up to 1 mo.), Deterministic unlock times, Enforce maximum coinbase amount, show_qr_code wallet command, CLSAG
+| 253,999                         | 2020-10-09 | Illiterate Illuminati  | v0.9.0.0               | v0.9.3.3                  | Dynamic coinbase unlock (up to 1 mo.), Deterministic unlock times, Enforce maximum coinbase amount, show_qr_code wallet command, CLSAG
 
 X's indicate that these details have not been determined as of commit date.
 

cmake/GitVersion.cmake

@@ -48,29 +48,21 @@ function (get_version_tag_from_git GIT)
         message(STATUS "You are currently on commit ${COMMIT}")
 
         # Get all the tags
-        execute_process(COMMAND "${GIT}" rev-list --tags --max-count=1 --abbrev-commit 
+        execute_process(COMMAND "${GIT}" tag -l --points-at HEAD
                         WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
                         RESULT_VARIABLE RET
-                        OUTPUT_VARIABLE TAGGEDCOMMIT
+                        OUTPUT_VARIABLE TAG
                         OUTPUT_STRIP_TRAILING_WHITESPACE)
 
-        if(NOT TAGGEDCOMMIT)
-            message(WARNING "Cannot determine most recent tag. Make sure that you are building either from a Git working tree or from a source archive.")
+        # Check if we're building that tagged commit or a different one
+        if(TAG)
+            message(STATUS "You are building a tagged release")
+            set(VERSIONTAG "release")
+            set(VERSION_IS_RELEASE "true")
+        else()
+            message(STATUS "You are ahead of or behind a tagged release")
             set(VERSIONTAG "${COMMIT}")
             set(VERSION_IS_RELEASE "false")
-        else()
-            message(STATUS "The most recent tag was at ${TAGGEDCOMMIT}")
-
-            # Check if we're building that tagged commit or a different one
-            if(COMMIT STREQUAL TAGGEDCOMMIT)
-                message(STATUS "You are building a tagged release")
-                set(VERSIONTAG "release")
-                set(VERSION_IS_RELEASE "true")
-            else()
-                message(STATUS "You are ahead of or behind a tagged release")
-                set(VERSIONTAG "${COMMIT}")
-                set(VERSION_IS_RELEASE "false")
-            endif()
         endif()	    
     endif()
 

cmake/SetClangTidy.cmake

@@ -0,0 +1,72 @@
+# Copyright (c) 2014-2020, The Monero Project
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# https://cmake.org/cmake/help/latest/variable/CMAKE_LANG_CLANG_TIDY.html
+# This module sets the following variables:
+# CMAKE_C_CLANG_TIDY
+# CMAKE_CXX_CLANG_TIDY
+# when clang-tidy is found in PATH. Afterwards, the code is being linted by the tool.
+# The checks to be enabled can be manipulated with the variable MONERO_CLANG_TIDY_CHECKS
+
+macro (monero_clang_tidy LANGUAGE)
+	set(TOOL_NAME "clang-tidy")
+	set(MONERO_CLANG_TIDY_MIN_VERSION "3.6")
+	if(${CMAKE_VERSION} VERSION_LESS "${MONERO_CLANG_TIDY_MIN_VERSION}")
+		message(FATAL_ERROR "Sorry, ${TOOL_NAME} is available for CMake from version ${MONERO_CLANG_TIDY_MIN_VERSION}")
+	else()
+		message(STATUS "Trying to enable ${TOOL_NAME}")
+		find_program(MONERO_CLANG_BIN ${TOOL_NAME})
+		if(NOT MONERO_CLANG_BIN)
+			message(FATAL_ERROR "${TOOL_NAME} not found! Try running: sudo apt install ${TOOL_NAME}")
+		else()
+			message(STATUS "Found ${MONERO_CLANG_BIN}")
+			set(MONERO_CLANG_TIDY_CHECKS
+			    -header-filter=.; 	# By default the headers are excluded. This line enables them.
+			    -checks=*; 		# Currently enabling all checks
+			    # An example of selectively enabling checks:
+			    #-checks=bugprone-*,cppcoreguidelines-avoid-goto # Have to be in one line :(
+			)
+			# Current list of checks is avaibale under:
+			# https://clang.llvm.org/extra/clang-tidy/
+			if (${LANGUAGE} STREQUAL "C")
+				set(CMAKE_C_CLANG_TIDY
+					${MONERO_CLANG_BIN}; # Mind the semicolon
+					${MONERO_CLANG_TIDY_CHECKS}
+				)
+			elseif (${LANGUAGE} STREQUAL "CXX")
+				set(CMAKE_CXX_CLANG_TIDY
+					${MONERO_CLANG_BIN}; # Mind the semicolon
+					${MONERO_CLANG_TIDY_CHECKS}
+				)
+			else()
+				message(FATAL_ERROR "${TOOL_NAME}: Unsupported language: ${LANGUAGE}")
+			endif()
+		endif()
+	endif()
+endmacro()
+

contrib/depends/packages/openssl.mk

@@ -1,36 +1,31 @@
 package=openssl
-$(package)_version=1.0.2r
-$(package)_download_path=https://ftp.openssl.org/source/old/1.0.2
+$(package)_version=1.1.1i
+$(package)_download_path=https://www.openssl.org/source
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
-$(package)_sha256_hash=ae51d08bba8a83958e894946f15303ff894d75c2b8bbd44a852b64e3fe11d0d6
-$(package)_patches=fix_arflags.patch
+$(package)_sha256_hash=e8be6a35fe41d10603c3cc635e93289ed00bf34b79671a3a4de64fcee00d5242
 
 define $(package)_set_vars
 $(package)_config_env=AR="$($(package)_ar)" ARFLAGS=$($(package)_arflags) RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)"
+$(package)_config_env_arm_android=ANDROID_NDK_HOME="$(host_prefix)/native" PATH="$(host_prefix)/native/bin" CC=clang AR=ar RANLIB=ranlib
+$(package)_config_env_aarch64_android=ANDROID_NDK_HOME="$(host_prefix)/native" PATH="$(host_prefix)/native/bin" CC=clang AR=ar RANLIB=ranlib
+$(package)_build_env_arm_android=ANDROID_NDK_HOME="$(host_prefix)/native"
+$(package)_build_env_aarch64_android=ANDROID_NDK_HOME="$(host_prefix)/native"
 $(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/openssl
 $(package)_config_opts+=no-capieng
 $(package)_config_opts+=no-dso
 $(package)_config_opts+=no-dtls1
 $(package)_config_opts+=no-ec_nistp_64_gcc_128
 $(package)_config_opts+=no-gost
-$(package)_config_opts+=no-gmp
 $(package)_config_opts+=no-heartbeats
-$(package)_config_opts+=no-jpake
-$(package)_config_opts+=no-krb5
-$(package)_config_opts+=no-libunbound
 $(package)_config_opts+=no-md2
 $(package)_config_opts+=no-rc5
 $(package)_config_opts+=no-rdrand
 $(package)_config_opts+=no-rfc3779
-$(package)_config_opts+=no-rsax
 $(package)_config_opts+=no-sctp
-$(package)_config_opts+=no-sha0
 $(package)_config_opts+=no-shared
 $(package)_config_opts+=no-ssl-trace
 $(package)_config_opts+=no-ssl2
 $(package)_config_opts+=no-ssl3
-$(package)_config_opts+=no-static_engine
-$(package)_config_opts+=no-store
 $(package)_config_opts+=no-unit-test
 $(package)_config_opts+=no-weak-ssl-ciphers
 $(package)_config_opts+=no-zlib
@@ -42,8 +37,8 @@ $(package)_config_opts_x86_64_linux=linux-x86_64
 $(package)_config_opts_i686_linux=linux-generic32
 $(package)_config_opts_arm_linux=linux-generic32
 $(package)_config_opts_aarch64_linux=linux-generic64
-$(package)_config_opts_arm_android=--static android-armv7 no-asm
-$(package)_config_opts_aarch64_android=--static android no-asm
+$(package)_config_opts_arm_android=--static android-arm
+$(package)_config_opts_aarch64_android=--static android-arm64
 $(package)_config_opts_riscv64_linux=linux-generic64
 $(package)_config_opts_mipsel_linux=linux-generic32
 $(package)_config_opts_mips_linux=linux-generic32
@@ -55,10 +50,8 @@ $(package)_config_opts_x86_64_freebsd=BSD-x86_64
 endef
 
 define $(package)_preprocess_cmds
-  sed -i.old "/define DATE/d" util/mkbuildinf.pl && \
-  sed -i.old "s|engines apps test|engines|" Makefile.org && \
-  sed -i -e "s/-mandroid //" Configure && \
-  patch < $($(package)_patch_dir)/fix_arflags.patch
+  sed -i.old 's|"engines", "apps", "test", "util", "tools", "fuzz"|"engines", "tools"|' Configure && \
+  sed -i -e 's|cflags --sysroot.*",|cflags",|' Configurations/15-android.conf
 endef
 
 define $(package)_config_cmds
@@ -70,7 +63,7 @@ define $(package)_build_cmds
 endef
 
 define $(package)_stage_cmds
-  $(MAKE) INSTALL_PREFIX=$($(package)_staging_dir) -j1 install_sw
+  $(MAKE) DESTDIR=$($(package)_staging_dir) -j1 install_sw
 endef
 
 define $(package)_postprocess_cmds

contrib/depends/patches/openssl/fix_arflags.patch

@@ -1,24 +0,0 @@
---- Makefile.org.O	2019-02-26 14:20:20.000000000 +0000
-+++ Makefile.org	2019-11-15 13:05:54.370086856 +0000
-@@ -63,8 +63,8 @@
- PEX_LIBS= 
- EX_LIBS= 
- EXE_EXT= 
--ARFLAGS=
--AR=ar $(ARFLAGS) r
-+ARFLAGS= r
-+AR=ar $(ARFLAGS)
- RANLIB= ranlib
- RC= windres
- NM= nm
---- Configure.O	2019-02-26 14:20:20.000000000 +0000
-+++ Configure	2019-11-16 07:43:14.933990774 +0000
-@@ -1251,7 +1251,7 @@
- my $shared_extension = $fields[$idx_shared_extension];
- my $ranlib = $ENV{'RANLIB'} || $fields[$idx_ranlib];
- my $ar = $ENV{'AR'} || "ar";
--my $arflags = $fields[$idx_arflags];
-+my $arflags = $ENV{'ARFLAGS'} || $fields[$idx_arflags];
- my $windres = $ENV{'RC'} || $ENV{'WINDRES'} || "windres";
- my $multilib = $fields[$idx_multilib];
- 

contrib/epee/include/byte_stream.h

@@ -58,7 +58,6 @@ namespace epee
     byte_buffer buffer_;        //! Beginning of buffer
     std::uint8_t* next_write_;  //! Current write position
     const std::uint8_t* end_;   //! End of buffer
-    std::size_t increase_size_; //! Minimum buffer size increase
 
     //! \post `requested <= available()`
     void overflow(const std::size_t requested);
@@ -75,29 +74,17 @@ namespace epee
     using char_type = std::uint8_t;
     using Ch = char_type;
 
-    //! \return Default minimum size increase on buffer overflow
-    static constexpr std::size_t default_increase() noexcept { return 4096; }
-
     //! Increase internal buffer by at least `byte_stream_increase` bytes.
     byte_stream() noexcept
-      : byte_stream(default_increase())
-    {}
-
-    //! Increase internal buffer by at least `increase` bytes.
-    explicit byte_stream(const std::size_t increase) noexcept
       : buffer_(nullptr),
         next_write_(nullptr),
-        end_(nullptr),
-        increase_size_(increase)
+        end_(nullptr)
     {}
 
     byte_stream(byte_stream&& rhs) noexcept;
     ~byte_stream() noexcept = default;
     byte_stream& operator=(byte_stream&& rhs) noexcept;
 
-    //! \return The minimum increase size on buffer overflow
-    std::size_t increase_size() const noexcept { return increase_size_; }
-
     const std::uint8_t* data() const noexcept { return buffer_.get(); }
     std::uint8_t* tellp() const noexcept { return next_write_; }
     std::size_t available() const noexcept { return end_ - next_write_; }
@@ -188,7 +175,7 @@ namespace epee
     void put_n(const std::uint8_t ch, const std::size_t count)
     {
       check(count);
-      std::memset(tellp(), count, ch);
+      std::memset(tellp(), ch, count);
       next_write_ += count;
     }
 

contrib/epee/include/net/abstract_http_client.h

@@ -70,7 +70,7 @@ namespace http
     virtual bool connect(std::chrono::milliseconds timeout) = 0;
     virtual bool disconnect() = 0;
     virtual bool is_connected(bool *ssl = NULL) = 0;
-    virtual bool invoke(const boost::string_ref uri, const boost::string_ref method, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0;
+    virtual bool invoke(const boost::string_ref uri, const boost::string_ref method, const boost::string_ref body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0;
     virtual bool invoke_get(const boost::string_ref uri, std::chrono::milliseconds timeout, const std::string& body = std::string(), const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0;
     virtual bool invoke_post(const boost::string_ref uri, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0;
     virtual uint64_t get_bytes_sent() const = 0;

contrib/epee/include/net/abstract_tcp_server2.inl

@@ -207,7 +207,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     buffer_ssl_init_fill = 0;
     if (is_income && m_ssl_support != epee::net_utils::ssl_support_t::e_ssl_support_disabled)
       socket().async_receive(boost::asio::buffer(buffer_),
-        boost::asio::socket_base::message_peek,
         strand_.wrap(
           std::bind(&connection<t_protocol_handler>::handle_receive, self,
             std::placeholders::_1,
@@ -334,6 +333,9 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     TRY_ENTRY();
     //_info("[sock " << socket().native_handle() << "] Async read calledback.");
     
+    if (m_was_shutdown)
+        return;
+
     if (!e)
     {
         double current_speed_down;
@@ -360,6 +362,9 @@ PRAGMA_WARNING_DISABLE_VS(4355)
 					CRITICAL_REGION_LOCAL(	epee::net_utils::network_throttle_manager::m_lock_get_global_throttle_in );
 					delay = epee::net_utils::network_throttle_manager::get_global_throttle_in().get_sleep_time_after_tick( bytes_transferred );
 				}
+
+				if (m_was_shutdown)
+					return;
 				
 				delay *= 0.5;
 				long int ms = (long int)(delay * 100);
@@ -431,6 +436,9 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     std::size_t bytes_transferred)
   {
     TRY_ENTRY();
+
+    if (m_was_shutdown) return;
+
     if (e)
     {
       // offload the error case
@@ -438,13 +446,11 @@ PRAGMA_WARNING_DISABLE_VS(4355)
       return;
     }
 
-    reset_timer(get_timeout_from_bytes_read(bytes_transferred), false);
-
     buffer_ssl_init_fill += bytes_transferred;
-    if (buffer_ssl_init_fill <= get_ssl_magic_size())
+    MTRACE("we now have " << buffer_ssl_init_fill << "/" << get_ssl_magic_size() << " bytes needed to detect SSL");
+    if (buffer_ssl_init_fill < get_ssl_magic_size())
     {
       socket().async_receive(boost::asio::buffer(buffer_.data() + buffer_ssl_init_fill, buffer_.size() - buffer_ssl_init_fill),
-        boost::asio::socket_base::message_peek,
         strand_.wrap(
           boost::bind(&connection<t_protocol_handler>::handle_receive, connection<t_protocol_handler>::shared_from_this(),
             boost::asio::placeholders::error,
@@ -470,7 +476,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     if (m_ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_enabled)
     {
       // Handshake
-      if (!handshake(boost::asio::ssl::stream_base::server))
+      if (!handshake(boost::asio::ssl::stream_base::server, boost::asio::const_buffer(buffer_.data(), buffer_ssl_init_fill)))
       {
         MERROR("SSL handshake failed");
         boost::interprocess::ipcdetail::atomic_write32(&m_want_close_connection, 1);
@@ -485,6 +491,11 @@ PRAGMA_WARNING_DISABLE_VS(4355)
         return;
       }
     }
+    else
+    {
+      handle_read(e, buffer_ssl_init_fill);
+      return;
+    }
 
     async_read_some(boost::asio::buffer(buffer_),
       strand_.wrap(
@@ -651,6 +662,8 @@ PRAGMA_WARNING_DISABLE_VS(4355)
         boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) );
         m_send_que_lock.lock();
         _dbg1("sleep for queue: " << ms);
+	if (m_was_shutdown)
+		return false;
 
         if (retry > retry_limit) {
             MWARNING("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection");
@@ -748,7 +761,8 @@ PRAGMA_WARNING_DISABLE_VS(4355)
   template<class t_protocol_handler>
   void connection<t_protocol_handler>::reset_timer(boost::posix_time::milliseconds ms, bool add)
   {
-    if (ms.total_milliseconds() < 0)
+    const auto tms = ms.total_milliseconds();
+    if (tms < 0 || (add && tms == 0))
     {
       MWARNING("Ignoring negative timeout " << ms);
       return;

contrib/epee/include/net/connection_basic.hpp

@@ -132,10 +132,10 @@ class connection_basic { // not-templated base class for rapid developmet of som
 		ssl_support_t get_ssl_support() const { return m_ssl_support; }
 		void disable_ssl() { m_ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_disabled; }
 
-		bool handshake(boost::asio::ssl::stream_base::handshake_type type)
+		bool handshake(boost::asio::ssl::stream_base::handshake_type type, boost::asio::const_buffer buffer = {})
 		{
 			//m_state != nullptr verified in constructor
-			return m_state->ssl_options().handshake(socket_, type);
+			return m_state->ssl_options().handshake(socket_, type, buffer);
 		}
 
 		template<typename MutableBufferSequence, typename ReadHandler>

contrib/epee/include/net/http_client.h

@@ -233,7 +233,7 @@ namespace net_utils
 			}
 
 			//---------------------------------------------------------------------------
-			inline bool invoke(const boost::string_ref uri, const boost::string_ref method, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) override
+			inline bool invoke(const boost::string_ref uri, const boost::string_ref method, const boost::string_ref body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) override
 			{
 				CRITICAL_REGION_LOCAL(m_lock);
 				if(!is_connected())

contrib/epee/include/net/http_server_handlers_map2.h

@@ -118,8 +118,10 @@
         return true; \
       } \
       uint64_t ticks2 = misc_utils::get_tick_count(); \
-      epee::serialization::store_t_to_binary(static_cast<command_type::response&>(resp), response_info.m_body); \
+      epee::byte_slice buffer; \
+      epee::serialization::store_t_to_binary(static_cast<command_type::response&>(resp), buffer, 64 * 1024); \
       uint64_t ticks3 = epee::misc_utils::get_tick_count(); \
+      response_info.m_body.assign(reinterpret_cast<const char*>(buffer.data()), buffer.size()); \
       response_info.m_mime_tipe = " application/octet-stream"; \
       response_info.m_header_info.m_content_type = " application/octet-stream"; \
       MDEBUG( s_pattern << "() processed with " << ticks1-ticks << "/"<< ticks2-ticks1 << "/" << ticks3-ticks2 << "ms"); \

contrib/epee/include/net/levin_base.h

@@ -31,7 +31,6 @@
 
 #include <cstdint>
 
-#include "byte_slice.h"
 #include "net_utils_base.h"
 #include "span.h"
 
@@ -39,6 +38,7 @@
 
 namespace epee
 {
+class byte_slice;
 namespace levin
 {
 #pragma pack(push)
@@ -72,7 +72,8 @@ namespace levin
 
 
 #define LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED 0
-#define LEVIN_DEFAULT_MAX_PACKET_SIZE 100000000      //100MB by default
+#define LEVIN_INITIAL_MAX_PACKET_SIZE  256*1024      // 256 KiB before handshake
+#define LEVIN_DEFAULT_MAX_PACKET_SIZE 100000000      //100MB by default after handshake
 
 #define LEVIN_PACKET_REQUEST			0x00000001
 #define LEVIN_PACKET_RESPONSE		0x00000002
@@ -86,7 +87,7 @@ namespace levin
   template<class t_connection_context = net_utils::connection_context_base>
   struct levin_commands_handler
   {
-    virtual int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, t_connection_context& context)=0;
+    virtual int invoke(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, t_connection_context& context)=0;
     virtual int notify(int command, const epee::span<const uint8_t> in_buff, t_connection_context& context)=0;
     virtual void callback(t_connection_context& context){};
 

contrib/epee/include/net/levin_protocol_handler_async.h

@@ -84,7 +84,8 @@ class async_protocol_handler_config
 
 public:
   typedef t_connection_context connection_context;
-  uint64_t m_max_packet_size; 
+  uint64_t m_initial_max_packet_size;
+  uint64_t m_max_packet_size;
   uint64_t m_invoke_timeout;
 
   int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, boost::uuids::uuid connection_id);
@@ -105,7 +106,7 @@ public:
   size_t get_in_connections_count();
   void set_handler(levin_commands_handler<t_connection_context>* handler, void (*destroy)(levin_commands_handler<t_connection_context>*) = NULL);
 
-  async_protocol_handler_config():m_pcommands_handler(NULL), m_pcommands_handler_destroy(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE), m_invoke_timeout(LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED)
+  async_protocol_handler_config():m_pcommands_handler(NULL), m_pcommands_handler_destroy(NULL), m_initial_max_packet_size(LEVIN_INITIAL_MAX_PACKET_SIZE), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE), m_invoke_timeout(LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED)
   {}
   ~async_protocol_handler_config() { set_handler(NULL, NULL); }
   void del_out_connections(size_t count);
@@ -162,6 +163,7 @@ public:
   net_utils::i_service_endpoint* m_pservice_endpoint; 
   config_type& m_config;
   t_connection_context& m_connection_context;
+  std::atomic<uint64_t> m_max_packet_size;
 
   net_utils::buffer m_cache_in_buffer;
   stream_state m_state;
@@ -289,7 +291,8 @@ public:
             m_current_head(bucket_head2()),
             m_pservice_endpoint(psnd_hndlr), 
             m_config(config), 
-            m_connection_context(conn_context), 
+            m_connection_context(conn_context),
+            m_max_packet_size(config.m_initial_max_packet_size),
             m_cache_in_buffer(4 * 1024),
             m_state(stream_state_head)
   {
@@ -399,13 +402,14 @@ public:
     }
 
     // these should never fail, but do runtime check for safety
-    CHECK_AND_ASSERT_MES(m_config.m_max_packet_size >= m_cache_in_buffer.size(), false, "Bad m_cache_in_buffer.size()");
-    CHECK_AND_ASSERT_MES(m_config.m_max_packet_size - m_cache_in_buffer.size() >= m_fragment_buffer.size(), false, "Bad m_cache_in_buffer.size() + m_fragment_buffer.size()");
+    const uint64_t max_packet_size = m_max_packet_size;
+    CHECK_AND_ASSERT_MES(max_packet_size >= m_cache_in_buffer.size(), false, "Bad m_cache_in_buffer.size()");
+    CHECK_AND_ASSERT_MES(max_packet_size - m_cache_in_buffer.size() >= m_fragment_buffer.size(), false, "Bad m_cache_in_buffer.size() + m_fragment_buffer.size()");
 
     // flipped to subtraction; prevent overflow since m_max_packet_size is variable and public
-    if(cb > m_config.m_max_packet_size - m_cache_in_buffer.size() - m_fragment_buffer.size())
+    if(cb > max_packet_size - m_cache_in_buffer.size() - m_fragment_buffer.size())
     {
-      MWARNING(m_connection_context << "Maximum packet size exceed!, m_max_packet_size = " << m_config.m_max_packet_size
+      MWARNING(m_connection_context << "Maximum packet size exceed!, m_max_packet_size = " << max_packet_size
                           << ", packet received " << m_cache_in_buffer.size() +  cb 
                           << ", connection will be closed.");
       return false;
@@ -430,7 +434,7 @@ public:
               //async call scenario
               boost::shared_ptr<invoke_response_handler_base> response_handler = m_invoke_response_handlers.front();
               response_handler->reset_timer();
-              MDEBUG(m_connection_context << "LEVIN_PACKET partial msg received. len=" << cb);
+              MDEBUG(m_connection_context << "LEVIN_PACKET partial msg received. len=" << cb << ", current total " << m_cache_in_buffer.size() << "/" << m_current_head.m_cb << " (" << (100.0f * m_cache_in_buffer.size() / (m_current_head.m_cb ? m_current_head.m_cb : 1)) << "%)");
             }
           }
           break;
@@ -465,6 +469,14 @@ public:
             temp = std::move(m_fragment_buffer);
             m_fragment_buffer.clear();
             std::memcpy(std::addressof(m_current_head), std::addressof(temp[0]), sizeof(bucket_head2));
+            const size_t max_bytes = m_connection_context.get_max_bytes(m_current_head.m_command);
+            if(m_current_head.m_cb > std::min<size_t>(max_packet_size, max_bytes))
+            {
+              MERROR(m_connection_context << "Maximum packet size exceed!, m_max_packet_size = " << std::min<size_t>(max_packet_size, max_bytes)
+                << ", packet header received " << m_current_head.m_cb << ", command " << m_current_head.m_command
+                << ", connection will be closed.");
+              return false;
+            }
             buff_to_invoke = {reinterpret_cast<const uint8_t*>(temp.data()) + sizeof(bucket_head2), temp.size() - sizeof(bucket_head2)};
           }
 
@@ -514,16 +526,19 @@ public:
           {
             if(m_current_head.m_have_to_return_data)
             {
-              std::string return_buff;
+              byte_slice return_buff;
               const uint32_t return_code = m_config.m_pcommands_handler->invoke(
                 m_current_head.m_command, buff_to_invoke, return_buff, m_connection_context
               );
 
+              // peer_id remains unset if dropped
+              if (m_current_head.m_command == m_connection_context.handshake_command() && m_connection_context.handshake_complete())
+                m_max_packet_size = m_config.m_max_packet_size;
+
               bucket_head2 head = make_header(m_current_head.m_command, return_buff.size(), LEVIN_PACKET_RESPONSE, false);
               head.m_return_code = SWAP32LE(return_code);
-              return_buff.insert(0, reinterpret_cast<const char*>(&head), sizeof(head));
 
-              if(!m_pservice_endpoint->do_send(byte_slice{std::move(return_buff)}))
+              if(!m_pservice_endpoint->do_send(byte_slice{{epee::as_byte_span(head), epee::to_span(return_buff)}}))
                 return false;
 
               MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb
@@ -577,10 +592,11 @@ public:
           m_cache_in_buffer.erase(sizeof(bucket_head2));
           m_state = stream_state_body;
           m_oponent_protocol_ver = m_current_head.m_protocol_version;
-          if(m_current_head.m_cb > m_config.m_max_packet_size)
+          const size_t max_bytes = m_connection_context.get_max_bytes(m_current_head.m_command);
+          if(m_current_head.m_cb > std::min<size_t>(max_packet_size, max_bytes))
           {
-            LOG_ERROR_CC(m_connection_context, "Maximum packet size exceed!, m_max_packet_size = " << m_config.m_max_packet_size 
-              << ", packet header received " << m_current_head.m_cb 
+            LOG_ERROR_CC(m_connection_context, "Maximum packet size exceed!, m_max_packet_size = " << std::min<size_t>(max_packet_size, max_bytes)
+              << ", packet header received " << m_current_head.m_cb << ", command " << m_current_head.m_command
               << ", connection will be closed.");
             return false;
           }
@@ -634,6 +650,9 @@ public:
       boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0);
       CRITICAL_REGION_BEGIN(m_invoke_response_handlers_lock);
 
+      if (command == m_connection_context.handshake_command())
+        m_max_packet_size = m_config.m_max_packet_size;
+
       if(!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true))
       {
         LOG_ERROR_CC(m_connection_context, "Failed to do_send");
@@ -675,6 +694,9 @@ public:
 
     boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0);
 
+    if (command == m_connection_context.handshake_command())
+      m_max_packet_size = m_config.m_max_packet_size;
+
     if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true))
     {
       LOG_ERROR_CC(m_connection_context, "Failed to send request");

contrib/epee/include/net/net_helper.h

@@ -44,6 +44,7 @@
 #include <boost/lambda/lambda.hpp>
 #include <boost/interprocess/detail/atomic.hpp>
 #include <boost/system/error_code.hpp>
+#include <boost/utility/string_ref.hpp>
 #include <functional>
 #include "net/net_utils_base.h"
 #include "net/net_ssl.h"
@@ -178,7 +179,7 @@ namespace net_utils
 					// SSL Options
 					if (m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_enabled || m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect)
 					{
-						if (!m_ssl_options.handshake(*m_ssl_socket, boost::asio::ssl::stream_base::client, addr, timeout))
+						if (!m_ssl_options.handshake(*m_ssl_socket, boost::asio::ssl::stream_base::client, {}, addr, timeout))
 						{
 							if (m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect)
 							{
@@ -280,7 +281,7 @@ namespace net_utils
 
 
 		inline 
-		bool send(const std::string& buff, std::chrono::milliseconds timeout)
+		bool send(const boost::string_ref buff, std::chrono::milliseconds timeout)
 		{
 
 			try
@@ -298,7 +299,7 @@ namespace net_utils
 				// object is used as a callback and will update the ec variable when the
 				// operation completes. The blocking_udp_client.cpp example shows how you
 				// can use boost::bind rather than boost::lambda.
-				async_write(buff.c_str(), buff.size(), ec);
+				async_write(buff.data(), buff.size(), ec);
 
 				// Block until the asynchronous operation has completed.
 				while (ec == boost::asio::error::would_block)

contrib/epee/include/net/net_parse_helpers.h

@@ -131,8 +131,8 @@ namespace net_utils
   inline
     bool parse_url_ipv6(const std::string url_str, http::url_content& content)
   {
-    STATIC_REGEXP_EXPR_1(rexp_match_uri, "^((.*?)://)?(\\[(.*)\\](:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal);
-    //                                     12         3   4      5 6        7
+    STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(\\[(.*)\\](:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal);
+    //                                     12            3   4      5 6        7
 
     content.port = 0;
     boost::smatch result;
@@ -175,8 +175,8 @@ namespace net_utils
 
     ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash=
     //STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal);
-    STATIC_REGEXP_EXPR_1(rexp_match_uri, "^((.*?)://)?(([^/:]*)(:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal);
-    //                                     12         34      5 6        7
+    STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(([^/:]*)(:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal);
+    //                                     12            34       5 6        7
     content.port = 0;
     boost::smatch result;	
     if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched))

contrib/epee/include/net/net_ssl.h

@@ -132,6 +132,7 @@ namespace net_utils
     bool handshake(
       boost::asio::ssl::stream<boost::asio::ip::tcp::socket> &socket,
       boost::asio::ssl::stream_base::handshake_type type,
+      boost::asio::const_buffer buffer = {},
       const std::string& host = {},
       std::chrono::milliseconds timeout = std::chrono::seconds(15)) const;
   };

contrib/epee/include/rolling_median.h

@@ -141,7 +141,6 @@ public:
 
   rolling_median_t(rolling_median_t &&m)
   {
-    free(data);
     memcpy(this, &m, sizeof(rolling_median_t));
     m.data = NULL;
   }

contrib/epee/include/storages/http_abstract_invoke.h

@@ -29,6 +29,7 @@
 #include <boost/utility/string_ref.hpp>
 #include <chrono>
 #include <string>
+#include "byte_slice.h"
 #include "portable_storage_template_helper.h"
 #include "net/http_base.h"
 #include "net/http_server_handlers_map2.h"
@@ -74,12 +75,12 @@ namespace epee
     template<class t_request, class t_response, class t_transport>
     bool invoke_http_bin(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref method = "POST")
     {
-      std::string req_param;
-      if(!serialization::store_t_to_binary(out_struct, req_param))
+      byte_slice req_param;
+      if(!serialization::store_t_to_binary(out_struct, req_param, 16 * 1024))
         return false;
 
       const http::http_response_info* pri = NULL;
-      if(!transport.invoke(uri, method, req_param, timeout, std::addressof(pri)))
+      if(!transport.invoke(uri, method, boost::string_ref{reinterpret_cast<const char*>(req_param.data()), req_param.size()}, timeout, std::addressof(pri)))
       {
         LOG_PRINT_L1("Failed to invoke http request to  " << uri);
         return false;
@@ -97,7 +98,12 @@ namespace epee
         return false;
       }
 
-      return serialization::load_t_from_binary(result_struct, epee::strspan<uint8_t>(pri->m_body));
+      static const constexpr epee::serialization::portable_storage::limits_t default_http_bin_limits = {
+        65536 * 3, // objects
+        65536 * 3, // fields
+        65536 * 3, // strings
+      };
+      return serialization::load_t_from_binary(result_struct, epee::strspan<uint8_t>(pri->m_body), &default_http_bin_limits);
     }
 
     template<class t_request, class t_response, class t_transport>

contrib/epee/include/storages/levin_abstract_invoke2.h

@@ -27,8 +27,10 @@
 #pragma once
 
 #include "portable_storage_template_helper.h"
+#include <boost/utility/string_ref.hpp>
 #include <boost/utility/value_init.hpp>
 #include <functional>
+#include "byte_slice.h"
 #include "span.h"
 #include "net/levin_base.h"
 
@@ -50,6 +52,11 @@ namespace
     snprintf(buf, sizeof(buf),  "command-%u", command);
     return on_levin_traffic(context, initiator, sent, error, bytes, buf);
   }
+  static const constexpr epee::serialization::portable_storage::limits_t default_levin_limits = {
+    8192, // objects
+    16384, // fields
+    16384, // strings
+  };
 }
 
 namespace epee
@@ -75,7 +82,7 @@ namespace epee
         return false;
       }
       serialization::portable_storage stg_ret;
-      if(!stg_ret.load_from_binary(buff_to_recv))
+      if(!stg_ret.load_from_binary(buff_to_recv, &default_levin_limits))
       {
         LOG_ERROR("Failed to load_from_binary on command " << command);
         return false;
@@ -110,18 +117,19 @@ namespace epee
       const boost::uuids::uuid &conn_id = context.m_connection_id;
       typename serialization::portable_storage stg;
       out_struct.store(stg);
-      std::string buff_to_send, buff_to_recv;
-      stg.store_to_binary(buff_to_send);
+      byte_slice buff_to_send;
+      std::string buff_to_recv;
+      stg.store_to_binary(buff_to_send, 16 * 1024);
 
       on_levin_traffic(context, true, true, false, buff_to_send.size(), command);
-      int res = transport.invoke(command, buff_to_send, buff_to_recv, conn_id);
+      int res = transport.invoke(command, boost::string_ref{reinterpret_cast<const char*>(buff_to_send.data()), buff_to_send.size()}, buff_to_recv, conn_id);
       if( res <=0 )
       {
         LOG_PRINT_L1("Failed to invoke command " << command << " return code " << res);
         return false;
       }
       typename serialization::portable_storage stg_ret;
-      if(!stg_ret.load_from_binary(buff_to_recv))
+      if(!stg_ret.load_from_binary(buff_to_recv, &default_levin_limits))
       {
         on_levin_traffic(context, true, false, true, buff_to_recv.size(), command);
         LOG_ERROR("Failed to load_from_binary on command " << command);
@@ -137,10 +145,10 @@ namespace epee
       const boost::uuids::uuid &conn_id = context.m_connection_id;
       typename serialization::portable_storage stg;
       const_cast<t_arg&>(out_struct).store(stg);//TODO: add true const support to searilzation
-      std::string buff_to_send;
-      stg.store_to_binary(buff_to_send);
+      byte_slice buff_to_send;
+      stg.store_to_binary(buff_to_send, 16 * 1024);
       on_levin_traffic(context, true, true, false, buff_to_send.size(), command);
-      int res = transport.invoke_async(command, epee::strspan<uint8_t>(buff_to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool 
+      int res = transport.invoke_async(command, epee::to_span(buff_to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool
       {
         t_result result_struct = AUTO_VAL_INIT(result_struct);
         if( code <=0 )
@@ -152,7 +160,7 @@ namespace epee
           return false;
         }
         serialization::portable_storage stg_ret;
-        if(!stg_ret.load_from_binary(buff))
+        if(!stg_ret.load_from_binary(buff, &default_levin_limits))
         {
           on_levin_traffic(context, true, false, true, buff.size(), command);
           LOG_ERROR("Failed to load_from_binary on command " << command);
@@ -184,11 +192,11 @@ namespace epee
       const boost::uuids::uuid &conn_id = context.m_connection_id;
       serialization::portable_storage stg;
       out_struct.store(stg);
-      std::string buff_to_send;
+      byte_slice buff_to_send;
       stg.store_to_binary(buff_to_send);
 
       on_levin_traffic(context, true, true, false, buff_to_send.size(), command);
-      int res = transport.notify(command, epee::strspan<uint8_t>(buff_to_send), conn_id);
+      int res = transport.notify(command, epee::to_span(buff_to_send), conn_id);
       if(res <=0 )
       {
         MERROR("Failed to notify command " << command << " return code " << res);
@@ -199,10 +207,10 @@ namespace epee
     //----------------------------------------------------------------------------------------------------
     //----------------------------------------------------------------------------------------------------
     template<class t_owner, class t_in_type, class t_out_type, class t_context, class callback_t>
-    int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, callback_t cb, t_context& context )
+    int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, callback_t cb, t_context& context )
     {
       serialization::portable_storage strg;
-      if(!strg.load_from_binary(in_buff))
+      if(!strg.load_from_binary(in_buff, &default_levin_limits))
       {
         on_levin_traffic(context, false, false, true, in_buff.size(), command);
         LOG_ERROR("Failed to load_from_binary in command " << command);
@@ -222,7 +230,7 @@ namespace epee
       serialization::portable_storage strg_out;
       static_cast<t_out_type&>(out_struct).store(strg_out);
 
-      if(!strg_out.store_to_binary(buff_out))
+      if(!strg_out.store_to_binary(buff_out, 32 * 1024))
       {
         LOG_ERROR("Failed to store_to_binary in command" << command);
         return -1;
@@ -236,7 +244,7 @@ namespace epee
     int buff_to_t_adapter(t_owner* powner, int command, const epee::span<const uint8_t> in_buff, callback_t cb, t_context& context)
     {
       serialization::portable_storage strg;
-      if(!strg.load_from_binary(in_buff))
+      if(!strg.load_from_binary(in_buff, &default_levin_limits))
       {
         on_levin_traffic(context, false, false, true, in_buff.size(), command);
         LOG_ERROR("Failed to load_from_binary in notify " << command);
@@ -254,7 +262,7 @@ namespace epee
     }
 
 #define CHAIN_LEVIN_INVOKE_MAP2(context_type) \
-  int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, context_type& context) \
+  int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, context_type& context) \
   { \
   bool handled = false; \
   return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \
@@ -263,13 +271,13 @@ namespace epee
 #define CHAIN_LEVIN_NOTIFY_MAP2(context_type) \
   int notify(int command, const epee::span<const uint8_t> in_buff, context_type& context) \
   { \
-  bool handled = false; std::string fake_str;\
+  bool handled = false; epee::byte_slice fake_str; \
   return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \
   } 
 
 
 #define CHAIN_LEVIN_INVOKE_MAP() \
-  int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, epee::net_utils::connection_context_base& context) \
+  int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, epee::net_utils::connection_context_base& context) \
   { \
   bool handled = false; \
   return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \
@@ -289,7 +297,7 @@ namespace epee
   } 
 
 #define BEGIN_INVOKE_MAP2(owner_type) \
-  template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, t_context& context, bool& handled) \
+  template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, t_context& context, bool& handled) \
   { \
   try { \
   typedef owner_type internal_owner_type_name;

contrib/epee/include/storages/portable_storage.h

@@ -32,7 +32,6 @@
 
 #include "misc_language.h"
 #include "portable_storage_base.h"
-#include "portable_storage_to_bin.h"
 #include "portable_storage_from_bin.h"
 #include "portable_storage_to_json.h"
 #include "portable_storage_from_json.h"
@@ -42,6 +41,7 @@
 
 namespace epee
 {
+  class byte_slice;
   namespace serialization
   {
     /************************************************************************/
@@ -54,6 +54,13 @@ namespace epee
       typedef epee::serialization::harray  harray;
       typedef storage_entry meta_entry;
 
+      struct limits_t
+      {
+        size_t n_objects;
+        size_t n_fields;
+        size_t n_strings; // not counting field names
+      };
+
       portable_storage(){}
       virtual ~portable_storage(){}
       hsection   open_section(const std::string& section_name,  hsection hparent_section, bool create_if_notexist = false);
@@ -83,9 +90,9 @@ namespace epee
       bool        delete_entry(const std::string& pentry_name, hsection hparent_section = nullptr);
 
       //-------------------------------------------------------------------------------
-      bool		store_to_binary(binarybuffer& target);
-      bool		load_from_binary(const epee::span<const uint8_t> target);
-      bool		load_from_binary(const std::string& target) { return load_from_binary(epee::strspan<uint8_t>(target)); }
+      bool		store_to_binary(byte_slice& target, std::size_t initial_buffer_size = 8192);
+      bool		load_from_binary(const epee::span<const uint8_t> target, const limits_t *limits = NULL);
+      bool		load_from_binary(const std::string& target, const limits_t *limits = NULL) { return load_from_binary(epee::strspan<uint8_t>(target), limits); }
       template<class trace_policy>
       bool		  dump_as_xml(std::string& targetObj, const std::string& root_name = "");
       bool		  dump_as_json(std::string& targetObj, size_t indent = 0, bool insert_newlines = true);
@@ -133,24 +140,8 @@ namespace epee
     {
       return false;//TODO: don't think i ever again will use xml - ambiguous and "overtagged" format
     }
-
     inline
-    bool portable_storage::store_to_binary(binarybuffer& target)
-    {
-      TRY_ENTRY();
-      std::stringstream ss;
-      storage_block_header sbh = AUTO_VAL_INIT(sbh);
-      sbh.m_signature_a = SWAP32LE(PORTABLE_STORAGE_SIGNATUREA);
-      sbh.m_signature_b = SWAP32LE(PORTABLE_STORAGE_SIGNATUREB);
-      sbh.m_ver = PORTABLE_STORAGE_FORMAT_VER;
-      ss.write((const char*)&sbh, sizeof(storage_block_header));
-      pack_entry_to_buff(ss, m_root);
-      target = ss.str();
-      return true;
-      CATCH_ENTRY("portable_storage::store_to_binary", false)
-    }
-    inline
-    bool portable_storage::load_from_binary(const epee::span<const uint8_t> source)
+    bool portable_storage::load_from_binary(const epee::span<const uint8_t> source, const limits_t *limits)
     {
       m_root.m_entries.clear();
       if(source.size() < sizeof(storage_block_header))
@@ -173,6 +164,8 @@ namespace epee
       }
       TRY_ENTRY();
       throwable_buffer_reader buf_reader(source.data()+sizeof(storage_block_header), source.size()-sizeof(storage_block_header));
+      if (limits)
+        buf_reader.set_limits(limits->n_objects, limits->n_fields, limits->n_strings);
       buf_reader.read(m_root);
       return true;//TODO:
       CATCH_ENTRY("portable_storage::load_from_binary", false);
@@ -282,6 +275,7 @@ namespace epee
       static_assert(std::is_rvalue_reference<entry_type&&>(), "unexpected copy of value");
       TRY_ENTRY();
       CHECK_AND_ASSERT(psection, nullptr);
+      CHECK_AND_ASSERT(!pentry_name.empty(), nullptr);
       auto ins_res = psection->m_entries.emplace(pentry_name, std::forward<entry_type>(entry));
       return &ins_res.first->second;
       CATCH_ENTRY("portable_storage::insert_new_entry_get_storage_entry", nullptr);

contrib/epee/include/storages/portable_storage_from_bin.h

@@ -29,6 +29,7 @@
 #pragma once 
 
 #include "misc_language.h"
+#include "misc_log_ex.h"
 #include "portable_storage_base.h"
 #include "portable_storage_bin_utils.h"
 
@@ -42,6 +43,24 @@ namespace epee
 {
   namespace serialization
   {
+    template<typename T>
+    struct ps_min_bytes {
+      static constexpr const size_t strict = 4096; // actual low bound
+    };
+    template<> struct ps_min_bytes<uint64_t> { static constexpr const size_t strict = 8; };
+    template<> struct ps_min_bytes<int64_t> { static constexpr const size_t strict = 8; };
+    template<> struct ps_min_bytes<uint32_t> { static constexpr const size_t strict = 4; };
+    template<> struct ps_min_bytes<int32_t> { static constexpr const size_t strict = 4; };
+    template<> struct ps_min_bytes<uint16_t> { static constexpr const size_t strict = 2; };
+    template<> struct ps_min_bytes<int16_t> { static constexpr const size_t strict = 2; };
+    template<> struct ps_min_bytes<uint8_t> { static constexpr const size_t strict = 1; };
+    template<> struct ps_min_bytes<int8_t> { static constexpr const size_t strict = 1; };
+    template<> struct ps_min_bytes<double> { static constexpr const size_t strict = 8; };
+    template<> struct ps_min_bytes<bool> { static constexpr const size_t strict = 1; };
+    template<> struct ps_min_bytes<std::string> { static constexpr const size_t strict = 2; };
+    template<> struct ps_min_bytes<section> { static constexpr const size_t strict = 1; };
+    template<> struct ps_min_bytes<array_entry> { static constexpr const size_t strict = 1; };
+
     struct throwable_buffer_reader
     {
       throwable_buffer_reader(const void* ptr, size_t sz);
@@ -61,6 +80,9 @@ namespace epee
       void read(section& sec);
       void read(std::string& str);
       void read(array_entry &ae);
+      template<class t_type>
+      size_t min_bytes() const;
+      void set_limits(size_t objects, size_t fields, size_t strings);
     private:
       struct recursuion_limitation_guard
       {
@@ -81,6 +103,13 @@ namespace epee
       const uint8_t* m_ptr;
       size_t m_count;
       size_t m_recursion_count;
+      size_t m_objects;
+      size_t m_fields;
+      size_t m_strings;
+
+      size_t max_objects;
+      size_t max_fields;
+      size_t max_strings;
     };
 
     inline throwable_buffer_reader::throwable_buffer_reader(const void* ptr, size_t sz)
@@ -92,6 +121,12 @@ namespace epee
       m_ptr = (uint8_t*)ptr;
       m_count = sz;
       m_recursion_count = 0;
+      m_objects = 0;
+      m_fields = 0;
+      m_strings = 0;
+      max_objects = std::numeric_limits<size_t>::max();
+      max_fields = std::numeric_limits<size_t>::max();
+      max_strings = std::numeric_limits<size_t>::max();
     }
     inline 
     void throwable_buffer_reader::read(void* target, size_t count)
@@ -108,6 +143,7 @@ namespace epee
       RECURSION_LIMITATION();
       uint8_t name_len = 0;
       read(name_len);
+      CHECK_AND_ASSERT_THROW_MES(name_len > 0, "Section name is missing");
       sce_name.resize(name_len);
       read((void*)sce_name.data(), name_len);
     }
@@ -138,7 +174,18 @@ namespace epee
       //for pod types
       array_entry_t<type_name> sa;
       size_t size = read_varint();
-      CHECK_AND_ASSERT_THROW_MES(size <= m_count, "Size sanity check failed");
+      CHECK_AND_ASSERT_THROW_MES(size <= m_count / ps_min_bytes<type_name>::strict, "Size sanity check failed");
+      if (std::is_same<type_name, section>())
+      {
+        CHECK_AND_ASSERT_THROW_MES(size <= max_objects - m_objects, "Too many objects");
+        m_objects += size;
+      }
+      else if (std::is_same<type_name, std::string>())
+      {
+        CHECK_AND_ASSERT_THROW_MES(size <= max_strings - m_strings, "Too many strings");
+        m_strings += size;
+      }
+
       sa.reserve(size);
       //TODO: add some optimization here later
       while(size--)
@@ -204,6 +251,8 @@ namespace epee
     inline storage_entry throwable_buffer_reader::read_se<std::string>()
     {
       RECURSION_LIMITATION();
+      CHECK_AND_ASSERT_THROW_MES(m_strings + 1 <= max_strings, "Too many strings");
+      m_strings += 1;
       return storage_entry(read<std::string>());
     }
 
@@ -212,6 +261,8 @@ namespace epee
     inline storage_entry throwable_buffer_reader::read_se<section>()
     {
       RECURSION_LIMITATION();
+      CHECK_AND_ASSERT_THROW_MES(m_objects < max_objects, "Too many objects");
+      ++m_objects;
       section s;//use extra variable due to vs bug, line "storage_entry se(section()); " can't be compiled in visual studio
       storage_entry se(std::move(s));
       section& section_entry = boost::get<section>(se);
@@ -263,12 +314,16 @@ namespace epee
       RECURSION_LIMITATION();
       sec.m_entries.clear();
       size_t count = read_varint();
+      CHECK_AND_ASSERT_THROW_MES(count <= max_fields - m_fields, "Too many object fields");
+      m_fields += count;
       while(count--)
       {
         //read section name string
         std::string sec_name;
         read_sec_name(sec_name);
-        sec.m_entries.emplace(std::move(sec_name), load_storage_entry());
+        const auto insert_loc = sec.m_entries.lower_bound(sec_name);
+        CHECK_AND_ASSERT_THROW_MES(insert_loc == sec.m_entries.end() || insert_loc->first != sec_name, "duplicate key: " << sec_name);
+        sec.m_entries.emplace_hint(insert_loc, std::move(sec_name), load_storage_entry());
       }
     }
     inline 
@@ -289,5 +344,12 @@ namespace epee
       RECURSION_LIMITATION();
       CHECK_AND_ASSERT_THROW_MES(false, "Reading array entry is not supported");
     }
+    inline
+    void throwable_buffer_reader::set_limits(size_t objects, size_t fields, size_t strings)
+    {
+      max_objects = objects;
+      max_fields = fields;
+      max_strings = strings;
+    }
   }
 }

contrib/epee/include/storages/portable_storage_template_helper.h

@@ -28,6 +28,7 @@
 
 #include <string>
 
+#include "byte_slice.h"
 #include "parserse_base_utils.h"
 #include "portable_storage.h"
 #include "file_io_utils.h"
@@ -84,10 +85,10 @@ namespace epee
     }
     //-----------------------------------------------------------------------------------------------------------
     template<class t_struct>
-    bool load_t_from_binary(t_struct& out, const epee::span<const uint8_t> binary_buff)
+    bool load_t_from_binary(t_struct& out, const epee::span<const uint8_t> binary_buff, const epee::serialization::portable_storage::limits_t *limits = NULL)
     {
       portable_storage ps;
-      bool rs = ps.load_from_binary(binary_buff);
+      bool rs = ps.load_from_binary(binary_buff, limits);
       if(!rs)
         return false;
 
@@ -111,18 +112,18 @@ namespace epee
     }
     //-----------------------------------------------------------------------------------------------------------
     template<class t_struct>
-    bool store_t_to_binary(t_struct& str_in, std::string& binary_buff, size_t indent = 0)
+    bool store_t_to_binary(t_struct& str_in, byte_slice& binary_buff, size_t initial_buffer_size = 8192)
     {
       portable_storage ps;
       str_in.store(ps);
-      return ps.store_to_binary(binary_buff);
+      return ps.store_to_binary(binary_buff, initial_buffer_size);
     }
     //-----------------------------------------------------------------------------------------------------------
     template<class t_struct>
-    std::string store_t_to_binary(t_struct& str_in, size_t indent = 0)
+    byte_slice store_t_to_binary(t_struct& str_in, size_t initial_buffer_size = 8192)
     {
-      std::string binary_buff;
-      store_t_to_binary(str_in, binary_buff, indent);
+      byte_slice binary_buff;
+      store_t_to_binary(str_in, binary_buff, initial_buffer_size);
       return binary_buff;
     }
   }

contrib/epee/include/storages/portable_storage_to_bin.h

@@ -211,6 +211,7 @@ namespace epee
       for(const section_pair& se: sec.m_entries)
       {
         CHECK_AND_ASSERT_THROW_MES(se.first.size() < std::numeric_limits<uint8_t>::max(), "storage_entry_name is too long: " << se.first.size() << ", val: " << se.first);
+        CHECK_AND_ASSERT_THROW_MES(!se.first.empty(), "storage_entry_name is empty");
         uint8_t len = static_cast<uint8_t>(se.first.size());
         strm.write((const char*)&len, sizeof(len));
         strm.write(se.first.data(), size_t(len));

contrib/epee/src/CMakeLists.txt

@@ -29,7 +29,7 @@
 
 add_library(epee STATIC byte_slice.cpp byte_stream.cpp hex.cpp abstract_http_client.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp
     wipeable_string.cpp levin_base.cpp memwipe.c connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp
-    int-util.cpp)
+    int-util.cpp portable_storage.cpp)
 
 if (USE_READLINE AND (GNU_READLINE_FOUND OR (DEPENDS AND NOT MINGW)))
   add_library(epee_readline STATIC readline_buffer.cpp)

contrib/epee/src/byte_stream.cpp

@@ -34,6 +34,11 @@
 
 #include <iostream>
 
+namespace
+{
+  constexpr const std::size_t minimum_increase = 4096;
+}
+
 namespace epee
 {
   void byte_stream::overflow(const std::size_t requested)
@@ -46,7 +51,7 @@ namespace epee
 
     const std::size_t len = size();
     const std::size_t cap = capacity();
-    const std::size_t increase = std::max(need, increase_size());
+    const std::size_t increase = std::max(std::max(need, cap), minimum_increase);
 
     next_write_ = nullptr;
     end_ = nullptr;
@@ -62,8 +67,7 @@ namespace epee
   byte_stream::byte_stream(byte_stream&& rhs) noexcept
     : buffer_(std::move(rhs.buffer_)),
       next_write_(rhs.next_write_),
-      end_(rhs.end_),
-      increase_size_(rhs.increase_size_)
+      end_(rhs.end_)
   {
     rhs.next_write_ = nullptr;
     rhs.end_ = nullptr;
@@ -76,7 +80,6 @@ namespace epee
       buffer_ = std::move(rhs.buffer_);
       next_write_ = rhs.next_write_;
       end_ = rhs.end_;
-      increase_size_ = rhs.increase_size_;
       rhs.next_write_ = nullptr;
       rhs.end_ = nullptr;
     }

contrib/epee/src/net_ssl.cpp

@@ -473,6 +473,7 @@ bool ssl_options_t::has_fingerprint(boost::asio::ssl::verify_context &ctx) const
 bool ssl_options_t::handshake(
   boost::asio::ssl::stream<boost::asio::ip::tcp::socket> &socket,
   boost::asio::ssl::stream_base::handshake_type type,
+  boost::asio::const_buffer buffer,
   const std::string& host,
   std::chrono::milliseconds timeout) const
 {
@@ -530,7 +531,7 @@ bool ssl_options_t::handshake(
   });
 
   boost::system::error_code ec = boost::asio::error::would_block;
-  socket.async_handshake(type, boost::lambda::var(ec) = boost::lambda::_1);
+  socket.async_handshake(type, boost::asio::buffer(buffer), boost::lambda::var(ec) = boost::lambda::_1);
   if (io_service.stopped())
   {
     io_service.reset();

contrib/epee/src/net_utils_base.cpp

@@ -6,6 +6,17 @@
 #include "string_tools.h"
 #include "net/local_ip.h"
 
+static inline uint32_t make_address_v4_from_v6(const boost::asio::ip::address_v6& a)
+{
+  const auto &bytes = a.to_bytes();
+  uint32_t v4 = 0;
+  v4 = (v4 << 8) | bytes[12];
+  v4 = (v4 << 8) | bytes[13];
+  v4 = (v4 << 8) | bytes[14];
+  v4 = (v4 << 8) | bytes[15];
+  return htonl(v4);
+}
+
 namespace epee { namespace net_utils
 {
 	bool ipv4_network_address::equal(const ipv4_network_address& other) const noexcept
@@ -83,8 +94,28 @@ namespace epee { namespace net_utils
 		network_address::interface const* const other_self = other.self.get();
 		if (self_ == other_self) return true;
 		if (!self_ || !other_self) return false;
-		if (typeid(*self_) != typeid(*other_self)) return false;
-		return self_->is_same_host(*other_self);
+		if (typeid(*self_) == typeid(*other_self))
+			return self_->is_same_host(*other_self);
+		const auto this_id = get_type_id();
+		if (this_id == ipv4_network_address::get_type_id() && other.get_type_id() == ipv6_network_address::get_type_id())
+		{
+			const boost::asio::ip::address_v6 &actual_ip = other.as<const epee::net_utils::ipv6_network_address>().ip();
+			if (actual_ip.is_v4_mapped())
+			{
+				const uint32_t v4ip = make_address_v4_from_v6(actual_ip);
+				return is_same_host(ipv4_network_address(v4ip, 0));
+			}
+		}
+		else if (this_id == ipv6_network_address::get_type_id() && other.get_type_id() == ipv4_network_address::get_type_id())
+		{
+			const boost::asio::ip::address_v6 &actual_ip = this->as<const epee::net_utils::ipv6_network_address>().ip();
+			if (actual_ip.is_v4_mapped())
+			{
+				const uint32_t v4ip = make_address_v4_from_v6(actual_ip);
+				return other.is_same_host(ipv4_network_address(v4ip, 0));
+			}
+		}
+		return false;
 	}
 
   std::string print_connection_context(const connection_context_base& ctx)

contrib/epee/src/portable_storage.cpp

@@ -0,0 +1,29 @@
+
+#include "byte_slice.h"
+#include "byte_stream.h"
+#include "misc_log_ex.h"
+#include "span.h"
+#include "storages/portable_storage.h"
+#include "storages/portable_storage_to_bin.h"
+
+namespace epee
+{
+namespace serialization
+{
+  bool portable_storage::store_to_binary(byte_slice& target, const std::size_t initial_buffer_size)
+  {
+    TRY_ENTRY();
+    byte_stream ss;
+    ss.reserve(initial_buffer_size);
+    storage_block_header sbh{};
+    sbh.m_signature_a = SWAP32LE(PORTABLE_STORAGE_SIGNATUREA);
+    sbh.m_signature_b = SWAP32LE(PORTABLE_STORAGE_SIGNATUREB);
+    sbh.m_ver = PORTABLE_STORAGE_FORMAT_VER;
+    ss.write(epee::as_byte_span(sbh));
+    pack_entry_to_buff(ss, m_root);
+    target = epee::byte_slice{std::move(ss)};
+    return true;
+    CATCH_ENTRY("portable_storage::store_to_binary", false)
+  }
+}
+}

contrib/epee/src/readline_buffer.cpp

@@ -6,6 +6,7 @@
 #include <boost/thread/lock_guard.hpp>
 #include <boost/algorithm/string.hpp>
 
+static bool same_as_last_line(const std::string&);
 static void install_line_handler();
 static void remove_line_handler();
 
@@ -175,8 +176,11 @@ static void handle_line(char* line)
     boost::trim_right(test_line);
     if(!test_line.empty())
     {
-      add_history(test_line.c_str());
-      history_set_pos(history_length);
+      if (!same_as_last_line(test_line))
+      {
+        add_history(test_line.c_str());
+        history_set_pos(history_length);
+      }
       if (test_line == "exit" || test_line == "q")
         exit = true;
     }
@@ -192,6 +196,16 @@ static void handle_line(char* line)
   return;
 }
 
+// same_as_last_line returns true, if the last line in the history is
+// equal to test_line.
+static bool same_as_last_line(const std::string& test_line)
+{
+  // Note that state->offset == state->length, when a new line was entered.
+  HISTORY_STATE* state = history_get_history_state();
+  return state->length > 0
+    && test_line.compare(state->entries[state->length-1]->line) == 0;
+}
+
 static char* completion_matches(const char* text, int state)
 {
   static size_t list_index;

contrib/fuzz_testing/fuzz.sh

@@ -14,8 +14,8 @@ then
   exit 1
 fi
 case "$type" in
-  block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof) ;;
-  *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof"; exit 1 ;;
+  block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof|utf8) ;;
+  *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof|utf8"; exit 1 ;;
 esac
 
 if test -d "fuzz-out/$type"

contrib/gitian/README.md

@@ -188,6 +188,7 @@ gpg --detach-sign ${VERSION}-linux/${GH_USER}/monero-linux-*-build.assert
 gpg --detach-sign ${VERSION}-win/${GH_USER}/monero-win-*-build.assert
 gpg --detach-sign ${VERSION}-osx/${GH_USER}/monero-osx-*-build.assert
 gpg --detach-sign ${VERSION}-android/${GH_USER}/monero-android-*-build.assert
+gpg --detach-sign ${VERSION}-freebsd/${GH_USER}/monero-freebsd-*-build.assert
 ```
 
 This will create a `.sig` file for each `.assert` file above (2 files for each platform).

src/CMakeLists.txt

@@ -75,6 +75,8 @@ function (monero_add_executable name)
     PROPERTY
       RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
   enable_stack_trace("${name}")
+  
+  monero_set_target_no_relink("${name}")
 endfunction ()
 
 function (monero_add_library name)
@@ -92,6 +94,7 @@ function (monero_add_library_with_deps)
   set(objlib obj_${MONERO_ADD_LIBRARY_NAME})
   add_library(${objlib} OBJECT ${MONERO_ADD_LIBRARY_SOURCES})
   add_library("${MONERO_ADD_LIBRARY_NAME}" $<TARGET_OBJECTS:${objlib}>)
+  monero_set_target_no_relink("${MONERO_ADD_LIBRARY_NAME}")
   if (MONERO_ADD_LIBRARY_DEPENDS)
     add_dependencies(${objlib} ${MONERO_ADD_LIBRARY_DEPENDS})
   endif()

src/blockchain_db/blockchain_db.h

@@ -1312,17 +1312,20 @@ public:
    * height. The number of blocks returned is variable, based on the max_size passed.
    *
    * @param start_height the height of the first block
-   * @param min_count the minimum number of blocks to return, if they exist
-   * @param max_count the maximum number of blocks to return
+   * @param min_block_count the minimum number of blocks to return, if they exist
+   * @param max_block_count the maximum number of blocks to return
+   * @param max_tx_count the maximum number of txes to return
    * @param max_size the maximum size of block/transaction data to return (will be exceeded by one blocks's worth at most, if min_count is met)
    * @param blocks the returned block/transaction data
    * @param pruned whether to return full or pruned tx data
    * @param skip_coinbase whether to return or skip coinbase transactions (they're in blocks regardless)
    * @param get_miner_tx_hash whether to calculate and return the miner (coinbase) tx hash
    *
+   * The call will return at least min_block_count if possible, even if this contravenes max_tx_count
+   *
    * @return true iff the blocks and transactions were found
    */
-  virtual bool get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const = 0;
+  virtual bool get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const = 0;
 
   /**
    * @brief fetches the prunable transaction blob with the given hash

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -3001,7 +3001,7 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const
 
   if (! tx_found)
   {
-    LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
+    LOG_PRINT_L3("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
     return false;
   }
 
@@ -3032,7 +3032,7 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h, uint64_t& tx_id) const
   bool ret = false;
   if (get_result == MDB_NOTFOUND)
   {
-    LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
+    LOG_PRINT_L3("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
   }
   else if (get_result)
     throw0(DB_ERROR(lmdb_error("DB error attempting to fetch transaction from hash", get_result).c_str()));
@@ -3171,7 +3171,7 @@ bool BlockchainLMDB::get_pruned_tx_blobs_from(const crypto::hash& h, size_t coun
   return true;
 }
 
-bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const
+bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -3185,14 +3185,15 @@ bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_count, si
     RCURSOR(txs_prunable);
   }
 
-  blocks.reserve(std::min<size_t>(max_count, 10000)); // guard against very large max count if only checking bytes
+  blocks.reserve(std::min<size_t>(max_block_count, 10000)); // guard against very large max count if only checking bytes
   const uint64_t blockchain_height = height();
   uint64_t size = 0;
+  size_t num_txes = 0;
   MDB_val_copy<uint64_t> key(start_height);
   MDB_val k, v, val_tx_id;
   uint64_t tx_id = ~0;
   MDB_cursor_op op = MDB_SET;
-  for (uint64_t h = start_height; h < blockchain_height && blocks.size() < max_count && (size < max_size || blocks.size() < min_count); ++h)
+  for (uint64_t h = start_height; h < blockchain_height && blocks.size() < max_block_count && (size < max_size || blocks.size() < min_block_count); ++h)
   {
     MDB_cursor_op op = h == start_height ? MDB_SET : MDB_NEXT;
     int result = mdb_cursor_get(m_cur_blocks, &key, &v, op);
@@ -3243,6 +3244,7 @@ bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_count, si
     op = MDB_NEXT;
 
     current_block.second.reserve(b.tx_hashes.size());
+    num_txes += b.tx_hashes.size() + (skip_coinbase ? 0 : 1);
     for (const auto &tx_hash: b.tx_hashes)
     {
       // get pruned data
@@ -3262,6 +3264,9 @@ bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_count, si
       current_block.second.push_back(std::make_pair(tx_hash, std::move(tx_blob)));
       size += current_block.second.back().second.size();
     }
+
+    if (blocks.size() >= min_block_count && num_txes >= max_tx_count)
+      break;
   }
 
   TXN_POSTFIX_RDONLY();

src/blockchain_db/lmdb/db_lmdb.h

@@ -257,7 +257,7 @@ public:
   virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const;
   virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const;
   virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<cryptonote::blobdata> &bd) const;
-  virtual bool get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const;
+  virtual bool get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const;
   virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const;
   virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const;
 

src/blockchain_db/testdb.h

@@ -70,7 +70,7 @@ public:
   virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; }
   virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; }
   virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<cryptonote::blobdata> &bd) const override { return false; }
-  virtual bool get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const override { return false; }
+  virtual bool get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const override { return false; }
   virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; }
   virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const override { return false; }
   virtual uint64_t get_block_height(const crypto::hash& h) const override { return 0; }

src/blocks/CMakeLists.txt

@@ -45,4 +45,4 @@ foreach(BLOB_NAME checkpoints)
     )
 endforeach()
 
-add_library(blocks STATIC blocks.cpp ${GENERATED_SOURCES})
+monero_add_library(blocks blocks.cpp ${GENERATED_SOURCES})

src/checkpoints/checkpoints.cpp

@@ -200,7 +200,7 @@ namespace cryptonote
       return true;
     }
     // make RPC call to daemon
-    // curl http://127.0.0.1:34568/json_rpc -d '{"jsonrpc":"2.0","id":"0","method":"get_block","params":{"height":247600}}' -H 'Content-Type: application/json'
+    // curl http://127.0.0.1:34568/json_rpc -d '{"jsonrpc":"2.0","id":"0","method":"get_block","params":{"height":307880}}' -H 'Content-Type: application/json'
     // "wide_cumulative_difficulty": "0x14eb4d0131fe8",
     ADD_CHECKPOINT2(1,       "97f4ce4d7879b3bea54dcec738cd2ebb7952b4e9bb9743262310cd5fec749340", "0x2");
     ADD_CHECKPOINT2(6969,    "aa7b66e8c461065139b55c29538a39c33ceda93e587f84d490ed573d80511c87", "0x118eef693fd");  //Hard fork to v8
@@ -214,6 +214,18 @@ namespace cryptonote
     ADD_CHECKPOINT2(253999,  "755a289fe8a68e96a0f69069ba4007b676ec87dce2e47dfb9647fe5691f49883", "0x172d026ef7fe8"); //Hard fork to v16
     ADD_CHECKPOINT2(254287,  "b37cb55abe73965b424f8028bf71bef98d069645077ffa52f0c134907b7734e3", "0x1746622f56668"); //Hard fork to v17
     ADD_CHECKPOINT2(256700,  "389a8ab95a80e84ec74639c1078bc67b33af208ef00f53bd9609cfc40efa7059", "0x185ace3c1bd68");
+    ADD_CHECKPOINT2(271600,  "9597cdbdc52ca57d7dbd8f9c0a23a73194ef2ebbcfdc75c21992672706108d43", "0x1e2d2d6a2a9e8");
+    ADD_CHECKPOINT2(278300,  "b10dcdf7a51651f60fbcc0447409773eef1458d2c706d9a61daf467571ac19c9", "0x20a83a16d3968");
+    ADD_CHECKPOINT2(282700,  "79c06cafd7cb5f76bcebbf8f1ae16203bb41fd75b284bcd0eb0b457991ab7d4a", "0x22e3baf142de8");
+    ADD_CHECKPOINT2(307686,  "dfd056b2739c132a07629409a59a028cb7414fac23e3419e79d2f49d66fc3af5", "0x305ba542e3ea8"); // "difficulty": 25800000
+    ADD_CHECKPOINT2(307692,  "d822cd72037f62824ec87c9dc11768b45dc2632f697fa372e1885789c90f37fc", "0x305e124633878"); // "difficulty": 1890000
+    ADD_CHECKPOINT2(307735,  "60970378aecdc0a78ccf5154edcc56f23aad8554b49e4716f820461a7588bfdc", "0x3070771b9ba58"); // "difficulty": 17900000
+    ADD_CHECKPOINT2(307742,  "0ed835bc9fcd949b5a184cf607dcc62ac4268c9e4cf220f8b09bcce58f10916b", "0x30732f1248978"); // "difficulty": 21300000
+    ADD_CHECKPOINT2(307750,  "7bcafbc757237125b70f569b181eb1b66c530b10d817d7b940f7a73dc827211c", "0x30766666b3d98"); // "difficulty": 10900000
+    ADD_CHECKPOINT2(307766,  "02fd6c7d6bae710cfa3efb08f50e4bc9a590f6ab61eabd87e5e951338c0c36f6", "0x307d2d47a7918"); // "difficulty": 2960000
+    ADD_CHECKPOINT2(307800,  "3594894b4231cfdfe911afed6552f9fb4cfe6048bacd0973a3a98623ec8548ce", "0x308b305ca7618");
+    ADD_CHECKPOINT2(307880,  "659274b698f680c6cae2716cbd4e15ad5def23b5de98e53734c4af2c2e74bb7a", "0x30af6e91e8018");
+    ADD_CHECKPOINT2(307883,  "9a8c35cd10963a14bba8a9628d1776df92fee5e3153b7249f5d15726efafaaea", "0x30b0965ba5a18");
     return true;
   }
 

src/common/powerof.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include <stdint.h>
+
+namespace tools
+{
+  template<uint64_t a, uint64_t b>
+  struct PowerOf
+  {
+    enum Data : uint64_t
+    {
+      // a^b = a * a^(b-1)
+      Value = a * PowerOf<a, b - 1>::Value,
+    };
+  };
+
+  template<uint64_t a>
+  struct PowerOf<a, 0>
+  {
+    enum Data : uint64_t
+    {
+      // a^0 = 1
+      Value = 1,
+    };
+  };
+}

src/common/util.cpp

@@ -871,10 +871,19 @@ std::string get_nix_version_display_string()
     return max_concurrency;
   }
 
+  bool is_privacy_preserving_network(const std::string &address)
+  {
+    if (boost::ends_with(address, ".onion"))
+      return true;
+    if (boost::ends_with(address, ".i2p"))
+      return true;
+    return false;
+  }
+
   bool is_local_address(const std::string &address)
   {
     // always assume Tor/I2P addresses to be untrusted by default
-    if (boost::ends_with(address, ".onion") || boost::ends_with(address, ".i2p"))
+    if (is_privacy_preserving_network(address))
     {
       MDEBUG("Address '" << address << "' is Tor/I2P, non local");
       return false;
@@ -1000,13 +1009,13 @@ std::string get_nix_version_display_string()
     for (char c: val)
     {
       if (c == '*')
-        newval += escape ? "*" : ".*";
+        newval += escape ? "*" : ".*", escape = false;
       else if (c == '?')
-        newval += escape ? "?" : ".";
+        newval += escape ? "?" : ".", escape = false;
       else if (c == '\\')
         newval += '\\', escape = !escape;
       else
-        newval += c;
+        newval += c, escape = false;
     }
     return newval;
   }

src/common/util.h

@@ -228,6 +228,7 @@ namespace tools
   unsigned get_max_concurrency();
 
   bool is_local_address(const std::string &address);
+  bool is_privacy_preserving_network(const std::string &address);
   int vercmp(const char *v0, const char *v1); // returns < 0, 0, > 0, similar to strcmp, but more human friendly than lexical - does not attempt to validate
 
   bool sha256sum(const uint8_t *data, size_t len, crypto::hash &hash);

src/crypto/crypto.cpp

@@ -123,13 +123,17 @@ namespace crypto {
   void random32_unbiased(unsigned char *bytes)
   {
     // l = 2^252 + 27742317777372353535851937790883648493.
-    // it fits 15 in 32 bytes
+    // l fits 15 times in 32 bytes (iow, 15 l is the highest multiple of l that fits in 32 bytes)
     static const unsigned char limit[32] = { 0xe3, 0x6a, 0x67, 0x72, 0x8b, 0xce, 0x13, 0x29, 0x8f, 0x30, 0x82, 0x8c, 0x0b, 0xa4, 0x10, 0x39, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0 };
-    do
+    while(1)
     {
       generate_random_bytes_thread_safe(32, bytes);
-    } while (!sc_isnonzero(bytes) && !less32(bytes, limit)); // should be good about 15/16 of the time
-    sc_reduce32(bytes);
+      if (!less32(bytes, limit))
+        continue;
+      sc_reduce32(bytes);
+      if (sc_isnonzero(bytes))
+        break;
+    }
   }
   /* generate a random 32-byte (256-bit) integer and copy it to res */
   static inline void random_scalar(ec_scalar &res) {

src/cryptonote_basic/CMakeLists.txt

@@ -38,6 +38,7 @@ endif()
 
 set(cryptonote_basic_sources
   account.cpp
+  connection_context.cpp
   cryptonote_basic_impl.cpp
   cryptonote_format_utils.cpp
   difficulty.cpp

src/cryptonote_basic/connection_context.cpp

@@ -0,0 +1,71 @@
+// Copyright (c) 2020, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "connection_context.h"
+
+#include "cryptonote_protocol/cryptonote_protocol_defs.h"
+#include "p2p/p2p_protocol_defs.h"
+
+namespace cryptonote
+{
+  std::size_t cryptonote_connection_context::get_max_bytes(const int command) noexcept
+  {
+    switch (command)
+    {
+    case nodetool::COMMAND_HANDSHAKE_T<cryptonote::CORE_SYNC_DATA>::ID:
+      return 65536;
+    case nodetool::COMMAND_TIMED_SYNC_T<cryptonote::CORE_SYNC_DATA>::ID:
+      return 65536;
+    case nodetool::COMMAND_PING::ID:
+      return 4096;
+    case nodetool::COMMAND_REQUEST_SUPPORT_FLAGS::ID:
+      return 4096;
+    case cryptonote::NOTIFY_NEW_BLOCK::ID:
+      return 1024 * 1024 * 128; // 128 MB (max packet is a bit less than 100 MB though)
+    case cryptonote::NOTIFY_NEW_TRANSACTIONS::ID:
+      return 1024 * 1024 * 128; // 128 MB (max packet is a bit less than 100 MB though)
+    case cryptonote::NOTIFY_REQUEST_GET_OBJECTS::ID:
+      return 1024 * 1024 * 2; // 2 MB
+    case cryptonote::NOTIFY_RESPONSE_GET_OBJECTS::ID:
+      return 1024 * 1024 * 128; // 128 MB (max packet is a bit less than 100 MB though)
+    case cryptonote::NOTIFY_REQUEST_CHAIN::ID:
+      return 512 * 1024; // 512 kB
+    case cryptonote::NOTIFY_RESPONSE_CHAIN_ENTRY::ID:
+      return 1024 * 1024 * 4; // 4 MB
+    case cryptonote::NOTIFY_NEW_FLUFFY_BLOCK::ID:
+      return 1024 * 1024 * 4; // 4 MB, but it does not includes transaction data
+    case cryptonote::NOTIFY_REQUEST_FLUFFY_MISSING_TX::ID:
+      return 1024 * 1024; // 1 MB
+    case cryptonote::NOTIFY_GET_TXPOOL_COMPLEMENT::ID:
+      return 1024 * 1024 * 4; // 4 MB
+    default:
+      break;
+    };
+    return std::numeric_limits<size_t>::max();
+  }
+} // cryptonote

src/cryptonote_basic/connection_context.h

@@ -31,6 +31,7 @@
 #pragma once
 #include <unordered_set>
 #include <atomic>
+#include <algorithm>
 #include <boost/date_time/posix_time/posix_time.hpp>
 #include "net/net_utils_base.h"
 #include "copyable_atomic.h"
@@ -38,12 +39,12 @@
 
 namespace cryptonote
 {
-
   struct cryptonote_connection_context: public epee::net_utils::connection_context_base
   {
     cryptonote_connection_context(): m_state(state_before_handshake), m_remote_blockchain_height(0), m_last_response_height(0),
         m_last_request_time(boost::date_time::not_a_date_time), m_callback_request_count(0),
-        m_last_known_hash(crypto::null_hash), m_pruning_seed(0), m_rpc_port(0), m_rpc_credits_per_hash(0),  m_anchor(false) {}
+        m_last_known_hash(crypto::null_hash), m_pruning_seed(0), m_rpc_port(0), m_rpc_credits_per_hash(0), m_anchor(false), m_score(0),
+        m_expect_response(0), m_expect_height(0), m_num_requested(0) {}
 
     enum state
     {
@@ -54,6 +55,12 @@ namespace cryptonote
       state_normal
     };
 
+    static constexpr int handshake_command() noexcept { return 1001; }
+    bool handshake_complete() const noexcept { return m_state != state_before_handshake; }
+
+    //! \return Maximum number of bytes permissible for `command`.
+    static size_t get_max_bytes(int command) noexcept;
+
     state m_state;
     std::vector<std::pair<crypto::hash, uint64_t>> m_needed_objects;
     std::unordered_set<crypto::hash> m_requested_objects;
@@ -66,7 +73,10 @@ namespace cryptonote
     uint16_t m_rpc_port;
     uint32_t m_rpc_credits_per_hash;
     bool m_anchor;
-    //size_t m_score;  TODO: add score calculations
+    int32_t m_score;
+    int m_expect_response;
+    uint64_t m_expect_height;
+    size_t m_num_requested;
   };
 
   inline std::string get_protocol_state_string(cryptonote_connection_context::state s)

src/cryptonote_basic/difficulty.cpp

@@ -375,7 +375,16 @@ namespace cryptonote {
     if (HEIGHT < 200 && m_nettype == TESTNET) { return 500; }
     assert(timestamps.size() == N+1);
 
-    uint64_t  L(0), next_D, i, this_timestamp(0), previous_timestamp(0), avg_D;
+    // hardcoding previously erroneously calculated difficulty entries
+    if(HEIGHT == 307686) return 25800000;
+    if(HEIGHT == 307692) return 1890000;
+    if(HEIGHT == 307735) return 17900000;
+    if(HEIGHT == 307742) return 21300000;
+    if(HEIGHT == 307750) return 10900000;
+    if(HEIGHT == 307766) return 2960000;
+
+    uint64_t  i, this_timestamp(0), previous_timestamp(0);
+    difficulty_type L(0), next_D, avg_D;
 
     previous_timestamp = timestamps[0]-T;
     for ( i = 1; i <= N; i++) {
@@ -386,10 +395,13 @@ namespace cryptonote {
       previous_timestamp = this_timestamp;
     }
     if (L < N*N*T/20 ) { L =  N*N*T/20; }
-    avg_D = static_cast<uint64_t>(( cumulative_difficulties[N] - cumulative_difficulties[0] )/ N);
+    avg_D = ( cumulative_difficulties[N] - cumulative_difficulties[0] )/ N;
 
     // Prevent round off error for small D and overflow for large D.
-    if (avg_D > 2000000*N*N*T) {
+    if (avg_D > 2000000*N*N*T && HEIGHT < 307800) {
+      next_D = (avg_D/(200*L))*(N*(N+1)*T*99);
+    }
+    else if (avg_D > uint64_t(-1)/(N*(N+1)*T*99) && HEIGHT > 307800) {
       next_D = (avg_D/(200*L))*(N*(N+1)*T*99);
     }
     else {    next_D = (avg_D*N*(N+1)*T*99)/(200*L);    }

src/cryptonote_config.h

@@ -97,6 +97,7 @@
 
 
 #define BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT          10000  //by default, blocks ids count in synchronizing
+#define BLOCKS_IDS_SYNCHRONIZING_MAX_COUNT              25000  //max blocks ids count in synchronizing
 #define BLOCKS_SYNCHRONIZING_DEFAULT_COUNT_PRE_V4       100    //by default, blocks count in blocks downloading
 #define BLOCKS_SYNCHRONIZING_DEFAULT_COUNT              20     //by default, blocks count in blocks downloading
 #define BLOCKS_SYNCHRONIZING_MAX_COUNT                  2048   //must be a power of 2, greater than 128, equal to SEEDHASH_EPOCH_BLOCKS
@@ -106,11 +107,11 @@
 
 
 #define CRYPTONOTE_DANDELIONPP_STEMS              2 // number of outgoing stem connections per epoch
-#define CRYPTONOTE_DANDELIONPP_FLUFF_PROBABILITY 10 // out of 100
+#define CRYPTONOTE_DANDELIONPP_FLUFF_PROBABILITY 20 // out of 100
 #define CRYPTONOTE_DANDELIONPP_MIN_EPOCH         10 // minutes
 #define CRYPTONOTE_DANDELIONPP_EPOCH_RANGE       30 // seconds
 #define CRYPTONOTE_DANDELIONPP_FLUSH_AVERAGE      5 // seconds average for poisson distributed fluff flush
-#define CRYPTONOTE_DANDELIONPP_EMBARGO_AVERAGE  173 // seconds (see tx_pool.cpp for more info)
+#define CRYPTONOTE_DANDELIONPP_EMBARGO_AVERAGE   39 // seconds (see tx_pool.cpp for more info)
 
 // see src/cryptonote_protocol/levin_notify.cpp
 #define CRYPTONOTE_NOISE_MIN_EPOCH                      5      // minutes
@@ -127,7 +128,8 @@
 
 #define CRYPTONOTE_MAX_FRAGMENTS                        20 // ~20 * NOISE_BYTES max payload size for covert/noise send
 
-#define COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT           1000
+#define COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT     1000
+#define COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT        20000
 
 #define P2P_LOCAL_WHITE_PEERLIST_LIMIT                  1000
 #define P2P_LOCAL_GRAY_PEERLIST_LIMIT                   5000
@@ -136,6 +138,7 @@
 #define P2P_DEFAULT_HANDSHAKE_INTERVAL                  60           //secondes
 #define P2P_DEFAULT_PACKET_MAX_SIZE                     50000000     //50000000 bytes maximum packet size
 #define P2P_DEFAULT_PEERS_IN_HANDSHAKE                  250
+#define P2P_MAX_PEERS_IN_HANDSHAKE                      250
 #define P2P_DEFAULT_CONNECTION_TIMEOUT                  5000       //5 seconds
 #define P2P_DEFAULT_SOCKS_CONNECT_TIMEOUT               45         // seconds
 #define P2P_DEFAULT_PING_CONNECTION_TIMEOUT             2000       //2 seconds
@@ -201,6 +204,8 @@
 
 #define RPC_CREDITS_PER_HASH_SCALE ((float)(1<<24))
 
+#define DNS_BLOCKLIST_LIFETIME (86400 * 8)
+
 // New constants are intended to go here
 namespace config
 {

src/cryptonote_core/blockchain.cpp

@@ -2780,7 +2780,7 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc
 // find split point between ours and foreign blockchain (or start at
 // blockchain height <req_start_block>), and return up to max_count FULL
 // blocks by reference.
-bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const
+bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_block_count, size_t max_tx_count) const
 {
   LOG_PRINT_L3("Blockchain::" << __func__);
   CRITICAL_REGION_LOCAL(m_blockchain_lock);
@@ -2805,8 +2805,8 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons
 
   db_rtxn_guard rtxn_guard(m_db);
   total_height = get_current_blockchain_height();
-  blocks.reserve(std::min(std::min(max_count, (size_t)10000), (size_t)(total_height - start_height)));
-  CHECK_AND_ASSERT_MES(m_db->get_blocks_from(start_height, 3, max_count, FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE, blocks, pruned, true, get_miner_tx_hash),
+  blocks.reserve(std::min(std::min(max_block_count, (size_t)10000), (size_t)(total_height - start_height)));
+  CHECK_AND_ASSERT_MES(m_db->get_blocks_from(start_height, 3, max_block_count, max_tx_count, FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE, blocks, pruned, true, get_miner_tx_hash),
       false, "Error getting blocks");
 
   return true;
@@ -2837,32 +2837,44 @@ void Blockchain::flush_invalid_blocks()
   m_invalid_blocks.clear();
 }
 //------------------------------------------------------------------
-bool Blockchain::have_block(const crypto::hash& id) const
+bool Blockchain::have_block_unlocked(const crypto::hash& id, int *where) const
 {
+  // WARNING: this function does not take m_blockchain_lock, and thus should only call read only
+  // m_db functions which do not depend on one another (ie, no getheight + gethash(height-1), as
+  // well as not accessing class members, even read only (ie, m_invalid_blocks). The caller must
+  // lock if it is otherwise needed.
   LOG_PRINT_L3("Blockchain::" << __func__);
-  CRITICAL_REGION_LOCAL(m_blockchain_lock);
 
   if(m_db->block_exists(id))
   {
     LOG_PRINT_L2("block " << id << " found in main chain");
+    if (where) *where = HAVE_BLOCK_MAIN_CHAIN;
     return true;
   }
 
   if(m_db->get_alt_block(id, NULL, NULL))
   {
     LOG_PRINT_L2("block " << id << " found in alternative chains");
+    if (where) *where = HAVE_BLOCK_ALT_CHAIN;
     return true;
   }
 
   if(m_invalid_blocks.count(id))
   {
     LOG_PRINT_L2("block " << id << " found in m_invalid_blocks");
+    if (where) *where = HAVE_BLOCK_INVALID;
     return true;
   }
 
   return false;
 }
 //------------------------------------------------------------------
+bool Blockchain::have_block(const crypto::hash& id, int *where) const
+{
+  CRITICAL_REGION_LOCAL(m_blockchain_lock);
+  return have_block_unlocked(id, where);
+}
+//------------------------------------------------------------------
 bool Blockchain::handle_block_to_main_chain(const block& bl, block_verification_context& bvc, bool notify/* = true*/)
 {
     LOG_PRINT_L3("Blockchain::" << __func__);
@@ -3220,14 +3232,6 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr
       }
     }
   }
-  else if (rv.type == rct::RCTTypeCLSAG)
-  {
-    CHECK_AND_ASSERT_MES(rv.p.CLSAGs.size() == tx.vin.size(), false, "Bad CLSAGs size");
-    for (size_t n = 0; n < tx.vin.size(); ++n)
-    {
-      rv.p.CLSAGs[n].I = rct::ki2rct(boost::get<txin_to_key>(tx.vin[n]).k_image);
-    }
-  }
   else
   {
     CHECK_AND_ASSERT_MES(false, false, "Unsupported rct tx type: " + boost::lexical_cast<std::string>(rv.type));
@@ -3710,19 +3714,6 @@ void Blockchain::check_ring_signature(const crypto::hash &tx_prefix_hash, const
   result = crypto::check_ring_signature(tx_prefix_hash, key_image, p_output_keys, sig.data()) ? 1 : 0;
 }
 
-//------------------------------------------------------------------
-uint64_t Blockchain::get_fee_quantization_mask()
-{
-  static uint64_t mask = 0;
-  if (mask == 0)
-  {
-    mask = 1;
-    for (size_t n = PER_KB_FEE_QUANTIZATION_DECIMALS; n < CRYPTONOTE_DISPLAY_DECIMAL_POINT; ++n)
-      mask *= 10;
-  }
-  return mask;
-}
-
 //------------------------------------------------------------------
 uint64_t Blockchain::get_dynamic_base_fee(uint64_t block_reward, size_t median_block_weight, uint8_t version)
 {
@@ -4828,6 +4819,8 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector
 
   CHECK_AND_ASSERT_MES(weights.empty() || weights.size() == hashes.size(), 0, "Unexpected weights size");
 
+  CRITICAL_REGION_LOCAL(m_blockchain_lock);
+
   // easy case: height >= hashes
   if (height >= m_blocks_hash_of_hashes.size() * HASH_OF_HASHES_STEP)
     return hashes.size();
@@ -5467,7 +5460,7 @@ void Blockchain::cancel()
 }
 
 #if defined(PER_BLOCK_CHECKPOINT)
-static const char expected_block_hashes_hash[] = "5b78fd258ef3078654d2b15c9c2d6baab371805b94a9aa748f615995b207d045";
+static const char expected_block_hashes_hash[] = "392961db5c3578616c40283e5469f5fae7f628a0b4a13c87db9339cad35f8e53";
 void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints)
 {
   if (get_checkpoints == nullptr || !m_fast_sync)

src/cryptonote_core/blockchain.h

@@ -51,6 +51,7 @@
 #include "string_tools.h"
 #include "rolling_median.h"
 #include "cryptonote_basic/cryptonote_basic.h"
+#include "common/powerof.h"
 #include "common/util.h"
 #include "cryptonote_protocol/cryptonote_protocol_defs.h"
 #include "rpc/core_rpc_server_commands_defs.h"
@@ -376,10 +377,12 @@ namespace cryptonote
      * for a block with the given hash
      *
      * @param id the hash to search for
+     * @param where the type of block, if non NULL
      *
      * @return true if the block is known, else false
      */
-    bool have_block(const crypto::hash& id) const;
+    bool have_block_unlocked(const crypto::hash& id, int *where = NULL) const;
+    bool have_block(const crypto::hash& id, int *where = NULL) const;
 
     /**
      * @brief gets the total number of transactions on the main chain
@@ -464,11 +467,12 @@ namespace cryptonote
      * @param total_height return-by-reference our current blockchain height
      * @param start_height return-by-reference the height of the first block returned
      * @param pruned whether to return full or pruned tx blobs
-     * @param max_count the max number of blocks to get
+     * @param max_block_count the max number of blocks to get
+     * @param max_tx_count the max number of txes to get (it can get overshot by the last block's number of txes minus 1)
      *
      * @return true if a block found in common or req_start_block specified, else false
      */
-    bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const;
+    bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_block_count, size_t max_tx_count) const;
 
     /**
      * @brief retrieves a set of blocks and their transactions, and possibly other transactions
@@ -591,7 +595,10 @@ namespace cryptonote
      *
      * @return the fee quantized mask
      */
-    static uint64_t get_fee_quantization_mask();
+    static uint64_t get_fee_quantization_mask()
+    {
+      return tools::PowerOf<10, CRYPTONOTE_DISPLAY_DECIMAL_POINT - PER_KB_FEE_QUANTIZATION_DECIMALS>::Value;
+    }
 
     /**
      * @brief get dynamic per kB or byte fee for a given block weight

src/cryptonote_core/cryptonote_core.cpp

@@ -1446,9 +1446,9 @@ namespace cryptonote
     return m_blockchain_storage.find_blockchain_supplement(qblock_ids, clip_pruned, resp);
   }
   //-----------------------------------------------------------------------------------------------
-  bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const
+  bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_block_count, size_t max_tx_count) const
   {
-    return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, pruned, get_miner_tx_hash, max_count);
+    return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, pruned, get_miner_tx_hash, max_block_count, max_tx_count);
   }
   //-----------------------------------------------------------------------------------------------
   bool core::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const
@@ -1550,6 +1550,11 @@ namespace cryptonote
     return true;
   }
   //-----------------------------------------------------------------------------------------------
+  bool core::is_synchronized() const
+  {
+    return m_pprotocol != nullptr && m_pprotocol->is_synchronized();
+  }
+  //-----------------------------------------------------------------------------------------------
   void core::on_synchronized()
   {
     m_miner.on_synchronized();
@@ -1660,9 +1665,14 @@ namespace cryptonote
     return m_mempool.get_transactions_count(include_sensitive_txes);
   }
   //-----------------------------------------------------------------------------------------------
-  bool core::have_block(const crypto::hash& id) const
+  bool core::have_block_unlocked(const crypto::hash& id, int *where) const
   {
-    return m_blockchain_storage.have_block(id);
+    return m_blockchain_storage.have_block_unlocked(id, where);
+  }
+  //-----------------------------------------------------------------------------------------------
+  bool core::have_block(const crypto::hash& id, int *where) const
+  {
+    return m_blockchain_storage.have_block(id, where);
   }
   //-----------------------------------------------------------------------------------------------
   bool core::parse_tx_from_blob(transaction& tx, crypto::hash& tx_hash, const blobdata& blob) const
@@ -1799,7 +1809,7 @@ namespace cryptonote
       m_starter_message_showed = true;
     }
 
-    m_txpool_auto_relayer.do_call(boost::bind(&core::relay_txpool_transactions, this));
+    relay_txpool_transactions(); // txpool handles periodic DB checking
     m_check_updates_interval.do_call(boost::bind(&core::check_updates, this));
     m_check_disk_space_interval.do_call(boost::bind(&core::check_disk_space, this));
     m_block_rate_interval.do_call(boost::bind(&core::check_block_rate, this));

src/cryptonote_core/cryptonote_core.h

@@ -55,6 +55,8 @@
 PUSH_WARNINGS
 DISABLE_VS_WARNINGS(4355)
 
+enum { HAVE_BLOCK_MAIN_CHAIN, HAVE_BLOCK_ALT_CHAIN, HAVE_BLOCK_INVALID };
+
 namespace cryptonote
 {
    struct test_options {
@@ -329,7 +331,7 @@ namespace cryptonote
       *
       * @note see Blockchain::get_current_blockchain_height()
       */
-     uint64_t get_current_blockchain_height() const;
+     virtual uint64_t get_current_blockchain_height() const final;
 
      /**
       * @brief get the hash and height of the most recent block
@@ -543,7 +545,8 @@ namespace cryptonote
       *
       * @note see Blockchain::have_block
       */
-     bool have_block(const crypto::hash& id) const;
+     bool have_block_unlocked(const crypto::hash& id, int *where = NULL) const;
+     bool have_block(const crypto::hash& id, int *where = NULL) const;
 
      /**
       * @copydoc Blockchain::get_short_chain_history
@@ -564,7 +567,7 @@ namespace cryptonote
       *
       * @note see Blockchain::find_blockchain_supplement(const uint64_t, const std::list<crypto::hash>&, std::vector<std::pair<cryptonote::blobdata, std::vector<transaction> > >&, uint64_t&, uint64_t&, size_t) const
       */
-     bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const;
+     bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_block_count, size_t max_tx_count) const;
 
      /**
       * @copydoc Blockchain::get_tx_outputs_gindexs
@@ -637,6 +640,13 @@ namespace cryptonote
       */
      std::string print_pool(bool short_format) const;
 
+     /**
+      * @brief gets the core synchronization status
+      *
+      * @return core synchronization status
+      */
+     virtual bool is_synchronized() const final;
+
      /**
       * @copydoc miner::on_synchronized
       *
@@ -663,7 +673,7 @@ namespace cryptonote
       *
       * @param target_blockchain_height the target height
       */
-     virtual uint64_t get_target_blockchain_height() const override;
+     uint64_t get_target_blockchain_height() const;
 
      /**
       * @brief returns the newest hardfork version known to the blockchain
@@ -1065,7 +1075,6 @@ namespace cryptonote
 
      epee::math_helper::once_a_time_seconds<60*60*12, false> m_store_blockchain_interval; //!< interval for manual storing of Blockchain, if enabled
      epee::math_helper::once_a_time_seconds<60*60*2, true> m_fork_moaner; //!< interval for checking HardFork status
-     epee::math_helper::once_a_time_seconds<60*2, false> m_txpool_auto_relayer; //!< interval for checking re-relaying txpool transactions
      epee::math_helper::once_a_time_seconds<60*60*12, true> m_check_updates_interval; //!< interval for checking for new versions
      epee::math_helper::once_a_time_seconds<60*10, true> m_check_disk_space_interval; //!< interval for checking for disk space
      epee::math_helper::once_a_time_seconds<90, false> m_block_rate_interval; //!< interval for checking block rate

src/cryptonote_core/i_core_events.h

@@ -39,7 +39,8 @@ namespace cryptonote
     virtual ~i_core_events() noexcept
     {}
 
-    virtual uint64_t get_target_blockchain_height() const = 0;
+    virtual uint64_t get_current_blockchain_height() const = 0;
+    virtual bool is_synchronized() const = 0;
     virtual void on_transactions_relayed(epee::span<const cryptonote::blobdata> tx_blobs, relay_method tx_relay) = 0;
   };
 }

src/cryptonote_core/tx_pool.cpp

@@ -75,11 +75,11 @@ namespace cryptonote
         not ideal since a blackhole is more likely to reveal earlier nodes in
         the chain.
 
-        This value was calculated with k=10, ep=0.10, and hop = 175 ms. A
+        This value was calculated with k=5, ep=0.10, and hop = 175 ms. A
         testrun from a recent Intel laptop took ~80ms to
         receive+parse+proces+send transaction. At least 50ms will be added to
         the latency if crossing an ocean. So 175ms is the fudge factor for
-        a single hop with 173s being the embargo timer. */
+        a single hop with 39s being the embargo timer. */
     constexpr const std::chrono::seconds dandelionpp_embargo_average{CRYPTONOTE_DANDELIONPP_EMBARGO_AVERAGE};
 
     //TODO: constants such as these should at least be in the header,
@@ -91,6 +91,9 @@ namespace cryptonote
     time_t const MAX_RELAY_TIME = (60 * 60 * 4); // at most that many seconds between resends
     float const ACCEPT_THRESHOLD = 1.0f;
 
+    //! Max DB check interval for relayable txes
+    constexpr const std::chrono::minutes max_relayable_check{2};
+
     constexpr const std::chrono::seconds forward_delay_average{CRYPTONOTE_FORWARD_DELAY_AVERAGE};
 
     // a kind of increasing backoff within min/max bounds
@@ -115,12 +118,21 @@ namespace cryptonote
       else
         return get_min_block_weight(version) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE;
     }
+
+    // external lock must be held for the comparison+set to work properly
+    void set_if_less(std::atomic<time_t>& next_check, const time_t candidate) noexcept
+    {
+      if (candidate < next_check.load(std::memory_order_relaxed))
+        next_check = candidate;
+    }
   }
   //---------------------------------------------------------------------------------
   //---------------------------------------------------------------------------------
-  tx_memory_pool::tx_memory_pool(Blockchain& bchs): m_blockchain(bchs), m_cookie(0), m_txpool_max_weight(DEFAULT_TXPOOL_MAX_WEIGHT), m_txpool_weight(0), m_mine_stem_txes(false)
+  tx_memory_pool::tx_memory_pool(Blockchain& bchs): m_blockchain(bchs), m_cookie(0), m_txpool_max_weight(DEFAULT_TXPOOL_MAX_WEIGHT), m_txpool_weight(0), m_mine_stem_txes(false), m_next_check(std::time(nullptr))
   {
-
+    // class code expects unsigned values throughout
+    if (m_next_check < time_t(0))
+      throw std::runtime_error{"Unexpected time_t (system clock) value"};
   }
   //---------------------------------------------------------------------------------
   bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, relay_method tx_relay, bool relayed, uint8_t version)
@@ -314,7 +326,10 @@ namespace cryptonote
           using clock = std::chrono::system_clock;
           auto last_relayed_time = std::numeric_limits<decltype(meta.last_relayed_time)>::max();
           if (tx_relay == relay_method::forward)
+          {
             last_relayed_time = clock::to_time_t(clock::now() + crypto::random_poisson_seconds{forward_delay_average}());
+            set_if_less(m_next_check, time_t(last_relayed_time));
+          }
           // else the `set_relayed` function will adjust the time accordingly later
 
           //update transactions container
@@ -728,16 +743,22 @@ namespace cryptonote
   }
   //---------------------------------------------------------------------------------
   //TODO: investigate whether boolean return is appropriate
-  bool tx_memory_pool::get_relayable_transactions(std::vector<std::tuple<crypto::hash, cryptonote::blobdata, relay_method>> &txs) const
+  bool tx_memory_pool::get_relayable_transactions(std::vector<std::tuple<crypto::hash, cryptonote::blobdata, relay_method>> &txs)
   {
-    std::vector<std::pair<crypto::hash, txpool_tx_meta_t>> change_timestamps;
+    using clock = std::chrono::system_clock;
+
     const uint64_t now = time(NULL);
+    if (uint64_t{std::numeric_limits<time_t>::max()} < now || time_t(now) < m_next_check)
+      return false;
+
+    uint64_t next_check = clock::to_time_t(clock::from_time_t(time_t(now)) + max_relayable_check);
+    std::vector<std::pair<crypto::hash, txpool_tx_meta_t>> change_timestamps;
 
     CRITICAL_REGION_LOCAL(m_transactions_lock);
     CRITICAL_REGION_LOCAL1(m_blockchain);
     LockedTXN lock(m_blockchain.get_db());
     txs.reserve(m_blockchain.get_txpool_tx_count());
-    m_blockchain.for_all_txpool_txes([this, now, &txs, &change_timestamps](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref *){
+    m_blockchain.for_all_txpool_txes([this, now, &txs, &change_timestamps, &next_check](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref *){
       // 0 fee transactions are never relayed
       if(!meta.pruned && meta.fee > 0 && !meta.do_not_relay)
       {
@@ -747,7 +768,10 @@ namespace cryptonote
           case relay_method::stem:
           case relay_method::forward:
             if (meta.last_relayed_time > now)
+            {
+              next_check = std::min(next_check, meta.last_relayed_time);
               return true; // continue to next tx
+            }
             change_timestamps.emplace_back(txid, meta);
             break;
           default:
@@ -792,6 +816,8 @@ namespace cryptonote
       elem.second.last_relayed_time = now + get_relay_delay(now, elem.second.receive_time);
       m_blockchain.update_txpool_tx(elem.first, elem.second);
     }
+
+    m_next_check = time_t(next_check);
     return true;
   }
   //---------------------------------------------------------------------------------
@@ -799,6 +825,7 @@ namespace cryptonote
   {
     crypto::random_poisson_seconds embargo_duration{dandelionpp_embargo_average};
     const auto now = std::chrono::system_clock::now();
+    uint64_t next_relay = uint64_t{std::numeric_limits<time_t>::max()};
 
     CRITICAL_REGION_LOCAL(m_transactions_lock);
     CRITICAL_REGION_LOCAL1(m_blockchain);
@@ -815,7 +842,10 @@ namespace cryptonote
           meta.relayed = true;
 
           if (meta.dandelionpp_stem)
+          {
             meta.last_relayed_time = std::chrono::system_clock::to_time_t(now + embargo_duration());
+            next_relay = std::min(next_relay, meta.last_relayed_time);
+          }
           else
             meta.last_relayed_time = std::chrono::system_clock::to_time_t(now);
 
@@ -829,6 +859,7 @@ namespace cryptonote
       }
     }
     lock.commit();
+    set_if_less(m_next_check, time_t(next_relay));
   }
   //---------------------------------------------------------------------------------
   size_t tx_memory_pool::get_transactions_count(bool include_sensitive) const

src/cryptonote_core/tx_pool.h

@@ -31,6 +31,7 @@
 #pragma once
 #include "include_base_utils.h"
 
+#include <atomic>
 #include <set>
 #include <tuple>
 #include <unordered_map>
@@ -329,11 +330,14 @@ namespace cryptonote
      *   isn't old enough that relaying it is considered harmful
      * Note a transaction can be "relayable" even if do_not_relay is true
      *
+     * This function will skip all DB checks if an insufficient amount of
+     * time since the last call.
+     *
      * @param txs return-by-reference the transactions and their hashes
      *
-     * @return true
+     * @return True if DB was checked, false if DB checks skipped.
      */
-    bool get_relayable_transactions(std::vector<std::tuple<crypto::hash, cryptonote::blobdata, relay_method>>& txs) const;
+    bool get_relayable_transactions(std::vector<std::tuple<crypto::hash, cryptonote::blobdata, relay_method>>& txs);
 
     /**
      * @brief tell the pool that certain transactions were just relayed
@@ -609,6 +613,9 @@ private:
     mutable std::unordered_map<crypto::hash, std::tuple<bool, tx_verification_context, uint64_t, crypto::hash>> m_input_cache;
 
     std::unordered_map<crypto::hash, transaction> m_parsed_tx_cache;
+
+    //! Next timestamp that a DB check for relayable txes is allowed
+    std::atomic<time_t> m_next_check;
   };
 }
 

src/cryptonote_protocol/block_queue.cpp

@@ -52,12 +52,12 @@ namespace std {
 namespace cryptonote
 {
 
-void block_queue::add_blocks(uint64_t height, std::vector<cryptonote::block_complete_entry> bcel, const boost::uuids::uuid &connection_id, float rate, size_t size)
+void block_queue::add_blocks(uint64_t height, std::vector<cryptonote::block_complete_entry> bcel, const boost::uuids::uuid &connection_id, const epee::net_utils::network_address &addr, float rate, size_t size)
 {
   boost::unique_lock<boost::recursive_mutex> lock(mutex);
   std::vector<crypto::hash> hashes;
   bool has_hashes = remove_span(height, &hashes);
-  blocks.insert(span(height, std::move(bcel), connection_id, rate, size));
+  blocks.insert(span(height, std::move(bcel), connection_id, addr, rate, size));
   if (has_hashes)
   {
     for (const crypto::hash &h: hashes)
@@ -69,11 +69,11 @@ void block_queue::add_blocks(uint64_t height, std::vector<cryptonote::block_comp
   }
 }
 
-void block_queue::add_blocks(uint64_t height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time)
+void block_queue::add_blocks(uint64_t height, uint64_t nblocks, const boost::uuids::uuid &connection_id, const epee::net_utils::network_address &addr, boost::posix_time::ptime time)
 {
   CHECK_AND_ASSERT_THROW_MES(nblocks > 0, "Empty span");
   boost::unique_lock<boost::recursive_mutex> lock(mutex);
-  blocks.insert(span(height, nblocks, connection_id, time));
+  blocks.insert(span(height, nblocks, connection_id, addr, time));
 }
 
 void block_queue::flush_spans(const boost::uuids::uuid &connection_id, bool all)
@@ -228,7 +228,7 @@ bool block_queue::have(const crypto::hash &hash) const
   return have_blocks.find(hash) != have_blocks.end();
 }
 
-std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, bool sync_pruned_blocks, uint32_t local_pruning_seed, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<std::pair<crypto::hash, uint64_t>> &block_hashes, boost::posix_time::ptime time)
+std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const epee::net_utils::network_address &addr, bool sync_pruned_blocks, uint32_t local_pruning_seed, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<std::pair<crypto::hash, uint64_t>> &block_hashes, boost::posix_time::ptime time)
 {
   boost::unique_lock<boost::recursive_mutex> lock(mutex);
 
@@ -305,7 +305,7 @@ std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_hei
     return std::make_pair(0, 0);
   }
   MDEBUG("Reserving span " << span_start_height << " - " << (span_start_height + span_length - 1) << " for " << connection_id);
-  add_blocks(span_start_height, span_length, connection_id, time);
+  add_blocks(span_start_height, span_length, connection_id, addr, time);
   set_span_hashes(span_start_height, connection_id, hashes);
   return std::make_pair(span_start_height, span_length);
 }
@@ -354,7 +354,7 @@ void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uui
   }
 }
 
-bool block_queue::get_next_span(uint64_t &height, std::vector<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled) const
+bool block_queue::get_next_span(uint64_t &height, std::vector<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, epee::net_utils::network_address &addr, bool filled) const
 {
   boost::unique_lock<boost::recursive_mutex> lock(mutex);
   if (blocks.empty())
@@ -367,6 +367,7 @@ bool block_queue::get_next_span(uint64_t &height, std::vector<cryptonote::block_
       height = i->start_block_height;
       bcel = i->blocks;
       connection_id = i->connection_id;
+      addr = i->origin;
       return true;
     }
   }

src/cryptonote_protocol/block_queue.h

@@ -36,6 +36,7 @@
 #include <unordered_set>
 #include <boost/thread/recursive_mutex.hpp>
 #include <boost/uuid/uuid.hpp>
+#include "net/net_utils_base.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
 #define MONERO_DEFAULT_LOG_CATEGORY "cn.block_queue"
@@ -57,19 +58,20 @@ namespace cryptonote
       float rate;
       size_t size;
       boost::posix_time::ptime time;
+      epee::net_utils::network_address origin{};
 
-      span(uint64_t start_block_height, std::vector<cryptonote::block_complete_entry> blocks, const boost::uuids::uuid &connection_id, float rate, size_t size):
-        start_block_height(start_block_height), blocks(std::move(blocks)), connection_id(connection_id), nblocks(this->blocks.size()), rate(rate), size(size), time() {}
-      span(uint64_t start_block_height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time):
-        start_block_height(start_block_height), connection_id(connection_id), nblocks(nblocks), rate(0.0f), size(0), time(time) {}
+      span(uint64_t start_block_height, std::vector<cryptonote::block_complete_entry> blocks, const boost::uuids::uuid &connection_id, const epee::net_utils::network_address &addr, float rate, size_t size):
+        start_block_height(start_block_height), blocks(std::move(blocks)), connection_id(connection_id), nblocks(this->blocks.size()), rate(rate), size(size), time(boost::date_time::min_date_time), origin(addr) {}
+      span(uint64_t start_block_height, uint64_t nblocks, const boost::uuids::uuid &connection_id, const epee::net_utils::network_address &addr, boost::posix_time::ptime time):
+        start_block_height(start_block_height), connection_id(connection_id), nblocks(nblocks), rate(0.0f), size(0), time(time), origin(addr) {}
 
       bool operator<(const span &s) const { return start_block_height < s.start_block_height; }
     };
     typedef std::set<span> block_map;
 
   public:
-    void add_blocks(uint64_t height, std::vector<cryptonote::block_complete_entry> bcel, const boost::uuids::uuid &connection_id, float rate, size_t size);
-    void add_blocks(uint64_t height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time = boost::date_time::min_date_time);
+    void add_blocks(uint64_t height, std::vector<cryptonote::block_complete_entry> bcel, const boost::uuids::uuid &connection_id, const epee::net_utils::network_address &addr, float rate, size_t size);
+    void add_blocks(uint64_t height, uint64_t nblocks, const boost::uuids::uuid &connection_id, const epee::net_utils::network_address &addr, boost::posix_time::ptime time = boost::date_time::min_date_time);
     void flush_spans(const boost::uuids::uuid &connection_id, bool all = false);
     void flush_stale_spans(const std::set<boost::uuids::uuid> &live_connections);
     bool remove_span(uint64_t start_block_height, std::vector<crypto::hash> *hashes = NULL);
@@ -78,12 +80,12 @@ namespace cryptonote
     void print() const;
     std::string get_overview(uint64_t blockchain_height) const;
     bool has_unpruned_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) const;
-    std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, bool sync_pruned_blocks, uint32_t local_pruning_seed, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<std::pair<crypto::hash, uint64_t>> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time());
+    std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const epee::net_utils::network_address &addr, bool sync_pruned_blocks, uint32_t local_pruning_seed, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<std::pair<crypto::hash, uint64_t>> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time());
     uint64_t get_next_needed_height(uint64_t blockchain_height) const;
     std::pair<uint64_t, uint64_t> get_next_span_if_scheduled(std::vector<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const;
     void reset_next_span_time(boost::posix_time::ptime t = boost::posix_time::microsec_clock::universal_time());
     void set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::vector<crypto::hash> hashes);
-    bool get_next_span(uint64_t &height, std::vector<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled = true) const;
+    bool get_next_span(uint64_t &height, std::vector<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, epee::net_utils::network_address &addr, bool filled = true) const;
     bool has_next_span(const boost::uuids::uuid &connection_id, bool &filled, boost::posix_time::ptime &time) const;
     bool has_next_span(uint64_t height, bool &filled, boost::posix_time::ptime &time, boost::uuids::uuid &connection_id) const;
     size_t get_data_size() const;

src/cryptonote_protocol/cryptonote_protocol_defs.h

@@ -298,6 +298,7 @@ namespace cryptonote
       uint64_t cumulative_difficulty_top64;
       std::vector<crypto::hash> m_block_ids;
       std::vector<uint64_t> m_block_weights;
+      cryptonote::blobdata first_block;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(start_height)
@@ -309,6 +310,7 @@ namespace cryptonote
           KV_SERIALIZE_OPT(cumulative_difficulty_top64, (uint64_t)0)
         KV_SERIALIZE_CONTAINER_POD_AS_BLOB(m_block_ids)
         KV_SERIALIZE_CONTAINER_POD_AS_BLOB(m_block_weights)
+        KV_SERIALIZE(first_block)
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<request_t> request;

src/cryptonote_protocol/cryptonote_protocol_handler.h

@@ -37,6 +37,7 @@
 #include <boost/program_options/variables_map.hpp>
 #include <string>
 
+#include "byte_slice.h"
 #include "math_helper.h"
 #include "storages/levin_abstract_invoke2.h"
 #include "warnings.h"
@@ -100,11 +101,11 @@ namespace cryptonote
     void set_p2p_endpoint(nodetool::i_p2p_endpoint<connection_context>* p2p);
     //bool process_handshake_data(const blobdata& data, cryptonote_connection_context& context);
     bool process_payload_sync_data(const CORE_SYNC_DATA& hshd, cryptonote_connection_context& context, bool is_inital);
-    bool get_payload_sync_data(blobdata& data);
+    bool get_payload_sync_data(epee::byte_slice& data);
     bool get_payload_sync_data(CORE_SYNC_DATA& hshd);
     bool on_callback(cryptonote_connection_context& context);
     t_core& get_core(){return m_core;}
-    bool is_synchronized(){return m_synchronized;}
+    virtual bool is_synchronized() const final { return !no_sync() && m_synchronized; }
     void log_connections();
     std::list<connection_info> get_connections();
     const block_queue &get_block_queue() const { return m_block_queue; }
@@ -116,6 +117,8 @@ namespace cryptonote
     std::string get_peers_overview() const;
     std::pair<uint32_t, uint32_t> get_next_needed_pruning_stripe() const;
     bool needs_new_sync_connections() const;
+    bool is_busy_syncing();
+
   private:
     //----------------- commands handlers ----------------------------------------------
     int handle_notify_new_block(int command, NOTIFY_NEW_BLOCK::request& arg, cryptonote_connection_context& context);
@@ -141,13 +144,15 @@ namespace cryptonote
     bool should_ask_for_pruned_data(cryptonote_connection_context& context, uint64_t first_block_height, uint64_t nblocks, bool check_block_weights) const;
     void drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans);
     void drop_connection_with_score(cryptonote_connection_context &context, unsigned int score, bool flush_all_spans);
+    void drop_connections(const epee::net_utils::network_address address);
     bool kick_idle_peers();
     bool check_standby_peers();
     bool update_sync_search();
     int try_add_next_blocks(cryptonote_connection_context &context);
     void notify_new_stripe(cryptonote_connection_context &context, uint32_t stripe);
-    void skip_unneeded_hashes(cryptonote_connection_context& context, bool check_block_queue) const;
+    size_t skip_unneeded_hashes(cryptonote_connection_context& context, bool check_block_queue) const;
     bool request_txpool_complement(cryptonote_connection_context &context);
+    void hit_score(cryptonote_connection_context &context, int32_t score);
 
     t_core& m_core;
 
@@ -160,9 +165,10 @@ namespace cryptonote
     std::atomic<bool> m_ask_for_txpool_complement;
     boost::mutex m_sync_lock;
     block_queue m_block_queue;
-    epee::math_helper::once_a_time_seconds<30> m_idle_peer_kicker;
+    epee::math_helper::once_a_time_seconds<8> m_idle_peer_kicker;
     epee::math_helper::once_a_time_milliseconds<100> m_standby_checker;
     epee::math_helper::once_a_time_seconds<101> m_sync_search_checker;
+    epee::math_helper::once_a_time_seconds<43> m_bad_peer_checker;
     std::atomic<unsigned int> m_max_out_peers;
     tools::PerformanceTimer m_sync_timer, m_add_timer;
     uint64_t m_last_add_end_time;
@@ -183,14 +189,16 @@ namespace cryptonote
     double get_avg_block_size();
     boost::circular_buffer<size_t> m_avg_buffer = boost::circular_buffer<size_t>(10);
 
+    boost::mutex m_bad_peer_check_lock;
+
     template<class t_parameter>
       bool post_notify(typename t_parameter::request& arg, cryptonote_connection_context& context)
       {
         LOG_PRINT_L2("[" << epee::net_utils::print_connection_context_short(context) << "] post " << typeid(t_parameter).name() << " -->");
-        std::string blob;
-        epee::serialization::store_t_to_binary(arg, blob);
+        epee::byte_slice blob;
+        epee::serialization::store_t_to_binary(arg, blob, 256 * 1024); // optimize for block responses
         //handler_response_blocks_now(blob.size()); // XXX
-        return m_p2p->invoke_notify_to_peer(t_parameter::ID, epee::strspan<uint8_t>(blob), context);
+        return m_p2p->invoke_notify_to_peer(t_parameter::ID, epee::to_span(blob), context);
       }
   };
 

src/cryptonote_protocol/cryptonote_protocol_handler.inl

@@ -68,10 +68,12 @@
 #define BLOCK_QUEUE_FORCE_DOWNLOAD_NEAR_BLOCKS 1000
 #define REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY (5 * 1000000) // microseconds
 #define REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD (30 * 1000000) // microseconds
-#define IDLE_PEER_KICK_TIME (600 * 1000000) // microseconds
+#define IDLE_PEER_KICK_TIME (240 * 1000000) // microseconds
+#define NON_RESPONSIVE_PEER_KICK_TIME (20 * 1000000) // microseconds
 #define PASSIVE_PEER_KICK_TIME (60 * 1000000) // microseconds
 #define DROP_ON_SYNC_WEDGE_THRESHOLD (30 * 1000000000ull) // nanoseconds
 #define LAST_ACTIVITY_STALL_THRESHOLD (2.0f) // seconds
+#define DROP_PEERS_ON_SCORE -2
 
 namespace cryptonote
 {
@@ -135,13 +137,16 @@ namespace cryptonote
     CHECK_AND_ASSERT_MES_CC( context.m_callback_request_count > 0, false, "false callback fired, but context.m_callback_request_count=" << context.m_callback_request_count);
     --context.m_callback_request_count;
 
-    if(context.m_state == cryptonote_connection_context::state_synchronizing)
+    if(context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time == boost::posix_time::not_a_date_time)
     {
       NOTIFY_REQUEST_CHAIN::request r = {};
       context.m_needed_objects.clear();
+      context.m_expect_height = m_core.get_current_blockchain_height();
       m_core.get_short_chain_history(r.block_ids);
       handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?)
       r.prune = m_sync_pruned_blocks;
+      context.m_last_request_time = boost::posix_time::microsec_clock::universal_time();
+      context.m_expect_response = NOTIFY_RESPONSE_CHAIN_ENTRY::ID;
       MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() );
       post_notify<NOTIFY_REQUEST_CHAIN>(r, context);
       MLOG_PEER_STATE("requesting chain");
@@ -326,6 +331,11 @@ namespace cryptonote
       }
     }
 
+    if (hshd.current_height < context.m_remote_blockchain_height)
+    {
+      MINFO(context << "Claims " << hshd.current_height << ", claimed " << context.m_remote_blockchain_height << " before");
+      hit_score(context, 1);
+    }
     context.m_remote_blockchain_height = hshd.current_height;
     context.m_pruning_seed = hshd.pruning_seed;
 #ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED
@@ -399,6 +409,7 @@ namespace cryptonote
     ++context.m_callback_request_count;
     m_p2p->request_callback(context);
     MLOG_PEER_STATE("requesting callback");
+    context.m_num_requested = 0;
     return true;
   }
   //------------------------------------------------------------------------------------------------------------------------
@@ -416,7 +427,7 @@ namespace cryptonote
   }
   //------------------------------------------------------------------------------------------------------------------------
     template<class t_core>
-    bool t_cryptonote_protocol_handler<t_core>::get_payload_sync_data(blobdata& data)
+    bool t_cryptonote_protocol_handler<t_core>::get_payload_sync_data(epee::byte_slice& data)
   {
     CORE_SYNC_DATA hsd = {};
     get_payload_sync_data(hsd);
@@ -427,10 +438,10 @@ namespace cryptonote
     template<class t_core>
     int t_cryptonote_protocol_handler<t_core>::handle_notify_new_block(int command, NOTIFY_NEW_BLOCK::request& arg, cryptonote_connection_context& context)
   {
-    MLOGIF_P2P_MESSAGE(crypto::hash hash; cryptonote::block b; bool ret = cryptonote::parse_and_validate_block_from_blob(arg.b.block, b, &hash);, ret, "Received NOTIFY_NEW_BLOCK " << hash << " (height " << arg.current_blockchain_height << ", " << arg.b.txs.size() << " txes)");
+    MLOGIF_P2P_MESSAGE(crypto::hash hash; cryptonote::block b; bool ret = cryptonote::parse_and_validate_block_from_blob(arg.b.block, b, &hash);, ret, context << "Received NOTIFY_NEW_BLOCK " << hash << " (height " << arg.current_blockchain_height << ", " << arg.b.txs.size() << " txes)");
     if(context.m_state != cryptonote_connection_context::state_normal)
       return 1;
-    if(!is_synchronized() || m_no_sync) // can happen if a peer connection goes to normal but another thread still hasn't finished adding queued blocks
+    if(!is_synchronized()) // can happen if a peer connection goes to normal but another thread still hasn't finished adding queued blocks
     {
       LOG_DEBUG_CC(context, "Received new block while syncing, ignored");
       return 1;
@@ -484,9 +495,12 @@ namespace cryptonote
       context.m_needed_objects.clear();
       context.m_state = cryptonote_connection_context::state_synchronizing;
       NOTIFY_REQUEST_CHAIN::request r = {};
+      context.m_expect_height = m_core.get_current_blockchain_height();
       m_core.get_short_chain_history(r.block_ids);
       r.prune = m_sync_pruned_blocks;
       handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?)
+      context.m_last_request_time = boost::posix_time::microsec_clock::universal_time();
+      context.m_expect_response = NOTIFY_RESPONSE_CHAIN_ENTRY::ID;
       MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() );
       post_notify<NOTIFY_REQUEST_CHAIN>(r, context);
       MLOG_PEER_STATE("requesting chain");
@@ -498,10 +512,10 @@ namespace cryptonote
   template<class t_core>
   int t_cryptonote_protocol_handler<t_core>::handle_notify_new_fluffy_block(int command, NOTIFY_NEW_FLUFFY_BLOCK::request& arg, cryptonote_connection_context& context)
   {
-    MLOGIF_P2P_MESSAGE(crypto::hash hash; cryptonote::block b; bool ret = cryptonote::parse_and_validate_block_from_blob(arg.b.block, b, &hash);, ret, "Received NOTIFY_NEW_FLUFFY_BLOCK " << hash << " (height " << arg.current_blockchain_height << ", " << arg.b.txs.size() << " txes)");
+    MLOGIF_P2P_MESSAGE(crypto::hash hash; cryptonote::block b; bool ret = cryptonote::parse_and_validate_block_from_blob(arg.b.block, b, &hash);, ret, context << "Received NOTIFY_NEW_FLUFFY_BLOCK " << hash << " (height " << arg.current_blockchain_height << ", " << arg.b.txs.size() << " txes)");
     if(context.m_state != cryptonote_connection_context::state_normal)
       return 1;
-    if(!is_synchronized() || m_no_sync) // can happen if a peer connection goes to normal but another thread still hasn't finished adding queued blocks
+    if(!is_synchronized()) // can happen if a peer connection goes to normal but another thread still hasn't finished adding queued blocks
     {
       LOG_DEBUG_CC(context, "Received new block while syncing, ignored");
       return 1;
@@ -535,6 +549,7 @@ namespace cryptonote
       }      
 
       std::vector<tx_blob_entry> have_tx;
+      have_tx.reserve(new_block.tx_hashes.size());
 
       // Instead of requesting missing transactions by hash like BTC, 
       // we do it by index (thanks to a suggestion from moneromooo) because
@@ -543,6 +558,7 @@ namespace cryptonote
       // Also, remember to pepper some whitespace changes around to bother
       // moneromooo ... only because I <3 him. 
       std::vector<uint64_t> need_tx_indices;
+      need_tx_indices.reserve(new_block.tx_hashes.size());
         
       transaction tx;
       crypto::hash tx_hash;
@@ -762,9 +778,12 @@ namespace cryptonote
           context.m_needed_objects.clear();
           context.m_state = cryptonote_connection_context::state_synchronizing;
           NOTIFY_REQUEST_CHAIN::request r = {};
+          context.m_expect_height = m_core.get_current_blockchain_height();
           m_core.get_short_chain_history(r.block_ids);
           handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?)
           r.prune = m_sync_pruned_blocks;
+          context.m_last_request_time = boost::posix_time::microsec_clock::universal_time();
+          context.m_expect_response = NOTIFY_RESPONSE_CHAIN_ENTRY::ID;
           MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() );
           post_notify<NOTIFY_REQUEST_CHAIN>(r, context);
           MLOG_PEER_STATE("requesting chain");
@@ -812,6 +831,7 @@ namespace cryptonote
     }
 
     std::vector<crypto::hash> txids;
+    txids.reserve(b.tx_hashes.size());
     NOTIFY_NEW_FLUFFY_BLOCK::request fluffy_response;
     fluffy_response.b.block = t_serializable_object_to_blob(b);
     fluffy_response.current_blockchain_height = arg.current_blockchain_height;
@@ -929,7 +949,7 @@ namespace cryptonote
     // while syncing, core will lock for a long time, so we ignore
     // those txes as they aren't really needed anyway, and avoid a
     // long block before replying
-    if(!is_synchronized() || m_no_sync)
+    if(!is_synchronized())
     {
       LOG_DEBUG_CC(context, "Received new tx while syncing, ignored");
       return 1;
@@ -1029,6 +1049,7 @@ namespace cryptonote
       drop_connection(context, false, false);
       return 1;
     }
+    context.m_last_request_time = boost::posix_time::microsec_clock::universal_time();
     MLOG_P2P_MESSAGE("-->>NOTIFY_RESPONSE_GET_OBJECTS: blocks.size()="
                      << rsp.blocks.size() << ", rsp.m_current_blockchain_height=" << rsp.current_blockchain_height
                      << ", missed_ids.size()=" << rsp.missed_ids.size());
@@ -1062,6 +1083,14 @@ namespace cryptonote
     boost::posix_time::ptime request_time = context.m_last_request_time;
     context.m_last_request_time = boost::date_time::not_a_date_time;
 
+    if (context.m_expect_response != NOTIFY_RESPONSE_GET_OBJECTS::ID)
+    {
+      LOG_ERROR_CCONTEXT("Got NOTIFY_RESPONSE_GET_OBJECTS out of the blue, dropping connection");
+      drop_connection(context, true, false);
+      return 1;
+    }
+    context.m_expect_response = 0;
+
     // calculate size of request
     size_t size = 0;
     size_t blocks_size = 0;
@@ -1107,6 +1136,11 @@ namespace cryptonote
       return 1;
     }
 
+    if (arg.current_blockchain_height < context.m_remote_blockchain_height)
+    {
+      MINFO(context << "Claims " << arg.current_blockchain_height << ", claimed " << context.m_remote_blockchain_height << " before");
+      hit_score(context, 1);
+    }
     context.m_remote_blockchain_height = arg.current_blockchain_height;
     if (context.m_remote_blockchain_height > m_core.get_target_blockchain_height())
       m_core.set_target_blockchain_height(context.m_remote_blockchain_height);
@@ -1141,7 +1175,16 @@ namespace cryptonote
         return 1;
       }
       if (start_height == std::numeric_limits<uint64_t>::max())
+      {
         start_height = boost::get<txin_gen>(b.miner_tx.vin[0]).height;
+        if (start_height > context.m_expect_height)
+        {
+          LOG_ERROR_CCONTEXT("sent block ahead of expected height, dropping connection");
+          drop_connection(context, false, false);
+          ++m_sync_bad_spans_downloaded;
+          return 1;
+        }
+      }
 
       auto req_it = context.m_requested_objects.find(block_hash);
       if(req_it == context.m_requested_objects.end())
@@ -1230,7 +1273,7 @@ namespace cryptonote
       const boost::posix_time::time_duration dt = now - request_time;
       const float rate = size * 1e6 / (dt.total_microseconds() + 1);
       MDEBUG(context << " adding span: " << arg.blocks.size() << " at height " << start_height << ", " << dt.total_microseconds()/1e6 << " seconds, " << (rate/1024) << " kB/s, size now " << (m_block_queue.get_data_size() + blocks_size) / 1048576.f << " MB");
-      m_block_queue.add_blocks(start_height, arg.blocks, context.m_connection_id, rate, blocks_size);
+      m_block_queue.add_blocks(start_height, arg.blocks, context.m_connection_id, context.m_remote_address, rate, blocks_size);
 
       const crypto::hash last_block_hash = cryptonote::get_block_hash(b);
       context.m_last_known_hash = last_block_hash;
@@ -1331,7 +1374,8 @@ namespace cryptonote
           uint64_t start_height;
           std::vector<cryptonote::block_complete_entry> blocks;
           boost::uuids::uuid span_connection_id;
-          if (!m_block_queue.get_next_span(start_height, blocks, span_connection_id))
+          epee::net_utils::network_address span_origin;
+          if (!m_block_queue.get_next_span(start_height, blocks, span_connection_id, span_origin))
           {
             MDEBUG(context << " no next span found, going back to download");
             break;
@@ -1429,6 +1473,7 @@ namespace cryptonote
           if (!m_core.prepare_handle_incoming_blocks(blocks, pblocks))
           {
             LOG_ERROR_CCONTEXT("Failure in prepare_handle_incoming_blocks");
+            drop_connections(span_origin);
             return 1;
           }
           if (!pblocks.empty() && pblocks.size() != blocks.size())
@@ -1468,6 +1513,7 @@ namespace cryptonote
             {
               if(tvc[i].m_verifivation_failed)
               {
+                drop_connections(span_origin);
                 if (!m_p2p->for_connection(span_connection_id, [&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t f)->bool{
                   cryptonote::transaction tx;
                   crypto::hash txid;
@@ -1509,6 +1555,7 @@ namespace cryptonote
 
             if(bvc.m_verifivation_failed)
             {
+              drop_connections(span_origin);
               if (!m_p2p->for_connection(span_connection_id, [&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t f)->bool{
                 LOG_PRINT_CCONTEXT_L1("Block verification failed, dropping connection");
                 drop_connection_with_score(context, bvc.m_bad_pow ? P2P_IP_FAILS_BEFORE_BLOCK : 1, true);
@@ -1528,6 +1575,7 @@ namespace cryptonote
             }
             if(bvc.m_marked_as_orphaned)
             {
+              drop_connections(span_origin);
               if (!m_p2p->for_connection(span_connection_id, [&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t f)->bool{
                 LOG_PRINT_CCONTEXT_L1("Block received at sync phase was marked as orphaned, dropping connection");
                 drop_connection(context, true, true);
@@ -1666,24 +1714,46 @@ skip:
   bool t_cryptonote_protocol_handler<t_core>::kick_idle_peers()
   {
     MTRACE("Checking for idle peers...");
+    std::vector<std::pair<boost::uuids::uuid, unsigned>> idle_peers;
     m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool
     {
       if (context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time != boost::date_time::not_a_date_time)
       {
         const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time();
         const boost::posix_time::time_duration dt = now - context.m_last_request_time;
-        if (dt.total_microseconds() > IDLE_PEER_KICK_TIME)
+        const auto ms = dt.total_microseconds();
+        if (ms > IDLE_PEER_KICK_TIME || (context.m_expect_response && ms > NON_RESPONSIVE_PEER_KICK_TIME))
         {
-          MINFO(context << " kicking idle peer, last update " << (dt.total_microseconds() / 1.e6) << " seconds ago");
-          LOG_PRINT_CCONTEXT_L2("requesting callback");
-          context.m_last_request_time = boost::date_time::not_a_date_time;
-          context.m_state = cryptonote_connection_context::state_standby; // we'll go back to adding, then (if we can't), download
-          ++context.m_callback_request_count;
-          m_p2p->request_callback(context);
+          if (context.m_score-- >= 0)
+          {
+            MINFO(context << " kicking idle peer, last update " << (dt.total_microseconds() / 1.e6) << " seconds ago, expecting " << (int)context.m_expect_response);
+            LOG_PRINT_CCONTEXT_L2("requesting callback");
+            context.m_last_request_time = boost::date_time::not_a_date_time;
+            context.m_expect_response = 0;
+            context.m_expect_height = 0;
+            context.m_state = cryptonote_connection_context::state_standby; // we'll go back to adding, then (if we can't), download
+            ++context.m_callback_request_count;
+            m_p2p->request_callback(context);
+          }
+          else
+          {
+            idle_peers.push_back(std::make_pair(context.m_connection_id, context.m_expect_response == 0 ? 1 : 5));
+          }
         }
       }
       return true;
     });
+
+    for (const auto &e: idle_peers)
+    {
+      const auto &uuid = e.first;
+      m_p2p->for_connection(uuid, [&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t f)->bool{
+        MINFO(ctx << "dropping idle peer with negative score");
+        drop_connection_with_score(ctx, e.second, false);
+        return true;
+      });
+    }
+
     return true;
   }
   //------------------------------------------------------------------------------------------------------------------------
@@ -1760,6 +1830,16 @@ skip:
       LOG_ERROR_CCONTEXT("Failed to handle NOTIFY_REQUEST_CHAIN.");
       return 1;
     }
+    if (r.m_block_ids.size() >= 2)
+    {
+      cryptonote::block b;
+      if (!m_core.get_block_by_hash(r.m_block_ids[1], b))
+      {
+        LOG_ERROR_CCONTEXT("Failed to handle NOTIFY_REQUEST_CHAIN: first block not found");
+        return 1;
+      }
+      r.first_block = cryptonote::block_to_blob(b);
+    }
     MLOG_P2P_MESSAGE("-->>NOTIFY_RESPONSE_CHAIN_ENTRY: m_start_height=" << r.start_height << ", m_total_height=" << r.total_height << ", m_block_ids.size()=" << r.m_block_ids.size());
     post_notify<NOTIFY_RESPONSE_CHAIN_ENTRY>(r, context);
     return 1;
@@ -1914,7 +1994,7 @@ skip:
   }
   //------------------------------------------------------------------------------------------------------------------------
   template<class t_core>
-  void t_cryptonote_protocol_handler<t_core>::skip_unneeded_hashes(cryptonote_connection_context& context, bool check_block_queue) const
+  size_t t_cryptonote_protocol_handler<t_core>::skip_unneeded_hashes(cryptonote_connection_context& context, bool check_block_queue) const
   {
     // take out blocks we already have
     size_t skip = 0;
@@ -1931,6 +2011,7 @@ skip:
       MDEBUG(context << "skipping " << skip << "/" << context.m_needed_objects.size() << " blocks");
       context.m_needed_objects = std::vector<std::pair<crypto::hash, uint64_t>>(context.m_needed_objects.begin() + skip, context.m_needed_objects.end());
     }
+    return skip;
   }
   //------------------------------------------------------------------------------------------------------------------------
   template<class t_core>
@@ -1944,7 +2025,7 @@ skip:
     if (local_stripe == 0)
       return false;
     // don't request pre-bulletprooof pruned blocks, we can't reconstruct their weight (yet)
-    static const uint64_t bp_fork_height = m_core.get_earliest_ideal_height_for_version(HF_VERSION_SMALLER_BP);
+    static const uint64_t bp_fork_height = m_core.get_earliest_ideal_height_for_version(HF_VERSION_SMALLER_BP + 1);
     if (first_block_height < bp_fork_height)
       return false;
     // assumes the span size is less or equal to the stripe size
@@ -1984,7 +2065,17 @@ skip:
         const size_t block_queue_size_threshold = m_block_download_max_size ? m_block_download_max_size : BLOCK_QUEUE_SIZE_THRESHOLD;
         bool queue_proceed = nspans < BLOCK_QUEUE_NSPANS_THRESHOLD || size < block_queue_size_threshold;
         // get rid of blocks we already requested, or already have
-        skip_unneeded_hashes(context, true);
+        if (skip_unneeded_hashes(context, true) && context.m_needed_objects.empty() && context.m_num_requested == 0)
+        {
+          if (context.m_remote_blockchain_height > m_block_queue.get_next_needed_height(bc_height))
+          {
+            MERROR(context << "Nothing we can request from this peer, and we did not request anything previously");
+            return false;
+          }
+          MDEBUG(context << "Nothing to get from this peer, and it's not ahead of us, all done");
+          context.m_state = cryptonote_connection_context::state_normal;
+          return true;
+        }
         uint64_t next_needed_height = m_block_queue.get_next_needed_height(bc_height);
         uint64_t next_block_height;
         if (context.m_needed_objects.empty())
@@ -2101,6 +2192,7 @@ skip:
           if (span.second > 0)
           {
             is_next = true;
+            req.blocks.reserve(hashes.size());
             for (const auto &hash: hashes)
             {
               req.blocks.push_back(hash);
@@ -2120,12 +2212,22 @@ skip:
           context.m_last_response_height = 0;
           goto skip;
         }
-        skip_unneeded_hashes(context, false);
+        if (skip_unneeded_hashes(context, false) && context.m_needed_objects.empty() && context.m_num_requested == 0)
+        {
+          if (context.m_remote_blockchain_height > m_block_queue.get_next_needed_height(m_core.get_current_blockchain_height()))
+          {
+            MERROR(context << "Nothing we can request from this peer, and we did not request anything previously");
+            return false;
+          }
+          MDEBUG(context << "Nothing to get from this peer, and it's not ahead of us, all done");
+          context.m_state = cryptonote_connection_context::state_normal;
+          return true;
+        }
 
         const uint64_t first_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1;
         static const uint64_t bp_fork_height = m_core.get_earliest_ideal_height_for_version(HF_VERSION_SMALLER_BP);
         bool sync_pruned_blocks = m_sync_pruned_blocks && first_block_height >= bp_fork_height && m_core.get_blockchain_pruning_seed();
-        span = m_block_queue.reserve_span(first_block_height, context.m_last_response_height, count_limit, context.m_connection_id, sync_pruned_blocks, m_core.get_blockchain_pruning_seed(), context.m_pruning_seed, context.m_remote_blockchain_height, context.m_needed_objects);
+        span = m_block_queue.reserve_span(first_block_height, context.m_last_response_height, count_limit, context.m_connection_id, context.m_remote_address, sync_pruned_blocks, m_core.get_blockchain_pruning_seed(), context.m_pruning_seed, context.m_remote_blockchain_height, context.m_needed_objects);
         MDEBUG(context << " span from " << first_block_height << ": " << span.first << "/" << span.second);
         if (span.second > 0)
         {
@@ -2149,6 +2251,7 @@ skip:
         if (span.second > 0)
         {
           is_next = true;
+          req.blocks.reserve(hashes.size());
           for (const auto &hash: hashes)
           {
             req.blocks.push_back(hash);
@@ -2182,6 +2285,7 @@ skip:
             return false;
           }
 
+          req.blocks.reserve(req.blocks.size() + span.second);
           for (size_t n = 0; n < span.second; ++n)
           {
             req.blocks.push_back(context.m_needed_objects[n].first);
@@ -2206,6 +2310,8 @@ skip:
           }
         }
         context.m_last_request_time = boost::posix_time::microsec_clock::universal_time();
+        context.m_expect_height = span.first;
+        context.m_expect_response = NOTIFY_RESPONSE_GET_OBJECTS::ID;
         MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size()
             << "requested blocks count=" << count << " / " << count_limit << " from " << span.first << ", first hash " << req.blocks.front());
         //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size());
@@ -2215,35 +2321,22 @@ skip:
           << "/" << tools::get_pruning_stripe(span.first + span.second - 1, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES)
           << ", ours " << tools::get_pruning_stripe(m_core.get_blockchain_pruning_seed()) << ", peer stripe " << tools::get_pruning_stripe(context.m_pruning_seed));
 
+        context.m_num_requested += req.blocks.size();
         post_notify<NOTIFY_REQUEST_GET_OBJECTS>(req, context);
         MLOG_PEER_STATE("requesting objects");
         return true;
       }
 
-      // if we're still around, we might be at a point where the peer is pruned, so we could either
-      // drop it to make space for other peers, or ask for a span further down the line
-      const uint32_t next_stripe = get_next_needed_pruning_stripe().first;
-      const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed);
-      const uint32_t local_stripe = tools::get_pruning_stripe(m_core.get_blockchain_pruning_seed());
-      if (!(m_sync_pruned_blocks && peer_stripe == local_stripe) && next_stripe && peer_stripe && next_stripe != peer_stripe)
+      // we can do nothing, so drop this peer to make room for others unless we think we've downloaded all we need
+      const uint64_t blockchain_height = m_core.get_current_blockchain_height();
+      if (std::max(blockchain_height, m_block_queue.get_next_needed_height(blockchain_height)) >= m_core.get_target_blockchain_height())
       {
-        // at this point, we have to either close the connection, or start getting blocks past the
-        // current point, or become dormant
-        MDEBUG(context << "this peer is pruned at seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) <<
-            ", next stripe needed is " << next_stripe);
-        if (!context.m_is_income)
-        {
-          if (should_drop_connection(context, next_stripe))
-          {
-            m_p2p->add_used_stripe_peer(context);
-            return false; // drop outgoing connections
-          }
-        }
-        // we'll get back stuck waiting for the go ahead
         context.m_state = cryptonote_connection_context::state_normal;
         MLOG_PEER_STATE("Nothing to do for now, switching to normal state");
         return true;
       }
+      MLOG_PEER_STATE("We can download nothing from this peer, dropping");
+      return false;
     }
 
 skip:
@@ -2260,8 +2353,8 @@ skip:
         uint64_t start_height;
         std::vector<cryptonote::block_complete_entry> blocks;
         boost::uuids::uuid span_connection_id;
-        bool filled = false;
-        if (m_block_queue.get_next_span(start_height, blocks, span_connection_id, filled) && filled)
+        epee::net_utils::network_address span_origin;
+        if (m_block_queue.get_next_span(start_height, blocks, span_connection_id, span_origin, true))
         {
           LOG_DEBUG_CC(context, "No other thread is adding blocks, resuming");
           MLOG_PEER_STATE("will try to add blocks next");
@@ -2277,6 +2370,7 @@ skip:
     {//we have to fetch more objects ids, request blockchain entry
 
       NOTIFY_REQUEST_CHAIN::request r = {};
+      context.m_expect_height = m_core.get_current_blockchain_height();
       m_core.get_short_chain_history(r.block_ids);
       CHECK_AND_ASSERT_MES(!r.block_ids.empty(), false, "Short chain history is empty");
 
@@ -2284,7 +2378,10 @@ skip:
       {
         // we'll want to start off from where we are on that peer, which may not be added yet
         if (context.m_last_known_hash != crypto::null_hash && r.block_ids.front() != context.m_last_known_hash)
+        {
+          context.m_expect_height = std::numeric_limits<uint64_t>::max();
           r.block_ids.push_front(context.m_last_known_hash);
+        }
       }
 
       handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?)
@@ -2296,6 +2393,7 @@ skip:
       //LOG_PRINT_CCONTEXT_L1("r = " << 200);
 
       context.m_last_request_time = boost::posix_time::microsec_clock::universal_time();
+      context.m_expect_response = NOTIFY_RESPONSE_CHAIN_ENTRY::ID;
       MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() << ", start_from_current_chain " << start_from_current_chain);
       post_notify<NOTIFY_REQUEST_CHAIN>(r, context);
       MLOG_PEER_STATE("requesting chain");
@@ -2321,7 +2419,7 @@ skip:
       }
       else
       {
-        MINFO(context << " we've reached this peer's blockchain height");
+        MINFO(context << " we've reached this peer's blockchain height (theirs " << context.m_remote_blockchain_height << ", our target " << m_core.get_target_blockchain_height());
       }
     }
     return true;
@@ -2418,6 +2516,20 @@ skip:
       << ", m_start_height=" << arg.start_height << ", m_total_height=" << arg.total_height);
     MLOG_PEER_STATE("received chain");
 
+    if (context.m_expect_response != NOTIFY_RESPONSE_CHAIN_ENTRY::ID)
+    {
+      LOG_ERROR_CCONTEXT("Got NOTIFY_RESPONSE_CHAIN_ENTRY out of the blue, dropping connection");
+      drop_connection(context, true, false);
+      return 1;
+    }
+    context.m_expect_response = 0;
+    if (arg.start_height + 1 > context.m_expect_height) // we expect an overlapping block
+    {
+      LOG_ERROR_CCONTEXT("Got NOTIFY_RESPONSE_CHAIN_ENTRY past expected height, dropping connection");
+      drop_connection(context, true, false);
+      return 1;
+    }
+
     context.m_last_request_time = boost::date_time::not_a_date_time;
 
     m_sync_download_chain_size += arg.m_block_ids.size() * sizeof(crypto::hash);
@@ -2442,12 +2554,17 @@ skip:
     }
     MDEBUG(context << "first block hash " << arg.m_block_ids.front() << ", last " << arg.m_block_ids.back());
 
-    if (arg.total_height >= CRYPTONOTE_MAX_BLOCK_NUMBER || arg.m_block_ids.size() >= CRYPTONOTE_MAX_BLOCK_NUMBER)
+    if (arg.total_height >= CRYPTONOTE_MAX_BLOCK_NUMBER || arg.m_block_ids.size() > BLOCKS_IDS_SYNCHRONIZING_MAX_COUNT)
     {
       LOG_ERROR_CCONTEXT("sent wrong NOTIFY_RESPONSE_CHAIN_ENTRY, with total_height=" << arg.total_height << " and block_ids=" << arg.m_block_ids.size());
       drop_connection(context, false, false);
       return 1;
     }
+    if (arg.total_height < context.m_remote_blockchain_height)
+    {
+      MINFO(context << "Claims " << arg.total_height << ", claimed " << context.m_remote_blockchain_height << " before");
+      hit_score(context, 1);
+    }
     context.m_remote_blockchain_height = arg.total_height;
     context.m_last_response_height = arg.start_height + arg.m_block_ids.size()-1;
     if(context.m_last_response_height > context.m_remote_blockchain_height)
@@ -2460,7 +2577,7 @@ skip:
     }
 
     uint64_t n_use_blocks = m_core.prevalidate_block_hashes(arg.start_height, arg.m_block_ids, arg.m_block_weights);
-    if (n_use_blocks + HASH_OF_HASHES_STEP <= arg.m_block_ids.size())
+    if (n_use_blocks == 0 || n_use_blocks + HASH_OF_HASHES_STEP <= arg.m_block_ids.size())
     {
       LOG_ERROR_CCONTEXT("Most blocks are invalid, dropping connection");
       drop_connection(context, true, false);
@@ -2468,13 +2585,77 @@ skip:
     }
 
     context.m_needed_objects.clear();
+    context.m_needed_objects.reserve(arg.m_block_ids.size());
     uint64_t added = 0;
+    std::unordered_set<crypto::hash> blocks_found;
+    bool first = true;
+    bool expect_unknown = false;
     for (size_t i = 0; i < arg.m_block_ids.size(); ++i)
     {
+      if (!blocks_found.insert(arg.m_block_ids[i]).second)
+      {
+        LOG_ERROR_CCONTEXT("Duplicate blocks in chain entry response, dropping connection");
+        drop_connection_with_score(context, 5, false);
+        return 1;
+      }
+      int where;
+      const bool have_block = m_core.have_block_unlocked(arg.m_block_ids[i], &where);
+      if (first)
+      {
+        if (!have_block && !m_block_queue.requested(arg.m_block_ids[i]) && !m_block_queue.have(arg.m_block_ids[i]))
+        {
+          LOG_ERROR_CCONTEXT("First block hash is unknown, dropping connection");
+          drop_connection_with_score(context, 5, false);
+          return 1;
+        }
+        if (!have_block)
+          expect_unknown = true;
+      }
+      if (!first)
+      {
+        // after the first, blocks may be known or unknown, but if they are known,
+        // they should be at the same height if on the main chain
+        if (have_block)
+        {
+          switch (where)
+          {
+            default:
+            case HAVE_BLOCK_INVALID:
+              LOG_ERROR_CCONTEXT("Block is invalid or known without known type, dropping connection");
+              drop_connection(context, true, false);
+              return 1;
+            case HAVE_BLOCK_MAIN_CHAIN:
+              if (expect_unknown)
+              {
+                LOG_ERROR_CCONTEXT("Block is on the main chain, but we did not expect a known block, dropping connection");
+                drop_connection_with_score(context, 5, false);
+                return 1;
+              }
+              if (m_core.get_block_id_by_height(arg.start_height + i) != arg.m_block_ids[i])
+              {
+                LOG_ERROR_CCONTEXT("Block is on the main chain, but not at the expected height, dropping connection");
+                drop_connection_with_score(context, 5, false);
+                return 1;
+              }
+              break;
+            case HAVE_BLOCK_ALT_CHAIN:
+              if (expect_unknown)
+              {
+                LOG_ERROR_CCONTEXT("Block is on the main chain, but we did not expect a known block, dropping connection");
+                drop_connection_with_score(context, 5, false);
+                return 1;
+              }
+              break;
+          }
+        }
+        else
+          expect_unknown = true;
+      }
       const uint64_t block_weight = arg.m_block_weights.empty() ? 0 : arg.m_block_weights[i];
       context.m_needed_objects.push_back(std::make_pair(arg.m_block_ids[i], block_weight));
       if (++added == n_use_blocks)
         break;
+      first = false;
     }
     context.m_last_response_height -= arg.m_block_ids.size() - n_use_blocks;
 
@@ -2488,6 +2669,7 @@ skip:
     if (arg.total_height > m_core.get_target_blockchain_height())
       m_core.set_target_blockchain_height(arg.total_height);
 
+    context.m_num_requested = 0;
     return 1;
   }
   //------------------------------------------------------------------------------------------------------------------------
@@ -2504,6 +2686,7 @@ skip:
     std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> fullConnections, fluffyConnections;
     m_p2p->for_each_connection([this, &exclude_context, &fullConnections, &fluffyConnections](connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)
     {
+      // peer_id also filters out connections before handshake
       if (peer_id && exclude_context.m_connection_id != context.m_connection_id && context.m_remote_address.get_zone() == epee::net_utils::zone::public_)
       {
         if(m_core.fluffy_blocks_enabled() && (support_flags & P2P_SUPPORT_FLAG_FLUFFY_BLOCKS))
@@ -2523,15 +2706,15 @@ skip:
     // send fluffy ones first, we want to encourage people to run that
     if (!fluffyConnections.empty())
     {
-      std::string fluffyBlob;
-      epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob);
-      m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, epee::strspan<uint8_t>(fluffyBlob), std::move(fluffyConnections));
+      epee::byte_slice fluffyBlob;
+      epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob, 32 * 1024);
+      m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, epee::to_span(fluffyBlob), std::move(fluffyConnections));
     }
     if (!fullConnections.empty())
     {
-      std::string fullBlob;
-      epee::serialization::store_t_to_binary(arg, fullBlob);
-      m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, epee::strspan<uint8_t>(fullBlob), std::move(fullConnections));
+      epee::byte_slice fullBlob;
+      epee::serialization::store_t_to_binary(arg, fullBlob, 128 * 1024);
+      m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, epee::to_span(fullBlob), std::move(fullConnections));
     }
 
     return true;
@@ -2564,6 +2747,19 @@ skip:
   }
   //------------------------------------------------------------------------------------------------------------------------
   template<class t_core>
+  void t_cryptonote_protocol_handler<t_core>::hit_score(cryptonote_connection_context &context, int32_t score)
+  {
+    if (score <= 0)
+    {
+      MERROR("Negative score hit");
+      return;
+    }
+    context.m_score -= score;
+    if (context.m_score <= DROP_PEERS_ON_SCORE)
+      drop_connection_with_score(context, 5, false);
+  }
+  //------------------------------------------------------------------------------------------------------------------------
+  template<class t_core>
   std::string t_cryptonote_protocol_handler<t_core>::get_peers_overview() const
   {
     std::stringstream ss;
@@ -2638,18 +2834,28 @@ skip:
   }
   //------------------------------------------------------------------------------------------------------------------------
   template<class t_core>
+  bool t_cryptonote_protocol_handler<t_core>::is_busy_syncing()
+  {
+    const boost::unique_lock<boost::mutex> sync{m_sync_lock, boost::try_to_lock};
+    return !sync.owns_lock();
+  }
+  //------------------------------------------------------------------------------------------------------------------------
+  template<class t_core>
   void t_cryptonote_protocol_handler<t_core>::drop_connection_with_score(cryptonote_connection_context &context, unsigned score, bool flush_all_spans)
   {
     LOG_DEBUG_CC(context, "dropping connection id " << context.m_connection_id << " (pruning seed " <<
         epee::string_tools::to_string_hex(context.m_pruning_seed) <<
         "), score " << score << ", flush_all_spans " << flush_all_spans);
 
-    if (score > 0)
-      m_p2p->add_host_fail(context.m_remote_address, score);
-
     m_block_queue.flush_spans(context.m_connection_id, flush_all_spans);
 
+    // copy since dropping the connection will invalidate the context, and thus the address
+    const auto remote_address = context.m_remote_address;
+
     m_p2p->drop_connection(context);
+
+    if (score > 0)
+      m_p2p->add_host_fail(remote_address, score);
   }
   //------------------------------------------------------------------------------------------------------------------------
   template<class t_core>
@@ -2659,6 +2865,29 @@ skip:
   }
   //------------------------------------------------------------------------------------------------------------------------
   template<class t_core>
+  void t_cryptonote_protocol_handler<t_core>::drop_connections(const epee::net_utils::network_address address)
+  {
+    MWARNING("dropping connections to " << address.str());
+
+    m_p2p->add_host_fail(address, 5);
+
+    std::vector<boost::uuids::uuid> drop;
+    m_p2p->for_each_connection([&](const connection_context& cntxt, nodetool::peerid_type peer_id, uint32_t support_flags) {
+      if (address.is_same_host(cntxt.m_remote_address))
+        drop.push_back(cntxt.m_connection_id);
+      return true;
+    });
+    for (const boost::uuids::uuid &id: drop)
+    {
+      m_block_queue.flush_spans(id, true);
+      m_p2p->for_connection(id, [&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t f)->bool{
+        drop_connection(context, true, false);
+        return true;
+      });
+    }
+  }
+  //------------------------------------------------------------------------------------------------------------------------
+  template<class t_core>
   void t_cryptonote_protocol_handler<t_core>::on_connection_close(cryptonote_connection_context &context)
   {
     uint64_t target = 0;

src/cryptonote_protocol/cryptonote_protocol_handler_common.h

@@ -40,6 +40,7 @@ namespace cryptonote
   /************************************************************************/
   struct i_cryptonote_protocol
   {
+    virtual bool is_synchronized() const = 0;
     virtual bool relay_block(NOTIFY_NEW_BLOCK::request& arg, cryptonote_connection_context& exclude_context)=0;
     virtual bool relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, const boost::uuids::uuid& source, epee::net_utils::zone zone, relay_method tx_relay)=0;
     //virtual bool request_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, cryptonote_connection_context& context)=0;
@@ -50,6 +51,10 @@ namespace cryptonote
   /************************************************************************/
   struct cryptonote_protocol_stub: public i_cryptonote_protocol
   {
+    virtual bool is_synchronized() const final
+    {
+      return false;
+    }
     virtual bool relay_block(NOTIFY_NEW_BLOCK::request& arg, cryptonote_connection_context& exclude_context)
     {
       return false;

src/cryptonote_protocol/levin_notify.cpp

@@ -36,6 +36,7 @@
 #include <stdexcept>
 #include <utility>
 
+#include "byte_slice.h"
 #include "common/expect.h"
 #include "common/varint.h"
 #include "cryptonote_config.h"
@@ -50,14 +51,6 @@
 #undef MONERO_DEFAULT_LOG_CATEGORY
 #define MONERO_DEFAULT_LOG_CATEGORY "net.p2p.tx"
 
-namespace
-{
-  int get_command_from_message(const cryptonote::blobdata &msg)
-  {
-    return msg.size() >= sizeof(epee::levin::bucket_head2) ? SWAP32LE(((epee::levin::bucket_head2*)msg.data())->m_command) : 0;
-  }
-}
-
 namespace cryptonote
 {
 namespace levin
@@ -105,8 +98,44 @@ namespace levin
       return std::chrono::steady_clock::duration{crypto::rand_range(rep(0), range.count())};
     }
 
-    //! \return Outgoing connections supporting fragments in `connections` filtered by remote blockchain height.
-    std::vector<boost::uuids::uuid> get_out_connections(connections& p2p, uint64_t min_blockchain_height)
+    uint64_t get_median_remote_height(connections& p2p)
+    {
+        std::vector<uint64_t> remote_heights;
+        remote_heights.reserve(connection_id_reserve_size);
+        p2p.foreach_connection([&remote_heights] (detail::p2p_context& context) {
+          if (!context.m_is_income)
+          {
+            remote_heights.emplace_back(context.m_remote_blockchain_height);
+          }
+          return true;
+        });
+
+        if (remote_heights.empty())
+        {
+          return 0;
+        }
+
+        const size_t n = remote_heights.size() / 2;
+        std::sort(remote_heights.begin(), remote_heights.end());
+        if (remote_heights.size() % 2 != 0)
+        {
+          return remote_heights[n];
+        }
+        return remote_heights[n-1];
+    }
+
+    uint64_t get_blockchain_height(connections& p2p, const i_core_events* core)
+    {
+      const uint64_t local_blockchain_height = core->get_current_blockchain_height();
+      if (core->is_synchronized())
+      {
+        return local_blockchain_height;
+      }
+      return std::max(local_blockchain_height, get_median_remote_height(p2p));
+    }
+
+    //! \return Outgoing connections supporting fragments in `connections` filtered by blockchain height.
+    std::vector<boost::uuids::uuid> get_out_connections(connections& p2p, uint64_t blockchain_height)
     {
       std::vector<boost::uuids::uuid> outs;
       outs.reserve(connection_id_reserve_size);
@@ -115,16 +144,22 @@ namespace levin
          the reserve call so a strand is not used. Investigate if there is lots
          of waiting in here. */
 
-      p2p.foreach_connection([&outs, min_blockchain_height] (detail::p2p_context& context) {
-        if (!context.m_is_income && context.m_remote_blockchain_height >= min_blockchain_height)
+      p2p.foreach_connection([&outs, blockchain_height] (detail::p2p_context& context) {
+        if (!context.m_is_income && context.m_remote_blockchain_height >= blockchain_height)
           outs.emplace_back(context.m_connection_id);
         return true;
       });
 
+      MDEBUG("Found " << outs.size() << " out connections having height >= " << blockchain_height);
       return outs;
     }
 
-    std::string make_tx_payload(std::vector<blobdata>&& txs, const bool pad, const bool fluff)
+    std::vector<boost::uuids::uuid> get_out_connections(connections& p2p, const i_core_events* core)
+    {
+      return get_out_connections(p2p, get_blockchain_height(p2p, core));
+    }
+
+    epee::byte_slice make_tx_payload(std::vector<blobdata>&& txs, const bool pad, const bool fluff)
     {
       NOTIFY_NEW_TRANSACTIONS::request request{};
       request.txs = std::move(txs);
@@ -146,7 +181,7 @@ namespace levin
           padding -= overhead;
         request._ = std::string(padding, ' ');
 
-        std::string arg_buff;
+        epee::byte_slice arg_buff;
         epee::serialization::store_t_to_binary(request, arg_buff);
 
         // we probably lowballed the payload size a bit, so added a but too much. Fix this now.
@@ -158,7 +193,7 @@ namespace levin
         // if the size of _ moved enough, we might lose byte in size encoding, we don't care
       }
 
-      std::string fullBlob;
+      epee::byte_slice fullBlob;
       if (!epee::serialization::store_t_to_binary(request, fullBlob))
         throw std::runtime_error{"Failed to serialize to epee binary format"};
 
@@ -167,12 +202,12 @@ namespace levin
 
     bool make_payload_send_txs(connections& p2p, std::vector<blobdata>&& txs, const boost::uuids::uuid& destination, const bool pad, const bool fluff)
     {
-      const cryptonote::blobdata blob = make_tx_payload(std::move(txs), pad, fluff);
+      const epee::byte_slice blob = make_tx_payload(std::move(txs), pad, fluff);
       p2p.for_connection(destination, [&blob](detail::p2p_context& context) {
-        on_levin_traffic(context, true, true, false, blob.size(), get_command_from_message(blob));
+        on_levin_traffic(context, true, true, false, blob.size(), NOTIFY_NEW_TRANSACTIONS::ID);
         return true;
       });
-      return p2p.notify(NOTIFY_NEW_TRANSACTIONS::ID, epee::strspan<std::uint8_t>(blob), destination);
+      return p2p.notify(NOTIFY_NEW_TRANSACTIONS::ID, epee::to_span(blob), destination);
     }
 
     /* The current design uses `asio::strand`s. The documentation isn't as clear
@@ -233,7 +268,7 @@ namespace levin
   {
     struct zone
     {
-      explicit zone(boost::asio::io_service& io_service, std::shared_ptr<connections> p2p, epee::byte_slice noise_in, bool is_public, bool pad_txs)
+      explicit zone(boost::asio::io_service& io_service, std::shared_ptr<connections> p2p, epee::byte_slice noise_in, epee::net_utils::zone zone, bool pad_txs)
         : p2p(std::move(p2p)),
           noise(std::move(noise_in)),
           next_epoch(io_service),
@@ -243,7 +278,7 @@ namespace levin
           channels(),
           connection_count(0),
           flush_callbacks(0),
-          is_public(is_public),
+          nzone(zone),
           pad_txs(pad_txs),
           fluffing(false)
       {
@@ -260,7 +295,7 @@ namespace levin
       std::deque<noise_channel> channels;  //!< Never touch after init; only update elements on `noise_channel.strand`
       std::atomic<std::size_t> connection_count; //!< Only update in strand, can be read at any time
       std::uint32_t flush_callbacks;             //!< Number of active fluff flush callbacks queued
-      const bool is_public;                      //!< Zone is public ipv4/ipv6 connections
+      const epee::net_utils::zone nzone;         //!< Zone is public ipv4/ipv6 connections, or i2p or tor
       const bool pad_txs;                        //!< Pad txs to the next boundary for privacy
       bool fluffing;                             //!< Zone is in Dandelion++ fluff epoch
     };
@@ -297,7 +332,8 @@ namespace levin
         if (!channel.connection.is_nil())
           channel.queue.push_back(std::move(message_));
         else if (destination_ == 0 && zone_->connection_count == 0)
-          MWARNING("Unable to send transaction(s) over anonymity network - no available outbound connections");
+          MWARNING("Unable to send transaction(s) to " << epee::net_utils::zone_to_string(zone_->nzone) <<
+			" - no available outbound connections");
       }
     };
 
@@ -399,7 +435,7 @@ namespace levin
         zone->p2p->foreach_connection([txs, now, &zone, &source, &in_duration, &out_duration, &next_flush] (detail::p2p_context& context)
         {
           // When i2p/tor, only fluff to outbound connections
-          if (source != context.m_connection_id && (zone->is_public || !context.m_is_income))
+          if (context.handshake_complete() && source != context.m_connection_id && (zone->nzone == epee::net_utils::zone::public_ || !context.m_is_income))
           {
             if (context.fluff_txs.empty())
               context.flush_time = now + (context.m_is_income ? in_duration() : out_duration());
@@ -526,7 +562,7 @@ namespace levin
             }
 
             // connection list may be outdated, try again
-            update_channels::run(zone_, get_out_connections(*zone_->p2p, core_->get_target_blockchain_height()));
+            update_channels::run(zone_, get_out_connections(*zone_->p2p, core_));
           }
 
           MERROR("Unable to send transaction(s) via Dandelion++ stem");
@@ -562,7 +598,7 @@ namespace levin
 
         assert(zone_->strand.running_in_this_thread());
 
-        if (zone_->is_public)
+        if (zone_->nzone == epee::net_utils::zone::public_)
           MDEBUG("Starting new Dandelion++ epoch: " << (fluffing_ ? "fluff" : "stem"));
 
         zone_->map = std::move(map_);
@@ -630,10 +666,12 @@ namespace levin
           {
             channel.active = nullptr;
             channel.connection = boost::uuids::nil_uuid();
+            auto height = get_blockchain_height(*zone_->p2p, core_);
 
-            auto connections = get_out_connections(*zone_->p2p, core_->get_target_blockchain_height());
+            auto connections = get_out_connections(*zone_->p2p, height);
             if (connections.empty())
-              MWARNING("Lost all outbound connections to anonymity network - currently unable to send transaction(s)");
+              MWARNING("Unable to send transaction(s) to " << epee::net_utils::zone_to_string(zone_->nzone) <<
+			" - no suitable outbound connections at height " << height);
 
             zone_->strand.post(update_channels{zone_, std::move(connections)});
           }
@@ -664,7 +702,7 @@ namespace levin
 
         const bool fluffing = crypto::rand_idx(unsigned(100)) < CRYPTONOTE_DANDELIONPP_FLUFF_PROBABILITY;
         const auto start = std::chrono::steady_clock::now();
-        auto connections = get_out_connections(*(zone_->p2p), core_->get_target_blockchain_height());
+        auto connections = get_out_connections(*(zone_->p2p), core_);
         zone_->strand.dispatch(
           change_channels{zone_, net::dandelionpp::connection_map{std::move(connections), count_}, fluffing}
         );
@@ -676,15 +714,15 @@ namespace levin
     };
   } // anonymous
 
-  notify::notify(boost::asio::io_service& service, std::shared_ptr<connections> p2p, epee::byte_slice noise, const bool is_public, const bool pad_txs, i_core_events& core)
-    : zone_(std::make_shared<detail::zone>(service, std::move(p2p), std::move(noise), is_public, pad_txs))
+  notify::notify(boost::asio::io_service& service, std::shared_ptr<connections> p2p, epee::byte_slice noise, epee::net_utils::zone zone, const bool pad_txs, i_core_events& core)
+    : zone_(std::make_shared<detail::zone>(service, std::move(p2p), std::move(noise), zone, pad_txs))
     , core_(std::addressof(core))
   {
     if (!zone_->p2p)
       throw std::logic_error{"cryptonote::levin::notify cannot have nullptr p2p argument"};
 
     const bool noise_enabled = !zone_->noise.empty();
-    if (noise_enabled || is_public)
+    if (noise_enabled || zone == epee::net_utils::zone::public_)
     {
       const auto now = std::chrono::steady_clock::now();
       const auto min_epoch = noise_enabled ? noise_min_epoch : dandelionpp_min_epoch;
@@ -715,7 +753,7 @@ namespace levin
       return;
 
     zone_->strand.dispatch(
-      update_channels{zone_, get_out_connections(*(zone_->p2p), core_->get_target_blockchain_height())}
+      update_channels{zone_, get_out_connections(*(zone_->p2p), core_)}
     );
   }
 
@@ -778,9 +816,9 @@ namespace levin
 
       // Padding is not useful when using noise mode. Send as stem so receiver
       // forwards in Dandelion++ mode.
-      const std::string payload = make_tx_payload(std::move(txs), false, false);
+      const epee::byte_slice payload = make_tx_payload(std::move(txs), false, false);
       epee::byte_slice message = epee::levin::make_fragmented_notify(
-        zone_->noise, NOTIFY_NEW_TRANSACTIONS::ID, epee::strspan<std::uint8_t>(payload)
+        zone_->noise, NOTIFY_NEW_TRANSACTIONS::ID, epee::to_span(payload)
       );
       if (CRYPTONOTE_MAX_FRAGMENTS * zone_->noise.size() < message.size())
       {
@@ -806,7 +844,7 @@ namespace levin
         case relay_method::stem:
         case relay_method::forward:
         case relay_method::local:
-          if (zone_->is_public)
+          if (zone_->nzone == epee::net_utils::zone::public_)
           {
             // this will change a local/forward tx to stem or fluff ...
             zone_->strand.dispatch(

src/cryptonote_protocol/levin_notify.h

@@ -85,7 +85,7 @@ namespace levin
     {}
 
     //! Construct an instance with available notification `zones`.
-    explicit notify(boost::asio::io_service& service, std::shared_ptr<connections> p2p, epee::byte_slice noise, bool is_public, bool pad_txs, i_core_events& core);
+    explicit notify(boost::asio::io_service& service, std::shared_ptr<connections> p2p, epee::byte_slice noise, epee::net_utils::zone zone, bool pad_txs, i_core_events& core);
 
     notify(const notify&) = delete;
     notify(notify&&) = default;

src/daemon/command_parser_executor.cpp

@@ -28,6 +28,7 @@
 
 #include "common/dns_utils.h"
 #include "common/command_line.h"
+#include "net/parse.h"
 #include "daemon/command_parser_executor.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
@@ -50,7 +51,7 @@ bool t_command_parser_executor::print_peer_list(const std::vector<std::string>&
 {
   if (args.size() > 3)
   {
-    std::cout << "use: print_pl [white] [gray] [<limit>] [pruned] [publicrpc]" << std::endl;
+    std::cout << "Invalid syntax: Too many parameters. For more details, use the help command." << std::endl;
     return true;
   }
 
@@ -79,7 +80,7 @@ bool t_command_parser_executor::print_peer_list(const std::vector<std::string>&
     }
     else if (!epee::string_tools::get_xtype_from_string(limit, args[i]))
     {
-      std::cout << "unexpected argument: " << args[i] << std::endl;
+      std::cout << "Invalid syntax: Unexpected parameter: " << args[i] << ". For more details, use the help command." << std::endl;
       return true;
     }
   }
@@ -90,56 +91,79 @@ bool t_command_parser_executor::print_peer_list(const std::vector<std::string>&
 
 bool t_command_parser_executor::print_peer_list_stats(const std::vector<std::string>& args)
 {
-  if (!args.empty()) return false;
+  if (!args.empty()) {
+    std::cout << "Invalid syntax: No parameters expected. For more details, use the help command." << std::endl;
+    return true;
+  }
 
   return m_executor.print_peer_list_stats();
 }
 
 bool t_command_parser_executor::save_blockchain(const std::vector<std::string>& args)
 {
-  if (!args.empty()) return false;
-
+  if (!args.empty()) {
+    std::cout << "Invalid syntax: No parameters expected. For more details, use the help command." << std::endl;
+    return true;
+  }
   return m_executor.save_blockchain();
 }
 
 bool t_command_parser_executor::show_hash_rate(const std::vector<std::string>& args)
 {
-  if (!args.empty()) return false;
+  if (!args.empty()) {
+    std::cout << "Invalid syntax: No parameters expected. For more details, use the help command." << std::endl;
+    return true;
+  }
 
   return m_executor.show_hash_rate();
 }
 
 bool t_command_parser_executor::hide_hash_rate(const std::vector<std::string>& args)
 {
-  if (!args.empty()) return false;
+  if (!args.empty()) {
+    std::cout << "Invalid syntax: No parameters expected. For more details, use the help command." << std::endl;
+    return true;
+  }
 
   return m_executor.hide_hash_rate();
 }
 
 bool t_command_parser_executor::show_difficulty(const std::vector<std::string>& args)
 {
-  if (!args.empty()) return false;
+  if (!args.empty()) {
+    std::cout << "Invalid syntax: No parameters expected. For more details, use the help command." << std::endl;
+    return true;
+  }
 
   return m_executor.show_difficulty();
 }
 
 bool t_command_parser_executor::show_status(const std::vector<std::string>& args)
 {
-  if (!args.empty()) return false;
+  if (!args.empty()) {
+    std::cout << "Invalid syntax: No parameters expected. For more details, use the help command." << std::endl;
+    return true;
+  }
 
   return m_executor.show_status();
 }
 
 bool t_command_parser_executor::print_connections(const std::vector<std::string>& args)
 {
-  if (!args.empty()) return false;
+  if (!args.empty()) {
+    std::cout << "Invalid syntax: No parameters expected. For more details, use the help command." << std::endl;
+    return true;
+  }
 
   return m_executor.print_connections();
 }
 
 bool t_command_parser_executor::print_net_stats(const std::vector<std::string>& args)
 {
-  if (!args.empty()) return false;
+  if (!args.empty()) {
+    std::cout << "Invalid syntax: No parameters expected. For more details, use the help command." << std::endl;
+    return true;
+  }
 
   return m_executor.print_net_stats();
 }
@@ -148,8 +172,8 @@ bool t_command_parser_executor::print_blockchain_info(const std::vector<std::str
 {
   if(!args.size())
   {
-    std::cout << "need block index parameter" << std::endl;
-    return false;
+    std::cout << "Invalid syntax: At least one parameter expected. For more details, use the help command." << std::endl;
+    return true;
   }
   uint64_t start_index = 0;
   uint64_t end_index = 0;
@@ -158,20 +182,20 @@ bool t_command_parser_executor::print_blockchain_info(const std::vector<std::str
     int64_t nblocks;
     if(!epee::string_tools::get_xtype_from_string(nblocks, args[0]))
     {
-      std::cout << "wrong number of blocks" << std::endl;
-      return false;
+      std::cout << "Invalid syntax: Wrong number of blocks. For more details, use the help command." << std::endl;
+      return true;
     }
     return m_executor.print_blockchain_info(nblocks, (uint64_t)-nblocks);
   }
   if(!epee::string_tools::get_xtype_from_string(start_index, args[0]))
   {
-    std::cout << "wrong starter block index parameter" << std::endl;
-    return false;
+    std::cout << "Invalid syntax: Wrong starter block index parameter. For more details, use the help command." << std::endl;
+    return true;
   }
   if(args.size() >1 && !epee::string_tools::get_xtype_from_string(end_index, args[1]))
   {
-    std::cout << "wrong end block index parameter" << std::endl;
-    return false;
+    std::cout << "Invalid syntax: Wrong end block index parameter. For more details, use the help command." << std::endl;
+    return true;
   }
 
   return m_executor.print_blockchain_info(start_index, end_index);
@@ -181,7 +205,7 @@ bool t_command_parser_executor::set_log_level(const std::vector<std::string>& ar
 {
   if(args.size() > 1)
   {
-    std::cout << "use: set_log [<log_level_number_0-4> | <categories>]" << std::endl;
+    std::cout << "Invalid syntax: Too many parameters. For more details, use the help command." << std::endl;
     return true;
   }
 
@@ -195,7 +219,7 @@ bool t_command_parser_executor::set_log_level(const std::vector<std::string>& ar
   {
     if(4 < l)
     {
-      std::cout << "wrong number range, use: set_log <log_level_number_0-4>" << std::endl;
+      std::cout << "Invalid syntax: Wrong number range, use: set_log <log_level_number_0-4>. For more details, use the help command." << std::endl;
       return true;
     }
     return m_executor.set_log_level(l);
@@ -208,7 +232,10 @@ bool t_command_parser_executor::set_log_level(const std::vector<std::string>& ar
 
 bool t_command_parser_executor::print_height(const std::vector<std::string>& args) 
 {
-  if (!args.empty()) return false;
+  if (!args.empty()) {
+    std::cout << "Invalid syntax: No parameters expected. For more details, use the help command." << std::endl;
+    return true;
+  }
 
   return m_executor.print_height();
 }
@@ -223,14 +250,14 @@ bool t_command_parser_executor::print_block(const std::vector<std::string>& args
       include_hex = true;
     else
     {
-      std::cout << "unexpected argument: " << args[i] << std::endl;
+      std::cout << "Invalid syntax: Unexpected parameter: " << args[i] << ". For more details, use the help command." << std::endl;
       return true;
     }
   }
   if (args.empty())
   {
-    std::cout << "expected: print_block (<block_hash> | <block_height>) [+hex]" << std::endl;
-    return false;
+    std::cout << "Invalid syntax: At least one parameter expected. For more details, use the help command." << std::endl;
+    return true;
   }
 
   const std::string& arg = args.front();
@@ -248,7 +275,7 @@ bool t_command_parser_executor::print_block(const std::vector<std::string>& args
     }
   }
 
-  return false;
+  return true;
 }
 
 bool t_command_parser_executor::print_transaction(const std::vector<std::string>& args)
@@ -267,13 +294,13 @@ bool t_command_parser_executor::print_transaction(const std::vector<std::string>
       include_json = true;
     else
     {
-      std::cout << "unexpected argument: " << args[i] << std::endl;
+      std::cout << "Invalid syntax: Unexpected parameter: " << args[i] << ". For more details, use the help command." << std::endl;
       return true;
     }
   }
   if (args.empty())
   {
-    std::cout << "expected: print_tx <transaction_hash> [+meta] [+hex] [+json]" << std::endl;
+    std::cout << "Invalid syntax: At least one parameter expected. For more details, use the help command." << std::endl;
     return true;
   }
 
@@ -291,7 +318,7 @@ bool t_command_parser_executor::is_key_image_spent(const std::vector<std::string
 {
   if (args.empty())
   {
-    std::cout << "expected: is_key_image_spent <key_image>" << std::endl;
+    std::cout << "Invalid syntax: At least one parameter expected. For more details, use the help command." << std::endl;
     return true;
   }
 
@@ -309,21 +336,30 @@ bool t_command_parser_executor::is_key_image_spent(const std::vector<std::string
 
 bool t_command_parser_executor::print_transaction_pool_long(const std::vector<std::string>& args)
 {
-  if (!args.empty()) return false;
+  if (!args.empty()) {
+    std::cout << "Invalid syntax: No parameters expected. For more details, use the help command." << std::endl;
+    return true;
+  }
 
   return m_executor.print_transaction_pool_long();
 }
 
 bool t_command_parser_executor::print_transaction_pool_short(const std::vector<std::string>& args)
 {
-  if (!args.empty()) return false;
+  if (!args.empty()) {
+    std::cout << "Invalid syntax: No parameters expected. For more details, use the help command." << std::endl;
+    return true;
+  }
 
   return m_executor.print_transaction_pool_short();
 }
 
 bool t_command_parser_executor::print_transaction_pool_stats(const std::vector<std::string>& args)
 {
-  if (!args.empty()) return false;
+  if (!args.empty()) {
+    std::cout << "Invalid syntax: No parameters expected. For more details, use the help command." << std::endl;
+    return true;
+  }
 
   return m_executor.print_transaction_pool_stats();
 }
@@ -332,7 +368,7 @@ bool t_command_parser_executor::start_mining(const std::vector<std::string>& arg
 {
   if(!args.size())
   {
-    std::cout << "Please specify a wallet address to mine for: start_mining <addr> [<threads>|auto]" << std::endl;
+    std::cout << "Invalid syntax: At least one parameter expected. For more details, use the help command." << std::endl;
     return true;
   }
 
@@ -353,7 +389,7 @@ bool t_command_parser_executor::start_mining(const std::vector<std::string>& arg
           {
             if(!cryptonote::get_account_address_from_str(info, cryptonote::STAGENET, address_str))
             {
-              std::cout << "target account address has wrong format" << std::endl;
+              std::cout << "Invalid syntax: Target account address has wrong format. For more details, use the help command." << std::endl;
               return true;
             }
             else
@@ -389,9 +425,10 @@ bool t_command_parser_executor::start_mining(const std::vector<std::string>& arg
   bool ignore_battery = false;  
   if(args.size() > 4)
   {
-    return false;
+    std::cout << "Invalid syntax: Too many parameters. For more details, use the help command." << std::endl;
+    return true;
   }
-  
+
   if(args.size() == 4)
   {
     if(args[3] == "true" || command_line::is_yes(args[3]) || args[3] == "1")
@@ -400,10 +437,11 @@ bool t_command_parser_executor::start_mining(const std::vector<std::string>& arg
     }
     else if(args[3] != "false" && !command_line::is_no(args[3]) && args[3] != "0")
     {
-      return false;
+      std::cout << "Invalid syntax: Invalid combination of parameters. For more details, use the help command." << std::endl;
+      return true;
     }
-  }  
-  
+  }
+
   if(args.size() >= 3)
   {
     if(args[2] == "true" || command_line::is_yes(args[2]) || args[2] == "1")
@@ -412,10 +450,11 @@ bool t_command_parser_executor::start_mining(const std::vector<std::string>& arg
     }
     else if(args[2] != "false" && !command_line::is_no(args[2]) && args[2] != "0")
     {
-      return false;
+      std::cout << "Invalid syntax: Invalid combination of parameters. For more details, use the help command." << std::endl;
+      return true;
     }
   }
-  
+
   if(args.size() >= 2)
   {
     if (args[1] == "auto" || args[1] == "autodetect")
@@ -436,7 +475,10 @@ bool t_command_parser_executor::start_mining(const std::vector<std::string>& arg
 
 bool t_command_parser_executor::stop_mining(const std::vector<std::string>& args)
 {
-  if (!args.empty()) return false;
+  if (!args.empty()) {
+    std::cout << "Invalid syntax: No parameters expected. For more details, use the help command." << std::endl;
+    return true;
+  }
 
   return m_executor.stop_mining();
 }
@@ -448,21 +490,31 @@ bool t_command_parser_executor::mining_status(const std::vector<std::string>& ar
 
 bool t_command_parser_executor::stop_daemon(const std::vector<std::string>& args)
 {
-  if (!args.empty()) return false;
+  if (!args.empty()) {
+    std::cout << "Invalid syntax: No parameters expected. For more details, use the help command." << std::endl;
+    return true;
+  }
 
   return m_executor.stop_daemon();
 }
 
 bool t_command_parser_executor::print_status(const std::vector<std::string>& args)
 {
-  if (!args.empty()) return false;
+  if (!args.empty()) {
+    std::cout << "Invalid syntax: No parameters expected. For more details, use the help command." << std::endl;
+    return true;
+  }
 
   return m_executor.print_status();
 }
 
 bool t_command_parser_executor::set_limit(const std::vector<std::string>& args)
 {
-  if(args.size()>1) return false;
+  if(args.size()>1) {
+    std::cout << "Invalid syntax: Too many parameters. For more details, use the help command." << std::endl;
+    return true;
+  }
+
   if(args.size()==0) {
     return m_executor.get_limit();
   }
@@ -471,8 +523,8 @@ bool t_command_parser_executor::set_limit(const std::vector<std::string>& args)
       limit = std::stoll(args[0]);
   }
   catch(const std::exception& ex) {
-      std::cout << "failed to parse argument" << std::endl;
-      return false;
+      std::cout << "Invalid syntax: Failed to parse limit. For more details, use the help command." << std::endl;
+      return true;
   }
 
   return m_executor.set_limit(limit, limit);
@@ -480,7 +532,11 @@ bool t_command_parser_executor::set_limit(const std::vector<std::string>& args)
 
 bool t_command_parser_executor::set_limit_up(const std::vector<std::string>& args)
 {
-  if(args.size()>1) return false;
+  if(args.size()>1) {
+    std::cout << "Invalid syntax: Too many parameters. For more details, use the help command." << std::endl;
+    return true;
+  }
+
   if(args.size()==0) {
     return m_executor.get_limit_up();
   }
@@ -489,8 +545,8 @@ bool t_command_parser_executor::set_limit_up(const std::vector<std::string>& arg
       limit = std::stoll(args[0]);
   }
   catch(const std::exception& ex) {
-      std::cout << "failed to parse argument" << std::endl;
-      return false;
+      std::cout << "Invalid syntax: Failed to parse limit. For more details, use the help command." << std::endl;
+      return true;
   }
 
   return m_executor.set_limit(0, limit);
@@ -498,7 +554,11 @@ bool t_command_parser_executor::set_limit_up(const std::vector<std::string>& arg
 
 bool t_command_parser_executor::set_limit_down(const std::vector<std::string>& args)
 {
-  if(args.size()>1) return false;
+  if(args.size()>1) {
+    std::cout << "Invalid syntax: Too many parameters. For more details, use the help command." << std::endl;
+    return true;
+  }
+
   if(args.size()==0) {
     return m_executor.get_limit_down();
   }
@@ -507,8 +567,8 @@ bool t_command_parser_executor::set_limit_down(const std::vector<std::string>& a
       limit = std::stoll(args[0]);
   }
   catch(const std::exception& ex) {
-      std::cout << "failed to parse argument" << std::endl;
-      return false;
+      std::cout << "Invalid syntax: Failed to parse limit. For more details, use the help command." << std::endl;
+      return true;
   }
 
   return m_executor.set_limit(limit, 0);
@@ -525,12 +585,13 @@ bool t_command_parser_executor::out_peers(const std::vector<std::string>& args)
 			set = true;
 		}
 	}
-	  
+
 	catch(const std::exception& ex) {
 		_erro("stoi exception");
-		return false;
+		std::cout << "Invalid syntax: Failed to parse number. For more details, use the help command." << std::endl;
+		return true;
 	}
-	
+
 	return m_executor.out_peers(set, limit);
 }
 
@@ -548,7 +609,8 @@ bool t_command_parser_executor::in_peers(const std::vector<std::string>& args)
 
 	catch(const std::exception& ex) {
 		_erro("stoi exception");
-		return false;
+		std::cout << "Invalid syntax: Failed to parse number." << std::endl;
+		return true;
 	}
 
 	return m_executor.in_peers(set, limit);
@@ -565,27 +627,37 @@ bool t_command_parser_executor::hard_fork_info(const std::vector<std::string>& a
       version = std::stoi(args[0]);
     }
     catch(const std::exception& ex) {
-        return false;
+        std::cout << "Invalid syntax: Failed to parse version number. For more details, use the help command." << std::endl;
+        return true;
+    }
+    if (version <= 0 || version > 255) {
+      std::cout << "Invalid syntax: Unknown version number. Must be between 0 and 255. For more details, use the help command." << std::endl;
+      return true;
     }
-    if (version <= 0 || version > 255)
-      return false;
   }
   else {
-    return false;
+    std::cout << "Invalid syntax: Too many parameters. For more details, use the help command." << std::endl;
+    return true;
   }
   return m_executor.hard_fork_info(version);
 }
 
 bool t_command_parser_executor::show_bans(const std::vector<std::string>& args)
 {
-  if (!args.empty()) return false;
+  if (!args.empty()) {
+    std::cout << "Invalid syntax: No parameters expected. For more details, use the help command." << std::endl;
+    return true;
+  }
+
   return m_executor.print_bans();
 }
 
 bool t_command_parser_executor::ban(const std::vector<std::string>& args)
 {
-  if (args.size() != 1 && args.size() != 2) return false;
-  std::string ip = args[0];
+  if (args.size() != 1 && args.size() != 2) {
+    std::cout << "Invalid syntax: Expects one or two parameters. For more details, use the help command." << std::endl;
+    return true;
+  }
   time_t seconds = P2P_IP_BLOCKTIME;
   if (args.size() > 1)
   {
@@ -595,33 +667,89 @@ bool t_command_parser_executor::ban(const std::vector<std::string>& args)
     }
     catch (const std::exception &e)
     {
-      return false;
+      std::cout << "Invalid syntax: Failed to parse seconds. For more details, use the help command." << std::endl;
+      return true;
     }
     if (seconds == 0)
     {
+      std::cout << "Seconds must be greater than 0." << std::endl;
+      return true;
+    }
+  }
+  if (boost::starts_with(args[0], "@"))
+  {
+    const std::string ban_list = args[0].substr(1);
+
+    try
+    {
+      const boost::filesystem::path ban_list_path(ban_list);
+      boost::system::error_code ec;
+      if (!boost::filesystem::exists(ban_list_path, ec))
+      {
+        std::cout << "Can't find ban list file " + ban_list + " - " + ec.message() << std::endl;
+        return true;
+      }
+
+      bool ret = true;
+      std::ifstream ifs(ban_list_path.string());
+      for (std::string line; std::getline(ifs, line); )
+      {
+        auto subnet = net::get_ipv4_subnet_address(line);
+        if (subnet)
+        {
+          ret &= m_executor.ban(subnet->str(), seconds);
+          continue;
+        }
+        const expect<epee::net_utils::network_address> parsed_addr = net::get_network_address(line, 0);
+        if (parsed_addr)
+        {
+          ret &= m_executor.ban(parsed_addr->host_str(), seconds);
+          continue;
+        }
+        std::cout << "Invalid IP address or IPv4 subnet: " << line << std::endl;
+      }
+      return ret;
+    }
+    catch (const std::exception &e)
+    {
+      std::cout << "Error loading ban list: " << e.what() << std::endl;
       return false;
     }
   }
-  return m_executor.ban(ip, seconds);
+  else
+  {
+    const std::string ip = args[0];
+    return m_executor.ban(ip, seconds);
+  }
 }
 
 bool t_command_parser_executor::unban(const std::vector<std::string>& args)
 {
-  if (args.size() != 1) return false;
+  if (args.size() != 1) {
+    std::cout << "Invalid syntax: One parameter expected. For more details, use the help command." << std::endl;
+    return true;
+  }
+
   std::string ip = args[0];
   return m_executor.unban(ip);
 }
 
 bool t_command_parser_executor::banned(const std::vector<std::string>& args)
 {
-  if (args.size() != 1) return false;
+  if (args.size() != 1) {
+    std::cout << "Invalid syntax: One parameter expected. For more details, use the help command." << std::endl;
+    return true;
+  }
   std::string address = args[0];
   return m_executor.banned(address);
 }
 
 bool t_command_parser_executor::flush_txpool(const std::vector<std::string>& args)
 {
-  if (args.size() > 1) return false;
+  if (args.size() > 1) {
+    std::cout << "Invalid syntax: Too many parameters. For more details, use the help command." << std::endl;
+    return true;
+  }
 
   std::string txid;
   if (args.size() == 1)
@@ -629,7 +757,7 @@ bool t_command_parser_executor::flush_txpool(const std::vector<std::string>& arg
     crypto::hash hash;
     if (!parse_hash256(args[0], hash))
     {
-      std::cout << "failed to parse tx id" << std::endl;
+      std::cout << "Invalid syntax: Failed to parse tx id. For more details, use the help command." << std::endl;
       return true;
     }
     txid = args[0];
@@ -662,7 +790,7 @@ bool t_command_parser_executor::output_histogram(const std::vector<std::string>&
     }
     else
     {
-      std::cout << "Invalid syntax: more than two non-amount parameters" << std::endl;
+      std::cout << "Invalid syntax: More than two non-amount parameters. For more details, use the help command." << std::endl;
       return true;
     }
   }
@@ -673,20 +801,21 @@ bool t_command_parser_executor::print_coinbase_tx_sum(const std::vector<std::str
 {
   if(!args.size())
   {
-    std::cout << "need block height parameter" << std::endl;
-    return false;
+    std::cout << "Invalid syntax: At least one parameter expected. For more details, use the help command." << std::endl;
+    return true;
   }
+
   uint64_t height = 0;
   uint64_t count = 0;
   if(!epee::string_tools::get_xtype_from_string(height, args[0]))
   {
-    std::cout << "wrong starter block height parameter" << std::endl;
-    return false;
+    std::cout << "Invalid syntax: Wrong starter block height parameter. For more details, use the help command." << std::endl;
+    return true;
   }
   if(args.size() >1 && !epee::string_tools::get_xtype_from_string(count, args[1]))
   {
     std::cout << "wrong count parameter" << std::endl;
-    return false;
+    return true;
   }
 
   return m_executor.print_coinbase_tx_sum(height, count);
@@ -696,8 +825,8 @@ bool t_command_parser_executor::alt_chain_info(const std::vector<std::string>& a
 {
   if(args.size() > 1)
   {
-    std::cout << "usage: alt_chain_info [block_hash|>N|-N]" << std::endl;
-    return false;
+    std::cout << "Invalid syntax: Too many parameters. For more details, use the help command." << std::endl;
+    return true;
   }
 
   std::string tip;
@@ -709,16 +838,16 @@ bool t_command_parser_executor::alt_chain_info(const std::vector<std::string>& a
     {
       if (!epee::string_tools::get_xtype_from_string(above, args[0].c_str() + 1))
       {
-        std::cout << "invalid above parameter" << std::endl;
-        return false;
+        std::cout << "Invalid syntax: Invalid above parameter. For more details, use the help command." << std::endl;
+        return true;
       }
     }
     else if (args[0].size() > 0 && args[0][0] == '-')
     {
       if (!epee::string_tools::get_xtype_from_string(last_blocks, args[0].c_str() + 1))
       {
-        std::cout << "invalid last_blocks parameter" << std::endl;
-        return false;
+        std::cout << "Invalid syntax: Invalid last_blocks parameter. For more details, use the help command." << std::endl;
+        return true;
       }
     }
     else
@@ -734,15 +863,15 @@ bool t_command_parser_executor::print_blockchain_dynamic_stats(const std::vector
 {
   if(args.size() != 1)
   {
-    std::cout << "Exactly one parameter is needed" << std::endl;
-    return false;
+    std::cout << "Invalid syntax: One parameter expected. For more details, use the help command." << std::endl;
+    return true;
   }
 
   uint64_t nblocks = 0;
   if(!epee::string_tools::get_xtype_from_string(nblocks, args[0]) || nblocks == 0)
   {
-    std::cout << "wrong number of blocks" << std::endl;
-    return false;
+    std::cout << "Invalid syntax: Wrong number of blocks. For more details, use the help command." << std::endl;
+    return true;
   }
 
   return m_executor.print_blockchain_dynamic_stats(nblocks);
@@ -750,10 +879,10 @@ bool t_command_parser_executor::print_blockchain_dynamic_stats(const std::vector
 
 bool t_command_parser_executor::update(const std::vector<std::string>& args)
 {
-  if(args.size() != 1)
+  if (args.size() != 1)
   {
-    std::cout << "Exactly one parameter is needed: check, download, or update" << std::endl;
-    return false;
+    std::cout << "Invalid syntax: One parameter expected. For more details, use the help command." << std::endl;
+    return true;
   }
 
   return m_executor.update(args.front());
@@ -761,13 +890,17 @@ bool t_command_parser_executor::update(const std::vector<std::string>& args)
 
 bool t_command_parser_executor::relay_tx(const std::vector<std::string>& args)
 {
-  if (args.size() != 1) return false;
+  if (args.size() != 1)
+  {
+    std::cout << "Invalid syntax: One parameter expected. For more details, use the help command." << std::endl;
+    return true;
+  }
 
   std::string txid;
   crypto::hash hash;
   if (!parse_hash256(args[0], hash))
   {
-    std::cout << "failed to parse tx id" << std::endl;
+    std::cout << "Invalid syntax: Failed to parse tx id. For more details, use the help command." << std::endl;
     return true;
   }
   txid = args[0];
@@ -776,7 +909,10 @@ bool t_command_parser_executor::relay_tx(const std::vector<std::string>& args)
 
 bool t_command_parser_executor::sync_info(const std::vector<std::string>& args)
 {
-  if (args.size() != 0) return false;
+  if (args.size() != 0) {
+    std::cout << "Invalid syntax: No parameters expected. For more details, use the help command." << std::endl;
+    return true;
+  }
 
   return m_executor.sync_info();
 }
@@ -785,8 +921,8 @@ bool t_command_parser_executor::pop_blocks(const std::vector<std::string>& args)
 {
   if (args.size() != 1)
   {
-    std::cout << "Exactly one parameter is needed" << std::endl;
-    return false;
+    std::cout << "Invalid syntax: One parameter expected. For more details, use the help command." << std::endl;
+    return true;
   }
 
   try
@@ -794,21 +930,24 @@ bool t_command_parser_executor::pop_blocks(const std::vector<std::string>& args)
     uint64_t nblocks = boost::lexical_cast<uint64_t>(args[0]);
     if (nblocks < 1)
     {
-      std::cout << "number of blocks must be greater than 0" << std::endl;
-      return false;
+      std::cout << "Invalid syntax: Number of blocks must be greater than 0. For more details, use the help command." << std::endl;
+      return true;
     }
     return m_executor.pop_blocks(nblocks);
   }
   catch (const boost::bad_lexical_cast&)
   {
-    std::cout << "number of blocks must be a number greater than 0" << std::endl;
+    std::cout << "Invalid syntax: Number of blocks must be a number greater than 0. For more details, use the help command." << std::endl;
   }
-  return false;
+  return true;
 }
 
 bool t_command_parser_executor::rpc_payments(const std::vector<std::string>& args)
 {
-  if (args.size() != 0) return false;
+  if (args.size() != 0) {
+    std::cout << "Invalid syntax: No parameters expected. For more details, use the help command." << std::endl;
+    return true;
+  }
 
   return m_executor.rpc_payments();
 }
@@ -820,7 +959,11 @@ bool t_command_parser_executor::version(const std::vector<std::string>& args)
 
 bool t_command_parser_executor::prune_blockchain(const std::vector<std::string>& args)
 {
-  if (args.size() > 1) return false;
+  if (args.size() > 1)
+  {
+    std::cout << "Invalid syntax: Too many parameters. For more details, use the help command." << std::endl;
+    return true;
+  }
 
   if (args.empty() || args[0] != "confirm")
   {
@@ -846,7 +989,8 @@ bool t_command_parser_executor::set_bootstrap_daemon(const std::vector<std::stri
   const size_t args_count = args.size();
   if (args_count < 1 || args_count > 3)
   {
-    return false;
+    std::cout << "Invalid syntax: Wrong number of parameters. For more details, use the help command." << std::endl;
+    return true;
   }
 
   return m_executor.set_bootstrap_daemon(

src/daemon/command_server.cpp

@@ -223,7 +223,8 @@ t_command_server::t_command_server(
     m_command_lookup.set_handler(
       "hard_fork_info"
     , std::bind(&t_command_parser_executor::hard_fork_info, &m_parser, p::_1)
-    , "Print the hard fork voting information."
+    , "hard_fork_info <version>"
+    , "Print the hard fork voting information. If given a version, prints whether is this version enabled."
     );
     m_command_lookup.set_handler(
       "bans"
@@ -233,8 +234,8 @@ t_command_server::t_command_server(
     m_command_lookup.set_handler(
       "ban"
     , std::bind(&t_command_parser_executor::ban, &m_parser, p::_1)
-    , "ban <IP> [<seconds>]"
-    , "Ban a given <IP> for a given amount of <seconds>."
+    , "ban [<IP>|@<filename>] [<seconds>]"
+    , "Ban a given <IP> or list of IPs from a file for a given amount of <seconds>."
     );
     m_command_lookup.set_handler(
       "unban"
@@ -314,6 +315,7 @@ t_command_server::t_command_server(
     m_command_lookup.set_handler(
       "prune_blockchain"
     , std::bind(&t_command_parser_executor::prune_blockchain, &m_parser, p::_1)
+    , "prune_blockchain [confirm]"
     , "Prune the blockchain."
     );
     m_command_lookup.set_handler(

src/daemon/main.cpp

@@ -66,10 +66,12 @@ uint16_t parse_public_rpc_port(const po::variables_map &vm)
   }
 
   std::string rpc_port_str;
+  std::string rpc_bind_address = command_line::get_arg(vm, cryptonote::rpc_args::descriptors().rpc_bind_ip);
   const auto &restricted_rpc_port = cryptonote::core_rpc_server::arg_rpc_restricted_bind_port;
   if (!command_line::is_arg_defaulted(vm, restricted_rpc_port))
   {
     rpc_port_str = command_line::get_arg(vm, restricted_rpc_port);
+    rpc_bind_address = command_line::get_arg(vm, cryptonote::rpc_args::descriptors().rpc_restricted_bind_ip);
   }
   else if (command_line::get_arg(vm, cryptonote::core_rpc_server::arg_restricted_rpc))
   {
@@ -86,7 +88,6 @@ uint16_t parse_public_rpc_port(const po::variables_map &vm)
     throw std::runtime_error("invalid RPC port " + rpc_port_str);
   }
 
-  const auto rpc_bind_address = command_line::get_arg(vm, cryptonote::rpc_args::descriptors().rpc_bind_ip);
   const auto address = net::get_network_address(rpc_bind_address, rpc_port);
   if (!address) {
     throw std::runtime_error("failed to parse RPC bind address");

src/daemon/rpc_command_executor.cpp

@@ -191,6 +191,9 @@ bool t_rpc_command_executor::print_peer_list(bool white, bool gray, size_t limit
   cryptonote::COMMAND_RPC_GET_PEER_LIST::response res;
 
   std::string failure_message = "Couldn't retrieve peer list";
+
+  req.include_blocked = true;
+
   if (m_is_rpc)
   {
     if (!m_rpc_client->rpc_request(req, res, "/get_peer_list", failure_message.c_str()))
@@ -237,6 +240,7 @@ bool t_rpc_command_executor::print_peer_list_stats() {
   std::string failure_message = "Couldn't retrieve peer list";
 
   req.public_only = false;
+  req.include_blocked = true;
 
   if (m_is_rpc)
   {
@@ -2271,6 +2275,7 @@ bool t_rpc_command_executor::sync_info()
       tools::success_msg_writer() << "Next needed pruning seed: " << res.next_needed_pruning_seed;
 
     tools::success_msg_writer() << std::to_string(res.peers.size()) << " peers";
+    tools::success_msg_writer() << "Remote Host                        Peer_ID   State   Prune_Seed          Height  DL kB/s, Queued Blocks / MB";
     for (const auto &p: res.peers)
     {
       std::string address = epee::string_tools::pad_string(p.info.address, 24);

src/p2p/net_node.cpp

@@ -149,6 +149,7 @@ namespace nodetool
     const command_line::arg_descriptor<std::string> arg_ban_list = {"ban-list", "Specify ban list file, one IP address per line"};
     const command_line::arg_descriptor<bool> arg_p2p_hide_my_port   =    {"hide-my-port", "Do not announce yourself as peerlist candidate", false, true};
     const command_line::arg_descriptor<bool> arg_no_sync = {"no-sync", "Don't synchronize the blockchain with other peers", false};
+    const command_line::arg_descriptor<bool> arg_enable_dns_blocklist = {"enable-dns-blocklist", "Apply realtime blocklist from DNS", false};
 
     const command_line::arg_descriptor<bool>        arg_no_igd  = {"no-igd", "Disable UPnP port mapping"};
     const command_line::arg_descriptor<std::string> arg_igd = {"igd", "UPnP port mapping (disabled, enabled, delayed)", "delayed"};

src/p2p/net_node.h

@@ -139,6 +139,7 @@ namespace nodetool
 
     typedef COMMAND_HANDSHAKE_T<typename t_payload_net_handler::payload_type> COMMAND_HANDSHAKE;
     typedef COMMAND_TIMED_SYNC_T<typename t_payload_net_handler::payload_type> COMMAND_TIMED_SYNC;
+    static_assert(p2p_connection_context::handshake_command() == COMMAND_HANDSHAKE::ID, "invalid handshake command id");
 
     typedef epee::net_utils::boosted_tcp_server<epee::levin::async_protocol_handler<p2p_connection_context>> net_server;
 
@@ -149,7 +150,7 @@ namespace nodetool
     {
       config_t()
         : m_net_config(),
-          m_peer_id(crypto::rand<uint64_t>()),
+          m_peer_id(1),
           m_support_flags(0)
       {}
 
@@ -278,7 +279,7 @@ namespace nodetool
     uint32_t get_max_out_public_peers() const;
     void change_max_in_public_peers(size_t count);
     uint32_t get_max_in_public_peers() const;
-    virtual bool block_host(const epee::net_utils::network_address &adress, time_t seconds = P2P_IP_BLOCKTIME);
+    virtual bool block_host(epee::net_utils::network_address address, time_t seconds = P2P_IP_BLOCKTIME, bool add_only = false);
     virtual bool unblock_host(const epee::net_utils::network_address &address);
     virtual bool block_subnet(const epee::net_utils::ipv4_network_subnet &subnet, time_t seconds = P2P_IP_BLOCKTIME);
     virtual bool unblock_subnet(const epee::net_utils::ipv4_network_subnet &subnet);
@@ -357,6 +358,7 @@ namespace nodetool
     bool peer_sync_idle_maker();
     bool do_handshake_with_peer(peerid_type& pi, p2p_connection_context& context, bool just_take_peerlist = false);
     bool do_peer_timed_sync(const epee::net_utils::connection_context_base& context, peerid_type peer_id);
+    bool update_dns_blocklist();
 
     bool make_new_connection_from_anchor_peerlist(const std::vector<anchor_peerlist_entry>& anchor_peerlist);
     bool make_new_connection_from_peerlist(network_zone& zone, bool use_white_list);
@@ -461,6 +463,7 @@ namespace nodetool
     epee::math_helper::once_a_time_seconds<60*30, false> m_peerlist_store_interval;
     epee::math_helper::once_a_time_seconds<60> m_gray_peerlist_housekeeping_interval;
     epee::math_helper::once_a_time_seconds<3600, false> m_incoming_connections_interval;
+    epee::math_helper::once_a_time_seconds<7000> m_dns_blocklist_interval;
 
     std::list<epee::net_utils::network_address>   m_priority_peers;
     std::vector<epee::net_utils::network_address> m_exclusive_peers;
@@ -502,6 +505,8 @@ namespace nodetool
     cryptonote::network_type m_nettype;
 
     epee::net_utils::ssl_support_t m_ssl_support;
+
+    bool m_enable_dns_blocklist;
   };
 
     const int64_t default_limit_up = P2P_DEFAULT_LIMIT_RATE_UP;      // kB/s
@@ -523,6 +528,7 @@ namespace nodetool
     extern const command_line::arg_descriptor<std::string> arg_ban_list;
     extern const command_line::arg_descriptor<bool> arg_p2p_hide_my_port;
     extern const command_line::arg_descriptor<bool> arg_no_sync;
+    extern const command_line::arg_descriptor<bool> arg_enable_dns_blocklist;
 
     extern const command_line::arg_descriptor<bool>        arg_no_igd;
     extern const command_line::arg_descriptor<std::string> arg_igd;

src/p2p/net_node.inl

@@ -37,6 +37,7 @@
 #include <boost/optional/optional.hpp>
 #include <boost/thread/thread.hpp>
 #include <boost/uuid/uuid_io.hpp>
+#include <boost/algorithm/string.hpp>
 #include <atomic>
 #include <functional>
 #include <limits>
@@ -120,6 +121,7 @@ namespace nodetool
     command_line::add_arg(desc, arg_ban_list);
     command_line::add_arg(desc, arg_p2p_hide_my_port);
     command_line::add_arg(desc, arg_no_sync);
+    command_line::add_arg(desc, arg_enable_dns_blocklist);
     command_line::add_arg(desc, arg_no_igd);
     command_line::add_arg(desc, arg_igd);
     command_line::add_arg(desc, arg_out_peers);
@@ -139,7 +141,9 @@ namespace nodetool
     if (storage)
       m_peerlist_storage = std::move(*storage);
 
-    m_network_zones[epee::net_utils::zone::public_].m_config.m_support_flags = P2P_SUPPORT_FLAGS;
+    network_zone& public_zone = m_network_zones[epee::net_utils::zone::public_];
+    public_zone.m_config.m_support_flags = P2P_SUPPORT_FLAGS;
+    public_zone.m_config.m_peer_id = crypto::rand<uint64_t>();
     m_first_connection_maker_call = true;
 
     CATCH_ENTRY_L0("node_server::init_config", false);
@@ -224,12 +228,13 @@ namespace nodetool
   }
   //-----------------------------------------------------------------------------------
   template<class t_payload_net_handler>
-  bool node_server<t_payload_net_handler>::block_host(const epee::net_utils::network_address &addr, time_t seconds)
+  bool node_server<t_payload_net_handler>::block_host(epee::net_utils::network_address addr, time_t seconds, bool add_only)
   {
     if(!addr.is_blockable())
       return false;
 
     const time_t now = time(nullptr);
+    bool added = false;
 
     CRITICAL_REGION_LOCAL(m_blocked_hosts_lock);
     time_t limit;
@@ -237,7 +242,15 @@ namespace nodetool
       limit = std::numeric_limits<time_t>::max();
     else
       limit = now + seconds;
-    m_blocked_hosts[addr.host_str()] = limit;
+    const std::string host_str = addr.host_str();
+    auto it = m_blocked_hosts.find(host_str);
+    if (it == m_blocked_hosts.end())
+    {
+      m_blocked_hosts[host_str] = limit;
+      added = true;
+    }
+    else if (it->second < limit || !add_only)
+      it->second = limit;
 
     // drop any connection to that address. This should only have to look into
     // the zone related to the connection, but really make sure everything is
@@ -253,17 +266,23 @@ namespace nodetool
         }
         return true;
       });
-      for (const auto &c: conns)
-        zone.second.m_net_server.get_config_object().close(c);
-
-      conns.clear();
 
       peerlist_entry pe{};
       pe.adr = addr;
       zone.second.m_peerlist.remove_from_peer_white(pe);
+      zone.second.m_peerlist.remove_from_peer_gray(pe);
+      zone.second.m_peerlist.remove_from_peer_anchor(addr);
+
+      for (const auto &c: conns)
+        zone.second.m_net_server.get_config_object().close(c);
+
+      conns.clear();
     }
 
-    MCLOG_CYAN(el::Level::Info, "global", "Host " << addr.host_str() << " blocked.");
+    if (added)
+      MCLOG_CYAN(el::Level::Info, "global", "Host " << host_str << " blocked.");
+    else
+      MINFO("Host " << host_str << " block time updated.");
     return true;
   }
   //-----------------------------------------------------------------------------------
@@ -402,7 +421,7 @@ namespace nodetool
     m_use_ipv6 = command_line::get_arg(vm, arg_p2p_use_ipv6);
     m_require_ipv4 = !command_line::get_arg(vm, arg_p2p_ignore_ipv4);
     public_zone.m_notifier = cryptonote::levin::notify{
-      public_zone.m_net_server.get_io_service(), public_zone.m_net_server.get_config_shared(), nullptr, true, pad_txs, m_payload_handler.get_core()
+      public_zone.m_net_server.get_io_service(), public_zone.m_net_server.get_config_shared(), nullptr, epee::net_utils::zone::public_, pad_txs, m_payload_handler.get_core()
     };
 
     if (command_line::has_arg(vm, arg_p2p_add_peer))
@@ -477,13 +496,19 @@ namespace nodetool
       std::istringstream iss(banned_ips);
       for (std::string line; std::getline(iss, line); )
       {
-        const expect<epee::net_utils::network_address> parsed_addr = net::get_network_address(line, 0);
-        if (!parsed_addr)
+        auto subnet = net::get_ipv4_subnet_address(line);
+        if (subnet)
         {
-          MERROR("Invalid IP address: " << line << " - " << parsed_addr.error());
+          block_subnet(*subnet, std::numeric_limits<time_t>::max());
           continue;
         }
-        block_host(*parsed_addr, std::numeric_limits<time_t>::max());
+        const expect<epee::net_utils::network_address> parsed_addr = net::get_network_address(line, 0);
+        if (parsed_addr)
+        {
+          block_host(*parsed_addr, std::numeric_limits<time_t>::max());
+          continue;
+        }
+        MERROR("Invalid IP address or IPv4 subnet: " << line);
       }
     }
 
@@ -493,6 +518,8 @@ namespace nodetool
     if (command_line::has_arg(vm, arg_no_sync))
       m_payload_handler.set_no_sync(true);
 
+    m_enable_dns_blocklist = command_line::get_arg(vm, arg_enable_dns_blocklist);
+
     if ( !set_max_out_peers(public_zone, command_line::get_arg(vm, arg_out_peers) ) )
       return false;
     else
@@ -545,7 +572,7 @@ namespace nodetool
       }
 
       zone.m_notifier = cryptonote::levin::notify{
-        zone.m_net_server.get_io_service(), zone.m_net_server.get_config_shared(), std::move(this_noise), false, pad_txs, m_payload_handler.get_core()
+        zone.m_net_server.get_io_service(), zone.m_net_server.get_config_shared(), std::move(this_noise), proxy.zone, pad_txs, m_payload_handler.get_core()
       };
     }
 
@@ -1099,11 +1126,12 @@ namespace nodetool
         pi = context.peer_id = rsp.node_data.peer_id;
         context.m_rpc_port = rsp.node_data.rpc_port;
         context.m_rpc_credits_per_hash = rsp.node_data.rpc_credits_per_hash;
-        network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone());
+        const auto azone = context.m_remote_address.get_zone();
+        network_zone& zone = m_network_zones.at(azone);
         zone.m_peerlist.set_peer_just_seen(rsp.node_data.peer_id, context.m_remote_address, context.m_pruning_seed, context.m_rpc_port, context.m_rpc_credits_per_hash);
 
         // move
-        if(rsp.node_data.peer_id == zone.m_config.m_peer_id)
+        if(azone == epee::net_utils::zone::public_ && rsp.node_data.peer_id == zone.m_config.m_peer_id)
         {
           LOG_DEBUG_CC(context, "Connection to self detected, dropping connection");
           hsh_result = false;
@@ -1160,8 +1188,9 @@ namespace nodetool
       if(!handle_remote_peerlist(rsp.local_peerlist_new, context))
       {
         LOG_WARNING_CC(context, "COMMAND_TIMED_SYNC: failed to handle_remote_peerlist(...), closing connection.");
+        const auto remote_address = context.m_remote_address;
         m_network_zones.at(context.m_remote_address.get_zone()).m_net_server.get_config_object().close(context.m_connection_id );
-        add_host_fail(context.m_remote_address);
+        add_host_fail(remote_address);
       }
       if(!context.m_is_income)
         m_network_zones.at(context.m_remote_address.get_zone()).m_peerlist.set_peer_just_seen(context.peer_id, context.m_remote_address, context.m_pruning_seed, context.m_rpc_port, context.m_rpc_credits_per_hash);
@@ -1195,50 +1224,51 @@ namespace nodetool
   template<class t_payload_net_handler>
   bool node_server<t_payload_net_handler>::is_peer_used(const peerlist_entry& peer)
   {
-    for(const auto& zone : m_network_zones)
-      if(zone.second.m_config.m_peer_id == peer.id)
-        return true;//dont make connections to ourself
+    const auto zone = peer.adr.get_zone();
+    const auto server = m_network_zones.find(zone);
+    if (server == m_network_zones.end())
+      return false;
+
+    const bool is_public = (zone == epee::net_utils::zone::public_);
+    if(is_public && server->second.m_config.m_peer_id == peer.id)
+      return true;//dont make connections to ourself
 
     bool used = false;
-    for(auto& zone : m_network_zones)
+    server->second.m_net_server.get_config_object().foreach_connection([&, is_public](const p2p_connection_context& cntxt)
     {
-      zone.second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt)
+      if((is_public && cntxt.peer_id == peer.id) || (!cntxt.m_is_income && peer.adr == cntxt.m_remote_address))
       {
-        if(cntxt.peer_id == peer.id || (!cntxt.m_is_income && peer.adr == cntxt.m_remote_address))
-        {
-          used = true;
-          return false;//stop enumerating
-        }
-        return true;
-      });
-
-      if(used)
-        return true;
-    }
-    return false;
+        used = true;
+        return false;//stop enumerating
+      }
+      return true;
+    });
+    return used;
   }
   //-----------------------------------------------------------------------------------
   template<class t_payload_net_handler>
   bool node_server<t_payload_net_handler>::is_peer_used(const anchor_peerlist_entry& peer)
   {
-    for(auto& zone : m_network_zones) {
-      if(zone.second.m_config.m_peer_id == peer.id) {
-          return true;//dont make connections to ourself
-      }
-      bool used = false;
-      zone.second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt)
+    const auto zone = peer.adr.get_zone();
+    const auto server = m_network_zones.find(zone);
+    if (server == m_network_zones.end())
+      return false;
+
+    const bool is_public = (zone == epee::net_utils::zone::public_);
+    if(is_public && server->second.m_config.m_peer_id == peer.id)
+      return true;//dont make connections to ourself
+
+    bool used = false;
+    server->second.m_net_server.get_config_object().foreach_connection([&, is_public](const p2p_connection_context& cntxt)
+    {
+      if((is_public && cntxt.peer_id == peer.id) || (!cntxt.m_is_income && peer.adr == cntxt.m_remote_address))
       {
-        if(cntxt.peer_id == peer.id || (!cntxt.m_is_income && peer.adr == cntxt.m_remote_address))
-        {
-          used = true;
-          return false;//stop enumerating
-        }
-        return true;
-      });
-      if (used)
-        return true;
-    }
-    return false;
+        used = true;
+        return false;//stop enumerating
+      }
+      return true;
+    });
+    return used;
   }
   //-----------------------------------------------------------------------------------
   template<class t_payload_net_handler>
@@ -1281,6 +1311,9 @@ namespace nodetool
     if (zone.m_connect == nullptr) // outgoing connections in zone not possible
       return false;
 
+    if (zone.m_our_address == na)
+      return false;
+
     if (zone.m_current_number_of_out_peers == zone.m_config.m_net_config.max_out_connection_count) // out peers limit
     {
       return false;
@@ -1324,7 +1357,7 @@ namespace nodetool
     if(just_take_peerlist)
     {
       zone.m_net_server.get_config_object().close(con->m_connection_id);
-      LOG_DEBUG_CC(*con, "CONNECTION HANDSHAKED OK AND CLOSED.");
+      MDEBUG(na.str() << "CONNECTION HANDSHAKED OK AND CLOSED.");
       return true;
     }
 
@@ -1386,7 +1419,7 @@ namespace nodetool
 
     zone.m_net_server.get_config_object().close(con->m_connection_id);
 
-    LOG_DEBUG_CC(*con, "CONNECTION HANDSHAKED OK AND CLOSED.");
+    MDEBUG(na.str() << "CONNECTION HANDSHAKED OK AND CLOSED.");
 
     return true;
   }
@@ -1608,6 +1641,9 @@ namespace nodetool
           peerid_to_string(pe.id) << " " << pe.adr.str() << ", pruning seed " << epee::string_tools::to_string_hex(pe.pruning_seed) <<
           " (stripe " << next_needed_pruning_stripe << " needed)");
 
+      if(zone.m_our_address == pe.adr)
+        continue;
+
       if(is_peer_used(pe)) {
         _note("Peer is used");
         continue;
@@ -1915,6 +1951,52 @@ namespace nodetool
     m_gray_peerlist_housekeeping_interval.do_call(boost::bind(&node_server<t_payload_net_handler>::gray_peerlist_housekeeping, this));
     m_peerlist_store_interval.do_call(boost::bind(&node_server<t_payload_net_handler>::store_config, this));
     m_incoming_connections_interval.do_call(boost::bind(&node_server<t_payload_net_handler>::check_incoming_connections, this));
+    m_dns_blocklist_interval.do_call(boost::bind(&node_server<t_payload_net_handler>::update_dns_blocklist, this));
+    return true;
+  }
+  //-----------------------------------------------------------------------------------
+  template<class t_payload_net_handler>
+  bool node_server<t_payload_net_handler>::update_dns_blocklist()
+  {
+    if (!m_enable_dns_blocklist)
+      return true;
+    if (m_nettype != cryptonote::MAINNET)
+      return true;
+
+    static const std::vector<std::string> dns_urls = {
+      "blocklist.moneropulse.se"
+    , "blocklist.moneropulse.org"
+    , "blocklist.moneropulse.net"
+    , "blocklist.moneropulse.no"
+    , "blocklist.moneropulse.fr"
+    , "blocklist.moneropulse.de"
+    , "blocklist.moneropulse.ch"
+    };
+
+    std::vector<std::string> records;
+    if (!tools::dns_utils::load_txt_records_from_dns(records, dns_urls))
+      return true;
+
+    unsigned good = 0, bad = 0;
+    for (const auto& record : records)
+    {
+      std::vector<std::string> ips;
+      boost::split(ips, record, boost::is_any_of(";"));
+      for (const auto &ip: ips)
+      {
+        const expect<epee::net_utils::network_address> parsed_addr = net::get_network_address(ip, 0);
+        if (!parsed_addr)
+        {
+          MWARNING("Invalid IP address from DNS blocklist: " << ip << " - " << parsed_addr.error());
+          ++bad;
+          continue;
+        }
+        block_host(*parsed_addr, DNS_BLOCKLIST_LIFETIME, true);
+        ++good;
+      }
+    }
+    if (good > 0)
+      MINFO(good << " addresses added to the blocklist");
     return true;
   }
   //-----------------------------------------------------------------------------------
@@ -2016,6 +2098,11 @@ namespace nodetool
   template<class t_payload_net_handler>
   bool node_server<t_payload_net_handler>::handle_remote_peerlist(const std::vector<peerlist_entry>& peerlist, const epee::net_utils::connection_context_base& context)
   {
+    if (peerlist.size() > P2P_MAX_PEERS_IN_HANDSHAKE)
+    {
+      MWARNING(context << "peer sent " << peerlist.size() << " peers, considered spamming");
+      return false;
+    }
     std::vector<peerlist_entry> peerlist_ = peerlist;
     if(!sanitize_peerlist(peerlist_))
       return false;
@@ -2032,7 +2119,10 @@ namespace nodetool
 
     LOG_DEBUG_CC(context, "REMOTE PEERLIST: remote peerlist size=" << peerlist_.size());
     LOG_TRACE_CC(context, "REMOTE PEERLIST: " << ENDL << print_peerlist_to_string(peerlist_));
-    return m_network_zones.at(context.m_remote_address.get_zone()).m_peerlist.merge_peerlist(peerlist_, [this](const peerlist_entry &pe) { return !is_addr_recently_failed(pe.adr); });
+    CRITICAL_REGION_LOCAL(m_blocked_hosts_lock);
+    return m_network_zones.at(context.m_remote_address.get_zone()).m_peerlist.merge_peerlist(peerlist_, [this](const peerlist_entry &pe) {
+      return !is_addr_recently_failed(pe.adr) && is_remote_host_allowed(pe.adr);
+    });
   }
   //-----------------------------------------------------------------------------------
   template<class t_payload_net_handler>
@@ -2189,7 +2279,7 @@ namespace nodetool
 
     const epee::net_utils::network_address na = context.m_remote_address;
     std::string ip;
-    uint32_t ipv4_addr;
+    uint32_t ipv4_addr = 0;
     boost::asio::ip::address_v6 ipv6_addr;
     bool is_ipv4;
     if (na.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id())
@@ -2352,12 +2442,14 @@ namespace nodetool
   template<class t_payload_net_handler>
   int node_server<t_payload_net_handler>::handle_handshake(int command, typename COMMAND_HANDSHAKE::request& arg, typename COMMAND_HANDSHAKE::response& rsp, p2p_connection_context& context)
   {
+    // copy since dropping the connection will invalidate the context, and thus the address
+    const auto remote_address = context.m_remote_address;
+
     if(arg.node_data.network_id != m_network_id)
     {
-
       LOG_INFO_CC(context, "WRONG NETWORK AGENT CONNECTED! id=" << arg.node_data.network_id);
       drop_connection(context);
-      add_host_fail(context.m_remote_address);
+      add_host_fail(remote_address);
       return 1;
     }
 
@@ -2365,7 +2457,7 @@ namespace nodetool
     {
       LOG_WARNING_CC(context, "COMMAND_HANDSHAKE came not from incoming connection");
       drop_connection(context);
-      add_host_fail(context.m_remote_address);
+      add_host_fail(remote_address);
       return 1;
     }
 
@@ -2376,11 +2468,12 @@ namespace nodetool
       return 1;
     }
 
-    network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone());
+    const auto azone = context.m_remote_address.get_zone();
+    network_zone& zone = m_network_zones.at(azone);
 
     // test only the remote end's zone, otherwise an attacker could connect to you on clearnet
     // and pass in a tor connection's peer id, and deduce the two are the same if you reject it
-    if(arg.node_data.peer_id == zone.m_config.m_peer_id)
+    if(azone == epee::net_utils::zone::public_ && arg.node_data.peer_id == zone.m_config.m_peer_id)
     {
       LOG_DEBUG_CC(context, "Connection to self detected, dropping connection");
       drop_connection(context);
@@ -2776,8 +2869,8 @@ namespace nodetool
     const uint32_t index = stripe - 1;
     CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex);
     MINFO("adding stripe " << stripe << " peer: " << context.m_remote_address.str());
-    std::remove_if(m_used_stripe_peers[index].begin(), m_used_stripe_peers[index].end(),
-        [&context](const epee::net_utils::network_address &na){ return context.m_remote_address == na; });
+    m_used_stripe_peers[index].erase(std::remove_if(m_used_stripe_peers[index].begin(), m_used_stripe_peers[index].end(),
+        [&context](const epee::net_utils::network_address &na){ return context.m_remote_address == na; }), m_used_stripe_peers[index].end());
     m_used_stripe_peers[index].push_back(context.m_remote_address);
   }
 
@@ -2790,8 +2883,8 @@ namespace nodetool
     const uint32_t index = stripe - 1;
     CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex);
     MINFO("removing stripe " << stripe << " peer: " << context.m_remote_address.str());
-    std::remove_if(m_used_stripe_peers[index].begin(), m_used_stripe_peers[index].end(),
-        [&context](const epee::net_utils::network_address &na){ return context.m_remote_address == na; });
+    m_used_stripe_peers[index].erase(std::remove_if(m_used_stripe_peers[index].begin(), m_used_stripe_peers[index].end(),
+        [&context](const epee::net_utils::network_address &na){ return context.m_remote_address == na; }), m_used_stripe_peers[index].end());
   }
 
   template<class t_payload_net_handler>

src/p2p/net_node_common.h

@@ -58,7 +58,7 @@ namespace nodetool
     virtual uint64_t get_public_connections_count()=0;
     virtual void for_each_connection(std::function<bool(t_connection_context&, peerid_type, uint32_t)> f)=0;
     virtual bool for_connection(const boost::uuids::uuid&, std::function<bool(t_connection_context&, peerid_type, uint32_t)> f)=0;
-    virtual bool block_host(const epee::net_utils::network_address &address, time_t seconds = 0)=0;
+    virtual bool block_host(epee::net_utils::network_address address, time_t seconds = 0, bool add_only = false)=0;
     virtual bool unblock_host(const epee::net_utils::network_address &address)=0;
     virtual std::map<std::string, time_t> get_blocked_hosts()=0;
     virtual std::map<epee::net_utils::ipv4_network_subnet, time_t> get_blocked_subnets()=0;
@@ -108,7 +108,7 @@ namespace nodetool
     {
       return false;
     }
-    virtual bool block_host(const epee::net_utils::network_address &address, time_t seconds)
+    virtual bool block_host(epee::net_utils::network_address address, time_t seconds, bool add_only)
     {
       return true;
     }

src/p2p/net_peerlist.cpp

@@ -288,6 +288,19 @@ namespace nodetool
     copy_peers(peers.gray, m_peers_gray.get<by_addr>());
     copy_peers(peers.anchor, m_peers_anchor.get<by_addr>());
   }
+
+  void peerlist_manager::evict_host_from_white_peerlist(const peerlist_entry& pr)
+  {
+    peers_indexed::index<by_time>::type& sorted_index=m_peers_white.get<by_time>();
+    auto i = sorted_index.begin();
+    while (i != sorted_index.end())
+    {
+      if (i->adr.is_same_host(pr.adr))
+        i = sorted_index.erase(i);
+      else
+        ++i;
+    }
+  }
 }
 
 BOOST_CLASS_VERSION(nodetool::peerlist_types, nodetool::CURRENT_PEERLIST_STORAGE_ARCHIVE_VER);

src/p2p/net_peerlist.h

@@ -109,6 +109,7 @@ namespace nodetool
     bool get_white_peer_by_index(peerlist_entry& p, size_t i);
     bool get_gray_peer_by_index(peerlist_entry& p, size_t i);
     template<typename F> bool foreach(bool white, const F &f);
+    void evict_host_from_white_peerlist(const peerlist_entry& pr);
     bool append_with_peer_white(const peerlist_entry& pr);
     bool append_with_peer_gray(const peerlist_entry& pr);
     bool append_with_peer_anchor(const anchor_peerlist_entry& ple);
@@ -345,6 +346,7 @@ namespace nodetool
     if(by_addr_it_wt == m_peers_white.get<by_addr>().end())
     {
       //put new record into white list
+      evict_host_from_white_peerlist(ple);
       m_peers_white.insert(ple);
       trim_white_peerlist();
     }else

src/ringct/bulletproofs.cc

@@ -826,7 +826,7 @@ bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs)
   proof_data.reserve(proofs.size());
   size_t inv_offset = 0;
   std::vector<rct::key> to_invert;
-  to_invert.reserve(11 * sizeof(proofs));
+  to_invert.reserve(11 * proofs.size());
   size_t max_logM = 0;
   for (const Bulletproof *p: proofs)
   {

src/rpc/CMakeLists.txt

@@ -66,7 +66,7 @@ set(rpc_pub_headers zmq_pub.h)
 
 set(daemon_rpc_server_headers)
 
-set(rpc_daemon_private_headers
+set(rpc_private_headers
   bootstrap_daemon.h
   core_rpc_server.h
   rpc_payment.h

src/rpc/bootstrap_daemon.cpp

@@ -45,12 +45,12 @@ namespace cryptonote
     return host + ":" + m_http_client.get_port();
   }
 
-  boost::optional<uint64_t> bootstrap_daemon::get_height()
+  boost::optional<std::pair<uint64_t, uint64_t>> bootstrap_daemon::get_height()
   {
-    cryptonote::COMMAND_RPC_GET_HEIGHT::request req;
-    cryptonote::COMMAND_RPC_GET_HEIGHT::response res;
+    cryptonote::COMMAND_RPC_GET_INFO::request req;
+    cryptonote::COMMAND_RPC_GET_INFO::response res;
 
-    if (!invoke_http_json("/getheight", req, res))
+    if (!invoke_http_json("/getinfo", req, res))
     {
       return boost::none;
     }
@@ -60,7 +60,7 @@ namespace cryptonote
       return boost::none;
     }
 
-    return res.height;
+    return {{res.height, res.target_height}};
   }
 
   bool bootstrap_daemon::handle_result(bool success, const std::string &status)

src/rpc/bootstrap_daemon.h

@@ -2,6 +2,7 @@
 
 #include <functional>
 #include <map>
+#include <utility>
 
 #include <boost/optional/optional.hpp>
 #include <boost/thread/mutex.hpp>
@@ -27,7 +28,7 @@ namespace cryptonote
       bool rpc_payment_enabled);
 
     std::string address() const noexcept;
-    boost::optional<uint64_t> get_height();
+    boost::optional<std::pair<uint64_t, uint64_t>> get_height();
     bool handle_result(bool success, const std::string &status);
 
     template <class t_request, class t_response>

src/rpc/core_rpc_server.cpp

@@ -190,6 +190,7 @@ namespace cryptonote
 
     request.gray = true;
     request.white = true;
+    request.include_blocked = false;
     if (!on_get_public_nodes(request, response) || response.status != CORE_RPC_STATUS_OK)
     {
       return {};
@@ -264,6 +265,18 @@ namespace cryptonote
     if (!rpc_config)
       return false;
 
+    std::string bind_ip_str = rpc_config->bind_ip;
+    std::string bind_ipv6_str = rpc_config->bind_ipv6_address;
+    if (restricted)
+    {
+      const auto restricted_rpc_port_arg = cryptonote::core_rpc_server::arg_rpc_restricted_bind_port;
+      const bool has_restricted_rpc_port_arg = !command_line::is_arg_defaulted(vm, restricted_rpc_port_arg);
+      if (has_restricted_rpc_port_arg && port == command_line::get_arg(vm, restricted_rpc_port_arg))
+      {
+        bind_ip_str = rpc_config->restricted_bind_ip;
+        bind_ipv6_str = rpc_config->restricted_bind_ipv6_address;
+      }
+    }
     disable_rpc_ban = rpc_config->disable_rpc_ban;
     std::string address = command_line::get_arg(vm, arg_rpc_payment_address);
     if (!address.empty() && allow_rpc_payment)
@@ -300,7 +313,7 @@ namespace cryptonote
     if (!m_rpc_payment)
     {
       uint32_t bind_ip;
-      bool ok = epee::string_tools::get_ip_int32_from_string(bind_ip, rpc_config->bind_ip);
+      bool ok = epee::string_tools::get_ip_int32_from_string(bind_ip, bind_ip_str);
       if (ok & !epee::net_utils::is_ip_loopback(bind_ip))
         MWARNING("The RPC server is accessible from the outside, but no RPC payment was setup. RPC access will be free for all.");
     }
@@ -322,8 +335,8 @@ namespace cryptonote
 
     auto rng = [](size_t len, uint8_t *ptr){ return crypto::rand(len, ptr); };
     return epee::http_server_impl_base<core_rpc_server, connection_context>::init(
-      rng, std::move(port), std::move(rpc_config->bind_ip),
-      std::move(rpc_config->bind_ipv6_address), std::move(rpc_config->use_ipv6), std::move(rpc_config->require_ipv4),
+      rng, std::move(port), std::move(bind_ip_str),
+      std::move(bind_ipv6_str), std::move(rpc_config->use_ipv6), std::move(rpc_config->require_ipv4),
       std::move(rpc_config->access_control_origins), std::move(http_login), std::move(rpc_config->ssl_options)
     );
   }
@@ -439,7 +452,7 @@ namespace cryptonote
     m_core.get_blockchain_top(res.height, top_hash);
     ++res.height; // turn top block height into blockchain height
     res.top_block_hash = string_tools::pod_to_hex(top_hash);
-    res.target_height = m_core.get_target_blockchain_height();
+    res.target_height = m_p2p.get_payload_object().is_synchronized() ? 0 : m_core.get_target_blockchain_height();
     store_difficulty(m_core.get_blockchain_storage().get_difficulty_for_next_block(), res.difficulty, res.wide_difficulty, res.difficulty_top64);
     res.target = m_core.get_blockchain_storage().get_difficulty_target();
     res.tx_count = m_core.get_blockchain_storage().get_total_transactions() - res.height; //without coinbase
@@ -486,6 +499,9 @@ namespace cryptonote
       res.database_size = round_up(res.database_size, 5ull* 1024 * 1024 * 1024);
     res.update_available = restricted ? false : m_core.is_update_available();
     res.version = restricted ? "" : MONERO_VERSION_FULL;
+    res.synchronized = check_core_ready();
+    res.busy_syncing = m_p2p.get_payload_object().is_busy_syncing();
+    res.synchronized = check_core_ready();
 
     res.status = CORE_RPC_STATUS_OK;
     return true;
@@ -550,12 +566,12 @@ namespace cryptonote
       }
     }
 
-    size_t max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT;
+    size_t max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT;
     if (m_rpc_payment)
     {
       max_blocks = res.credits / COST_PER_BLOCK;
-      if (max_blocks > COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)
-        max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT;
+      if (max_blocks > COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT)
+        max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT;
       if (max_blocks == 0)
       {
         res.status = CORE_RPC_STATUS_PAYMENT_REQUIRED;
@@ -564,7 +580,7 @@ namespace cryptonote
     }
 
     std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > > bs;
-    if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, max_blocks))
+    if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, max_blocks, COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT))
     {
       res.status = "Failed";
       add_host_fail(ctx);
@@ -1142,7 +1158,7 @@ namespace cryptonote
     if (!restricted)
     {
       boost::shared_lock<boost::shared_mutex> lock(m_bootstrap_daemon_mutex);
-      if (m_bootstrap_daemon.get() != nullptr)
+      if (m_should_use_bootstrap_daemon)
       {
         skip_validation = !check_core_ready();
       }
@@ -1151,6 +1167,10 @@ namespace cryptonote
         CHECK_CORE_READY();
       }
     }
+    else
+    {
+      CHECK_CORE_READY();
+    }
 
     CHECK_PAYMENT_MIN1(req, res, COST_PER_TX_RELAY, false);
 
@@ -1365,6 +1385,8 @@ namespace cryptonote
 
     for (auto & entry : white_list)
     {
+      if (!req.include_blocked && m_p2p.is_host_blocked(entry.adr, NULL))
+        continue;
       if (entry.adr.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id())
         res.white_list.emplace_back(entry.id, entry.adr.as<epee::net_utils::ipv4_network_address>().ip(),
             entry.adr.as<epee::net_utils::ipv4_network_address>().port(), entry.last_seen, entry.pruning_seed, entry.rpc_port, entry.rpc_credits_per_hash);
@@ -1377,6 +1399,8 @@ namespace cryptonote
 
     for (auto & entry : gray_list)
     {
+      if (!req.include_blocked && m_p2p.is_host_blocked(entry.adr, NULL))
+        continue;
       if (entry.adr.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id())
         res.gray_list.emplace_back(entry.id, entry.adr.as<epee::net_utils::ipv4_network_address>().ip(),
             entry.adr.as<epee::net_utils::ipv4_network_address>().port(), entry.last_seen, entry.pruning_seed, entry.rpc_port, entry.rpc_credits_per_hash);
@@ -1395,8 +1419,10 @@ namespace cryptonote
   {
     RPC_TRACKER(get_public_nodes);
 
+    COMMAND_RPC_GET_PEER_LIST::request peer_list_req;
     COMMAND_RPC_GET_PEER_LIST::response peer_list_res;
-    const bool success = on_get_peer_list(COMMAND_RPC_GET_PEER_LIST::request(), peer_list_res, ctx);
+    peer_list_req.include_blocked = req.include_blocked;
+    const bool success = on_get_peer_list(peer_list_req, peer_list_res, ctx);
     res.status = peer_list_res.status;
     if (!success)
     {      
@@ -1988,34 +2014,37 @@ namespace cryptonote
     }
 
     auto current_time = std::chrono::system_clock::now();
-    if (!m_p2p.get_payload_object().no_sync() &&
-        current_time - m_bootstrap_height_check_time > std::chrono::seconds(30))  // update every 30s
+    if (current_time - m_bootstrap_height_check_time > std::chrono::seconds(30))  // update every 30s
     {
       {
         boost::upgrade_to_unique_lock<boost::shared_mutex> lock(upgrade_lock);
         m_bootstrap_height_check_time = current_time;
       }
 
-      boost::optional<uint64_t> bootstrap_daemon_height = m_bootstrap_daemon->get_height();
-      if (!bootstrap_daemon_height)
+      boost::optional<std::pair<uint64_t, uint64_t>> bootstrap_daemon_height_info = m_bootstrap_daemon->get_height();
+      if (!bootstrap_daemon_height_info)
       {
         MERROR("Failed to fetch bootstrap daemon height");
         return false;
       }
 
-      uint64_t target_height = m_core.get_target_blockchain_height();
-      if (*bootstrap_daemon_height < target_height)
+      const uint64_t bootstrap_daemon_height = bootstrap_daemon_height_info->first;
+      const uint64_t bootstrap_daemon_target_height = bootstrap_daemon_height_info->second;
+      if (bootstrap_daemon_height < bootstrap_daemon_target_height)
       {
         MINFO("Bootstrap daemon is out of sync");
         return m_bootstrap_daemon->handle_result(false, {});
       }
 
-      uint64_t top_height = m_core.get_current_blockchain_height();
-      m_should_use_bootstrap_daemon = top_height + 10 < *bootstrap_daemon_height;
-      MINFO((m_should_use_bootstrap_daemon ? "Using" : "Not using") << " the bootstrap daemon (our height: " << top_height << ", bootstrap daemon's height: " << *bootstrap_daemon_height << ")");
+      if (!m_p2p.get_payload_object().no_sync())
+      {
+        uint64_t top_height = m_core.get_current_blockchain_height();
+        m_should_use_bootstrap_daemon = top_height + 10 < bootstrap_daemon_height;
+        MINFO((m_should_use_bootstrap_daemon ? "Using" : "Not using") << " the bootstrap daemon (our height: " << top_height << ", bootstrap daemon's height: " << bootstrap_daemon_height << ")");
 
-      if (!m_should_use_bootstrap_daemon)
-        return false;
+        if (!m_should_use_bootstrap_daemon)
+          return false;
+      }
     }
 
     if (mode == invoke_http_mode::JON)
@@ -2886,7 +2915,7 @@ namespace cryptonote
     crypto::hash top_hash;
     m_core.get_blockchain_top(res.height, top_hash);
     ++res.height; // turn top block height into blockchain height
-    res.target_height = m_core.get_target_blockchain_height();
+    res.target_height = m_p2p.get_payload_object().is_synchronized() ? 0 : m_core.get_target_blockchain_height();
     res.next_needed_pruning_seed = m_p2p.get_payload_object().get_next_needed_pruning_stripe().second;
 
     for (const auto &c: m_p2p.get_payload_object().get_connections())
@@ -2895,11 +2924,7 @@ namespace cryptonote
     block_queue.foreach([&](const cryptonote::block_queue::span &span) {
       const std::string span_connection_id = epee::string_tools::pod_to_hex(span.connection_id);
       uint32_t speed = (uint32_t)(100.0f * block_queue.get_speed(span.connection_id) + 0.5f);
-      std::string address = "";
-      for (const auto &c: m_p2p.get_payload_object().get_connections())
-        if (c.connection_id == span_connection_id)
-          address = c.address;
-      res.spans.push_back({span.start_block_height, span.nblocks, span_connection_id, (uint32_t)(span.rate + 0.5f), speed, span.size, address});
+      res.spans.push_back({span.start_block_height, span.nblocks, span_connection_id, (uint32_t)(span.rate + 0.5f), speed, span.size, span.origin.str()});
       return true;
     });
     res.overview = block_queue.get_overview(res.height);

src/rpc/core_rpc_server_commands_defs.h

@@ -88,7 +88,7 @@ namespace cryptonote
 // advance which version they will stop working with
 // Don't go over 32767 for any of these
 #define CORE_RPC_VERSION_MAJOR 3
-#define CORE_RPC_VERSION_MINOR 2
+#define CORE_RPC_VERSION_MINOR 5
 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor))
 #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR)
 
@@ -684,7 +684,9 @@ namespace cryptonote
       bool was_bootstrap_ever_used;
       uint64_t database_size;
       bool update_available;
+      bool busy_syncing;
       std::string version;
+      bool synchronized;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE_PARENT(rpc_access_response_base)
@@ -723,7 +725,9 @@ namespace cryptonote
         KV_SERIALIZE(was_bootstrap_ever_used)
         KV_SERIALIZE(database_size)
         KV_SERIALIZE(update_available)
+        KV_SERIALIZE(busy_syncing)
         KV_SERIALIZE(version)
+        KV_SERIALIZE(synchronized)
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<response_t> response;
@@ -1193,10 +1197,12 @@ namespace cryptonote
     struct request_t: public rpc_request_base
     {
       bool public_only;
+      bool include_blocked;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE_PARENT(rpc_request_base)
         KV_SERIALIZE_OPT(public_only, true)
+        KV_SERIALIZE_OPT(include_blocked, false)
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<request_t> request;
@@ -1242,11 +1248,13 @@ namespace cryptonote
     {
       bool gray;
       bool white;
+      bool include_blocked;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE_PARENT(rpc_request_base)
         KV_SERIALIZE_OPT(gray, false)
         KV_SERIALIZE_OPT(white, true)
+        KV_SERIALIZE_OPT(include_blocked, false)
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<request_t> request;

src/rpc/daemon_handler.cpp

@@ -128,7 +128,7 @@ namespace rpc
   {
     std::vector<std::pair<std::pair<blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, blobdata> > > > blocks;
 
-    if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, blocks, res.current_height, res.start_height, req.prune, true, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT))
+    if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, blocks, res.current_height, res.start_height, req.prune, true, COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT, COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT))
     {
       res.status = Message::STATUS_FAILED;
       res.error_details = "core::find_blockchain_supplement() returned false";

src/rpc/rpc_args.cpp

@@ -91,6 +91,8 @@ namespace cryptonote
   rpc_args::descriptors::descriptors()
      : rpc_bind_ip({"rpc-bind-ip", rpc_args::tr("Specify IP to bind RPC server"), "127.0.0.1"})
      , rpc_bind_ipv6_address({"rpc-bind-ipv6-address", rpc_args::tr("Specify IPv6 address to bind RPC server"), "::1"})
+     , rpc_restricted_bind_ip({"rpc-restricted-bind-ip", rpc_args::tr("Specify IP to bind restricted RPC server"), "127.0.0.1"})
+     , rpc_restricted_bind_ipv6_address({"rpc-restricted-bind-ipv6-address", rpc_args::tr("Specify IPv6 address to bind restricted RPC server"), "::1"})
      , rpc_use_ipv6({"rpc-use-ipv6", rpc_args::tr("Allow IPv6 for RPC"), false})
      , rpc_ignore_ipv4({"rpc-ignore-ipv4", rpc_args::tr("Ignore unsuccessful IPv4 bind for RPC"), false})
      , rpc_login({"rpc-login", rpc_args::tr("Specify username[:password] required for RPC server"), "", true})
@@ -113,6 +115,8 @@ namespace cryptonote
     const descriptors arg{};
     command_line::add_arg(desc, arg.rpc_bind_ip);
     command_line::add_arg(desc, arg.rpc_bind_ipv6_address);
+    command_line::add_arg(desc, arg.rpc_restricted_bind_ip);
+    command_line::add_arg(desc, arg.rpc_restricted_bind_ipv6_address);
     command_line::add_arg(desc, arg.rpc_use_ipv6);
     command_line::add_arg(desc, arg.rpc_ignore_ipv4);
     command_line::add_arg(desc, arg.rpc_login);
@@ -136,6 +140,8 @@ namespace cryptonote
     
     config.bind_ip = command_line::get_arg(vm, arg.rpc_bind_ip);
     config.bind_ipv6_address = command_line::get_arg(vm, arg.rpc_bind_ipv6_address);
+    config.restricted_bind_ip = command_line::get_arg(vm, arg.rpc_restricted_bind_ip);
+    config.restricted_bind_ipv6_address = command_line::get_arg(vm, arg.rpc_restricted_bind_ipv6_address);
     config.use_ipv6 = command_line::get_arg(vm, arg.rpc_use_ipv6);
     config.require_ipv4 = !command_line::get_arg(vm, arg.rpc_ignore_ipv4);
     config.disable_rpc_ban = command_line::get_arg(vm, arg.disable_rpc_ban);
@@ -188,6 +194,34 @@ namespace cryptonote
         return boost::none;
       }
     }
+    if (!config.restricted_bind_ip.empty())
+    {
+      // always parse IP here for error consistency
+      boost::system::error_code ec{};
+      boost::asio::ip::address::from_string(config.restricted_bind_ip, ec);
+      if (ec)
+      {
+        LOG_ERROR(tr("Invalid IP address given for --") << arg.rpc_restricted_bind_ip.name);
+        return boost::none;
+      }
+    }
+    if (!config.restricted_bind_ipv6_address.empty())
+    {
+      // allow square braces, but remove them here if present
+      if (config.restricted_bind_ipv6_address.find('[') != std::string::npos)
+      {
+        config.restricted_bind_ipv6_address = config.restricted_bind_ipv6_address.substr(1, config.restricted_bind_ipv6_address.size() - 2);
+      }
+
+      // always parse IP here for error consistency
+      boost::system::error_code ec{};
+      boost::asio::ip::address::from_string(config.restricted_bind_ipv6_address, ec);
+      if (ec)
+      {
+        LOG_ERROR(tr("Invalid IP address given for --") << arg.rpc_restricted_bind_ipv6_address.name);
+        return boost::none;
+      }
+    }
 
     const char *env_rpc_login = nullptr;
     const bool has_rpc_arg = command_line::has_arg(vm, arg.rpc_login);

src/rpc/rpc_args.h

@@ -53,6 +53,8 @@ namespace cryptonote
 
       const command_line::arg_descriptor<std::string> rpc_bind_ip;
       const command_line::arg_descriptor<std::string> rpc_bind_ipv6_address;
+      const command_line::arg_descriptor<std::string> rpc_restricted_bind_ip;
+      const command_line::arg_descriptor<std::string> rpc_restricted_bind_ipv6_address;
       const command_line::arg_descriptor<bool> rpc_use_ipv6;
       const command_line::arg_descriptor<bool> rpc_ignore_ipv4;
       const command_line::arg_descriptor<std::string> rpc_login;
@@ -81,6 +83,8 @@ namespace cryptonote
 
     std::string bind_ip;
     std::string bind_ipv6_address;
+    std::string restricted_bind_ip;
+    std::string restricted_bind_ipv6_address;
     bool use_ipv6;
     bool require_ipv4;
     std::vector<std::string> access_control_origins;

src/rpc/zmq_pub.cpp

@@ -221,7 +221,7 @@ namespace
   void add_subscriptions(std::array<std::size_t, N>& subs, const epee::span<const context<T>> range, context<T> const* const first)
   {
     assert(range.size() <= N);
-    assert(range.begin() - first <= N - range.size());
+    assert((unsigned long)(range.begin() - first) <= N - range.size());
 
     for (const auto& ctx : range)
     {
@@ -234,7 +234,7 @@ namespace
   void remove_subscriptions(std::array<std::size_t, N>& subs, const epee::span<const context<T>> range, context<T> const* const first)
   {
     assert(range.size() <= N);
-    assert(range.begin() - first <= N - range.size());
+    assert((unsigned long)(range.begin() - first) <= N - range.size());
 
     for (const auto& ctx : range)
     {

src/simplewallet/simplewallet.cpp

@@ -143,6 +143,7 @@ typedef cryptonote::simple_wallet sw;
 #define CREDITS_TARGET 50000
 #define MAX_PAYMENT_DIFF 10000
 #define MIN_PAYMENT_RATE 0.01f // per hash
+#define MAX_MNEW_ADDRESSES 1000
 
 enum TransferType {
   Transfer,
@@ -181,7 +182,7 @@ namespace
   const command_line::arg_descriptor< std::vector<std::string> > arg_command = {"command", ""};
 
   const char* USAGE_START_MINING("start_mining [<number_of_threads>] [bg_mining] [ignore_battery]");
-  const char* USAGE_SET_DAEMON("set_daemon <host>[:<port>] [trusted|untrusted]");
+  const char* USAGE_SET_DAEMON("set_daemon <host>[:<port>] [trusted|untrusted|this-is-probably-a-spy-node]");
   const char* USAGE_SHOW_BALANCE("balance [detail]");
   const char* USAGE_INCOMING_TRANSFERS("incoming_transfers [available|unavailable] [verbose] [uses] [index=<N1>[,<N2>[,...]]]");
   const char* USAGE_PAYMENTS("payments <PID_1> [<PID_2> ... <PID_N>]");
@@ -203,7 +204,7 @@ namespace
                             "  account tag <tag_name> <account_index_1> [<account_index_2> ...]\n"
                             "  account untag <account_index_1> [<account_index_2> ...]\n"
                             "  account tag_description <tag_name> <description>");
-  const char* USAGE_ADDRESS("address [ new <label text with white spaces allowed> | all | <index_min> [<index_max>] | label <index> <label text with white spaces allowed> | device [<index>] | one-off <account> <subaddress>]");
+  const char* USAGE_ADDRESS("address [ new <label text with white spaces allowed> | mnew <amount of new addresses> | all | <index_min> [<index_max>] | label <index> <label text with white spaces allowed> | device [<index>] | one-off <account> <subaddress>]");
   const char* USAGE_INTEGRATED_ADDRESS("integrated_address [device] [<payment_id> | <address>]");
   const char* USAGE_ADDRESS_BOOK("address_book [(add (<address>|<integrated address>) [<description possibly with whitespaces>])|(delete <index>)]");
   const char* USAGE_SET_VARIABLE("set <option> [<value>]");
@@ -2286,6 +2287,7 @@ bool simple_wallet::public_nodes(const std::vector<std::string> &args)
   {
     fail_msg_writer() << tr("Error retrieving public node list: ") << e.what();
   }
+  message_writer(console_color_red, true) << tr("Most of these nodes are probably spies. You should not use them unless connecting via Tor or I2P");
   return true;
 }
 
@@ -3222,7 +3224,7 @@ simple_wallet::simple_wallet()
   m_cmd_binder.set_handler("set_daemon",
                            boost::bind(&simple_wallet::on_command, this, &simple_wallet::set_daemon, _1),
                            tr(USAGE_SET_DAEMON),
-                           tr("Set another daemon to connect to."));
+                           tr("Set another daemon to connect to. If it's not yours, it'll probably spy on you."));
   m_cmd_binder.set_handler("save_bc",
                            boost::bind(&simple_wallet::on_command, this, &simple_wallet::save_bc, _1),
                            tr("Save the current blockchain data."));
@@ -3302,7 +3304,7 @@ simple_wallet::simple_wallet()
   m_cmd_binder.set_handler("address",
                            boost::bind(&simple_wallet::on_command, this, &simple_wallet::print_address, _1),
                            tr(USAGE_ADDRESS),
-                           tr("If no arguments are specified or <index> is specified, the wallet shows the default or specified address. If \"all\" is specified, the wallet shows all the existing addresses in the currently selected account. If \"new \" is specified, the wallet creates a new address with the provided label text (which can be empty). If \"label\" is specified, the wallet sets the label of the address specified by <index> to the provided label text. If \"one-off\" is specified, the address for the specified index is generated and displayed, and remembered by the wallet"));
+                           tr("If no arguments are specified or <index> is specified, the wallet shows the default or specified address. If \"all\" is specified, the wallet shows all the existing addresses in the currently selected account. If \"new \" is specified, the wallet creates a new address with the provided label text (which can be empty). If \"mnew\" is specified, the wallet creates as many new addresses as specified by the argument; the default label is set for the new addresses. If \"label\" is specified, the wallet sets the label of the address specified by <index> to the provided label text. If \"one-off\" is specified, the address for the specified index is generated and displayed, and remembered by the wallet"));
   m_cmd_binder.set_handler("integrated_address",
                            boost::bind(&simple_wallet::on_command, this, &simple_wallet::print_integrated_address, _1),
                            tr(USAGE_INTEGRATED_ADDRESS),
@@ -4743,9 +4745,14 @@ bool simple_wallet::try_connect_to_daemon(bool silent, uint32_t* version)
   if (!m_wallet->check_connection(version))
   {
     if (!silent)
-      fail_msg_writer() << tr("wallet failed to connect to daemon: ") << m_wallet->get_daemon_address() << ". " <<
-        tr("Daemon either is not started or wrong port was passed. "
-        "Please make sure daemon is running or change the daemon address using the 'set_daemon' command.");
+    {
+      if (m_wallet->is_offline())
+        fail_msg_writer() << tr("wallet failed to connect to daemon, because it is set to offline mode");
+      else
+        fail_msg_writer() << tr("wallet failed to connect to daemon: ") << m_wallet->get_daemon_address() << ". " <<
+          tr("Daemon either is not started or wrong port was passed. "
+          "Please make sure daemon is running or change the daemon address using the 'set_daemon' command.");
+    }
     return false;
   }
   if (!m_allow_mismatched_daemon_version && ((*version >> 16) != CORE_RPC_VERSION_MAJOR))
@@ -5507,21 +5514,51 @@ bool simple_wallet::set_daemon(const std::vector<std::string>& args)
     } else {
       daemon_url = args[0];
     }
-    LOCK_IDLE_SCOPE();
-    m_wallet->init(daemon_url);
 
+    epee::net_utils::http::url_content parsed{};
+    const bool r = epee::net_utils::parse_url(daemon_url, parsed);
+    if (!r)
+    {
+      fail_msg_writer() << tr("Failed to parse address");
+      return true;
+    }
+
+    std::string trusted;
     if (args.size() == 2)
     {
       if (args[1] == "trusted")
-        m_wallet->set_trusted_daemon(true);
+        trusted = "trusted";
       else if (args[1] == "untrusted")
-        m_wallet->set_trusted_daemon(false);
+        trusted = "untrusted";
+      else if (args[1] == "this-is-probably-a-spy-node")
+        trusted = "this-is-probably-a-spy-node";
       else
       {
-        fail_msg_writer() << tr("Expected trusted or untrusted, got ") << args[1] << ": assuming untrusted";
-        m_wallet->set_trusted_daemon(false);
+        fail_msg_writer() << tr("Expected trusted, untrusted or this-is-probably-a-spy-node got ") << args[1];
+        return true;
       }
     }
+
+    if (!tools::is_privacy_preserving_network(parsed.host) && !tools::is_local_address(parsed.host))
+    {
+      if (trusted == "untrusted" || trusted == "")
+      {
+        fail_msg_writer() << tr("This is not Tor/I2P address, and is not a trusted daemon.");
+        fail_msg_writer() << tr("Either use your own trusted node, connect via Tor or I2P, or pass this-is-probably-a-spy-node and be spied on.");
+        return true;
+      }
+
+      if (parsed.schema != "https")
+        message_writer(console_color_red) << tr("Warning: connecting to a non-local daemon without SSL, passive adversaries will be able to spy on you.");
+    }
+
+    LOCK_IDLE_SCOPE();
+    m_wallet->init(daemon_url);
+
+    if (!trusted.empty())
+    {
+      m_wallet->set_trusted_daemon(trusted == "trusted");
+    }
     else
     {
       m_wallet->set_trusted_daemon(false);
@@ -9246,7 +9283,7 @@ bool simple_wallet::run()
 
   refresh_main(0, ResetNone, true);
 
-  m_auto_refresh_enabled = m_wallet->auto_refresh();
+  m_auto_refresh_enabled = !m_wallet->is_offline() && m_wallet->auto_refresh();
   m_idle_thread = boost::thread([&]{wallet_idle_thread();});
 
   message_writer(console_color_green, false) << "Background refresh thread started";
@@ -9508,6 +9545,31 @@ bool simple_wallet::print_address(const std::vector<std::string> &args/* = std::
     print_address_sub(m_wallet->get_num_subaddresses(m_current_subaddress_account) - 1);
     m_wallet->device_show_address(m_current_subaddress_account, m_wallet->get_num_subaddresses(m_current_subaddress_account) - 1, boost::none);
   }
+  else if (local_args[0] == "mnew")
+  {
+    local_args.erase(local_args.begin());
+    if (local_args.size() != 1)
+    {
+      fail_msg_writer() << tr("Expected exactly one argument for the amount of new addresses");
+      return true;
+    }
+    uint32_t n;
+    if (!epee::string_tools::get_xtype_from_string(n, local_args[0]))
+    {
+      fail_msg_writer() << tr("failed to parse the amount of new addresses: ") << local_args[0];
+      return true;
+    }
+    if (n > MAX_MNEW_ADDRESSES)
+    {
+      fail_msg_writer() << tr("the amount of new addresses must be lower or equal to ") << MAX_MNEW_ADDRESSES;
+      return true;
+    }
+    for (uint32_t i = 0; i < n; ++i)
+    {
+      m_wallet->add_subaddress(m_current_subaddress_account, tr("(Untitled address)"));
+      print_address_sub(m_wallet->get_num_subaddresses(m_current_subaddress_account) - 1);
+    }
+  }
   else if (local_args[0] == "one-off")
   {
     local_args.erase(local_args.begin());

src/version.cpp.in

@@ -1,5 +1,5 @@
 #define DEF_MONERO_VERSION_TAG "@VERSIONTAG@"
-#define DEF_MONERO_VERSION "0.9.1.0"
+#define DEF_MONERO_VERSION "0.9.3.3"
 #define DEF_MONERO_RELEASE_NAME "Illiterate Illuminati"
 #define DEF_MONERO_VERSION_FULL DEF_MONERO_VERSION "-" DEF_MONERO_VERSION_TAG
 #define DEF_MONERO_VERSION_IS_RELEASE @VERSION_IS_RELEASE@

src/wallet/api/transaction_history.cpp

@@ -212,6 +212,10 @@ void TransactionHistoryImpl::refresh()
         ti->m_label = pd.m_subaddr_indices.size() == 1 ? m_wallet->m_wallet->get_subaddress_label({pd.m_subaddr_account, *pd.m_subaddr_indices.begin()}) : "";
         ti->m_timestamp = pd.m_timestamp;
         ti->m_confirmations = 0;
+        for (const auto &d : pd.m_dests)
+        {
+            ti->m_transfers.push_back({d.amount, d.address(m_wallet->m_wallet->nettype(), pd.m_payment_id)});
+        }        
         m_history.push_back(ti);
     }
     

src/wallet/wallet2.cpp

@@ -2963,7 +2963,8 @@ void wallet2::update_pool_state(std::vector<std::tuple<cryptonote::transaction,
   MTRACE("update_pool_state got pool");
 
   // remove any pending tx that's not in the pool
-  constexpr const std::chrono::seconds tx_propagation_timeout{CRYPTONOTE_DANDELIONPP_EMBARGO_AVERAGE * 3 / 2};
+  // TODO: set tx_propagation_timeout to CRYPTONOTE_DANDELIONPP_EMBARGO_AVERAGE * 3 / 2 after v15 hardfork
+  constexpr const std::chrono::seconds tx_propagation_timeout{500};
   const auto now = std::chrono::system_clock::now();
   std::unordered_map<crypto::hash, wallet2::unconfirmed_transfer_details>::iterator it = m_unconfirmed_txs.begin();
   while (it != m_unconfirmed_txs.end())
@@ -3772,7 +3773,7 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable
 //----------------------------------------------------------------------------------------------------
 boost::optional<wallet2::keys_file_data> wallet2::get_keys_file_data(const epee::wipeable_string& password, bool watch_only)
 {
-  std::string account_data;
+  epee::byte_slice account_data;
   std::string multisig_signers;
   std::string multisig_derivations;
   cryptonote::account_base account = m_account;
@@ -3799,7 +3800,7 @@ boost::optional<wallet2::keys_file_data> wallet2::get_keys_file_data(const epee:
   rapidjson::Document json;
   json.SetObject();
   rapidjson::Value value(rapidjson::kStringType);
-  value.SetString(account_data.c_str(), account_data.length());
+  value.SetString(reinterpret_cast<const char*>(account_data.data()), account_data.size());
   json.AddMember("key_data", value, json.GetAllocator());
   if (!seed_language.empty())
   {
@@ -3970,13 +3971,12 @@ boost::optional<wallet2::keys_file_data> wallet2::get_keys_file_data(const epee:
   rapidjson::StringBuffer buffer;
   rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
   json.Accept(writer);
-  account_data = buffer.GetString();
 
   // Encrypt the entire JSON object.
   std::string cipher;
-  cipher.resize(account_data.size());
+  cipher.resize(buffer.GetSize());
   keys_file_data.get().iv = crypto::rand<crypto::chacha_iv>();
-  crypto::chacha20(account_data.data(), account_data.size(), key, keys_file_data.get().iv, &cipher[0]);
+  crypto::chacha20(buffer.GetString(), buffer.GetSize(), key, keys_file_data.get().iv, &cipher[0]);
   keys_file_data.get().account_data = cipher;
   return keys_file_data;
 }
@@ -8218,12 +8218,12 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
       const transfer_details &td = m_transfers[idx];
       const uint64_t amount = td.is_rct() ? 0 : td.amount();
       std::unordered_set<uint64_t> seen_indices;
-      uint64_t approx_blockchain_height = m_nettype == TESTNET ? 0 : (time(NULL) - 1522624244)/317;
+      uint64_t approx_blockchain_height = m_nettype == TESTNET ? 0 : (time(NULL) - 1522624244)/313;
       uint64_t unlock_height = td.m_block_height + std::max<uint64_t>(CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE, CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS);
       if (td.m_tx.unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER && td.m_tx.unlock_time > unlock_height)
         unlock_height = td.m_tx.unlock_time;
       uint64_t blocks_to_unlock = unlock_height > approx_blockchain_height ? unlock_height - approx_blockchain_height : 0;
-      size_t requested_outputs_count = base_requested_outputs_count + (td.is_rct() ? blocks_to_unlock - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE : 0);
+      size_t requested_outputs_count = base_requested_outputs_count + (td.is_rct() ? blocks_to_unlock - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE : 0) + 288;
       size_t start = req.outputs.size();
       bool use_histogram = amount != 0 || !has_rct_distribution;
 
@@ -8539,12 +8539,12 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>>
     for(size_t idx: selected_transfers)
     {
       const transfer_details &td = m_transfers[idx];
-      uint64_t approx_blockchain_height = m_nettype == TESTNET ? 0 : (time(NULL) - 1522624244)/317;
+      uint64_t approx_blockchain_height = m_nettype == TESTNET ? 0 : (time(NULL) - 1522624244)/313;
       uint64_t unlock_height = td.m_block_height + std::max<uint64_t>(CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE, CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS);
       if (td.m_tx.unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER && td.m_tx.unlock_time > unlock_height)
         unlock_height = td.m_tx.unlock_time;
       uint64_t blocks_to_unlock = unlock_height > approx_blockchain_height ? unlock_height - approx_blockchain_height : 0;
-      size_t requested_outputs_count = base_requested_outputs_count + (td.is_rct() ? blocks_to_unlock - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE : 0);
+      size_t requested_outputs_count = base_requested_outputs_count + (td.is_rct() ? blocks_to_unlock - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE : 0) + 288;
       outs.push_back(std::vector<get_outs_entry>());
       outs.back().reserve(fake_outputs_count + 1);
       const rct::key mask = td.is_rct() ? rct::commit(td.amount(), td.m_mask) : rct::zeroCommit(td.amount());
@@ -12124,7 +12124,7 @@ uint64_t wallet2::get_daemon_blockchain_target_height(string &err)
 
 uint64_t wallet2::get_approximate_blockchain_height() const
 {
-  uint64_t approx_blockchain_height = m_nettype == TESTNET ? 0 : (time(NULL) - 1522624244)/315;
+  uint64_t approx_blockchain_height = m_nettype == TESTNET ? 0 : (time(NULL) - 1522624244)/313;
   LOG_PRINT_L2("Calculated blockchain height: " << approx_blockchain_height);
   return approx_blockchain_height;
 }
@@ -13376,6 +13376,20 @@ size_t wallet2::import_multisig(std::vector<cryptonote::blobdata> blobs)
       loaded = true;
     }
     CHECK_AND_ASSERT_THROW_MES(loaded, "Failed to load output data");
+
+    for (const auto &e: i)
+    {
+      for (const auto &lr: e.m_LR)
+      {
+        CHECK_AND_ASSERT_THROW_MES(rct::isInMainSubgroup(lr.m_L), "Multisig value is not in the main subgroup");
+        CHECK_AND_ASSERT_THROW_MES(rct::isInMainSubgroup(lr.m_R), "Multisig value is not in the main subgroup");
+      }
+      for (const auto &ki: e.m_partial_key_images)
+      {
+        CHECK_AND_ASSERT_THROW_MES(rct::isInMainSubgroup(rct::ki2rct(ki)), "Multisig partial key image is not in the main subgroup");
+      }
+    }
+
     MINFO(boost::format("%u outputs found") % boost::lexical_cast<std::string>(i.size()));
     info.push_back(std::move(i));
   }
@@ -14135,6 +14149,7 @@ std::vector<cryptonote::public_node> wallet2::get_public_nodes(bool white_only)
 
   req.white = true;
   req.gray = !white_only;
+  req.include_blocked = false;
 
   {
     const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};

src/wallet/wallet2.h

@@ -1546,6 +1546,7 @@ private:
     void finish_rescan_bc_keep_key_images(uint64_t transfer_height, const crypto::hash &hash);
     void enable_dns(bool enable) { m_use_dns = enable; }
     void set_offline(bool offline = true);
+    bool is_offline() const { return m_offline; }
 
     uint64_t credits() const { return m_rpc_payment_state.credits; }
     void credit_report(uint64_t &expected_spent, uint64_t &discrepancy) const { expected_spent = m_rpc_payment_state.expected_spent; discrepancy = m_rpc_payment_state.discrepancy; }

tests/CMakeLists.txt

@@ -80,17 +80,22 @@ file(COPY
   data/signed_monero_tx
   DESTINATION data)
 
-add_subdirectory(core_tests)
-add_subdirectory(fuzz)
-add_subdirectory(crypto)
-add_subdirectory(functional_tests)
-add_subdirectory(performance_tests)
-add_subdirectory(core_proxy)
-add_subdirectory(unit_tests)
-add_subdirectory(difficulty)
-add_subdirectory(block_weight)
-add_subdirectory(hash)
-add_subdirectory(net_load_tests)
+if (CMAKE_BUILD_TYPE STREQUAL "fuzz" OR OSSFUZZ)
+  add_subdirectory(fuzz)
+else ()
+  add_subdirectory(core_tests)
+  add_subdirectory(fuzz)
+  add_subdirectory(crypto)
+  add_subdirectory(functional_tests)
+  add_subdirectory(performance_tests)
+  add_subdirectory(core_proxy)
+  add_subdirectory(unit_tests)
+  add_subdirectory(difficulty)
+  add_subdirectory(block_weight)
+  add_subdirectory(hash)
+  add_subdirectory(net_load_tests)
+endif()
+
 if (BUILD_GUI_DEPS)
   add_subdirectory(libwallet_api_tests)
 endif()
@@ -106,7 +111,7 @@ set(hash_targets_sources
 
 set(hash_targets_headers)
 
-add_executable(hash-target-tests
+monero_add_minimal_executable(hash-target-tests
   ${hash_targets_sources}
   ${hash_targets_headers})
 target_link_libraries(hash-target-tests
@@ -149,7 +154,7 @@ foreach(BENCH IN LISTS MONERO_WALLET_CRYPTO_BENCH)
 endforeach ()
 
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/benchmark.h.in" "${MONERO_GENERATED_HEADERS_DIR}/tests/benchmark.h")
-add_executable(monero-wallet-crypto-bench benchmark.cpp ${BENCH_OBJECTS})
+monero_add_minimal_executable(monero-wallet-crypto-bench benchmark.cpp ${BENCH_OBJECTS})
 target_link_libraries(monero-wallet-crypto-bench cncrypto)
 
 add_test(NAME wallet-crypto-bench COMMAND monero-wallet-crypto-bench)

tests/block_weight/CMakeLists.txt

@@ -31,7 +31,7 @@ set(block_weight_sources
 
 set(block_weight_headers)
 
-add_executable(block_weight
+monero_add_minimal_executable(block_weight
   ${block_weight_sources}
   ${block_weight_headers})
 target_link_libraries(block_weight

tests/core_proxy/CMakeLists.txt

@@ -32,7 +32,7 @@ set(core_proxy_sources
 set(core_proxy_headers
   core_proxy.h)
 
-add_executable(core_proxy
+monero_add_minimal_executable(core_proxy
   ${core_proxy_sources}
   ${core_proxy_headers})
 target_link_libraries(core_proxy

tests/core_proxy/core_proxy.cpp

@@ -245,12 +245,17 @@ bool tests::proxy_core::init(const boost::program_options::variables_map& /*vm*/
     return true;
 }
 
-bool tests::proxy_core::have_block(const crypto::hash& id) {
+bool tests::proxy_core::have_block_unlocked(const crypto::hash& id, int *where) {
     if (m_hash2blkidx.end() == m_hash2blkidx.find(id))
         return false;
+    if (where) *where = HAVE_BLOCK_MAIN_CHAIN;
     return true;
 }
 
+bool tests::proxy_core::have_block(const crypto::hash& id, int *where) {
+    return have_block_unlocked(id, where);
+}
+
 void tests::proxy_core::build_short_history(std::list<crypto::hash> &m_history, const crypto::hash &m_start) {
     m_history.push_front(get_block_hash(m_genesis));
     /*std::unordered_map<crypto::hash, tests::block_index>::const_iterator cit = m_hash2blkidx.find(m_lastblk);

tests/core_proxy/core_proxy.h

@@ -66,14 +66,16 @@ namespace tests
       
 
   public:
+    virtual bool is_synchronized() const final { return true; }
     void on_synchronized(){}
     void safesyncmode(const bool){}
-    uint64_t get_current_blockchain_height(){return 1;}
+    virtual uint64_t get_current_blockchain_height() const final {return 1;}
     void set_target_blockchain_height(uint64_t) {}
     bool init(const boost::program_options::variables_map& vm);
     bool deinit(){return true;}
     bool get_short_chain_history(std::list<crypto::hash>& ids);
-    bool have_block(const crypto::hash& id);
+    bool have_block(const crypto::hash& id, int *where = NULL);
+    bool have_block_unlocked(const crypto::hash& id, int *where = NULL);
     void get_blockchain_top(uint64_t& height, crypto::hash& top_id);
     bool handle_incoming_tx(const cryptonote::tx_blob_entry& tx_blob, cryptonote::tx_verification_context& tvc, cryptonote::relay_method tx_relay, bool relayed);
     bool handle_incoming_txs(const std::vector<cryptonote::tx_blob_entry>& tx_blobs, std::vector<cryptonote::tx_verification_context>& tvc, cryptonote::relay_method tx_relay, bool relayed);
@@ -111,5 +113,6 @@ namespace tests
     bool prune_blockchain(uint32_t pruning_seed) const { return true; }
     bool get_txpool_complement(const std::vector<crypto::hash> &hashes, std::vector<cryptonote::blobdata> &txes) { return false; }
     bool get_pool_transaction_hashes(std::vector<crypto::hash>& txs, bool include_unrelayed_txes = true) const { return false; }
+    crypto::hash get_block_id_by_height(uint64_t height) const { return crypto::null_hash; }
   };
 }

tests/core_tests/CMakeLists.txt

@@ -68,7 +68,7 @@ set(core_tests_headers
   rct2.h
   wallet_tools.h)
 
-add_executable(core_tests
+monero_add_minimal_executable(core_tests
   ${core_tests_sources}
   ${core_tests_headers})
 target_link_libraries(core_tests

tests/crypto/CMakeLists.txt

@@ -37,7 +37,7 @@ set(crypto_sources
 set(crypto_headers
   crypto-tests.h)
 
-add_executable(cncrypto-tests
+monero_add_minimal_executable(cncrypto-tests
   ${crypto_sources}
   ${crypto_headers})
 target_link_libraries(cncrypto-tests
@@ -53,7 +53,7 @@ add_test(
   NAME    cncrypto
   COMMAND cncrypto-tests "${CMAKE_CURRENT_SOURCE_DIR}/tests.txt")
 
-add_executable(cnv4-jit-tests cnv4-jit.c)
+monero_add_minimal_executable(cnv4-jit-tests cnv4-jit.c)
 target_link_libraries(cnv4-jit-tests
   PRIVATE
     common