feat: add method for fetching both clockhain and target height

A reorg can end up causing an output's position in the chain to
move. Since the wallet doesn't update the RingDB on reorg, it
may refer to the output's stale position in the chain.

This seems a reasonable solution rather than introducing complex
logic to update the stale ring member's value on rerog, since
RingDB can be deprecated with FCMP++.

.github/FUNDING.yml

@@ -1 +0,0 @@
-custom: https://www.getmonero.org/get-started/contributing/

.github/workflows/build.yml

@@ -1,184 +0,0 @@
-name: ci/gh-actions/cli
-
-on:
-  push:
-  pull_request:
-    paths-ignore:
-      - 'docs/**'
-      - '**/README.md'
-
-# The below variables reduce repetitions across similar targets
-env:
-  REMOVE_BUNDLED_BOOST : rm -rf /usr/local/share/boost
-  BUILD_DEFAULT_LINUX: |
-        cmake -S . -B build -D ARCH="default" -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=Release && cmake --build build -j3
-  APT_INSTALL_LINUX: 'sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler ccache'
-  APT_SET_CONF: |
-        echo "Acquire::Retries \"3\";"         | sudo tee -a /etc/apt/apt.conf.d/80-custom
-        echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
-        echo "Acquire::ftp::Timeout \"120\";"  | sudo tee -a /etc/apt/apt.conf.d/80-custom
-  CCACHE_SETTINGS: |
-        ccache --max-size=150M
-        ccache --set-config=compression=true
-
-jobs:
-  build-macos:
-    runs-on: macOS-latest
-    env:
-      CCACHE_TEMPDIR: /tmp/.ccache-temp
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-    - uses: actions/cache@v3
-      with:
-        path: /Users/runner/Library/Caches/ccache
-        key: ccache-${{ runner.os }}-build-${{ github.sha }}
-        restore-keys: ccache-${{ runner.os }}-build-
-    - name: install dependencies
-      run: HOMEBREW_NO_AUTO_UPDATE=1 brew install boost hidapi openssl zmq libpgm miniupnpc expat libunwind-headers protobuf ccache
-    - name: build
-      run: |
-        ${{env.CCACHE_SETTINGS}}
-        make -j3
-
-  build-windows:
-    runs-on: windows-latest
-    env:
-      CCACHE_TEMPDIR: C:\Users\runneradmin\.ccache-temp
-      CCACHE_DIR: C:\Users\runneradmin\.ccache
-    defaults:
-      run:
-        shell: msys2 {0}
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-    - uses: actions/cache@v3
-      with:
-        path: C:\Users\runneradmin\.ccache
-        key: ccache-${{ runner.os }}-build-${{ github.sha }}
-        restore-keys: ccache-${{ runner.os }}-build-
-    - uses: msys2/setup-msys2@v2
-      with:
-        update: true
-        install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf-c mingw-w64-x86_64-libusb mingw-w64-x86_64-unbound git
-    - name: build
-      run: |
-        ${{env.CCACHE_SETTINGS}}
-        make release-static-win64 -j2
-
-# See the OS labels and monitor deprecations here:
-# https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
-
-  build-ubuntu:
-    runs-on: ${{ matrix.os }}
-    env:
-      CCACHE_TEMPDIR: /tmp/.ccache-temp
-    strategy:
-      matrix:
-        os: [ubuntu-22.04, ubuntu-20.04]
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-    - uses: actions/cache@v3
-      with:
-        path: ~/.ccache
-        key: ccache-${{ runner.os }}-build-${{ matrix.os }}-${{ github.sha }}
-        restore-keys: ccache-${{ runner.os }}-build-${{ matrix.os }}
-    - name: remove bundled boost
-      run: ${{env.REMOVE_BUNDLED_BOOST}}
-    - name: set apt conf
-      run: ${{env.APT_SET_CONF}}
-    - name: update apt
-      run: sudo apt update
-    - name: install monero dependencies
-      run: ${{env.APT_INSTALL_LINUX}}
-    - name: build
-      run: |
-        ${{env.CCACHE_SETTINGS}}
-        ${{env.BUILD_DEFAULT_LINUX}}
-
-  libwallet-ubuntu:
-    runs-on: ubuntu-20.04
-    env:
-      CCACHE_TEMPDIR: /tmp/.ccache-temp
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-    - uses: actions/cache@v3
-      with:
-        path: ~/.ccache
-        key: ccache-${{ runner.os }}-libwallet-${{ github.sha }}
-        restore-keys: ccache-${{ runner.os }}-libwallet-
-    - name: remove bundled boost
-      run: ${{env.REMOVE_BUNDLED_BOOST}}
-    - name: set apt conf
-      run: ${{env.APT_SET_CONF}}
-    - name: update apt
-      run: sudo apt update
-    - name: install monero dependencies
-      run: ${{env.APT_INSTALL_LINUX}}
-    - name: build
-      run: |
-        ${{env.CCACHE_SETTINGS}}
-        cmake .
-        make wallet_api -j3
-
-  test-ubuntu:
-    needs: build-ubuntu
-    runs-on: ubuntu-20.04
-    env:
-      CCACHE_TEMPDIR: /tmp/.ccache-temp
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: recursive
-    - name: ccache
-      uses: actions/cache@v3
-      with:
-        path: ~/.ccache
-        key: ccache-${{ runner.os }}-build-ubuntu-latest-${{ github.sha }}
-        restore-keys: ccache-${{ runner.os }}-build-ubuntu-latest
-    - name: remove bundled boost
-      run: ${{env.REMOVE_BUNDLED_BOOST}}
-    - name: set apt conf
-      run: ${{env.APT_SET_CONF}}
-    - name: update apt
-      run: sudo apt update
-    - name: install monero dependencies
-      run: ${{env.APT_INSTALL_LINUX}}
-    - name: install Python dependencies
-      run: pip install requests psutil monotonic zmq deepdiff
-    - name: tests
-      env:
-        CTEST_OUTPUT_ON_FAILURE: ON
-        DNS_PUBLIC: tcp://9.9.9.9
-      run: |
-        ${{env.CCACHE_SETTINGS}}
-        ${{env.BUILD_DEFAULT_LINUX}}
-        cmake --build build --target test
-
-# ARCH="default" (not "native") ensures, that a different execution host can execute binaries compiled elsewhere.
-# BUILD_SHARED_LIBS=ON speeds up the linkage part a bit, reduces size, and is the only place where the dynamic linkage is tested.
-
-  source-archive:
-    runs-on: ubuntu-20.04
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        submodules: recursive
-    - name: archive
-      run: |
-        pip install git-archive-all
-        export VERSION="monero-$(git describe)"
-        export OUTPUT="$VERSION.tar"
-        echo "OUTPUT=$OUTPUT" >> $GITHUB_ENV
-        /home/runner/.local/bin/git-archive-all --prefix "$VERSION/" --force-submodules "$OUTPUT"
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ env.OUTPUT }}
-        path: /home/runner/work/monero/monero/${{ env.OUTPUT }}

.github/workflows/depends.yml

@@ -1,106 +0,0 @@
-name: ci/gh-actions/depends
-
-on:
-  push:
-  pull_request:
-    paths-ignore:
-      - 'docs/**'
-      - '**/README.md'
-
-env:
-  APT_SET_CONF: |
-        echo "Acquire::Retries \"3\";"         | sudo tee -a /etc/apt/apt.conf.d/80-custom
-        echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
-        echo "Acquire::ftp::Timeout \"120\";"  | sudo tee -a /etc/apt/apt.conf.d/80-custom
-  CCACHE_SETTINGS: |
-        ccache --max-size=150M
-        ccache --set-config=compression=true
-
-jobs:
-  build-cross:
-    runs-on: ubuntu-20.04
-    env:
-      CCACHE_TEMPDIR: /tmp/.ccache-temp
-    strategy:
-      fail-fast: false
-      matrix:
-        toolchain:
-          - name: "RISCV 64bit"
-            host: "riscv64-linux-gnu"
-            packages: "python3 gperf g++-riscv64-linux-gnu"
-          - name: "ARM v7"
-            host: "arm-linux-gnueabihf"
-            packages: "python3 gperf g++-arm-linux-gnueabihf"
-          - name: "ARM v8"
-            host: "aarch64-linux-gnu"
-            packages: "python3 gperf g++-aarch64-linux-gnu"
-          - name: "i686 Win"
-            host: "i686-w64-mingw32"
-            packages: "python3 g++-mingw-w64-i686"
-          - name: "i686 Linux"
-            host: "i686-pc-linux-gnu"
-            packages: "gperf cmake g++-multilib python3-zmq"
-          - name: "Win64"
-            host: "x86_64-w64-mingw32"
-            packages: "cmake python3 g++-mingw-w64-x86-64"
-          - name: "x86_64 Linux"
-            host: "x86_64-unknown-linux-gnu"
-            packages: "gperf cmake python3-zmq libdbus-1-dev libharfbuzz-dev"
-          - name: "Cross-Mac x86_64"
-            host: "x86_64-apple-darwin11"
-            packages: "cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git"
-          - name: "Cross-Mac aarch64"
-            host: "aarch64-apple-darwin11"
-            packages: "cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git"
-          - name: "x86_64 Freebsd"
-            host: "x86_64-unknown-freebsd"
-            packages: "clang-8 gperf cmake python3-zmq libdbus-1-dev libharfbuzz-dev"
-    name: ${{ matrix.toolchain.name }}
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        submodules: recursive
-# Most volatile cache
-    - name: ccache
-      uses: actions/cache@v3
-      with:
-        path: ~/.ccache
-        key: ccache-${{ matrix.toolchain.host }}-${{ github.sha }}
-        restore-keys: ccache-${{ matrix.toolchain.host }}-
-# Less volatile cache
-    - name: depends cache
-      uses: actions/cache@v3
-      with:
-        path: contrib/depends/built
-        key: depends-${{ matrix.toolchain.host }}-${{ hashFiles('contrib/depends/packages/*') }}
-        restore-keys: |
-          depends-${{ matrix.toolchain.host }}-${{ hashFiles('contrib/depends/packages/*') }}
-          depends-${{ matrix.toolchain.host }}-
-# Static cache
-    - name: OSX SDK cache
-      uses: actions/cache@v3
-      with:
-        path: contrib/depends/sdk-sources
-        key: sdk-${{ matrix.toolchain.host }}-${{ matrix.toolchain.osx_sdk }}
-        restore-keys: sdk-${{ matrix.toolchain.host }}-${{ matrix.toolchain.osx_sdk }}
-    - name: set apt conf
-      run: ${{env.APT_SET_CONF}}
-    - name: install dependencies
-      run: sudo apt update; sudo apt -y install build-essential libtool cmake autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache ${{ matrix.toolchain.packages }}
-    - name: prepare w64-mingw32
-      if: ${{ matrix.toolchain.host == 'x86_64-w64-mingw32' || matrix.toolchain.host == 'i686-w64-mingw32' }}
-      run: |
-        sudo update-alternatives --set ${{ matrix.toolchain.host }}-g++ $(which ${{ matrix.toolchain.host }}-g++-posix)
-        sudo update-alternatives --set ${{ matrix.toolchain.host }}-gcc $(which ${{ matrix.toolchain.host }}-gcc-posix)
-    - name: build
-      run: |
-        ${{env.CCACHE_SETTINGS}}
-        make depends target=${{ matrix.toolchain.host }} -j2
-    - uses: actions/upload-artifact@v3
-      if: ${{ matrix.toolchain.host == 'x86_64-w64-mingw32' || matrix.toolchain.host == 'x86_64-apple-darwin11' || matrix.toolchain.host == 'x86_64-unknown-linux-gnu' }}
-      with:
-        name: ${{ matrix.toolchain.name }}
-        path: |
-          /home/runner/work/monero/monero/build/${{ matrix.toolchain.host }}/release/bin/monero-wallet-cli*
-          /home/runner/work/monero/monero/build/${{ matrix.toolchain.host }}/release/bin/monerod*

.github/workflows/gitian.yml

@@ -1,49 +0,0 @@
-name: ci/gh-actions/gitian
-
-on:
-  push:
-    tags:
-      - '*'
-
-jobs:
-  build-gitian:
-    runs-on: ubuntu-20.04
-    strategy:
-      fail-fast: false
-      matrix:
-        operating-system:
-          - name: "Linux"
-            option: "l"
-          - name: "Windows"
-            option: "w"
-          - name: "Android"
-            option: "a"
-          - name: "FreeBSD"
-            option: "f"
-          - name: "macOS"
-            option: "m"
-    name: ${{ matrix.operating-system.name }}
-    steps:
-    - name: prepare
-      run: |
-        sudo apt update
-        curl -O https://raw.githubusercontent.com/monero-project/monero/${{ github.ref_name }}/contrib/gitian/gitian-build.py
-        chmod +x gitian-build.py
-    - name: setup
-      run: |
-        ./gitian-build.py --setup --docker github-actions ${{ github.ref_name }}
-    - name: build
-      run: |
-        ./gitian-build.py --docker --detach-sign --no-commit --build -j 3 -o ${{ matrix.operating-system.option }} github-actions ${{ github.ref_name }}
-    - name: post build
-      run: |
-        cd out/${{ github.ref_name }}
-        shasum -a256 *
-        echo \`\`\` >> $GITHUB_STEP_SUMMARY
-        shasum -a256 * >> $GITHUB_STEP_SUMMARY
-        echo \`\`\` >> $GITHUB_STEP_SUMMARY
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ${{ matrix.operating-system.name }}
-        path: |
-          out/${{ github.ref_name }}/*

.gitmodules

@@ -5,9 +5,14 @@
 	path = external/rapidjson
 	url = https://github.com/Tencent/rapidjson
 [submodule "external/trezor-common"]
-	active = false
 	path = external/trezor-common
 	url = https://github.com/trezor/trezor-common.git
+[submodule "external/utf8proc"]
+	path = external/utf8proc
+	url = https://github.com/JuliaStrings/utf8proc.git
+[submodule "external/polyseed"]
+	path = external/polyseed
+	url = https://github.com/tevador/polyseed.git
 [submodule "external/supercop"]
 	path = external/supercop
 	url = https://github.com/monero-project/supercop

CMakeLists.txt

@@ -219,7 +219,7 @@ function(forbid_undefined_symbols)
     file(MAKE_DIRECTORY "${TEST_PROJECT}")
     file(WRITE "${TEST_PROJECT}/CMakeLists.txt"
       [=[
-cmake_minimum_required(VERSION 3.1)
+cmake_minimum_required(VERSION 3.5)
 project(test)
 option(EXPECT_SUCCESS "" ON)
 file(WRITE "${CMAKE_SOURCE_DIR}/incorrect_source.cpp" "void undefined_symbol(); void symbol() { undefined_symbol(); }")
@@ -367,9 +367,11 @@ if(NOT MANUAL_SUBMODULES)
     message(STATUS "Checking submodules")
     check_submodule(external/miniupnp)
     check_submodule(external/rapidjson)
-    #check_submodule(external/trezor-common)
+    check_submodule(external/trezor-common)
     check_submodule(external/randomwow)
     check_submodule(external/supercop)
+    check_submodule(external/polyseed)
+    check_submodule(external/utf8proc)
   endif()
 endif()
 
@@ -406,7 +408,7 @@ option(BOOST_IGNORE_SYSTEM_PATHS "Ignore boost system paths for local boost inst
 set_property(GLOBAL PROPERTY USE_FOLDERS ON)
 enable_testing()
 
-option(BUILD_DOCUMENTATION "Build the Doxygen documentation." OFF)
+option(BUILD_DOCUMENTATION "Build the Doxygen documentation." ON)
 option(BUILD_TESTS "Build tests." OFF)
 if (CMAKE_BUILD_TYPE STREQUAL "Debug")
   set(DEFAULT_BUILD_DEBUG_UTILITIES ON)
@@ -459,7 +461,7 @@ endif()
 # elseif(CMAKE_SYSTEM_NAME MATCHES ".*BSDI.*")
 #   set(BSDI TRUE)
 
-include_directories(external/rapidjson/include external/easylogging++ src contrib/epee/include external external/supercop/include)
+include_directories(external/rapidjson/include external/easylogging++ src contrib/epee/include external external/supercop/include external/polyseed/include external/utf8proc)
 
 if(APPLE)
   cmake_policy(SET CMP0042 NEW)
@@ -1116,7 +1118,19 @@ if(MINGW)
   if(DEPENDS)
     set(ICU_LIBRARIES icuio icui18n icuuc icudata icutu iconv)
   else()
-    set(ICU_LIBRARIES icuio icuin icuuc icudt icutu iconv)
+    # This is an extremely ugly hack to get around Boost not being built with static ICU.
+    # We reported the issue, we are waiting for upstream to fix this issue: https://github.com/boostorg/boost/issues/1079#issue-3384962885
+    # This hack links shared ICU libs to avoid linker errors we get in MSYS2 compilation (undefined symbols to ICU).
+    set(OLD_LIB_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
+    set(CMAKE_FIND_LIBRARY_SUFFIXES ".dll.a")
+    find_library(ICUIO_LIBRARIES NAMES icuio REQUIRED)
+    find_library(ICUIN_LIBRARIES NAMES icuin REQUIRED)
+    find_library(ICUUC_LIBRARIES NAMES icuuc REQUIRED)
+    find_library(ICUDT_LIBRARIES NAMES icudt REQUIRED)
+    find_library(ICUTU_LIBRARIES NAMES icutu REQUIRED)
+    find_library(ICONV_LIBRARIES NAMES iconv REQUIRED)
+    set(ICU_LIBRARIES ${ICUIO_LIBRARIES} ${ICUIN_LIBRARIES} ${ICUUC_LIBRARIES} ${ICUDT_LIBRARIES} ${ICUTU_LIBRARIES} ${ICONV_LIBRARIES})
+    set(CMAKE_FIND_LIBRARY_SUFFIXES ${OLD_LIB_SUFFIXES})
   endif()
 elseif(APPLE OR OPENBSD OR ANDROID)
   set(EXTRA_LIBRARIES "")

Dockerfile

@@ -41,26 +41,26 @@ RUN set -ex && \
     rm -rf /var/lib/apt
 COPY --from=builder /src/build/x86_64-linux-gnu/release/bin /usr/local/bin/
 
-# Create wownero user
-RUN adduser --system --group --disabled-password wownero && \
-	mkdir -p /wallet /home/wownero/.wownero && \
-	chown -R wownero:wownero /home/wownero/.wownero && \
-	chown -R wownero:wownero /wallet
+# Create monero user
+RUN adduser --system --group --disabled-password monero && \
+	mkdir -p /wallet /home/monero/.bitmonero && \
+	chown -R monero:monero /home/monero/.bitmonero && \
+	chown -R monero:monero /wallet
 
 # Contains the blockchain
-VOLUME /home/wownero/.wownero
+VOLUME /home/monero/.bitmonero
 
 # Generate your wallet via accessing the container and run:
 # cd /wallet
-# wownero-wallet-cli
+# monero-wallet-cli
 VOLUME /wallet
 
-EXPOSE 34567
-EXPOSE 34568
+EXPOSE 18080
+EXPOSE 18081
 
-# switch to user wownero
-USER wownero
+# switch to user monero
+USER monero
 
-ENTRYPOINT ["wownerod"]
-CMD ["--p2p-bind-ip=0.0.0.0", "--p2p-bind-port=34567", "--rpc-bind-ip=0.0.0.0", "--rpc-bind-port=34568", "--non-interactive", "--confirm-external-bind"]
+ENTRYPOINT ["monerod"]
+CMD ["--p2p-bind-ip=0.0.0.0", "--p2p-bind-port=18080", "--rpc-bind-ip=0.0.0.0", "--rpc-bind-port=18081", "--non-interactive", "--confirm-external-bind"]
 

Makefile

@@ -44,7 +44,7 @@ else
   deldirs  := $(builddir)/debug $(builddir)/release $(builddir)/fuzz
 endif
 
-all: release-all
+all: release-minimal
 
 depends:
 	cd contrib/depends && $(MAKE) HOST=$(target) && cd ../.. && mkdir -p build/$(target)/release
@@ -100,11 +100,18 @@ release-test:
 
 release-all:
 	mkdir -p $(builddir)/release
-	cd $(builddir)/release && cmake -D BUILD_TESTS=OFF -D USE_DEVICE_TREZOR=OFF -D CMAKE_BUILD_TYPE=Release $(topdir) && $(MAKE)
+	cd $(builddir)/release && cmake -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=Release $(topdir) && $(MAKE)
 
 release-static:
 	mkdir -p $(builddir)/release
-	cd $(builddir)/release && cmake -D BUILD_TESTS=OFF -D USE_DEVICE_TREZOR=OFF -D STATIC=ON -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Release $(topdir) && $(MAKE)
+	cd $(builddir)/release && cmake -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Release $(topdir) && $(MAKE)
+
+release-minimal:
+	@echo "Starting minimal Wownero build... for full build, run: make release-all"
+	mkdir -p $(builddir)/release
+	cd $(builddir)/release && cmake -Wno-dev -D BUILD_TAG="minimal" -D CMAKE_BUILD_TYPE=Release -D BUILD_TESTS=OFF -D BUILD_DOCUMENTATION=OFF -D BUILD_DEBUG_UTILITIES=OFF -D USE_DEVICE_TREZOR=OFF -D TREZOR_DEBUG=OFF -D BUILD_GUI_DEPS=OFF $(topdir) && $(MAKE) daemon simplewallet wallet_rpc_server
+	@echo "\n===== Build complete =====\n"
+	@echo "Binaries are available at: cd $(builddir)/release/bin\n"
 
 coverage:
 	mkdir -p $(builddir)/debug
@@ -130,6 +137,23 @@ release-static-android-armv8:
 	cd $(builddir)/release/translations && cmake ../../../translations && $(MAKE)
 	cd $(builddir)/release && CC=aarch64-linux-android-clang CXX=aarch64-linux-android-clang++ cmake -D BUILD_TESTS=OFF -D ARCH="armv8-a" -D STATIC=ON -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Release -D ANDROID=true -D BUILD_TAG="android-armv8" -D CMAKE_SYSTEM_NAME="Android" -D CMAKE_ANDROID_STANDALONE_TOOLCHAIN="${ANDROID_STANDALONE_TOOLCHAIN_PATH}" -D CMAKE_ANDROID_ARCH_ABI="arm64-v8a" ../.. && $(MAKE)
 
+release-static-android-armv7-wallet_api:
+	mkdir -p $(builddir)/release
+	cd $(builddir)/release && CC=arm-linux-androideabi-clang CXX=arm-linux-androideabi-clang++ cmake -D USE_DEVICE_TREZOR=OFF -D BUILD_GUI_DEPS=1 -D BUILD_TESTS=OFF -D ARCH="armv7-a" -D STATIC=ON -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release -D ANDROID=true -D BUILD_TAG="android-armv7" -D CMAKE_SYSTEM_NAME="Android" -D CMAKE_ANDROID_STANDALONE_TOOLCHAIN="${ANDROID_STANDALONE_TOOLCHAIN_PATH}" -D CMAKE_ANDROID_ARM_MODE=ON -D CMAKE_ANDROID_ARCH_ABI="armeabi-v7a" -D NO_AES=true ../.. && $(MAKE) wallet_api
+
+release-static-android-armv8-wallet_api:
+	mkdir -p $(builddir)/release
+	cd $(builddir)/release && CC=aarch64-linux-android-clang CXX=aarch64-linux-android-clang++ cmake -D USE_DEVICE_TREZOR=OFF -D BUILD_GUI_DEPS=1 -D BUILD_TESTS=OFF -D ARCH="armv8-a" -D STATIC=ON -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release -D ANDROID=true -D BUILD_TAG="android-armv8" -D CMAKE_SYSTEM_NAME="Android" -D CMAKE_ANDROID_STANDALONE_TOOLCHAIN="${ANDROID_STANDALONE_TOOLCHAIN_PATH}" -D CMAKE_ANDROID_ARCH_ABI="arm64-v8a" ../.. && $(MAKE) wallet_api
+
+release-static-android-x86_64-wallet_api:
+	mkdir -p $(builddir)/release
+	cd $(builddir)/release && CC=x86_64-linux-android-clang CXX=x86_64-linux-android-clang++ cmake -D USE_DEVICE_TREZOR=OFF -D BUILD_GUI_DEPS=1 -D BUILD_TESTS=OFF -D ARCH="x86-64" -D STATIC=ON -D BUILD_64=ON -D CMAKE_BUILD_TYPE=release -D ANDROID=true -D BUILD_TAG="android-x86_64" -D CMAKE_SYSTEM_NAME="Android" -D CMAKE_ANDROID_STANDALONE_TOOLCHAIN="${ANDROID_STANDALONE_TOOLCHAIN_PATH}" -D CMAKE_ANDROID_ARCH_ABI="x86_64" ../.. && $(MAKE) wallet_api
+
+release-static-android-x86-wallet_api:
+	mkdir -p $(builddir)/release
+	cd $(builddir)/release && CC=i686-linux-android-clang CXX=i686-linux-android-clang++ cmake -D USE_DEVICE_TREZOR=OFF -D BUILD_GUI_DEPS=1 -D BUILD_TESTS=OFF -D ARCH="i686" -D STATIC=ON -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release -D ANDROID=true -D BUILD_TAG="android-x86" -D CMAKE_SYSTEM_NAME="Android" -D CMAKE_ANDROID_STANDALONE_TOOLCHAIN="${ANDROID_STANDALONE_TOOLCHAIN_PATH}" -D CMAKE_ANDROID_ARCH_ABI="x86" ../.. && $(MAKE) wallet_api
+
+
 release-static-linux-armv8:
 	mkdir -p $(builddir)/release
 	cd $(builddir)/release && cmake -D BUILD_TESTS=OFF -D ARCH="armv8-a" -D STATIC=ON -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Release -D BUILD_TAG="linux-armv8" $(topdir) && $(MAKE)

README.md

@@ -1,6 +1,6 @@
 # ~~Mo~~Wownero -  Such privacy! Many coins! Wow!
 
-[<img src="https://suchwow.xyz/data/suchwow/image/to23moqn.jpeg">](https://suchwow.xyz/s/oh-really-9eda16b2/item)
+[<img src="https://suchwow.xyz/uploads/J28THOqDiNkV.jpg">](https://suchwow.xyz/meme/157)
 
 ## Introduction
 
@@ -10,44 +10,39 @@ Unlike Opposing Projects.
 
 ## Resources
 
-- IRC: [OFTC #wownero](https://webchat.oftc.net/?channels=wownero)
 - Web: [wownero.org](https://wownero.org)
 - Twitter: [@w0wn3r0](https://twitter.com/w0wn3r0)
 - Reddit: [/r/wownero](https://www.reddit.com/r/wownero)
 - Mail: [wownero@wownero.org](mailto:wownero@wownero.org)
 - Git: [codeberg.org/wownero/wownero](https://codeberg.org/wownero/wownero)
+- Matrix General Chat Room: [#wownero-gen:wowne.ro](https://matrix.to/#/#wownero-gen:wowne.ro)
+- IRC: [OFTC #wownero](https://webchat.oftc.net/?channels=wownero)
 - Discord: [discord.gg/ykZyAzJhDK](https://discord.com/invite/ykZyAzJhDK)
 - Telegram: [t.me/wownero](https://t.me/wownero)
 - Public Node Status: [monero.fail](https://monero.fail/?chain=wownero&network=mainnet)
-- Wownero Memes: [suchwow.xyz](https://suchwow.xyz/posts/top)
+- Wownero Memes: [suchwow.xyz](https://suchwow.xyz)
 - Market Info: [coinmarketcap.com](https://coinmarketcap.com/currencies/wownero), [coingecko.com](https://www.coingecko.com/en/coins/wownero/usd)
 
 ## Exchanges
 
-- [NonKYC](https://nonkyc.io/market/WOW_BTC)
+- [BasicSwap DEX](https://basicswapdex.com) ([Installation Guide](https://academy.particl.io/en/latest/basicswap-guides/basicswapguides_installation.html))
+- [NonLogs](https://nonlogs.com/trade/WOW-BTC)
 - [AltQuick](https://altquick.com/market/Wownero)
-- [Majestic Bank](https://majesticbank.sc)
-- [TradeOgre](https://tradeogre.com/exchange/BTC-WOW)
 
 ## Wallets
 
 - Wonero CLI Wallet: [codeberg.org/wownero/wownero](https://codeberg.org/wownero/wownero/releases)
-- Wowlet Desktop Wallet: [codeberg.org/wownero/wowlet](https://codeberg.org/wownero/wowlet/releases)
 - Stack Wallet iOS & Android Mobile Wallet: [stackwallet.com](https://stackwallet.com)
 - Cake Wallet [cakewallet.com](https://cakewallet.com)
 
 ## Blockchain Explorers
 
-- https://explorer.suchwow.xyz
+- https://explore.wownero.com
 
 ## Supporting the project
 
 Wownero is a 100% community-sponsored endeavor. Supporting services are also graciously provided by sponsors:
 
-[<img src="https://git.wownero.com/wownero/meta/raw/branch/master/images/macstadium.png"
-alt="MacStadium"
-height="100">](https://www.macstadium.com)
-
 Developers are volunteers doing this mostly for shits and giggles. If you would like to support our shenanigans and stimulant addictions, please consider donating to the dev slush fund.
 
 ### Donation Addresses
@@ -89,7 +84,7 @@ Dates are provided in the format YYYY-MM-DD.
 | -                             | 2020-06-28 | Hallucinogenic Hypnotoad  | v0.8.0.0                | v0.8.0.2                    | Dandelion++ support
 | 253,999                       | 2020-10-09 | Illiterate Illuminati  | v0.9.0.0                | v0.9.3.3                    | Dynamic coinbase unlock (up to 1 mo.), Deterministic unlock times, Enforce maximum coinbase amount, show_qr_code wallet command, CLSAG
 | 331,170                       | 2021-07-04 | Junkie Jeff  | v0.10.0.0               | v0.10.2.0                   | Bulletproofs+, Miner Block Header Signing, Vote by Block, Change coinbase unlock time to 1 day, Reset difficulty and switch back to Monero's difficulty algorithm
-| 514,000                       | 2023-04-01 | Kunty Karen  | v0.11.0.0               | v0.11.3.0                   | View tags, fee changes, adjusted dynamic block weight algorithm, multisig security fixes, RPC broadcast node donation sub-address, Limit tx_extra max size to ~1kb, 12-hour difficulty adjustment window
+| 514,000                       | 2023-04-01 | Kunty Karen  | v0.11.0.0               | v0.11.4.0                   | View tags, fee changes, adjusted dynamic block weight algorithm, multisig security fixes, RPC broadcast node donation sub-address, Limit tx_extra max size to ~1kb, 12-hour difficulty adjustment window
 
 X's indicate that these details have not been determined as of commit date.
 
@@ -101,14 +96,6 @@ Packages are available for
 
         yay -S wownero-git
 
-* Gentoo - Russian hacking tool
-
-        emerge --noreplace eselect-repository
-        eselect repository enable monero
-        emaint sync -r monero
-        echo '*/*::monero ~amd64' >> /etc/portage/package.accept_keywords
-        emerge net-p2p/wownero
-
 * NixOS
 
         nix-shell -p wownero
@@ -120,92 +107,41 @@ Packaging for your favorite distribution would be a welcome contribution!
 
 ## Building from Source
 
-* Docker
-
-        git clone https://codeberg.org/wownero/wownero && cd wownero
-        docker build -t git-wow:master -m 4g .
-        docker run -it -p 34567:34567 -p 34568:34568 -w /home/wownero/build/release/bin git-wow:master bash
-
 * Arch Linux/Manjaro
 
-        sudo pacman -Syu --needed base-devel cmake boost openssl zeromq libpgm unbound libsodium libunwind xz readline expat gtest python3 ccache doxygen graphviz qt5-tools hidapi libusb protobuf systemd gcc13
+        sudo pacman -Syu --needed base-devel cmake boost openssl zeromq libpgm unbound libsodium libunwind xz readline expat gtest python3 ccache doxygen graphviz qt5-tools hidapi libusb protobuf systemd
+        
         git clone https://codeberg.org/wownero/wownero && cd wownero
-        export CC=gcc-13 CXX=g++-13
-        make -j2
+        
+        make
 
 * Debian/Ubuntu
 
-        sudo apt update && sudo apt install build-essential cmake pkg-config libssl-dev libzmq3-dev libunbound-dev libsodium-dev libunwind8-dev liblzma-dev libreadline6-dev libexpat1-dev libpgm-dev qttools5-dev-tools libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler libudev-dev libboost-chrono-dev libboost-date-time-dev libboost-filesystem-dev libboost-locale-dev libboost-program-options-dev libboost-regex-dev libboost-serialization-dev libboost-system-dev libboost-thread-dev python3 ccache doxygen graphviz gcc-13 g++-13
+        sudo apt update && sudo apt install build-essential cmake pkg-config libssl-dev libzmq3-dev libunbound-dev libsodium-dev libunwind8-dev liblzma-dev libreadline6-dev libexpat1-dev libpgm-dev qttools5-dev-tools libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler libudev-dev libboost-chrono-dev libboost-date-time-dev libboost-filesystem-dev libboost-locale-dev libboost-program-options-dev libboost-regex-dev libboost-serialization-dev libboost-system-dev libboost-thread-dev python3 ccache doxygen graphviz
+        
         git clone https://codeberg.org/wownero/wownero && cd wownero
-        CC="gcc-13" CXX="g++-13" make -j2
-
+        
+        make
 
 ## Running Binaries
 
-The build places the binary in `bin/` sub-directory within the build directory
-from which cmake was invoked (repository root by default). To run in the
+The build places the binary in `/build/<OS>/<BRANCH>/release/bin` sub-directory. To run in the
 foreground:
 
-    ./bin/wownerod
+    ./wownerod
 
-To list all available options, run `./bin/wownerod --help`.  Options can be
-specified either on the command line or in a configuration file passed by the
-`--config-file` argument.  To specify an option in the configuration file, add
-a line with the syntax `argumentname=value`, where `argumentname` is the name
-of the argument without the leading dashes, for example, `log-level=1`.
+To list all available options, run `./wownerod --help`.  Options can be specified either on the command line or in a configuration file passed by the `--config-file` argument.  To specify an option in the configuration file, add a line with the syntax `argumentname=value`, where `argumentname` is the name of the argument without the leading dashes, for example, `log-level=1`.
 
 To run in background:
 
-    ./bin/wownerod --log-file wownerod.log --detach
-
-To run as a systemd service, copy
-[wownerod.service](utils/systemd/wownerod.service) to `/etc/systemd/system/` and
-[wow.conf](utils/conf/wow.conf) to `/etc/`. The [example
-service](utils/systemd/wownerod.service) assumes that the user `wownero` exists
-and its home is the data directory specified in the [example
-config](wow.conf).
+    ./wownerod --detach
 
 Once node is synced to network, run the CLI wallet by entering:
 
-    ./bin/wownero-wallet-cli
+    ./wownero-wallet-cli
 
 Type `help` in CLI wallet to see standard commands (for advanced options, type `help_advanced`).
 
-## Tor Anonymity Network
+Copyright (c) 2014-2025 The Monero Project.
 
-### Ubuntu
-
-* `sudo apt-get update && sudo apt-get install tor -y`
-* `sudo nano /etc/tor/torrc`
-
-add the following:
-```
-HiddenServiceDir /var/lib/tor/wownero/
-HiddenServicePort 34569 127.0.0.1:34569
-HiddenServicePort 34566 127.0.0.1:34566
-HiddenServiceVersion 3
-```
-save and close nano
-
-* `sudo /etc/init.d/tor restart && sudo systemctl enable tor`
-* copy [wow.conf](utils/conf/wow.conf) file and save it in same directory as `wownerod`.
-* start wownerod like this:
-
-```
-./wownerod --config-file=wow.conf
-```
-
-* `sudo cat /var/lib/tor/wownero/hostname`
-
-Copy your onion address and share node with others [here](https://monero.fail/?crypto=wownero).
-
-To share your node over p2p, uncomment first line of wownerod.conf and add your onion address.
-
-### Access remote Tor node from CLI wallet
-
-```
-./wownero-wallet-cli --proxy 127.0.0.1:9050 --daemon-address iy6ry6uudpzvbd72zsipepukp6nsazjdu72n52vg3isfnxqn342flzad.onion:34568
-```
-
-Copyright (c) 2014-2024 The Monero Project.
 Portions Copyright (c) 2012-2013 The Cryptonote developers.

cmake/CheckLinkerFlag.cmake

@@ -6,7 +6,7 @@ macro(CHECK_LINKER_FLAG flag VARIABLE)
       message(STATUS "Looking for ${flag} linker flag")
     endif()
 
-    set(_cle_source ${CMAKE_SOURCE_DIR}/cmake/CheckLinkerFlag.c)
+    set(_cle_source ${monero_SOURCE_DIR}/cmake/CheckLinkerFlag.c)
 
     set(saved_CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
     set(CMAKE_C_FLAGS "${flag}")

cmake/CheckTrezor.cmake

@@ -55,6 +55,10 @@ if (USE_DEVICE_TREZOR)
         set(Protobuf_FOUND 1)  # override found if all rquired info was provided by variables
     endif()
 
+    if (Protobuf_VERSION VERSION_GREATER_EQUAL 22.0)
+        add_definitions(-DPROTOBUF_HAS_ABSEIL)
+    endif()
+
     if(TREZOR_DEBUG)
         set(USE_DEVICE_TREZOR_DEBUG 1)
     endif()

contrib/depends/Makefile

@@ -21,18 +21,24 @@ host_toolchain:=$(HOST)-
 endif
 
 ifneq ($(DEBUG),)
-release_type=Debug
+release_type=debug
 else
-release_type=Release
+release_type=release
 endif
 
 ifneq ($(TESTS),)
 build_tests=ON
-release_type=Debug
+release_type=debug
 else
 build_tests=OFF
 endif
 
+ifeq ($(release_type),debug)
+cmake_release_type=Debug
+else
+cmake_release_type=Release
+endif
+
 base_build_dir=$(BASEDIR)/work/build
 base_staging_dir=$(BASEDIR)/work/staging
 base_download_dir=$(BASEDIR)/work/download
@@ -179,7 +185,7 @@ $(host_prefix)/share/toolchain.cmake : toolchain.cmake.in $(host_prefix)/.stamp_
             -e 's|@LDFLAGS@|$(strip $(host_LDFLAGS) $(host_$(release_type)_LDFLAGS))|' \
             -e 's|@allow_host_packages@|$(ALLOW_HOST_PACKAGES)|' \
             -e 's|@debug@|$(DEBUG)|' \
-            -e 's|@release_type@|$(release_type)|' \
+            -e 's|@release_type@|$(cmake_release_type)|' \
             -e 's|@build_tests@|$(build_tests)|' \
             -e 's|@depends@|$(host_cmake)|' \
             -e 's|@prefix@|$($(host_arch)_$(host_os)_prefix)|'\

contrib/depends/packages/boost.mk

@@ -1,15 +1,15 @@
 package=boost                                                                                                                                                                                                                      
-$(package)_version=1_64_0
-$(package)_download_path=https://downloads.sourceforge.net/project/boost/boost/1.64.0/
-$(package)_file_name=$(package)_$($(package)_version).tar.bz2
-$(package)_sha256_hash=7bcc5caace97baa948931d712ea5f37038dbb1c5d89b43ad4def4ed7cb683332
+$(package)_version=1.69.0
+$(package)_download_path=https://archives.boost.io/release/$($(package)_version)/source/
+$(package)_file_name=$(package)_$(subst .,_,$($(package)_version)).tar.gz
+$(package)_sha256_hash=9a2c2819310839ea373f42d69e733c339b4e9a19deab6bfec448281554aa4dbb
 $(package)_dependencies=libiconv
 $(package)_patches=fix_aroptions.patch fix_arm_arch.patch
 
 define $(package)_set_vars
 $(package)_config_opts_release=variant=release
 $(package)_config_opts_debug=variant=debug
-$(package)_config_opts=--layout=tagged --build-type=complete --user-config=user-config.jam
+$(package)_config_opts+=--layout=system --user-config=user-config.jam
 $(package)_config_opts+=threading=multi link=static -sNO_BZIP2=1 -sNO_ZLIB=1
 $(package)_config_opts_linux=threadapi=pthread runtime-link=shared
 $(package)_config_opts_android=threadapi=pthread runtime-link=static target-os=android

contrib/epee/include/file_io_utils.h

@@ -38,7 +38,6 @@ namespace file_io_utils
     bool is_file_exist(const std::string& path);
     bool save_string_to_file(const std::string& path_to_file, const std::string& str);
     bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size = 1000000000);
-    bool get_file_size(const std::string& path_to_file, uint64_t &size);
 }
 }
 

contrib/epee/include/md5_l.h

@@ -1,96 +0,0 @@
-/*
- * libEtPan! -- a mail stuff library
- *
- * Copyright (C) 2001, 2005 - DINH Viet Hoa
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. Neither the name of the libEtPan! project nor the names of its
- *    contributors may be used to endorse or promote products derived
- *    from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-/*
- * $Id: md5.h,v 1.1.1.1 2005/03/18 20:17:27 zautrix Exp $
- */
-
-/* MD5.H - header file for MD5C.C
- */
-
-/* Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All
-rights reserved.
-
-License to copy and use this software is granted provided that it
-is identified as the "RSA Data Security, Inc. MD5 Message-Digest
-Algorithm" in all material mentioning or referencing this software
-or this function.
-
-License is also granted to make and use derivative works provided
-that such works are identified as "derived from the RSA Data
-Security, Inc. MD5 Message-Digest Algorithm" in all material
-mentioning or referencing the derived work.
-
-RSA Data Security, Inc. makes no representations concerning either
-the merchantability of this software or the suitability of this
-software for any particular purpose. It is provided "as is"
-without express or implied warranty of any kind.
-These notices must be retained in any copies of any part of this
-documentation and/or software.
- */
-#ifndef MD5_H
-#define MD5_H
-
-
-#include "md5global.h"
-
-namespace md5
-{
-	/* MD5 context. */
-	typedef struct {
-		UINT4 state[4];                                   /* state (ABCD) */
-		UINT4 count[2];        /* number of bits, modulo 2^64 (lsb first) */
-		unsigned char buffer[64];                         /* input buffer */
-	} MD5_CTX;
-
-	static void MD5Init(MD5_CTX * context);
-	static void MD5Update( MD5_CTX *context, const unsigned char *input, unsigned int inputLen );
-	static void MD5Final ( unsigned char digest[16], MD5_CTX *context );
-
-	
-	inline bool md5( unsigned char *input, int ilen, unsigned char output[16] )
-	{
-		MD5_CTX ctx;
-
-		MD5Init( &ctx );
-		MD5Update( &ctx, input, ilen );
-		MD5Final( output, &ctx);
-
-		memwipe( &ctx, sizeof( MD5_CTX ));
-		return true;
-	}
-
-
-}
-
-#include "md5_l.inl"
-
-#endif

contrib/epee/include/md5_l.inl

@@ -1,352 +0,0 @@
-/*
-* libEtPan! -- a mail stuff library
-*
-* Copyright (C) 2001, 2005 - DINH Viet Hoa
-* All rights reserved.
-*
-* Redistribution and use in source and binary forms, with or without
-* modification, are permitted provided that the following conditions
-* are met:
-* 1. Redistributions of source code must retain the above copyright
-*    notice, this list of conditions and the following disclaimer.
-* 2. Redistributions in binary form must reproduce the above copyright
-*    notice, this list of conditions and the following disclaimer in the
-*    documentation and/or other materials provided with the distribution.
-* 3. Neither the name of the libEtPan! project nor the names of its
-*    contributors may be used to endorse or promote products derived
-*    from this software without specific prior written permission.
-*
-* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
-* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-* ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
-* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-* SUCH DAMAGE.
-*/
-
-/*
-* $Id: md5.c,v 1.1.1.1 2005/03/18 20:17:27 zautrix Exp $
-*/
-
-/* MD5C.C - RSA Data Security, Inc., MD5 message-digest algorithm
-*/
-
-/* Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All
-rights reserved.
-
-License to copy and use this software is granted provided that it
-is identified as the "RSA Data Security, Inc. MD5 Message-Digest
-Algorithm" in all material mentioning or referencing this software
-or this function.
-
-License is also granted to make and use derivative works provided
-that such works are identified as "derived from the RSA Data
-Security, Inc. MD5 Message-Digest Algorithm" in all material
-mentioning or referencing the derived work.
-
-RSA Data Security, Inc. makes no representations concerning either
-the merchantability of this software or the suitability of this
-software for any particular purpose. It is provided "as is"
-without express or implied warranty of any kind.
-
-These notices must be retained in any copies of any part of this
-documentation and/or software.
-*/
-
-#ifdef _WIN32
-# include <winsock2.h>
-#else
-# include <arpa/inet.h>
-#endif
-#include "md5global.h"
-#include "md5_l.h"
-
-namespace md5
-{
-	/* Constants for MD5Transform routine.
-	*/
-
-#define S11 7
-#define S12 12
-#define S13 17
-#define S14 22
-#define S21 5
-#define S22 9
-#define S23 14
-#define S24 20
-#define S31 4
-#define S32 11
-#define S33 16
-#define S34 23
-#define S41 6
-#define S42 10
-#define S43 15
-#define S44 21
-
-	static void MD5_memcpy (POINTER output, POINTER input, unsigned int len)
-	{
-		unsigned int i; 
-
-		for (i = 0; i < len; i++) 
-			output[i] = input[i]; 
-	}
-
-	static void MD5Transform (UINT4 state[4], unsigned char block[64]);
-
-	static unsigned char* PADDING()
-	{
-		static unsigned char local_PADDING[64] = {
-			0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-			0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 
-		};
-
-		return local_PADDING;
-
-	}
-
-
-
-	/* F, G, H and I are basic MD5 functions.
-
-	*/
-#ifdef I
-	/* This might be defined via NANA */
-#undef I
-#endif
-
-#define MD5_M_F(x, y, z) (((x) & (y)) | ((~x) & (z)))
-#define MD5_M_G(x, y, z) (((x) & (z)) | ((y) & (~z)))
-#define MD5_M_H(x, y, z) ((x) ^ (y) ^ (z))
-#define MD5_M_I(x, y, z) ((y) ^ ((x) | (~z)))
-
-	/* ROTATE_LEFT rotates x left n bits.
-
-	*/
-
-#define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32-(n))))
-
-	/* FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4.
-	Rotation is separate from addition to prevent recomputation.
-	*/
-
-#define FF(a, b, c, d, x, s, ac) { (a) += MD5_M_F ((b), (c), (d)) + (x) + (UINT4)(ac); (a) = ROTATE_LEFT ((a), (s));        (a) += (b);        } 
-#define GG(a, b, c, d, x, s, ac) {        (a) += MD5_M_G ((b), (c), (d)) + (x) + (UINT4)(ac);        (a) = ROTATE_LEFT ((a), (s));        (a) += (b);         } 
-#define HH(a, b, c, d, x, s, ac) {        (a) += MD5_M_H ((b), (c), (d)) + (x) + (UINT4)(ac);        (a) = ROTATE_LEFT ((a), (s));        (a) += (b);        } 
-#define II(a, b, c, d, x, s, ac) {        (a) += MD5_M_I ((b), (c), (d)) + (x) + (UINT4)(ac);        (a) = ROTATE_LEFT ((a), (s));        (a) += (b);        } 
-
-	/* MD5 initialization. Begins an MD5 operation, writing a new context.
-	*/
-
-	static void MD5Init(MD5_CTX * context)
-	{
-		context->count[0] = context->count[1] = 0; 
-
-		/* Load magic initialization constants.
-
-		*/
-		context->state[0] = 0x67452301; 
-		context->state[1] = 0xefcdab89; 
-		context->state[2] = 0x98badcfe; 
-		context->state[3] = 0x10325476; 
-	}
-
-	/* MD5 block update operation. Continues an MD5 message-digest
-	operation, processing another message block, and updating the context. 
-	*/
-
-	static void MD5Update( MD5_CTX *context, const unsigned char *input, unsigned int inputLen )
-	{
-		unsigned int i, index, partLen; 
-
-		/* Compute number of bytes mod 64 */
-		index = (unsigned int)((context->count[0] >> 3) & 0x3F);
-
-		/* Update number of bits */
-		if ((context->count[0] += ((UINT4)inputLen << 3))
-			< ((UINT4)inputLen << 3))
-			context->count[1]++;
-		context->count[1] += ((UINT4)inputLen >> 29);
-
-		partLen = 64 - index; 
-
-		/* Transform as many times as possible.
-
-		*/
-		if (inputLen >= partLen)
-		{ 
-			MD5_memcpy( (POINTER)&context->buffer[index], (POINTER)input, partLen );
-			MD5Transform( context->state, context->buffer ); 
-
-			for (i = partLen; i + 63 < inputLen; i += 64) 
-				MD5Transform (context->state, (unsigned char*)&input[i]); 
-
-			index = 0; 
-		} 
-		else 
-			i = 0; 
-
-		/* Buffer remaining input */
-		MD5_memcpy( (POINTER)&context->buffer[index], (POINTER)&input[i], inputLen-i );
-
-	}
-
-	/* Encodes input (UINT4) into output (unsigned char). Assumes len is
-	a multiple of 4. 
-
-	*/
-
-	static void Encode (unsigned char *output, UINT4 *input, unsigned int len)
-	{
-		unsigned int i, j; 
-
-		for (i = 0, j = 0; j < len; i++, j += 4) { 
-			output[j] = (unsigned char)(input[i] & 0xff); 
-			output[j+1] = (unsigned char)((input[i] >> 8) & 0xff); 
-			output[j+2] = (unsigned char)((input[i] >> 16) & 0xff); 
-			output[j+3] = (unsigned char)((input[i] >> 24) & 0xff); 
-		} 
-	}
-
-	/* Decodes input (unsigned char) into output (UINT4). Assumes len is
-	a multiple of 4. 
-
-	*/
-
-	static void Decode (UINT4 *output, unsigned char *input, unsigned int len)
-	{
-		unsigned int i, j; 
-
-		for (i = 0, j = 0; j < len; i++, j += 4) 
-			output[i] = ((UINT4)input[j]) | (((UINT4)input[j+1]) << 8) | (((UINT4)input[j+2]) << 16)
-			| (((UINT4)input[j+3]) << 24); 
-	}
-
-	/* MD5 finalization. Ends an MD5 message-digest operation, writing the
-	the message digest and zeroizing the context. 
-
-	*/
-
-	static void MD5Final ( unsigned char digest[16], MD5_CTX *context )
-	{
-		unsigned char bits[8]; 
-		unsigned int index, padLen; 
-
-		/* Save number of bits */
-		Encode (bits, context->count, 8);
-
-		/* Pad out to 56 mod 64.
-
-		*/
-		index = (unsigned int)((context->count[0] >> 3) & 0x3f); 
-		padLen = (index < 56) ? (56 - index) : (120 - index); 
-		MD5Update (context, PADDING(), padLen); 
-
-		/* Append length (before padding) */
-		MD5Update (context, bits, 8);
-
-		/* Store state in digest */
-		Encode (digest, context->state, 16);
-
-		/* Zeroize sensitive information.
-
-		*/
-		memwipe ((POINTER)context, sizeof (*context));
-	}
-
-	/* MD5 basic transformation. Transforms state based on block.
-
-	*/
-
-	static void MD5Transform (UINT4 state[4], unsigned char block[64])
-	{
-		UINT4 a = state[0], b = state[1], c = state[2], d = state[3], x[16]; 
-
-		Decode (x, block, 64); 
-
-		/* Round 1 */
-		FF (a, b, c, d, x[ 0], S11, 0xd76aa478); /* 1 */
-		FF (d, a, b, c, x[ 1], S12, 0xe8c7b756); /* 2 */
-		FF (c, d, a, b, x[ 2], S13, 0x242070db); /* 3 */
-		FF (b, c, d, a, x[ 3], S14, 0xc1bdceee); /* 4 */
-		FF (a, b, c, d, x[ 4], S11, 0xf57c0faf); /* 5 */
-		FF (d, a, b, c, x[ 5], S12, 0x4787c62a); /* 6 */
-		FF (c, d, a, b, x[ 6], S13, 0xa8304613); /* 7 */
-		FF (b, c, d, a, x[ 7], S14, 0xfd469501); /* 8 */
-		FF (a, b, c, d, x[ 8], S11, 0x698098d8); /* 9 */
-		FF (d, a, b, c, x[ 9], S12, 0x8b44f7af); /* 10 */
-		FF (c, d, a, b, x[10], S13, 0xffff5bb1); /* 11 */
-		FF (b, c, d, a, x[11], S14, 0x895cd7be); /* 12 */
-		FF (a, b, c, d, x[12], S11, 0x6b901122); /* 13 */
-		FF (d, a, b, c, x[13], S12, 0xfd987193); /* 14 */
-		FF (c, d, a, b, x[14], S13, 0xa679438e); /* 15 */
-		FF (b, c, d, a, x[15], S14, 0x49b40821); /* 16 */
-
-		/* Round 2 */
-		GG (a, b, c, d, x[ 1], S21, 0xf61e2562); /* 17 */
-		GG (d, a, b, c, x[ 6], S22, 0xc040b340); /* 18 */
-		GG (c, d, a, b, x[11], S23, 0x265e5a51); /* 19 */
-		GG (b, c, d, a, x[ 0], S24, 0xe9b6c7aa); /* 20 */
-		GG (a, b, c, d, x[ 5], S21, 0xd62f105d); /* 21 */
-		GG (d, a, b, c, x[10], S22,  0x2441453); /* 22 */
-		GG (c, d, a, b, x[15], S23, 0xd8a1e681); /* 23 */
-		GG (b, c, d, a, x[ 4], S24, 0xe7d3fbc8); /* 24 */
-		GG (a, b, c, d, x[ 9], S21, 0x21e1cde6); /* 25 */
-		GG (d, a, b, c, x[14], S22, 0xc33707d6); /* 26 */
-		GG (c, d, a, b, x[ 3], S23, 0xf4d50d87); /* 27 */
-		GG (b, c, d, a, x[ 8], S24, 0x455a14ed); /* 28 */ 
-		GG (a, b, c, d, x[13], S21, 0xa9e3e905); /* 29 */ 
-		GG (d, a, b, c, x[ 2], S22, 0xfcefa3f8); /* 30 */ 
-		GG (c, d, a, b, x[ 7], S23, 0x676f02d9); /* 31 */ 
-		GG (b, c, d, a, x[12], S24, 0x8d2a4c8a); /* 32 */ 
-
-		/* Round 3 */
-		HH (a, b, c, d, x[ 5], S31, 0xfffa3942); /* 33 */
-		HH (d, a, b, c, x[ 8], S32, 0x8771f681); /* 34 */
-		HH (c, d, a, b, x[11], S33, 0x6d9d6122); /* 35 */
-		HH (b, c, d, a, x[14], S34, 0xfde5380c); /* 36 */
-		HH (a, b, c, d, x[ 1], S31, 0xa4beea44); /* 37 */
-		HH (d, a, b, c, x[ 4], S32, 0x4bdecfa9); /* 38 */
-		HH (c, d, a, b, x[ 7], S33, 0xf6bb4b60); /* 39 */
-		HH (b, c, d, a, x[10], S34, 0xbebfbc70); /* 40 */
-		HH (a, b, c, d, x[13], S31, 0x289b7ec6); /* 41 */
-		HH (d, a, b, c, x[ 0], S32, 0xeaa127fa); /* 42 */
-		HH (c, d, a, b, x[ 3], S33, 0xd4ef3085); /* 43 */
-		HH (b, c, d, a, x[ 6], S34,  0x4881d05); /* 44 */
-		HH (a, b, c, d, x[ 9], S31, 0xd9d4d039); /* 45 */
-		HH (d, a, b, c, x[12], S32, 0xe6db99e5); /* 46 */
-		HH (c, d, a, b, x[15], S33, 0x1fa27cf8); /* 47 */
-		HH (b, c, d, a, x[ 2], S34, 0xc4ac5665); /* 48 */
-
-		/* Round 4 */
-		II (a, b, c, d, x[ 0], S41, 0xf4292244); /* 49 */
-		II (d, a, b, c, x[ 7], S42, 0x432aff97); /* 50 */
-		II (c, d, a, b, x[14], S43, 0xab9423a7); /* 51 */
-		II (b, c, d, a, x[ 5], S44, 0xfc93a039); /* 52 */
-		II (a, b, c, d, x[12], S41, 0x655b59c3); /* 53 */
-		II (d, a, b, c, x[ 3], S42, 0x8f0ccc92); /* 54 */
-		II (c, d, a, b, x[10], S43, 0xffeff47d); /* 55 */
-		II (b, c, d, a, x[ 1], S44, 0x85845dd1); /* 56 */
-		II (a, b, c, d, x[ 8], S41, 0x6fa87e4f); /* 57 */
-		II (d, a, b, c, x[15], S42, 0xfe2ce6e0); /* 58 */
-		II (c, d, a, b, x[ 6], S43, 0xa3014314); /* 59 */
-		II (b, c, d, a, x[13], S44, 0x4e0811a1); /* 60 */
-		II (a, b, c, d, x[ 4], S41, 0xf7537e82); /* 61 */
-		II (d, a, b, c, x[11], S42, 0xbd3af235); /* 62 */
-		II (c, d, a, b, x[ 2], S43, 0x2ad7d2bb); /* 63 */
-		II (b, c, d, a, x[ 9], S44, 0xeb86d391); /* 64 */
-
-		state[0] += a; 
-		state[1] += b; 
-		state[2] += c; 
-		state[3] += d; 
-
-		/* Zeroize sensitive information.
-		*/
-		memwipe ((POINTER)x, sizeof (x));
-	}
-}

contrib/epee/include/md5global.h

@@ -1,77 +0,0 @@
-/*
- * libEtPan! -- a mail stuff library
- *
- * Copyright (C) 2001, 2005 - DINH Viet Hoa
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. Neither the name of the libEtPan! project nor the names of its
- *    contributors may be used to endorse or promote products derived
- *    from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-/*
- * $Id: md5global.h,v 1.1.1.1 2005/03/18 20:17:28 zautrix Exp $
- */
-
-/* GLOBAL.H - RSAREF types and constants
- */
-
-#ifndef MD5GLOBAL_H
-#define MD5GLOBAL_H
-
-namespace md5
-{
-
-
-	/* PROTOTYPES should be set to one if and only if the compiler supports
-	function argument prototyping.
-	The following makes PROTOTYPES default to 0 if it has not already
-	been defined with C compiler flags.
-	*/
-#ifndef PROTOTYPES
-#define PROTOTYPES 0
-#endif
-
-	/* POINTER defines a generic pointer type */
-	typedef unsigned char *POINTER;
-
-	/* UINT2 defines a two byte word */
-	typedef unsigned short int UINT2;
-
-	/* UINT4 defines a four byte word */
-	//typedef unsigned long int UINT4;
-	typedef unsigned int UINT4;
-
-	/* PROTO_LIST is defined depending on how PROTOTYPES is defined above.
-	If using PROTOTYPES, then PROTO_LIST returns the list, otherwise it
-	returns an empty list.
-	*/
-#if PROTOTYPES
-#define PROTO_LIST(list) list
-#else
-#define PROTO_LIST(list) ()
-#endif
-
-}
-
-#endif

contrib/epee/include/net/abstract_tcp_server2.h

@@ -47,6 +47,7 @@
 #include <condition_variable>
 
 #include <boost/asio.hpp>
+#include <boost/asio/post.hpp>
 #include <boost/asio/ssl.hpp>
 #include <boost/asio/strand.hpp>
 #include <boost/asio/steady_timer.hpp>
@@ -64,6 +65,7 @@
 #define MONERO_DEFAULT_LOG_CATEGORY "net"
 
 #define ABSTRACT_SERVER_SEND_QUE_MAX_COUNT 1000
+#define ABSTRACT_SERVER_SEND_QUE_MAX_BYTES_DEFAULT 100 * 1024 * 1024
 
 namespace epee
 {
@@ -76,6 +78,13 @@ namespace net_utils
   protected:
     virtual ~i_connection_filter(){}
   };
+
+  struct i_connection_limit
+  {
+    virtual bool is_host_limit(const epee::net_utils::network_address &address)=0;
+  protected:
+    virtual ~i_connection_limit(){}
+  };
   
 
   /************************************************************************/
@@ -100,8 +109,8 @@ namespace net_utils
     using ec_t = boost::system::error_code;
     using handshake_t = boost::asio::ssl::stream_base::handshake_type;
 
-    using io_context_t = boost::asio::io_service;
-    using strand_t = boost::asio::io_service::strand;
+    using io_context_t = boost::asio::io_context;
+    using strand_t = io_context_t::strand;
     using socket_t = boost::asio::ip::tcp::socket;
 
     using network_throttle_t = epee::net_utils::network_throttle;
@@ -162,6 +171,7 @@ namespace net_utils
         } read;
         struct {
           std::deque<epee::byte_slice> queue;
+          std::size_t total_bytes;
           bool wait_consume;
         } write;
       };
@@ -260,23 +270,33 @@ namespace net_utils
     struct shared_state : connection_basic_shared_state, t_protocol_handler::config_type
     {
       shared_state()
-        : connection_basic_shared_state(), t_protocol_handler::config_type(), pfilter(nullptr), stop_signal_sent(false)
+        : connection_basic_shared_state(),
+          t_protocol_handler::config_type(),
+          pfilter(nullptr),
+          plimit(nullptr),
+          response_soft_limit(ABSTRACT_SERVER_SEND_QUE_MAX_BYTES_DEFAULT), 
+          stop_signal_sent(false)
       {}
 
       i_connection_filter* pfilter;
+      i_connection_limit* plimit;
+      std::size_t response_soft_limit;
       bool stop_signal_sent;
     };
 
-    /// Construct a connection with the given io_service.
-    explicit connection( boost::asio::io_service& io_service,
+    /// Construct a connection with the given io_context.
+    explicit connection( io_context_t& io_context,
                         std::shared_ptr<shared_state> state,
 			t_connection_type connection_type,
-			epee::net_utils::ssl_support_t ssl_support);
+			epee::net_utils::ssl_support_t ssl_support,
+      t_connection_context&& initial = t_connection_context{});
 
-    explicit connection( boost::asio::ip::tcp::socket&& sock,
+    explicit connection( io_context_t& io_context,
+      boost::asio::ip::tcp::socket&& sock,
 			 std::shared_ptr<shared_state> state,
 			t_connection_type connection_type,
-			epee::net_utils::ssl_support_t ssl_support);
+			epee::net_utils::ssl_support_t ssl_support,
+      t_connection_context&& initial = t_connection_context{});
 
 
 
@@ -306,7 +326,7 @@ namespace net_utils
     virtual bool close();
     virtual bool call_run_once_service_io();
     virtual bool request_callback();
-    virtual boost::asio::io_service& get_io_service();
+    virtual io_context_t& get_io_context();
     virtual bool add_ref();
     virtual bool release();
     //------------------------------------------------------
@@ -336,7 +356,7 @@ namespace net_utils
     /// serve up files from the given directory.
 
     boosted_tcp_server(t_connection_type connection_type);
-    explicit boosted_tcp_server(boost::asio::io_service& external_io_service, t_connection_type connection_type);
+    explicit boosted_tcp_server(boost::asio::io_context& external_io_context, t_connection_type connection_type);
     ~boosted_tcp_server();
     
     std::map<std::string, t_connection_type> server_type_map;
@@ -349,7 +369,7 @@ namespace net_utils
 	const std::string port_ipv6 = "", const std::string address_ipv6 = "::", bool use_ipv6 = false, bool require_ipv4 = true,
 	ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect);
 
-    /// Run the server's io_service loop.
+    /// Run the server's io_context loop.
     bool run_server(size_t threads_count, bool wait = true, const boost::thread::attributes& attrs = boost::thread::attributes());
 
     /// wait for service workers stop
@@ -369,6 +389,8 @@ namespace net_utils
     size_t get_threads_count(){return m_threads_count;}
 
     void set_connection_filter(i_connection_filter* pfilter);
+    void set_connection_limit(i_connection_limit* plimit);
+    void set_response_soft_limit(std::size_t limit);
 
     void set_default_remote(epee::net_utils::network_address remote)
     {
@@ -379,7 +401,7 @@ namespace net_utils
     try_connect_result_t try_connect(connection_ptr new_connection_l, const std::string& adr, const std::string& port, boost::asio::ip::tcp::socket &sock_, const boost::asio::ip::tcp::endpoint &remote_endpoint, const std::string &bind_ip, uint32_t conn_timeout, epee::net_utils::ssl_support_t ssl_support);
     bool connect(const std::string& adr, const std::string& port, uint32_t conn_timeot, t_connection_context& cn, const std::string& bind_ip = "0.0.0.0", epee::net_utils::ssl_support_t ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_autodetect);
     template<class t_callback>
-    bool connect_async(const std::string& adr, const std::string& port, uint32_t conn_timeot, const t_callback &cb, const std::string& bind_ip = "0.0.0.0", epee::net_utils::ssl_support_t ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_autodetect);
+    bool connect_async(const std::string& adr, const std::string& port, uint32_t conn_timeot, const t_callback &cb, const std::string& bind_ip = "0.0.0.0", epee::net_utils::ssl_support_t ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_autodetect, t_connection_context&& initial = t_connection_context{});
 
     boost::asio::ssl::context& get_ssl_context() noexcept
     {
@@ -409,7 +431,7 @@ namespace net_utils
       return connections_count;
     }
 
-    boost::asio::io_service& get_io_service(){return io_service_;}
+    boost::asio::io_context& get_io_context(){return io_context_;}
 
     struct idle_callback_conext_base
     {
@@ -417,7 +439,7 @@ namespace net_utils
 
       virtual bool call_handler(){return true;}
 
-      idle_callback_conext_base(boost::asio::io_service& io_serice):
+      idle_callback_conext_base(boost::asio::io_context& io_serice):
                                                           m_timer(io_serice)
       {}
       boost::asio::deadline_timer m_timer;
@@ -426,7 +448,7 @@ namespace net_utils
     template <class t_handler>
     struct idle_callback_conext: public idle_callback_conext_base
     {
-      idle_callback_conext(boost::asio::io_service& io_serice, t_handler& h, uint64_t period):
+      idle_callback_conext(boost::asio::io_context& io_serice, t_handler& h, uint64_t period):
                                                     idle_callback_conext_base(io_serice),
                                                     m_handler(h)
       {this->m_period = period;}
@@ -442,7 +464,7 @@ namespace net_utils
     template<class t_handler>
     bool add_idle_handler(t_handler t_callback, uint64_t timeout_ms)
       {
-        boost::shared_ptr<idle_callback_conext<t_handler>> ptr(new idle_callback_conext<t_handler>(io_service_, t_callback, timeout_ms));
+        boost::shared_ptr<idle_callback_conext<t_handler>> ptr(new idle_callback_conext<t_handler>(io_context_, t_callback, timeout_ms));
         //needed call handler here ?...
         ptr->m_timer.expires_from_now(boost::posix_time::milliseconds(ptr->m_period));
         ptr->m_timer.async_wait(boost::bind(&boosted_tcp_server<t_protocol_handler>::global_timer_handler<t_handler>, this, ptr));
@@ -461,14 +483,14 @@ namespace net_utils
     }
 
     template<class t_handler>
-    bool async_call(t_handler t_callback)
+    bool async_call(t_handler&& t_callback)
     {
-      io_service_.post(t_callback);
+      boost::asio::post(io_context_, std::forward<t_handler>(t_callback));
       return true;
     }
 
   private:
-    /// Run the server's io_service loop.
+    /// Run the server's io_context loop.
     bool worker_thread();
     /// Handle completion of an asynchronous accept operation.
     void handle_accept_ipv4(const boost::system::error_code& e);
@@ -479,18 +501,18 @@ namespace net_utils
 
     const std::shared_ptr<typename connection<t_protocol_handler>::shared_state> m_state;
 
-    /// The io_service used to perform asynchronous operations.
+    /// The io_context used to perform asynchronous operations.
     struct worker
     {
       worker()
-        : io_service(), work(io_service)
+        : io_context(), work(io_context.get_executor())
       {}
 
-      boost::asio::io_service io_service;
-      boost::asio::io_service::work work;
+      boost::asio::io_context io_context;
+      boost::asio::executor_work_guard<boost::asio::io_context::executor_type> work;
     };
-    std::unique_ptr<worker> m_io_service_local_instance;
-    boost::asio::io_service& io_service_;    
+    std::unique_ptr<worker> m_io_context_local_instance;
+    boost::asio::io_context& io_context_;    
 
     /// Acceptor used to listen for incoming connections.
     boost::asio::ip::tcp::acceptor acceptor_;

contrib/epee/include/net/abstract_tcp_server2.inl

@@ -31,11 +31,12 @@
 // 
 
 
-
+#include <boost/asio/post.hpp>
 #include <boost/foreach.hpp>
 #include <boost/uuid/random_generator.hpp>
 #include <boost/chrono.hpp>
 #include <boost/utility/value_init.hpp>
+#include <boost/asio/bind_executor.hpp>
 #include <boost/asio/deadline_timer.hpp>
 #include <boost/date_time/posix_time/posix_time.hpp> // TODO
 #include <boost/thread/condition_variable.hpp> // TODO
@@ -145,23 +146,19 @@ namespace net_utils
     if (m_state.timers.general.wait_expire) {
       m_state.timers.general.cancel_expire = true;
       m_state.timers.general.reset_expire = true;
-      ec_t ec;
-      m_timers.general.expires_from_now(
+      m_timers.general.expires_after(
         std::min(
-          duration + (add ? m_timers.general.expires_from_now() : duration_t{}),
+          duration + (add ? (m_timers.general.expiry() - std::chrono::steady_clock::now()) : duration_t{}),
           get_default_timeout()
-        ),
-        ec
+        )
       );
     }
     else {
-      ec_t ec;
-      m_timers.general.expires_from_now(
+      m_timers.general.expires_after(
         std::min(
-          duration + (add ? m_timers.general.expires_from_now() : duration_t{}),
+          duration + (add ? (m_timers.general.expiry() - std::chrono::steady_clock::now()) : duration_t{}),
           get_default_timeout()
-        ),
-        ec
+        )
       );
       async_wait_timer();
     }
@@ -202,8 +199,7 @@ namespace net_utils
       return;
     m_state.timers.general.cancel_expire = true;
     m_state.timers.general.reset_expire = false;
-    ec_t ec;
-    m_timers.general.cancel(ec);
+    m_timers.general.cancel();
   }
 
   template<typename T>
@@ -225,7 +221,8 @@ namespace net_utils
           m_state.data.read.buffer.size()
         ),
         boost::asio::transfer_exactly(epee::net_utils::get_ssl_magic_size()),
-        m_strand.wrap(
+        boost::asio::bind_executor(
+          m_strand,
           [this, self](const ec_t &ec, size_t bytes_transferred){
             std::lock_guard<std::mutex> guard(m_state.lock);
             m_state.socket.wait_read = false;
@@ -246,7 +243,8 @@ namespace net_utils
             ) {
               m_state.ssl.enabled = false;
               m_state.socket.handle_read = true;
-              connection_basic::strand_.post(
+              boost::asio::post(
+                connection_basic::strand_,
                 [this, self, bytes_transferred]{
                   bool success = m_handler.handle_recv(
                     reinterpret_cast<char *>(m_state.data.read.buffer.data()),
@@ -304,7 +302,8 @@ namespace net_utils
     static_cast<shared_state&>(
       connection_basic::get_state()
     ).ssl_options().configure(connection_basic::socket_, handshake);
-    m_strand.post(
+    boost::asio::post(
+      m_strand,
       [this, self, on_handshake]{
         connection_basic::socket_.async_handshake(
           handshake,
@@ -313,7 +312,7 @@ namespace net_utils
             m_state.ssl.forced ? 0 :
             epee::net_utils::get_ssl_magic_size()
           ),
-          m_strand.wrap(on_handshake)
+          boost::asio::bind_executor(m_strand, on_handshake)
         );
       }
     );
@@ -328,7 +327,7 @@ namespace net_utils
       return;
     }
     auto self = connection<T>::shared_from_this();
-    if (m_connection_type != e_connection_type_RPC) {
+    if (speed_limit_is_enabled()) {
       auto calc_duration = []{
         CRITICAL_REGION_LOCAL(
           network_throttle_manager_t::m_lock_get_global_throttle_in
@@ -345,8 +344,7 @@ namespace net_utils
       };
       const auto duration = calc_duration();
       if (duration > duration_t{}) {
-        ec_t ec;
-        m_timers.throttle.in.expires_from_now(duration, ec);
+        m_timers.throttle.in.expires_after(duration);
         m_state.timers.throttle.in.wait_expire = true;
         m_timers.throttle.in.async_wait([this, self](const ec_t &ec){
           std::lock_guard<std::mutex> guard(m_state.lock);
@@ -382,7 +380,7 @@ namespace net_utils
             m_conn_context.m_max_speed_down,
             speed
           );
-          {
+          if (speed_limit_is_enabled()) {
             CRITICAL_REGION_LOCAL(
               network_throttle_manager_t::m_lock_get_global_throttle_in
             );
@@ -401,7 +399,8 @@ namespace net_utils
         // writes until the connection terminates without deadlocking waiting
         // for handle_recv.
         m_state.socket.handle_read = true;
-        connection_basic::strand_.post(
+        boost::asio::post(
+          connection_basic::strand_,
           [this, self, bytes_transferred]{
             bool success = m_handler.handle_recv(
               reinterpret_cast<char *>(m_state.data.read.buffer.data()),
@@ -428,17 +427,18 @@ namespace net_utils
           m_state.data.read.buffer.data(),
           m_state.data.read.buffer.size()
         ),
-        m_strand.wrap(on_read)
+        boost::asio::bind_executor(m_strand, on_read)
       );
     else
-      m_strand.post(
+      boost::asio::post(
+        m_strand,
         [this, self, on_read]{
           connection_basic::socket_.async_read_some(
             boost::asio::buffer(
               m_state.data.read.buffer.data(),
               m_state.data.read.buffer.size()
             ),
-            m_strand.wrap(on_read)
+            boost::asio::bind_executor(m_strand, on_read)
           );
         }
       );
@@ -454,7 +454,7 @@ namespace net_utils
       return;
     }
     auto self = connection<T>::shared_from_this();
-    if (m_connection_type != e_connection_type_RPC) {
+    if (speed_limit_is_enabled()) {
       auto calc_duration = [this]{
         CRITICAL_REGION_LOCAL(
           network_throttle_manager_t::m_lock_get_global_throttle_out
@@ -473,8 +473,7 @@ namespace net_utils
       };
       const auto duration = calc_duration();
       if (duration > duration_t{}) {
-        ec_t ec;
-        m_timers.throttle.out.expires_from_now(duration, ec);
+        m_timers.throttle.out.expires_after(duration);
         m_state.timers.throttle.out.wait_expire = true;
         m_timers.throttle.out.async_wait([this, self](const ec_t &ec){
           std::lock_guard<std::mutex> guard(m_state.lock);
@@ -498,10 +497,12 @@ namespace net_utils
       if (m_state.socket.cancel_write) {
         m_state.socket.cancel_write = false;
         m_state.data.write.queue.clear();
+        m_state.data.write.total_bytes = 0;
         state_status_check();
       }
       else if (ec.value()) {
         m_state.data.write.queue.clear();
+        m_state.data.write.total_bytes = 0;
         interrupt();
       }
       else {
@@ -513,7 +514,7 @@ namespace net_utils
             m_conn_context.m_max_speed_down,
             speed
           );
-          {
+          if (speed_limit_is_enabled()) {
             CRITICAL_REGION_LOCAL(
               network_throttle_manager_t::m_lock_get_global_throttle_out
             );
@@ -526,8 +527,11 @@ namespace net_utils
 
           start_timer(get_default_timeout(), true);
         }
-        assert(bytes_transferred == m_state.data.write.queue.back().size());
+        const std::size_t byte_count = m_state.data.write.queue.back().size();
+        assert(bytes_transferred == byte_count);
         m_state.data.write.queue.pop_back();
+        m_state.data.write.total_bytes -=
+          std::min(m_state.data.write.total_bytes, byte_count);
         m_state.condition.notify_all();
         start_write();
       }
@@ -539,10 +543,11 @@ namespace net_utils
           m_state.data.write.queue.back().data(),
           m_state.data.write.queue.back().size()
         ),
-        m_strand.wrap(on_write)
+        boost::asio::bind_executor(m_strand, on_write)
       );
     else
-      m_strand.post(
+      boost::asio::post(
+        m_strand,
         [this, self, on_write]{
           boost::asio::async_write(
             connection_basic::socket_,
@@ -550,7 +555,7 @@ namespace net_utils
               m_state.data.write.queue.back().data(),
               m_state.data.write.queue.back().size()
             ),
-            m_strand.wrap(on_write)
+            boost::asio::bind_executor(m_strand, on_write)
           );
         }
       );
@@ -587,10 +592,11 @@ namespace net_utils
         terminate();
       }
     };
-    m_strand.post(
+    boost::asio::post(
+      m_strand,
       [this, self, on_shutdown]{
         connection_basic::socket_.async_shutdown(
-          m_strand.wrap(on_shutdown)
+          boost::asio::bind_executor(m_strand, on_shutdown)
         );
       }
     );
@@ -605,15 +611,13 @@ namespace net_utils
       wait_socket = m_state.socket.cancel_handshake = true;
     if (m_state.timers.throttle.in.wait_expire) {
       m_state.timers.throttle.in.cancel_expire = true;
-      ec_t ec;
-      m_timers.throttle.in.cancel(ec);
+      m_timers.throttle.in.cancel();
     }
     if (m_state.socket.wait_read)
       wait_socket = m_state.socket.cancel_read = true;
     if (m_state.timers.throttle.out.wait_expire) {
       m_state.timers.throttle.out.cancel_expire = true;
-      ec_t ec;
-      m_timers.throttle.out.cancel(ec);
+      m_timers.throttle.out.cancel();
     }
     if (m_state.socket.wait_write)
       wait_socket = m_state.socket.cancel_write = true;
@@ -671,8 +675,9 @@ namespace net_utils
       return;
     if (m_state.timers.throttle.out.wait_expire)
       return;
-    if (m_state.socket.wait_write)
-      return;
+    // \NOTE See on_terminating() comments
+    //if (m_state.socket.wait_write)
+    // return;
     if (m_state.socket.wait_shutdown)
       return;
     if (m_state.protocol.wait_init)
@@ -730,8 +735,13 @@ namespace net_utils
       return;
     if (m_state.timers.throttle.out.wait_expire)
       return;
-    if (m_state.socket.wait_write)
-      return;
+    // Writes cannot be canceled due to `async_write` being a "composed"
+    // handler. ASIO has new cancellation routines, not available in 1.66, to
+    // handle this situation. The problem is that if cancel is called after an
+    // intermediate handler is queued, the op will not check the cancel flag in
+    // our code, and will instead queue up another write.
+    //if (m_state.socket.wait_write)
+    //  return;
     if (m_state.socket.wait_shutdown)
       return;
     if (m_state.protocol.wait_init)
@@ -758,6 +768,8 @@ namespace net_utils
     std::lock_guard<std::mutex> guard(m_state.lock);
     if (m_state.status != status_t::RUNNING || m_state.socket.wait_handshake)
       return false;
+    if (std::numeric_limits<std::size_t>::max() - m_state.data.write.total_bytes < message.size())
+      return false;
 
     // Wait for the write queue to fall below the max. If it doesn't after a
     // randomized delay, drop the connection.
@@ -775,7 +787,14 @@ namespace net_utils
           std::uniform_int_distribution<>(5000, 6000)(rng)
         );
       };
-      if (m_state.data.write.queue.size() <= ABSTRACT_SERVER_SEND_QUE_MAX_COUNT)
+
+      // The bytes check intentionally does not include incoming message size.
+      // This allows for a soft overflow; a single http response will never fail
+      // this check, but multiple responses could. Clients can avoid this case
+      // by reading the entire response before making another request. P2P
+      // should never hit the MAX_BYTES check (when using default values).
+      if (m_state.data.write.queue.size() <= ABSTRACT_SERVER_SEND_QUE_MAX_COUNT &&
+          m_state.data.write.total_bytes <= static_cast<shared_state&>(connection_basic::get_state()).response_soft_limit)
         return true;
       m_state.data.write.wait_consume = true;
       bool success = m_state.condition.wait_for(
@@ -784,14 +803,23 @@ namespace net_utils
         [this]{
           return (
             m_state.status != status_t::RUNNING ||
-            m_state.data.write.queue.size() <=
-              ABSTRACT_SERVER_SEND_QUE_MAX_COUNT
+            (
+              m_state.data.write.queue.size() <=
+                ABSTRACT_SERVER_SEND_QUE_MAX_COUNT &&
+              m_state.data.write.total_bytes <=
+                static_cast<shared_state&>(connection_basic::get_state()).response_soft_limit
+            )
           );
         }
       );
       m_state.data.write.wait_consume = false;
       if (!success) {
-        terminate();
+        // synchronize with intermediate writes on `m_strand`
+        auto self = connection<T>::shared_from_this();
+        boost::asio::post(m_strand, [this, self] {
+          std::lock_guard<std::mutex> guard(m_state.lock);
+          terminate();
+        });
         return false;
       }
       else
@@ -817,7 +845,9 @@ namespace net_utils
     ) {
       if (!wait_consume())
         return false;
+      const std::size_t byte_count = message.size();
       m_state.data.write.queue.emplace_front(std::move(message));
+      m_state.data.write.total_bytes += byte_count;
       start_write();
     }
     else {
@@ -827,6 +857,7 @@ namespace net_utils
         m_state.data.write.queue.emplace_front(
           message.take_slice(CHUNK_SIZE)
         );
+        m_state.data.write.total_bytes += m_state.data.write.queue.front().size();
         start_write();
       }
     }
@@ -860,7 +891,7 @@ namespace net_utils
           ipv4_network_address{
             uint32_t{
               boost::asio::detail::socket_ops::host_to_network_long(
-                endpoint.address().to_v4().to_ulong()
+                endpoint.address().to_v4().to_uint()
               )
             },
             endpoint.port()
@@ -873,6 +904,13 @@ namespace net_utils
     ).pfilter;
     if (filter && !filter->is_remote_host_allowed(*real_remote))
       return false;
+
+    auto *limit = static_cast<shared_state&>(
+      connection_basic::get_state()
+    ).plimit;
+    if (is_income && limit && limit->is_host_limit(*real_remote))
+      return false;
+
     ec_t ec;
     #if !defined(_WIN32) || !defined(__i686)
     connection_basic::socket_.next_layer().set_option(
@@ -935,28 +973,34 @@ namespace net_utils
     io_context_t &io_context,
     std::shared_ptr<shared_state> shared_state,
     t_connection_type connection_type,
-    ssl_support_t ssl_support
+    ssl_support_t ssl_support,
+    t_connection_context&& initial
   ):
     connection(
-      std::move(socket_t{io_context}),
+      io_context,
+      socket_t{io_context},
       std::move(shared_state),
       connection_type,
-      ssl_support
+      ssl_support,
+      std::move(initial)
     )
   {
   }
 
   template<typename T>
   connection<T>::connection(
+    io_context_t &io_context,
     socket_t &&socket,
     std::shared_ptr<shared_state> shared_state,
     t_connection_type connection_type,
-    ssl_support_t ssl_support
+    ssl_support_t ssl_support,
+    t_connection_context&& initial
   ):
-    connection_basic(std::move(socket), shared_state, ssl_support),
+    connection_basic(io_context, std::move(socket), shared_state, ssl_support),
     m_handler(this, *shared_state, m_conn_context),
     m_connection_type(connection_type),
-    m_io_context{GET_IO_SERVICE(connection_basic::socket_)},
+    m_io_context{io_context},
+    m_conn_context(std::move(initial)),
     m_strand{m_io_context},
     m_timers{m_io_context}
   {
@@ -1022,7 +1066,7 @@ namespace net_utils
   template<typename T>
   bool connection<T>::speed_limit_is_enabled() const
   {
-    return m_connection_type != e_connection_type_RPC;
+    return m_connection_type == e_connection_type_P2P;
   }
 
   template<typename T>
@@ -1075,7 +1119,7 @@ namespace net_utils
       return false;
     auto self = connection<T>::shared_from_this();
     ++m_state.protocol.wait_callback;
-    connection_basic::strand_.post([this, self]{
+    boost::asio::post(connection_basic::strand_, [this, self]{
       m_handler.handle_qued_callback();
       std::lock_guard<std::mutex> guard(m_state.lock);
       --m_state.protocol.wait_callback;
@@ -1088,7 +1132,7 @@ namespace net_utils
   }
 
   template<typename T>
-  typename connection<T>::io_context_t &connection<T>::get_io_service()
+  typename connection<T>::io_context_t &connection<T>::get_io_context()
   {
     return m_io_context;
   }
@@ -1128,10 +1172,10 @@ namespace net_utils
   template<class t_protocol_handler>
   boosted_tcp_server<t_protocol_handler>::boosted_tcp_server( t_connection_type connection_type ) :
     m_state(std::make_shared<typename connection<t_protocol_handler>::shared_state>()),
-    m_io_service_local_instance(new worker()),
-    io_service_(m_io_service_local_instance->io_service),
-    acceptor_(io_service_),
-    acceptor_ipv6(io_service_),
+    m_io_context_local_instance(new worker()),
+    io_context_(m_io_context_local_instance->io_context),
+    acceptor_(io_context_),
+    acceptor_ipv6(io_context_),
     default_remote(),
     m_stop_signal_sent(false), m_port(0), 
     m_threads_count(0),
@@ -1145,11 +1189,11 @@ namespace net_utils
   }
 
   template<class t_protocol_handler>
-  boosted_tcp_server<t_protocol_handler>::boosted_tcp_server(boost::asio::io_service& extarnal_io_service, t_connection_type connection_type) :
+  boosted_tcp_server<t_protocol_handler>::boosted_tcp_server(boost::asio::io_context& extarnal_io_context, t_connection_type connection_type) :
     m_state(std::make_shared<typename connection<t_protocol_handler>::shared_state>()),
-    io_service_(extarnal_io_service),
-    acceptor_(io_service_),
-    acceptor_ipv6(io_service_),
+    io_context_(extarnal_io_context),
+    acceptor_(io_context_),
+    acceptor_ipv6(io_context_),
     default_remote(),
     m_stop_signal_sent(false), m_port(0),
     m_threads_count(0),
@@ -1196,24 +1240,27 @@ namespace net_utils
 
     std::string ipv4_failed = "";
     std::string ipv6_failed = "";
+
+    boost::asio::ip::tcp::resolver resolver(io_context_);
+
     try
     {
-      boost::asio::ip::tcp::resolver resolver(io_service_);
-      boost::asio::ip::tcp::resolver::query query(address, boost::lexical_cast<std::string>(port), boost::asio::ip::tcp::resolver::query::canonical_name);
-      boost::asio::ip::tcp::endpoint endpoint = *resolver.resolve(query);
-      acceptor_.open(endpoint.protocol());
+      const auto results = resolver.resolve(
+        address, boost::lexical_cast<std::string>(port), boost::asio::ip::tcp::resolver::canonical_name
+      );
+      acceptor_.open(results.begin()->endpoint().protocol());
 #if !defined(_WIN32)
       acceptor_.set_option(boost::asio::ip::tcp::acceptor::reuse_address(true));
 #endif
-      acceptor_.bind(endpoint);
+      acceptor_.bind(*results.begin());
       acceptor_.listen();
       boost::asio::ip::tcp::endpoint binded_endpoint = acceptor_.local_endpoint();
       m_port = binded_endpoint.port();
       MDEBUG("start accept (IPv4)");
-      new_connection_.reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, m_state->ssl_options().support));
+      new_connection_.reset(new connection<t_protocol_handler>(io_context_, m_state, m_connection_type, m_state->ssl_options().support));
       acceptor_.async_accept(new_connection_->socket(),
-	boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept_ipv4, this,
-	boost::asio::placeholders::error));
+            boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept_ipv4, this,
+              boost::asio::placeholders::error));
     }
     catch (const std::exception &e)
     {
@@ -1234,23 +1281,25 @@ namespace net_utils
       try
       {
         if (port_ipv6 == 0) port_ipv6 = port; // default arg means bind to same port as ipv4
-        boost::asio::ip::tcp::resolver resolver(io_service_);
-        boost::asio::ip::tcp::resolver::query query(address_ipv6, boost::lexical_cast<std::string>(port_ipv6), boost::asio::ip::tcp::resolver::query::canonical_name);
-        boost::asio::ip::tcp::endpoint endpoint = *resolver.resolve(query);
-        acceptor_ipv6.open(endpoint.protocol());
+
+        const auto results = resolver.resolve(
+          address_ipv6, boost::lexical_cast<std::string>(port_ipv6), boost::asio::ip::tcp::resolver::canonical_name
+        );
+
+        acceptor_ipv6.open(results.begin()->endpoint().protocol());
 #if !defined(_WIN32)
         acceptor_ipv6.set_option(boost::asio::ip::tcp::acceptor::reuse_address(true));
 #endif
         acceptor_ipv6.set_option(boost::asio::ip::v6_only(true));
-        acceptor_ipv6.bind(endpoint);
+        acceptor_ipv6.bind(*results.begin());
         acceptor_ipv6.listen();
         boost::asio::ip::tcp::endpoint binded_endpoint = acceptor_ipv6.local_endpoint();
         m_port_ipv6 = binded_endpoint.port();
         MDEBUG("start accept (IPv6)");
-        new_connection_ipv6.reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, m_state->ssl_options().support));
+        new_connection_ipv6.reset(new connection<t_protocol_handler>(io_context_, m_state, m_connection_type, m_state->ssl_options().support));
         acceptor_ipv6.async_accept(new_connection_ipv6->socket(),
-            boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept_ipv6, this,
-              boost::asio::placeholders::error));
+          boost::bind(&boosted_tcp_server<t_protocol_handler>::handle_accept_ipv6, this,
+          boost::asio::placeholders::error));
       }
       catch (const std::exception &e)
       {
@@ -1314,7 +1363,7 @@ namespace net_utils
     {
       try
       {
-        io_service_.run();
+        io_context_.run();
         return true;
       }
       catch(const std::exception& ex)
@@ -1349,6 +1398,20 @@ namespace net_utils
   }
   //---------------------------------------------------------------------------------
   template<class t_protocol_handler>
+  void boosted_tcp_server<t_protocol_handler>::set_connection_limit(i_connection_limit* plimit)
+  {
+    assert(m_state != nullptr); // always set in constructor
+    m_state->plimit = plimit;
+  }
+  //---------------------------------------------------------------------------------
+  template<class t_protocol_handler>
+  void boosted_tcp_server<t_protocol_handler>::set_response_soft_limit(const std::size_t limit)
+  {
+    assert(m_state != nullptr); // always set in constructor
+    m_state->response_soft_limit = limit;
+  }
+  //---------------------------------------------------------------------------------
+  template<class t_protocol_handler>
   bool boosted_tcp_server<t_protocol_handler>::run_server(size_t threads_count, bool wait, const boost::thread::attributes& attrs)
   {
     TRY_ENTRY();
@@ -1358,7 +1421,7 @@ namespace net_utils
     while(!m_stop_signal_sent)
     {
 
-      // Create a pool of threads to run all of the io_services.
+      // Create a pool of threads to run all of the io_contexts.
       CRITICAL_REGION_BEGIN(m_threads_lock);
       for (std::size_t i = 0; i < threads_count; ++i)
       {
@@ -1450,7 +1513,7 @@ namespace net_utils
     }
     connections_.clear();
     connections_mutex.unlock();
-    io_service_.stop();
+    io_context_.stop();
     CATCH_ENTRY_L0("boosted_tcp_server<t_protocol_handler>::send_stop_signal()", void());
   }
   //---------------------------------------------------------------------------------
@@ -1481,6 +1544,7 @@ namespace net_utils
       accept_function_pointer = &boosted_tcp_server<t_protocol_handler>::handle_accept_ipv6;
     }
 
+    bool accept_started = false;
     try
     {
     if (!e)
@@ -1497,10 +1561,11 @@ namespace net_utils
         (*current_new_connection)->setRpcStation(); // hopefully this is not needed actually
       }
       connection_ptr conn(std::move((*current_new_connection)));
-      (*current_new_connection).reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, conn->get_ssl_support()));
+      (*current_new_connection).reset(new connection<t_protocol_handler>(io_context_, m_state, m_connection_type, conn->get_ssl_support()));
       current_acceptor->async_accept((*current_new_connection)->socket(),
           boost::bind(accept_function_pointer, this,
             boost::asio::placeholders::error));
+      accept_started = true;
 
       boost::asio::socket_base::keep_alive opt(true);
       conn->socket().set_option(opt);
@@ -1526,13 +1591,15 @@ namespace net_utils
     catch (const std::exception &e)
     {
       MERROR("Exception in boosted_tcp_server<t_protocol_handler>::handle_accept: " << e.what());
+      if (accept_started)
+        return;
     }
 
     // error path, if e or exception
     assert(m_state != nullptr); // always set in constructor
     _erro("Some problems at accept: " << e.message() << ", connections_count = " << m_state->sock_count);
     misc_utils::sleep_no_w(100);
-    (*current_new_connection).reset(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, (*current_new_connection)->get_ssl_support()));
+    (*current_new_connection).reset(new connection<t_protocol_handler>(io_context_, m_state, m_connection_type, (*current_new_connection)->get_ssl_support()));
     current_acceptor->async_accept((*current_new_connection)->socket(),
         boost::bind(accept_function_pointer, this,
           boost::asio::placeholders::error));
@@ -1541,9 +1608,9 @@ namespace net_utils
   template<class t_protocol_handler>
   bool boosted_tcp_server<t_protocol_handler>::add_connection(t_connection_context& out, boost::asio::ip::tcp::socket&& sock, network_address real_remote, epee::net_utils::ssl_support_t ssl_support)
   {
-    if(std::addressof(get_io_service()) == std::addressof(GET_IO_SERVICE(sock)))
+    if(std::addressof(get_io_context()) == std::addressof(sock.get_executor().context()))
     {
-      connection_ptr conn(new connection<t_protocol_handler>(std::move(sock), m_state, m_connection_type, ssl_support));
+      connection_ptr conn(new connection<t_protocol_handler>(io_context_, std::move(sock), m_state, m_connection_type, ssl_support));
       if(conn->start(false, 1 < m_threads_count, std::move(real_remote)))
       {
         conn->get_context(out);
@@ -1553,7 +1620,7 @@ namespace net_utils
     }
     else
     {
-	MWARNING(out << " was not added, socket/io_service mismatch");
+	MWARNING(out << " was not added, socket/io_context mismatch");
     }
     return false;
   }
@@ -1566,7 +1633,7 @@ namespace net_utils
     sock_.open(remote_endpoint.protocol());
     if(bind_ip != "0.0.0.0" && bind_ip != "0" && bind_ip != "" )
     {
-      boost::asio::ip::tcp::endpoint local_endpoint(boost::asio::ip::address::from_string(bind_ip.c_str()), 0);
+      boost::asio::ip::tcp::endpoint local_endpoint(boost::asio::ip::make_address(bind_ip), 0);
       boost::system::error_code ec;
       sock_.bind(local_endpoint, ec);
       if (ec)
@@ -1661,7 +1728,7 @@ namespace net_utils
   {
     TRY_ENTRY();
 
-    connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, ssl_support) );
+    connection_ptr new_connection_l(new connection<t_protocol_handler>(io_context_, m_state, m_connection_type, ssl_support) );
     connections_mutex.lock();
     connections_.insert(new_connection_l);
     MDEBUG("connections_ size now " << connections_.size());
@@ -1671,14 +1738,16 @@ namespace net_utils
 
     bool try_ipv6 = false;
 
-    boost::asio::ip::tcp::resolver resolver(io_service_);
-    boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), adr, port, boost::asio::ip::tcp::resolver::query::canonical_name);
+    boost::asio::ip::tcp::resolver resolver(io_context_);
+    boost::asio::ip::tcp::resolver::results_type results{};
     boost::system::error_code resolve_error;
-    boost::asio::ip::tcp::resolver::iterator iterator;
+
     try
     {
       //resolving ipv4 address as ipv6 throws, catch here and move on
-      iterator = resolver.resolve(query, resolve_error);
+      results = resolver.resolve(
+        boost::asio::ip::tcp::v4(), adr, port, boost::asio::ip::tcp::resolver::canonical_name, resolve_error
+      );
     }
     catch (const boost::system::system_error& e)
     {
@@ -1696,8 +1765,7 @@ namespace net_utils
 
     std::string bind_ip_to_use;
 
-    boost::asio::ip::tcp::resolver::iterator end;
-    if(iterator == end)
+    if(results.empty())
     {
       if (!m_use_ipv6)
       {
@@ -1717,11 +1785,11 @@ namespace net_utils
 
     if (try_ipv6)
     {
-      boost::asio::ip::tcp::resolver::query query6(boost::asio::ip::tcp::v6(), adr, port, boost::asio::ip::tcp::resolver::query::canonical_name);
+      results = resolver.resolve(
+        boost::asio::ip::tcp::v6(), adr, port, boost::asio::ip::tcp::resolver::canonical_name, resolve_error
+      );
 
-      iterator = resolver.resolve(query6, resolve_error);
-
-      if(iterator == end)
+      if(results.empty())
       {
         _erro("Failed to resolve " << adr);
         return false;
@@ -1741,6 +1809,8 @@ namespace net_utils
 
     }
 
+    const auto iterator = results.begin();
+
     MDEBUG("Trying to connect to " << adr << ":" << port << ", bind_ip = " << bind_ip_to_use);
 
     //boost::asio::ip::tcp::endpoint remote_endpoint(boost::asio::ip::address::from_string(addr.c_str()), port);
@@ -1767,7 +1837,6 @@ namespace net_utils
     if (r)
     {
       new_connection_l->get_context(conn_context);
-      //new_connection_l.reset(new connection<t_protocol_handler>(io_service_, m_config, m_sock_count, m_pfilter));
     }
     else
     {
@@ -1783,10 +1852,10 @@ namespace net_utils
   }
   //---------------------------------------------------------------------------------
   template<class t_protocol_handler> template<class t_callback>
-  bool boosted_tcp_server<t_protocol_handler>::connect_async(const std::string& adr, const std::string& port, uint32_t conn_timeout, const t_callback &cb, const std::string& bind_ip, epee::net_utils::ssl_support_t ssl_support)
+  bool boosted_tcp_server<t_protocol_handler>::connect_async(const std::string& adr, const std::string& port, uint32_t conn_timeout, const t_callback &cb, const std::string& bind_ip, epee::net_utils::ssl_support_t ssl_support, t_connection_context&& initial)
   {
     TRY_ENTRY();    
-    connection_ptr new_connection_l(new connection<t_protocol_handler>(io_service_, m_state, m_connection_type, ssl_support) );
+    connection_ptr new_connection_l(new connection<t_protocol_handler>(io_context_, m_state, m_connection_type, ssl_support, std::move(initial)) );
     connections_mutex.lock();
     connections_.insert(new_connection_l);
     MDEBUG("connections_ size now " << connections_.size());
@@ -1796,14 +1865,16 @@ namespace net_utils
     
     bool try_ipv6 = false;
 
-    boost::asio::ip::tcp::resolver resolver(io_service_);
-    boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), adr, port, boost::asio::ip::tcp::resolver::query::canonical_name);
+    boost::asio::ip::tcp::resolver resolver(io_context_);
+    boost::asio::ip::tcp::resolver::results_type results{};
     boost::system::error_code resolve_error;
-    boost::asio::ip::tcp::resolver::iterator iterator;
+
     try
     {
       //resolving ipv4 address as ipv6 throws, catch here and move on
-      iterator = resolver.resolve(query, resolve_error);
+      results = resolver.resolve(
+        boost::asio::ip::tcp::v4(), adr, port, boost::asio::ip::tcp::resolver::canonical_name, resolve_error
+      );
     }
     catch (const boost::system::system_error& e)
     {
@@ -1819,8 +1890,7 @@ namespace net_utils
       throw;
     }
 
-    boost::asio::ip::tcp::resolver::iterator end;
-    if(iterator == end)
+    if(results.empty())
     {
       if (!try_ipv6)
       {
@@ -1835,24 +1905,23 @@ namespace net_utils
 
     if (try_ipv6)
     {
-      boost::asio::ip::tcp::resolver::query query6(boost::asio::ip::tcp::v6(), adr, port, boost::asio::ip::tcp::resolver::query::canonical_name);
+      results = resolver.resolve(
+        boost::asio::ip::tcp::v6(), adr, port, boost::asio::ip::tcp::resolver::canonical_name, resolve_error
+      );
 
-      iterator = resolver.resolve(query6, resolve_error);
-
-      if(iterator == end)
+      if(results.empty())
       {
         _erro("Failed to resolve " << adr);
         return false;
       }
     }
 
-
-    boost::asio::ip::tcp::endpoint remote_endpoint(*iterator);
+    boost::asio::ip::tcp::endpoint remote_endpoint(*results.begin());
      
     sock_.open(remote_endpoint.protocol());
     if(bind_ip != "0.0.0.0" && bind_ip != "0" && bind_ip != "" )
     {
-      boost::asio::ip::tcp::endpoint local_endpoint(boost::asio::ip::address::from_string(bind_ip.c_str()), 0);
+      boost::asio::ip::tcp::endpoint local_endpoint(boost::asio::ip::make_address(bind_ip.c_str()), 0);
       boost::system::error_code ec;
       sock_.bind(local_endpoint, ec);
       if (ec)
@@ -1864,7 +1933,7 @@ namespace net_utils
       }
     }
     
-    boost::shared_ptr<boost::asio::deadline_timer> sh_deadline(new boost::asio::deadline_timer(io_service_));
+    boost::shared_ptr<boost::asio::deadline_timer> sh_deadline(new boost::asio::deadline_timer(io_context_));
     //start deadline
     sh_deadline->expires_from_now(boost::posix_time::milliseconds(conn_timeout));
     sh_deadline->async_wait([=](const boost::system::error_code& error)

contrib/epee/include/net/connection_basic.hpp

@@ -112,21 +112,20 @@ class connection_basic { // not-templated base class for rapid developmet of som
     std::deque<byte_slice> m_send_que;
     volatile bool m_is_multithreaded;
     /// Strand to ensure the connection's handlers are not called concurrently.
-    boost::asio::io_service::strand strand_;
+    boost::asio::io_context::strand strand_;
     /// Socket for the connection.
     boost::asio::ssl::stream<boost::asio::ip::tcp::socket> socket_;
     ssl_support_t m_ssl_support;
 
 	public:
 		// first counter is the ++/-- count of current sockets, the other socket_number is only-increasing ++ number generator
-		connection_basic(boost::asio::ip::tcp::socket&& socket, std::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support);
-		connection_basic(boost::asio::io_service &io_service, std::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support);
+		connection_basic(boost::asio::io_context &context, boost::asio::ip::tcp::socket&& sock, std::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support);
+		connection_basic(boost::asio::io_context &context, std::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support);
 
 		virtual ~connection_basic() noexcept(false);
 
                 //! \return `shared_state` object passed in construction (ptr never changes).
 		connection_basic_shared_state& get_state() noexcept { return *m_state; /* verified in constructor */ }
-		connection_basic(boost::asio::io_service& io_service, std::atomic<long> &ref_sock_count, std::atomic<long> &sock_number, ssl_support_t ssl);
 
 		boost::asio::ip::tcp::socket& socket() { return socket_.next_layer(); }
 		ssl_support_t get_ssl_support() const { return m_ssl_support; }
@@ -135,7 +134,7 @@ class connection_basic { // not-templated base class for rapid developmet of som
 		bool handshake(boost::asio::ssl::stream_base::handshake_type type, boost::asio::const_buffer buffer = {})
 		{
 			//m_state != nullptr verified in constructor
-			return m_state->ssl_options().handshake(socket_, type, buffer);
+			return m_state->ssl_options().handshake(strand_.context(), socket_, type, buffer);
 		}
 
 		template<typename MutableBufferSequence, typename ReadHandler>

contrib/epee/include/net/http_protocol_handler.h

@@ -32,6 +32,7 @@
 
 #include <boost/optional/optional.hpp>
 #include <string>
+#include <unordered_map>
 #include "net_utils_base.h"
 #include "http_auth.h"
 #include "http_base.h"
@@ -54,8 +55,13 @@ namespace net_utils
 		{
 			std::string m_folder;
 			std::vector<std::string> m_access_control_origins;
+			std::unordered_map<std::string, std::size_t> m_connections;
 			boost::optional<login> m_user;
 			size_t m_max_content_length{std::numeric_limits<size_t>::max()};
+			std::size_t m_connection_count{0};
+			std::size_t m_max_public_ip_connections{3};
+			std::size_t m_max_private_ip_connections{25};
+			std::size_t m_max_connections{100};
 			critical_section m_lock;
 		};
 
@@ -70,7 +76,7 @@ namespace net_utils
 			typedef http_server_config config_type;
 
 			simple_http_connection_handler(i_service_endpoint* psnd_hndlr, config_type& config, t_connection_context& conn_context);
-			virtual ~simple_http_connection_handler(){}
+			virtual ~simple_http_connection_handler();
 
 			bool release_protocol()
 			{
@@ -86,10 +92,7 @@ namespace net_utils
 			{
 				return true;
 			}
-			bool after_init_connection()
-			{
-				return true;
-			}
+			bool after_init_connection();
 			virtual bool handle_recv(const void* ptr, size_t cb);
 			virtual bool handle_request(const http::http_request_info& query_info, http_response_info& response);
 
@@ -146,6 +149,7 @@ namespace net_utils
 		protected:
 			i_service_endpoint* m_psnd_hndlr; 
 			t_connection_context& m_conn_context;
+			bool m_initialized;
 		};
 
 		template<class t_connection_context>
@@ -212,10 +216,6 @@ namespace net_utils
 			}
 			void handle_qued_callback()
 			{}
-			bool after_init_connection()
-			{
-				return true;
-			}
 
 		private:
 			//simple_http_connection_handler::config_type m_stub_config;

contrib/epee/include/net/http_protocol_handler.inl

@@ -208,11 +208,46 @@ namespace net_utils
 		m_newlines(0),
 		m_bytes_read(0),
 		m_psnd_hndlr(psnd_hndlr),
-		m_conn_context(conn_context)
+		m_conn_context(conn_context),
+		m_initialized(false)
 	{
 
 	}
 	//--------------------------------------------------------------------------------------------
+	template<class t_connection_context>
+	simple_http_connection_handler<t_connection_context>::~simple_http_connection_handler()
+	{
+	  try
+	  {
+	    if (m_initialized)
+	    {
+	      CRITICAL_REGION_LOCAL(m_config.m_lock);
+	      if (m_config.m_connection_count)
+	        --m_config.m_connection_count;
+	      auto elem = m_config.m_connections.find(m_conn_context.m_remote_address.host_str());
+	      if (elem != m_config.m_connections.end())
+	      {
+	        if (elem->second == 1 || elem->second == 0)
+	          m_config.m_connections.erase(elem);
+	        else
+	          --(elem->second);
+	      }
+	    }
+	  }
+	  catch (...)
+	  {}
+	}
+	//--------------------------------------------------------------------------------------------
+	template<class t_connection_context>
+	bool simple_http_connection_handler<t_connection_context>::after_init_connection()
+	{
+	  CRITICAL_REGION_LOCAL(m_config.m_lock);
+	  ++m_config.m_connections[m_conn_context.m_remote_address.host_str()];
+	  ++m_config.m_connection_count;
+	  m_initialized = true;
+	  return true;
+	}
+	//--------------------------------------------------------------------------------------------
     template<class t_connection_context>
 	bool simple_http_connection_handler<t_connection_context>::set_ready_state()
 	{

contrib/epee/include/net/http_server_handlers_map2.h

@@ -71,7 +71,7 @@
     else if((query_info.m_URI == s_pattern) && (cond)) \
     { \
       handled = true; \
-      uint64_t ticks = misc_utils::get_tick_count(); \
+      uint64_t ticks = epee::misc_utils::get_tick_count(); \
       boost::value_initialized<command_type::request> req; \
       bool parse_res = epee::serialization::load_t_from_json(static_cast<command_type::request&>(req), query_info.m_body); \
       if (!parse_res) \
@@ -107,7 +107,7 @@
     else if(query_info.m_URI == s_pattern) \
     { \
       handled = true; \
-      uint64_t ticks = misc_utils::get_tick_count(); \
+      uint64_t ticks = epee::misc_utils::get_tick_count(); \
       boost::value_initialized<command_type::request> req; \
       bool parse_res = epee::serialization::load_t_from_binary(static_cast<command_type::request&>(req), epee::strspan<uint8_t>(query_info.m_body)); \
       if (!parse_res) \
@@ -117,7 +117,7 @@
          response_info.m_response_comment = "Bad request"; \
          return true; \
       } \
-      uint64_t ticks1 = misc_utils::get_tick_count(); \
+      uint64_t ticks1 = epee::misc_utils::get_tick_count(); \
       boost::value_initialized<command_type::response> resp;\
       MINFO(m_conn_context << "calling " << s_pattern); \
       bool res = false; \
@@ -129,7 +129,7 @@
         response_info.m_response_comment = "Internal Server Error"; \
         return true; \
       } \
-      uint64_t ticks2 = misc_utils::get_tick_count(); \
+      uint64_t ticks2 = epee::misc_utils::get_tick_count(); \
       epee::byte_slice buffer; \
       epee::serialization::store_t_to_binary(static_cast<command_type::response&>(resp), buffer, 64 * 1024); \
       uint64_t ticks3 = epee::misc_utils::get_tick_count(); \

contrib/epee/include/net/http_server_impl_base.h

@@ -33,6 +33,7 @@
 #include <boost/thread.hpp>
 #include <boost/bind/bind.hpp>
 
+#include "cryptonote_config.h"
 #include "net/abstract_tcp_server2.h"
 #include "http_protocol_handler.h"
 #include "net/http_server_handlers_map2.h"
@@ -44,7 +45,8 @@ namespace epee
 {
 
   template<class t_child_class, class t_connection_context = epee::net_utils::connection_context_base>
-  class http_server_impl_base: public net_utils::http::i_http_server_handler<t_connection_context>
+  class http_server_impl_base: public net_utils::http::i_http_server_handler<t_connection_context>,
+                                      net_utils::i_connection_limit
   {
 
   public:
@@ -52,7 +54,7 @@ namespace epee
         : m_net_server(epee::net_utils::e_connection_type_RPC)
     {}
 
-    explicit http_server_impl_base(boost::asio::io_service& external_io_service)
+    explicit http_server_impl_base(boost::asio::io_context& external_io_service)
         : m_net_server(external_io_service)
     {}
 
@@ -60,8 +62,16 @@ namespace epee
       const std::string& bind_ipv6_address = "::", bool use_ipv6 = false, bool require_ipv4 = true,
       std::vector<std::string> access_control_origins = std::vector<std::string>(),
       boost::optional<net_utils::http::login> user = boost::none,
-      net_utils::ssl_options_t ssl_options = net_utils::ssl_support_t::e_ssl_support_autodetect)
+      net_utils::ssl_options_t ssl_options = net_utils::ssl_support_t::e_ssl_support_autodetect,
+      const std::size_t max_public_ip_connections = DEFAULT_RPC_MAX_CONNECTIONS_PER_PUBLIC_IP,
+      const std::size_t max_private_ip_connections = DEFAULT_RPC_MAX_CONNECTIONS_PER_PRIVATE_IP,
+      const std::size_t max_connections = DEFAULT_RPC_MAX_CONNECTIONS,
+      const std::size_t response_soft_limit = DEFAULT_RPC_SOFT_LIMIT_SIZE)
     {
+      if (max_connections < max_public_ip_connections)
+        throw std::invalid_argument{"Max public IP connections cannot be more than max connections"};
+      if (max_connections < max_private_ip_connections)
+        throw std::invalid_argument{"Max private IP connections cannot be more than max connections"};
 
       //set self as callback handler
       m_net_server.get_config_object().m_phandler = static_cast<t_child_class*>(this);
@@ -75,6 +85,11 @@ namespace epee
       m_net_server.get_config_object().m_access_control_origins = std::move(access_control_origins);
 
       m_net_server.get_config_object().m_user = std::move(user);
+      m_net_server.get_config_object().m_max_public_ip_connections = max_public_ip_connections;
+      m_net_server.get_config_object().m_max_private_ip_connections = max_private_ip_connections;
+      m_net_server.get_config_object().m_max_connections = max_connections;
+      m_net_server.set_response_soft_limit(response_soft_limit);
+      m_net_server.set_connection_limit(this);
 
       MGINFO("Binding on " << bind_ip << " (IPv4):" << bind_port);
       if (use_ipv6)
@@ -131,6 +146,26 @@ namespace epee
     }
 
   protected: 
+
+    virtual bool is_host_limit(const net_utils::network_address& na) override final
+    {
+      auto& config = m_net_server.get_config_object();
+      CRITICAL_REGION_LOCAL(config.m_lock);
+      if (config.m_max_connections <= config.m_connection_count)
+        return true;
+
+      const bool is_private = na.is_loopback() || na.is_local();
+      const auto elem = config.m_connections.find(na.host_str());
+      if (elem != config.m_connections.end())
+      {
+        if (is_private)
+          return config.m_max_private_ip_connections <= elem->second;
+        else
+          return config.m_max_public_ip_connections <= elem->second;
+      }
+      return false;
+    }
+
     net_utils::boosted_tcp_server<net_utils::http::http_custom_handler<t_connection_context> > m_net_server;
   };
 }

contrib/epee/include/net/levin_protocol_handler_async.h

@@ -200,7 +200,7 @@ public:
   struct anvoke_handler: invoke_response_handler_base
   {
     anvoke_handler(const callback_t& cb, uint64_t timeout,  async_protocol_handler& con, int command)
-      :m_cb(cb), m_timeout(timeout), m_con(con), m_timer(con.m_pservice_endpoint->get_io_service()), m_timer_started(false),
+      :m_cb(cb), m_timeout(timeout), m_con(con), m_timer(con.m_pservice_endpoint->get_io_context()), m_timer_started(false),
       m_cancel_timer_called(false), m_timer_cancelled(false), m_command(command)
     {
       if(m_con.start_outer_call())

contrib/epee/include/net/net_helper.h

@@ -34,7 +34,7 @@
 #include <atomic>
 #include <string>
 #include <boost/version.hpp>
-#include <boost/asio/io_service.hpp>
+#include <boost/asio/io_context.hpp>
 #include <boost/asio/ip/tcp.hpp>
 #include <boost/asio/read.hpp>
 #include <boost/asio/ssl.hpp>
@@ -158,11 +158,11 @@ namespace net_utils
     inline
 			try_connect_result_t try_connect(const std::string& addr, const std::string& port, std::chrono::milliseconds timeout)
 		{
-				m_deadline.expires_from_now(timeout);
+				m_deadline.expires_after(timeout);
 				boost::unique_future<boost::asio::ip::tcp::socket> connection = m_connector(addr, port, m_deadline);
 				for (;;)
 				{
-					m_io_service.reset();
+					m_io_service.restart();
 					m_io_service.run_one();
 
 					if (connection.is_ready())
@@ -178,7 +178,7 @@ namespace net_utils
 					// SSL Options
 					if (m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_enabled || m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect)
 					{
-						if (!m_ssl_options.handshake(*m_ssl_socket, boost::asio::ssl::stream_base::client, {}, addr, timeout))
+						if (!m_ssl_options.handshake(m_io_service, *m_ssl_socket, boost::asio::ssl::stream_base::client, {}, addr, timeout))
 						{
 							if (m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect)
 							{
@@ -285,7 +285,7 @@ namespace net_utils
 
 			try
 			{
-				m_deadline.expires_from_now(timeout);
+				m_deadline.expires_after(timeout);
 
 				// Set up the variable that receives the result of the asynchronous
 				// operation. The error code is set to would_block to signal that the
@@ -303,7 +303,7 @@ namespace net_utils
 				// Block until the asynchronous operation has completed.
 				while (ec == boost::asio::error::would_block)
 				{
-					m_io_service.reset();
+					m_io_service.restart();
 					m_io_service.run_one(); 
 				}
 
@@ -409,7 +409,7 @@ namespace net_utils
 				// Set a deadline for the asynchronous operation. Since this function uses
 				// a composed operation (async_read_until), the deadline applies to the
 				// entire operation, rather than individual reads from the socket.
-				m_deadline.expires_from_now(timeout);
+				m_deadline.expires_after(timeout);
 
 				// Set up the variable that receives the result of the asynchronous
 				// operation. The error code is set to would_block to signal that the
@@ -436,7 +436,7 @@ namespace net_utils
 				// Block until the asynchronous operation has completed.
 				while (ec == boost::asio::error::would_block && !m_shutdowned)
 				{
-					m_io_service.reset();
+					m_io_service.restart();
 					m_io_service.run_one(); 
 				}
 
@@ -495,7 +495,7 @@ namespace net_utils
 				// Set a deadline for the asynchronous operation. Since this function uses
 				// a composed operation (async_read_until), the deadline applies to the
 				// entire operation, rather than individual reads from the socket.
-				m_deadline.expires_from_now(timeout);
+				m_deadline.expires_after(timeout);
 
 				// Set up the variable that receives the result of the asynchronous
 				// operation. The error code is set to would_block to signal that the
@@ -580,7 +580,7 @@ namespace net_utils
 			return true;
 		}
 		
-		boost::asio::io_service& get_io_service()
+		boost::asio::io_context& get_io_service()
 		{
 			return m_io_service;
 		}
@@ -607,7 +607,7 @@ namespace net_utils
 			// Check whether the deadline has passed. We compare the deadline against
 			// the current time since a new asynchronous operation may have moved the
 			// deadline before this actor had a chance to run.
-			if (m_deadline.expires_at() <= std::chrono::steady_clock::now())
+			if (m_deadline.expiry() <= std::chrono::steady_clock::now())
 			{
 				// The deadline has passed. The socket is closed so that any outstanding
 				// asynchronous operations are cancelled. This allows the blocked
@@ -628,11 +628,11 @@ namespace net_utils
 		void shutdown_ssl() {
 			// ssl socket shutdown blocks if server doesn't respond. We close after 2 secs
 			boost::system::error_code ec = boost::asio::error::would_block;
-			m_deadline.expires_from_now(std::chrono::milliseconds(2000));
+			m_deadline.expires_after(std::chrono::milliseconds(2000));
 			m_ssl_socket->async_shutdown(boost::lambda::var(ec) = boost::lambda::_1);
 			while (ec == boost::asio::error::would_block)
 			{
-				m_io_service.reset();
+				m_io_service.restart();
 				m_io_service.run_one();
 			}
 			// Ignore "short read" error
@@ -676,7 +676,7 @@ namespace net_utils
 		}
 		
 	protected:
-		boost::asio::io_service m_io_service;
+		boost::asio::io_context m_io_service;
 		boost::asio::ssl::context m_ctx;
 		std::shared_ptr<boost::asio::ssl::stream<boost::asio::ip::tcp::socket>> m_ssl_socket;
 		std::function<connect_func> m_connector;
@@ -688,119 +688,6 @@ namespace net_utils
 		std::atomic<uint64_t> m_bytes_sent;
 		std::atomic<uint64_t> m_bytes_received;
 	};
-
-
-	/************************************************************************/
-	/*                                                                      */
-	/************************************************************************/
-	class async_blocked_mode_client: public blocked_mode_client
-	{
-	public:
-		async_blocked_mode_client():m_send_deadline(blocked_mode_client::m_io_service)
-		{
-
-			// No deadline is required until the first socket operation is started. We
-			// set the deadline to positive infinity so that the actor takes no action
-			// until a specific deadline is set.
-			m_send_deadline.expires_at(boost::posix_time::pos_infin);
-
-			// Start the persistent actor that checks for deadline expiry.
-			check_send_deadline();
-		}
-		~async_blocked_mode_client()
-		{
-			m_send_deadline.cancel();
-		}
-		
-		bool shutdown()
-		{
-			blocked_mode_client::shutdown();
-			m_send_deadline.cancel();
-			return true;
-		}
-
-		inline 
-			bool send(const void* data, size_t sz)
-		{
-			try
-			{
-				/*
-				m_send_deadline.expires_from_now(boost::posix_time::milliseconds(m_reciev_timeout));
-
-				// Set up the variable that receives the result of the asynchronous
-				// operation. The error code is set to would_block to signal that the
-				// operation is incomplete. Asio guarantees that its asynchronous
-				// operations will never fail with would_block, so any other value in
-				// ec indicates completion.
-				boost::system::error_code ec = boost::asio::error::would_block;
-
-				// Start the asynchronous operation itself. The boost::lambda function
-				// object is used as a callback and will update the ec variable when the
-				// operation completes. The blocking_udp_client.cpp example shows how you
-				// can use boost::bind rather than boost::lambda.
-				boost::asio::async_write(m_socket, boost::asio::buffer(data, sz), boost::lambda::var(ec) = boost::lambda::_1);
-
-				// Block until the asynchronous operation has completed.
-				while(ec == boost::asio::error::would_block)
-				{
-					m_io_service.run_one();
-				}*/
-				
-				boost::system::error_code ec;
-
-				size_t writen = write(data, sz, ec);
-				
-				if (!writen || ec)
-				{
-					LOG_PRINT_L3("Problems at write: " << ec.message());
-					return false;
-				}else
-				{
-					m_send_deadline.expires_at(boost::posix_time::pos_infin);
-				}
-			}
-
-			catch(const boost::system::system_error& er)
-			{
-				LOG_ERROR("Some problems at connect, message: " << er.what());
-				return false;
-			}
-			catch(...)
-			{
-				LOG_ERROR("Some fatal problems.");
-				return false;
-			}
-
-			return true;
-		}
-
-
-	private:
-
-		boost::asio::deadline_timer m_send_deadline;
-
-		void check_send_deadline()
-		{
-			// Check whether the deadline has passed. We compare the deadline against
-			// the current time since a new asynchronous operation may have moved the
-			// deadline before this actor had a chance to run.
-			if (m_send_deadline.expires_at() <= boost::asio::deadline_timer::traits_type::now())
-			{
-				// The deadline has passed. The socket is closed so that any outstanding
-				// asynchronous operations are cancelled. This allows the blocked
-				// connect(), read_line() or write_line() functions to return.
-				LOG_PRINT_L3("Timed out socket");
-				m_ssl_socket->next_layer().close();
-
-				// There is no longer an active deadline. The expiry is set to positive
-				// infinity so that the actor takes no action until a new deadline is set.
-				m_send_deadline.expires_at(boost::posix_time::pos_infin);
-			}
-
-			// Put the actor back to sleep.
-			m_send_deadline.async_wait(boost::bind(&async_blocked_mode_client::check_send_deadline, this));
-		}
-	};
 }
 }
 

contrib/epee/include/net/net_ssl.h

@@ -34,6 +34,7 @@
 #include <string>
 #include <vector>
 #include <boost/utility/string_ref.hpp>
+#include <boost/asio/io_context.hpp>
 #include <boost/asio/ip/tcp.hpp>
 #include <boost/asio/ssl.hpp>
 #include <boost/filesystem/path.hpp>
@@ -125,6 +126,7 @@ namespace net_utils
         \note It is strongly encouraged that clients using `system_ca`
           verification provide a non-empty `host` for rfc2818 verification.
 
+        \param io_context associated with `socket`.
         \param socket Used in SSL handshake and verification
         \param type Client or server
         \param host This parameter is only used when
@@ -136,6 +138,7 @@ namespace net_utils
         \return True if the SSL handshake completes with peer verification
           settings. */
     bool handshake(
+      boost::asio::io_context& io_context,
       boost::asio::ssl::stream<boost::asio::ip::tcp::socket> &socket,
       boost::asio::ssl::stream_base::handshake_type type,
       boost::asio::const_buffer buffer = {},

contrib/epee/include/net/net_utils_base.h

@@ -30,7 +30,7 @@
 #define _NET_UTILS_BASE_H_
 
 #include <boost/uuid/uuid.hpp>
-#include <boost/asio/io_service.hpp>
+#include <boost/asio/io_context.hpp>
 #include <boost/asio/ip/address_v6.hpp>
 #include <typeinfo>
 #include <type_traits>
@@ -47,10 +47,12 @@
 #define MAKE_IP( a1, a2, a3, a4 )	(a1|(a2<<8)|(a3<<16)|(((uint32_t)a4)<<24))
 #endif
 
+/* Use the below function carefully. The executor and io_context are slightly
+  different concepts. */
 #if BOOST_VERSION >= 107000
-#define GET_IO_SERVICE(s) ((boost::asio::io_context&)(s).get_executor().context())
+  #define MONERO_GET_EXECUTOR(type) type . get_executor()
 #else
-#define GET_IO_SERVICE(s) ((s).get_io_service())
+  #define MONERO_GET_EXECUTOR(type) type . get_io_context()
 #endif
 
 namespace net
@@ -443,7 +445,7 @@ namespace net_utils
     virtual bool send_done()=0;
     virtual bool call_run_once_service_io()=0;
     virtual bool request_callback()=0;
-    virtual boost::asio::io_service& get_io_service()=0;
+    virtual boost::asio::io_context& get_io_context()=0;
     //protect from deletion connection object(with protocol instance) during external call "invoke"
     virtual bool add_ref()=0;
     virtual bool release()=0;

contrib/epee/include/net/network_throttle-detail.hpp

@@ -46,13 +46,13 @@ namespace net_utils
 
 
 class network_throttle : public i_network_throttle {
-	private:
+	public:
 		struct packet_info {
 			size_t m_size; // octets sent. Summary for given small-window (e.g. for all packaged in 1 second)
 			packet_info();
 		};
 
-
+	private:
 		network_speed_bps m_target_speed;
 		size_t m_network_add_cost; // estimated add cost of headers 
 		size_t m_network_minimal_segment; // estimated minimal cost of sending 1 byte to round up to

contrib/epee/include/serialization/keyvalue_serialization_overloads.h

@@ -26,6 +26,8 @@
 
 #pragma once
 
+#include <cstddef>
+#include <cstdint>
 #include <set>
 #include <list>
 #include <vector>

contrib/epee/include/wipeable_string.h

@@ -34,6 +34,7 @@
 #include <string>
 #include "memwipe.h"
 #include "fnv1.h"
+#include "serialization/keyvalue_serialization.h"
 
 namespace epee
 {
@@ -75,6 +76,12 @@ namespace epee
     bool operator!=(const wipeable_string &other) const noexcept { return buffer != other.buffer; }
     wipeable_string &operator=(wipeable_string &&other);
     wipeable_string &operator=(const wipeable_string &other);
+    char& operator[](size_t idx);
+    const char& operator[](size_t idx) const;
+
+    BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE_CONTAINER_POD_AS_BLOB(buffer)
+    END_KV_SERIALIZE_MAP()
 
   private:
     void grow(size_t sz, size_t reserved = 0);

contrib/epee/src/abstract_http_client.cpp

@@ -1,3 +1,4 @@
+#include <math.h>
 #include "net/abstract_http_client.h"
 #include "net/http_base.h"
 #include "net/net_parse_helpers.h"
@@ -135,6 +136,13 @@ namespace http
     http::url_content parsed{};
     const bool r = parse_url(address, parsed);
     CHECK_AND_ASSERT_MES(r, false, "failed to parse url: " << address);
+    if (parsed.port == 0)
+    {
+      if (parsed.schema == "http")
+        parsed.port = 80;
+      else if (parsed.schema == "https")
+        parsed.port = 443;
+    }
     set_server(std::move(parsed.host), std::to_string(parsed.port), std::move(user), std::move(ssl_options));
     return true;
   }

contrib/epee/src/connection_basic.cpp

@@ -46,12 +46,6 @@
 // TODO:
 #include "net/network_throttle-detail.hpp"
 
-#if BOOST_VERSION >= 107000
-#define GET_IO_SERVICE(s) ((boost::asio::io_context&)(s).get_executor().context())
-#else
-#define GET_IO_SERVICE(s) ((s).get_io_service())
-#endif
-
 #undef MONERO_DEFAULT_LOG_CATEGORY
 #define MONERO_DEFAULT_LOG_CATEGORY "net.conn"
 
@@ -127,12 +121,12 @@ connection_basic_pimpl::connection_basic_pimpl(const std::string &name) : m_thro
 int connection_basic_pimpl::m_default_tos;
 
 // methods:
-connection_basic::connection_basic(boost::asio::ip::tcp::socket&& sock, std::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support)
+connection_basic::connection_basic(boost::asio::io_context &io_context, boost::asio::ip::tcp::socket&& sock, std::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support)
 	:
 	m_state(std::move(state)),
 	mI( new connection_basic_pimpl("peer") ),
-	strand_(GET_IO_SERVICE(sock)),
-	socket_(GET_IO_SERVICE(sock), get_context(m_state.get())),
+	strand_(io_context),
+	socket_(io_context, get_context(m_state.get())),
 	m_want_close_connection(false),
 	m_was_shutdown(false),
 	m_is_multithreaded(false),
@@ -152,12 +146,12 @@ connection_basic::connection_basic(boost::asio::ip::tcp::socket&& sock, std::sha
 	_note("Spawned connection #"<<mI->m_peer_number<<" to " << remote_addr_str << " currently we have sockets count:" << m_state->sock_count);
 }
 
-connection_basic::connection_basic(boost::asio::io_service &io_service, std::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support)
+connection_basic::connection_basic(boost::asio::io_context &io_context, std::shared_ptr<connection_basic_shared_state> state, ssl_support_t ssl_support)
 	:
 	m_state(std::move(state)),
 	mI( new connection_basic_pimpl("peer") ),
-	strand_(io_service),
-	socket_(io_service, get_context(m_state.get())),
+	strand_(io_context),
+	socket_(io_context, get_context(m_state.get())),
 	m_want_close_connection(false),
 	m_was_shutdown(false),
 	m_is_multithreaded(false),

contrib/epee/src/file_io_utils.cpp

@@ -149,40 +149,5 @@ namespace file_io_utils
 		}
 #endif
 	}
-
-
-	bool get_file_size(const std::string& path_to_file, uint64_t &size)
-	{
-#ifdef _WIN32
-                std::wstring wide_path;
-                try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; }
-                HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
-                if (file_handle == INVALID_HANDLE_VALUE)
-                    return false;
-                LARGE_INTEGER file_size;
-                BOOL result = GetFileSizeEx(file_handle, &file_size);
-                CloseHandle(file_handle);
-                if (result) {
-                    size = file_size.QuadPart;
-                }
-                return size;
-#else
-		try
-		{
-			std::ifstream fstream;
-			fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit);
-			fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate);
-			size = fstream.tellg();
-			fstream.close();
-			return true;
-		}
-
-		catch(...)
-		{
-			return false;
-		}
-#endif
-	}
-
 }
 }

contrib/epee/src/http_auth.cpp

@@ -63,11 +63,11 @@
 #include <cassert>
 #include <iterator>
 #include <limits>
+#include <openssl/evp.h>
 #include <tuple>
 #include <type_traits>
 
 #include "hex.h"
-#include "md5_l.h"
 #include "string_coding.h"
 
 /* This file uses the `u8` prefix and specifies all chars by ASCII numeric
@@ -114,8 +114,8 @@ namespace
       void operator()(const T& arg) const
       {
         const boost::iterator_range<const char*> data(boost::as_literal(arg));
-        md5::MD5Update(
-          std::addressof(ctx),
+        EVP_DigestUpdate(
+          ctx,
           reinterpret_cast<const std::uint8_t*>(data.begin()),
           data.size()
         );
@@ -126,25 +126,25 @@ namespace
       }
       void operator()(const epee::wipeable_string& arg) const
       {
-        md5::MD5Update(
-          std::addressof(ctx),
+        EVP_DigestUpdate(
+          ctx,
           reinterpret_cast<const std::uint8_t*>(arg.data()),
           arg.size()
         );
       }
 
-      md5::MD5_CTX& ctx;
+      EVP_MD_CTX *ctx;
     };
 
     template<typename... T>
     std::array<char, 32> operator()(const T&... args) const
     {      
-      md5::MD5_CTX ctx{};
-      md5::MD5Init(std::addressof(ctx));
-      boost::fusion::for_each(std::tie(args...), update{ctx});
+      std::unique_ptr<EVP_MD_CTX, decltype(&EVP_MD_CTX_free)> ctx(EVP_MD_CTX_new(), &EVP_MD_CTX_free);
+      EVP_DigestInit(ctx.get(), EVP_md5());
+      boost::fusion::for_each(std::tie(args...), update{ctx.get()});
 
       std::array<std::uint8_t, 16> digest{{}};
-      md5::MD5Final(digest.data(), std::addressof(ctx));
+      EVP_DigestFinal(ctx.get(), digest.data(), NULL);
       return epee::to_hex::array(digest);
     }
   };

contrib/epee/src/net_helper.cpp

@@ -4,22 +4,38 @@ namespace epee
 {
 namespace net_utils
 {
+	namespace
+	{
+		struct new_connection
+		{
+			boost::promise<boost::asio::ip::tcp::socket> result_;
+			boost::asio::ip::tcp::socket socket_;
+
+			template<typename T>
+			explicit new_connection(T&& executor)
+			  : result_(), socket_(std::forward<T>(executor))
+			{}
+		};
+	}
+
 	boost::unique_future<boost::asio::ip::tcp::socket>
 	direct_connect::operator()(const std::string& addr, const std::string& port, boost::asio::steady_timer& timeout) const
 	{
 		// Get a list of endpoints corresponding to the server name.
 		//////////////////////////////////////////////////////////////////////////
-		boost::asio::ip::tcp::resolver resolver(GET_IO_SERVICE(timeout));
-		boost::asio::ip::tcp::resolver::query query(boost::asio::ip::tcp::v4(), addr, port, boost::asio::ip::tcp::resolver::query::canonical_name);
+		boost::asio::ip::tcp::resolver resolver(MONERO_GET_EXECUTOR(timeout));
 
 		bool try_ipv6 = false;
-		boost::asio::ip::tcp::resolver::iterator iterator;
-		boost::asio::ip::tcp::resolver::iterator end;
+		boost::asio::ip::tcp::resolver::results_type results{};
 		boost::system::error_code resolve_error;
+
 		try
 		{
-			iterator = resolver.resolve(query, resolve_error);
-			if(iterator == end) // Documentation states that successful call is guaranteed to be non-empty
+			results = resolver.resolve(
+				boost::asio::ip::tcp::v4(), addr, port, boost::asio::ip::tcp::resolver::canonical_name, resolve_error
+			);
+
+			if (results.empty())
 			{
 				// if IPv4 resolution fails, try IPv6.  Unintentional outgoing IPv6 connections should only
 				// be possible if for some reason a hostname was given and that hostname fails IPv4 resolution,
@@ -37,27 +53,20 @@ namespace net_utils
 			}
 			try_ipv6 = true;
 		}
+
 		if (try_ipv6)
 		{
-			boost::asio::ip::tcp::resolver::query query6(boost::asio::ip::tcp::v6(), addr, port, boost::asio::ip::tcp::resolver::query::canonical_name);
-			iterator = resolver.resolve(query6);
-			if (iterator == end)
+			results = resolver.resolve(
+				boost::asio::ip::tcp::v6(), addr, port, boost::asio::ip::tcp::resolver::canonical_name
+			);
+			if (results.empty())
 				throw boost::system::system_error{boost::asio::error::fault, "Failed to resolve " + addr};
 		}
 
 		//////////////////////////////////////////////////////////////////////////
 
-		struct new_connection
-		{
-			boost::promise<boost::asio::ip::tcp::socket> result_;
-			boost::asio::ip::tcp::socket socket_;
 
-			explicit new_connection(boost::asio::io_service& io_service)
-			  : result_(), socket_(io_service)
-			{}
-		};
-
-		const auto shared = std::make_shared<new_connection>(GET_IO_SERVICE(timeout));
+		const auto shared = std::make_shared<new_connection>(MONERO_GET_EXECUTOR(timeout));
 		timeout.async_wait([shared] (boost::system::error_code error)
 		{
 			if (error != boost::system::errc::operation_canceled && shared && shared->socket_.is_open())
@@ -66,7 +75,7 @@ namespace net_utils
 				shared->socket_.close();
 			}
 		});
-		shared->socket_.async_connect(*iterator, [shared] (boost::system::error_code error)
+		shared->socket_.async_connect(*results.begin(), [shared] (boost::system::error_code error)
 		{
 			if (shared)
 			{

contrib/epee/src/net_parse_helpers.cpp

@@ -92,7 +92,13 @@ namespace net_utils
     }
     return true;
   }
-  
+
+  static bool parse_port(const std::string& port_str, uint64_t& out_port)
+  {
+    out_port = 0;
+    return boost::conversion::try_lexical_convert(port_str, out_port) && out_port <= 65535;
+  }
+
   bool parse_uri(const std::string uri, http::uri_content& content)
   {
 
@@ -153,7 +159,8 @@ namespace net_utils
     }
     if(result[6].matched)
     {
-      content.port = boost::lexical_cast<uint64_t>(result[6]);
+      if (!parse_port(result[6].str(), content.port))
+        return false;
     }
     if(result[7].matched)
     {
@@ -191,7 +198,8 @@ namespace net_utils
     }
     if(result[6].matched)
     {
-      content.port = boost::lexical_cast<uint64_t>(result[6]);
+      if (!parse_port(result[6].str(), content.port))
+        return false;
     }
     if(result[7].matched)
     {

contrib/epee/src/net_ssl.cpp

@@ -29,6 +29,7 @@
 
 #include <string.h>
 #include <thread>
+#include <boost/asio/post.hpp>
 #include <boost/asio/ssl.hpp>
 #include <boost/cerrno.hpp>
 #include <boost/filesystem/operations.hpp>
@@ -45,6 +46,13 @@
 #undef MONERO_DEFAULT_LOG_CATEGORY
 #define MONERO_DEFAULT_LOG_CATEGORY "net.ssl"
 
+
+#if BOOST_VERSION >= 107300
+  #define MONERO_HOSTNAME_VERIFY boost::asio::ssl::host_name_verification
+#else
+  #define MONERO_HOSTNAME_VERIFY boost::asio::ssl::rfc2818_verification
+#endif
+
 // openssl genrsa -out /tmp/KEY 4096
 // openssl req -new -key /tmp/KEY -out /tmp/REQ
 // openssl x509 -req -days 999999 -sha256 -in /tmp/REQ -signkey /tmp/KEY -out /tmp/CERT
@@ -526,7 +534,7 @@ void ssl_options_t::configure(
       // preverified means it passed system or user CA check. System CA is never loaded
       // when fingerprints are whitelisted.
       const bool verified = preverified &&
-        (verification != ssl_verification_t::system_ca || host.empty() || boost::asio::ssl::rfc2818_verification(host)(preverified, ctx));
+        (verification != ssl_verification_t::system_ca || host.empty() || MONERO_HOSTNAME_VERIFY(host)(preverified, ctx));
 
       if (!verified && !has_fingerprint(ctx))
       {
@@ -544,6 +552,7 @@ void ssl_options_t::configure(
 }
 
 bool ssl_options_t::handshake(
+  boost::asio::io_context& io_context,
   boost::asio::ssl::stream<boost::asio::ip::tcp::socket> &socket,
   boost::asio::ssl::stream_base::handshake_type type,
   boost::asio::const_buffer buffer,
@@ -555,12 +564,11 @@ bool ssl_options_t::handshake(
   auto start_handshake = [&]{
     using ec_t = boost::system::error_code;
     using timer_t = boost::asio::steady_timer;
-    using strand_t = boost::asio::io_service::strand;
+    using strand_t = boost::asio::io_context::strand;
     using socket_t = boost::asio::ip::tcp::socket;
 
-    auto &io_context = GET_IO_SERVICE(socket);
     if (io_context.stopped())
-      io_context.reset();
+      io_context.restart();
     strand_t strand(io_context);
     timer_t deadline(io_context, timeout);
 
@@ -595,13 +603,13 @@ bool ssl_options_t::handshake(
       state.result = ec;
       if (!state.cancel_handshake) {
         state.cancel_timer = true;
-        ec_t ec;
-        deadline.cancel(ec);
+        deadline.cancel();
       }
     };
 
     deadline.async_wait(on_timer);
-    strand.post(
+    boost::asio::post(
+      strand,
       [&]{
         socket.async_handshake(
           type,

contrib/epee/src/network_throttle-detail.cpp

@@ -46,7 +46,7 @@
 #include "misc_log_ex.h" 
 #include <boost/chrono.hpp>
 #include "misc_language.h"
-#include <sstream>
+#include <fstream>
 #include <iomanip>
 #include <algorithm>
 
@@ -186,6 +186,23 @@ void network_throttle::handle_trafic_exact(size_t packet_size)
 	_handle_trafic_exact(packet_size, packet_size);
 }
 
+namespace
+{
+	struct output_history
+	{
+		const boost::circular_buffer< network_throttle::packet_info >& history;
+	};
+
+	std::ostream& operator<<(std::ostream& out, const output_history& source)
+	{
+		out << '[';
+		for (auto sample: source.history)
+			out << sample.m_size << ' ';
+		out << ']';
+		return out;
+	}
+}
+
 void network_throttle::_handle_trafic_exact(size_t packet_size, size_t orginal_size)
 {
 	tick();
@@ -196,14 +213,11 @@ void network_throttle::_handle_trafic_exact(size_t packet_size, size_t orginal_s
 	m_total_packets++;
 	m_total_bytes += packet_size;
 
-	std::ostringstream oss; oss << "["; 	for (auto sample: m_history) oss << sample.m_size << " ";	 oss << "]" << std::ends;
-	std::string history_str = oss.str();
-
 	MTRACE("Throttle " << m_name << ": packet of ~"<<packet_size<<"b " << " (from "<<orginal_size<<" b)"
         << " Speed AVG=" << std::setw(4) <<  ((long int)(cts .average/1024)) <<"[w="<<cts .window<<"]"
         <<           " " << std::setw(4) <<  ((long int)(cts2.average/1024)) <<"[w="<<cts2.window<<"]"
 				<<" / " << " Limit="<< ((long int)(m_target_speed/1024)) <<" KiB/sec "
-				<< " " << history_str
+				<< " " << output_history{m_history}
 		);
 }
 
@@ -289,8 +303,6 @@ void network_throttle::calculate_times(size_t packet_size, calculate_times_struc
     }
 
 	if (dbg) {
-		std::ostringstream oss; oss << "["; 	for (auto sample: m_history) oss << sample.m_size << " ";	 oss << "]" << std::ends;
-		std::string history_str = oss.str();
 		MTRACE((cts.delay > 0 ? "SLEEP" : "")
 			<< "dbg " << m_name << ": " 
 			<< "speed is A=" << std::setw(8) <<cts.average<<" vs "
@@ -300,7 +312,7 @@ void network_throttle::calculate_times(size_t packet_size, calculate_times_struc
 			<< "E="<< std::setw(8) << E << " (Enow="<<std::setw(8)<<Enow<<") "
             << "M=" << std::setw(8) << M <<" W="<< std::setw(8) << cts.window << " "
             << "R=" << std::setw(8) << cts.recomendetDataSize << " Wgood" << std::setw(8) << Wgood << " "
-			<< "History: " << std::setw(8) << history_str << " "
+			<< "History: " << std::setw(8) << output_history{m_history} << " "
 			<< "m_last_sample_time=" << std::setw(8) << m_last_sample_time
 		);
 

contrib/epee/src/string_tools.cpp

@@ -163,11 +163,15 @@ namespace string_tools
 
   void set_module_name_and_folder(const std::string& path_to_process_)
   {
-    boost::filesystem::path path_to_process = path_to_process_;
+    boost::filesystem::path path_to_process;
 
 #ifdef _WIN32
-    path_to_process = get_current_module_path();
-#endif 
+    // Convert to wide string to avoid codecvt errors with Unicode paths
+    std::wstring wpath = epee::string_tools::utf8_to_utf16(get_current_module_path());
+    path_to_process = boost::filesystem::path(wpath);
+#else
+    path_to_process = boost::filesystem::path(path_to_process_);
+#endif
 
     get_current_module_name() = path_to_process.filename().string();
     get_current_module_folder() = path_to_process.parent_path().string();

contrib/epee/src/wipeable_string.cpp

@@ -261,4 +261,14 @@ wipeable_string &wipeable_string::operator=(const wipeable_string &other)
   return *this;
 }
 
+char& wipeable_string::operator[](size_t idx) {
+  CHECK_AND_ASSERT_THROW_MES(idx < buffer.size(), "Index out of bounds");
+  return buffer[idx];
+}
+
+const char& wipeable_string::operator[](size_t idx) const {
+  CHECK_AND_ASSERT_THROW_MES(idx < buffer.size(), "Index out of bounds");
+  return buffer[idx];
+}
+
 }

contrib/gitian/DOCKRUN.md

@@ -57,7 +57,7 @@ The dockrun.sh script will do everything to build the binaries. Just specify the
 version to build as its only argument, e.g.
 
 ```bash
-VERSION=v0.18.3.4
+VERSION=v0.18.4.2
 ./dockrun.sh $VERSION
 ```
 

contrib/gitian/README.md

@@ -133,7 +133,7 @@ Common setup part:
 su - gitianuser
 
 GH_USER=YOUR_GITHUB_USER_NAME
-VERSION=v0.18.3.4
+VERSION=v0.18.4.2
 ```
 
 Where `GH_USER` is your GitHub user name and `VERSION` is the version tag you want to build. 

contrib/gitian/gitian-android.yml

@@ -120,8 +120,8 @@ script: |
     cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DCMAKE_BUILD_TYPE=Release
     make ${MAKEOPTS}
     chmod 755 bin/*
-    cp ../utils/conf/wow.conf bin
-    chmod 644 bin/wow.conf
+    cp ../LICENSE ../README.md ../docs/ANONYMITY_NETWORKS.md bin
+    chmod 644 bin/LICENSE bin/*.md
     DISTNAME=wownero-${i}-${version}
     mv bin ${DISTNAME}
     find ${DISTNAME}/ | sort | tar --no-recursion --owner=0 --group=0 -c -T - | bzip2 -9 > ${OUTDIR}/${DISTNAME}.tar.bz2

contrib/gitian/gitian-freebsd.yml

@@ -117,8 +117,8 @@ script: |
     cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_SKIP_RPATH=ON
     make ${MAKEOPTS}
     chmod 755 bin/*
-    cp ../utils/conf/wow.conf bin
-    chmod 644 bin/wow.conf
+    cp ../LICENSE ../README.md ../docs/ANONYMITY_NETWORKS.md bin
+    chmod 644 bin/LICENSE bin/*.md
     DISTNAME=wownero-${i}-${version}
     mv bin ${DISTNAME}
     find ${DISTNAME}/ | sort | tar --no-recursion --owner=0 --group=0 -c -T - | bzip2 -9 > ${OUTDIR}/${DISTNAME}.tar.bz2

contrib/gitian/gitian-linux.yml

@@ -169,8 +169,8 @@ script: |
     cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=${BACKCOMPAT_OPTION} -DCMAKE_SKIP_RPATH=ON
     make ${MAKEOPTS}
     chmod 755 bin/*
-    cp ../utils/conf/wow.conf bin
-    chmod 644 bin/wow.conf
+    cp ../LICENSE ../README.md ../docs/ANONYMITY_NETWORKS.md bin
+    chmod 644 bin/LICENSE bin/*.md
     DISTNAME=wownero-${i}-${version}
     mv bin ${DISTNAME}
     find ${DISTNAME}/ | sort | tar --no-recursion --owner=0 --group=0 -c -T - | bzip2 -9 > ${OUTDIR}/${DISTNAME}.tar.bz2

contrib/gitian/gitian-osx.yml

@@ -108,8 +108,8 @@ script: |
     cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake
     make ${MAKEOPTS}
     chmod 755 bin/*
-    cp ../utils/conf/wow.conf bin
-    chmod 644 bin/wow.conf
+    cp ../LICENSE ../README.md ../docs/ANONYMITY_NETWORKS.md bin
+    chmod 644 bin/LICENSE bin/*.md
     DISTNAME=wownero-${i}-${version}
     mv bin ${DISTNAME}
     find ${DISTNAME}/ | sort | tar --no-recursion --owner=0 --group=0 -c -T - | bzip2 -9 > ${OUTDIR}/${DISTNAME}.tar.bz2

contrib/gitian/gitian-win.yml

@@ -127,7 +127,7 @@ script: |
     mkdir build && cd build
     cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake
     make ${MAKEOPTS}
-    cp ../utils/conf/wow.conf bin
+    cp ../LICENSE ../README.md ../docs/ANONYMITY_NETWORKS.md bin
     DISTNAME=wownero-${i}-${version}
     mv bin ${DISTNAME}
     find ${DISTNAME}/ | sort | zip -X@ ${OUTDIR}/${DISTNAME}.zip

external/CMakeLists.txt

@@ -39,6 +39,7 @@ find_package(Miniupnpc REQUIRED)
 
 message(STATUS "Using in-tree miniupnpc")
 set(UPNPC_NO_INSTALL TRUE CACHE BOOL "Disable miniupnp installation" FORCE)
+set(UPNPC_BUILD_SHARED OFF CACHE BOOL "Disable building shared library" FORCE)
 add_subdirectory(miniupnp/miniupnpc)
 set_property(TARGET libminiupnpc-static PROPERTY FOLDER "external")
 set_property(TARGET libminiupnpc-static PROPERTY POSITION_INDEPENDENT_CODE ON)
@@ -70,3 +71,5 @@ add_subdirectory(db_drivers)
 add_subdirectory(easylogging++)
 add_subdirectory(qrcodegen)
 add_subdirectory(randomwow EXCLUDE_FROM_ALL)
+add_subdirectory(polyseed EXCLUDE_FROM_ALL)
+add_subdirectory(utf8proc EXCLUDE_FROM_ALL)

external/easylogging++/easylogging++.h

@@ -3260,12 +3260,12 @@ class Writer : base::NoCopy {
          const char* func, base::DispatchAction dispatchAction = base::DispatchAction::NormalLog,
          base::type::VerboseLevel verboseLevel = 0) :
     m_msg(nullptr), m_level(level), m_color(color), m_file(file), m_line(line), m_func(func), m_verboseLevel(verboseLevel),
-    m_logger(nullptr), m_proceed(false), m_dispatchAction(dispatchAction) {
+    m_logger(nullptr), m_proceed(false), m_dispatchAction(dispatchAction), m_sync(ELPP->lock()) {
   }
 
   Writer(LogMessage* msg, base::DispatchAction dispatchAction = base::DispatchAction::NormalLog) :
     m_msg(msg), m_level(msg != nullptr ? msg->level() : Level::Unknown),
-    m_line(0), m_logger(nullptr), m_proceed(false), m_dispatchAction(dispatchAction) {
+    m_line(0), m_logger(nullptr), m_proceed(false), m_dispatchAction(dispatchAction), m_sync(ELPP->lock()) {
   }
 
   virtual ~Writer(void) {
@@ -3323,6 +3323,7 @@ class Writer : base::NoCopy {
   base::MessageBuilder m_messageBuilder;
   base::DispatchAction m_dispatchAction;
   std::vector<std::string> m_loggerIds;
+  base::threading::ScopedLock m_sync;
   friend class el::Helpers;
 
   void initializeLogger(const std::string& loggerId, bool lookup = true, bool needLock = true);

src/CMakeLists.txt

@@ -95,6 +95,7 @@ add_subdirectory(net)
 add_subdirectory(hardforks)
 add_subdirectory(blockchain_db)
 add_subdirectory(mnemonics)
+add_subdirectory(polyseed)
 add_subdirectory(rpc)
 if(NOT IOS)
   add_subdirectory(serialization)
@@ -106,15 +107,15 @@ endif()
 add_subdirectory(cryptonote_protocol)
 if(NOT IOS)
   add_subdirectory(simplewallet)
+  add_subdirectory(gen_multisig)
+  add_subdirectory(gen_ssl_cert)
   add_subdirectory(daemonizer)
   add_subdirectory(daemon)
+  add_subdirectory(blockchain_utilities)
 endif()
 
 if(BUILD_DEBUG_UTILITIES)
   add_subdirectory(debug_utilities)
-  add_subdirectory(blockchain_utilities)
-  add_subdirectory(gen_multisig)
-  add_subdirectory(gen_ssl_cert)
 endif()
 
 if(PER_BLOCK_CHECKPOINT)

src/blockchain_db/blockchain_db.cpp

@@ -245,7 +245,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair
       if (tx.version > 1)
       {
         commitment = tx.rct_signatures.outPk[i].mask;
-        if (rct::is_rct_bulletproof_plus(tx.rct_signatures.type))
+        if (rct::is_rct_bp_plus_legacy(tx.rct_signatures.type))
           commitment = rct::scalarmult8(commitment);
       }
       amount_output_indices[i] = add_output(tx_hash, tx.vout[i], i, tx.unlock_time,

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -28,13 +28,17 @@
 #include "db_lmdb.h"
 
 #include <boost/filesystem.hpp>
+#include <boost/filesystem/fstream.hpp>
 #include <boost/format.hpp>
 #include <boost/circular_buffer.hpp>
 #include <memory>  // std::unique_ptr
 #include <cstring>  // memcpy
 
+#ifdef WIN32
+#include <winioctl.h>
+#endif
+
 #include "string_tools.h"
-#include "file_io_utils.h"
 #include "common/util.h"
 #include "common/pruning.h"
 #include "cryptonote_basic/cryptonote_format_utils.h"
@@ -1321,6 +1325,54 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions): BlockchainDB()
   m_hardfork = nullptr;
 }
 
+#ifdef WIN32
+static bool disable_ntfs_compression(const boost::filesystem::path& filepath)
+{
+  DWORD file_attributes = ::GetFileAttributesW(filepath.c_str());
+  if (file_attributes == INVALID_FILE_ATTRIBUTES)
+  {
+    MERROR("Failed to get " << filepath.string() << " file attributes. Error: " << ::GetLastError());
+    return false;
+  }
+  
+  if (!(file_attributes & FILE_ATTRIBUTE_COMPRESSED))
+    return true; // not compressed
+
+  LOG_PRINT_L1("Disabling NTFS compression for " << filepath.string());
+  HANDLE file_handle = ::CreateFileW(
+    filepath.c_str(),
+    GENERIC_READ | GENERIC_WRITE,
+    FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+    nullptr,
+    OPEN_EXISTING,
+    boost::filesystem::is_directory(filepath) ? FILE_FLAG_BACKUP_SEMANTICS : 0, // Needed to open handles to directories
+    nullptr
+  );
+
+  if (file_handle == INVALID_HANDLE_VALUE)
+  {
+    MERROR("Failed to open handle: " << filepath.string() << ". Error: " << ::GetLastError());
+    return false;
+  }
+
+  USHORT compression_state = COMPRESSION_FORMAT_NONE;
+  DWORD bytes_returned;
+  BOOL ok = ::DeviceIoControl(
+    file_handle,
+    FSCTL_SET_COMPRESSION,
+    &compression_state,
+    sizeof(compression_state),
+    nullptr,
+    0,
+    &bytes_returned,
+    nullptr
+  );
+
+  ::CloseHandle(file_handle);
+  return ok;
+}
+#endif
+
 void BlockchainLMDB::open(const std::string& filename, const int db_flags)
 {
   int result;
@@ -1347,6 +1399,18 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags)
     throw DB_ERROR("Database could not be opened");
   }
 
+#ifdef WIN32
+  // ensure NTFS compression is disabled on the directory and database file to avoid corruption of the blockchain 
+  if (!disable_ntfs_compression(filename))
+    LOG_PRINT_L0("Failed to disable NTFS compression on folder: " << filename << ". Error: " << ::GetLastError());
+  boost::filesystem::path datafile(filename);
+  datafile /= CRYPTONOTE_BLOCKCHAINDATA_FILENAME;
+  if (!boost::filesystem::exists(datafile))
+    boost::filesystem::ofstream(datafile).close(); // create the file to see if NTFS compression is enabled beforehand
+  if (!disable_ntfs_compression(datafile))
+    throw DB_ERROR("Database file is NTFS compressed and compression could not be disabled");
+#endif
+
   boost::optional<bool> is_hdd_result = tools::is_hdd(filename.c_str());
   if (is_hdd_result)
   {
@@ -4500,12 +4564,11 @@ bool BlockchainLMDB::is_read_only() const
 
 uint64_t BlockchainLMDB::get_database_size() const
 {
-  uint64_t size = 0;
   boost::filesystem::path datafile(m_folder);
   datafile /= CRYPTONOTE_BLOCKCHAINDATA_FILENAME;
-  if (!epee::file_io_utils::get_file_size(datafile.string(), size))
-    size = 0;
-  return size;
+  boost::system::error_code ec{};
+  const boost::uintmax_t size = boost::filesystem::file_size(datafile, ec);
+  return (ec ? 0 : static_cast<uint64_t>(size));
 }
 
 void BlockchainLMDB::fixup()

src/blockchain_utilities/blockchain_import.cpp

@@ -174,7 +174,9 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block
     for(auto& tx_blob: block_entry.txs)
     {
       tx_verification_context tvc = AUTO_VAL_INIT(tvc);
-      core.handle_incoming_tx(tx_blob, tvc, relay_method::block, true);
+      CHECK_AND_ASSERT_THROW_MES(tx_blob.prunable_hash == crypto::null_hash,
+        "block entry must not contain pruned txs");
+      core.handle_incoming_tx(tx_blob.blob, tvc, relay_method::block, true);
       if(tvc.m_verifivation_failed)
       {
         cryptonote::transaction transaction;
@@ -190,8 +192,9 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block
     // process block
 
     block_verification_context bvc = {};
+    pool_supplement ps{};
 
-    core.handle_incoming_block(block_entry.block, pblocks.empty() ? NULL : &pblocks[blockidx++], bvc, false); // <--- process block
+    core.handle_incoming_block(block_entry.block, pblocks.empty() ? NULL : &pblocks[blockidx++], bvc, ps, false); // <--- process block
 
     if(bvc.m_verifivation_failed)
     {

src/checkpoints/checkpoints.cpp

@@ -184,17 +184,10 @@ namespace cryptonote
   {
     if (nettype == TESTNET)
     {
-      ADD_CHECKPOINT2(0,     "48ca7cd3c8de5b6a4d53d2861fbdaedca141553559f9be9520068053cda8430b", "0x1");
-      ADD_CHECKPOINT2(1000000, "46b690b710a07ea051bc4a6b6842ac37be691089c0f7758cfeec4d5fc0b4a258", "0x7aaad7153");
-      ADD_CHECKPOINT2(1058600, "12904f6b4d9e60fd875674e07147d2c83d6716253f046af7b894c3e81da7e1bd", "0x971efd119");
-      ADD_CHECKPOINT2(1450000, "87562ca6786f41556b8d5b48067303a57dc5ca77155b35199aedaeca1550f5a0", "0xa639e2930e");
       return true;
     }
     if (nettype == STAGENET)
     {
-      ADD_CHECKPOINT2(0,       "76ee3cc98646292206cd3e86f74d88b4dcc1d937088645e9b0cbca84b7ce74eb", "0x1");
-      ADD_CHECKPOINT2(10000,   "1f8b0ce313f8b9ba9a46108bfd285c45ad7c2176871fd41c3a690d4830ce2fd5", "0x1d73ba");
-      ADD_CHECKPOINT2(550000,  "409f68cddd8e74b37469b41c1e61250d81c5776b42264f416d5d27c4626383ed", "0x5f3d4d03e");
       return true;
     }
     ADD_CHECKPOINT2(1,       "97f4ce4d7879b3bea54dcec738cd2ebb7952b4e9bb9743262310cd5fec749340", "0x2");
@@ -233,7 +226,8 @@ namespace cryptonote
     ADD_CHECKPOINT2(489400,  "b14f49eae77398117ea93435676100d8b655a804689f73a5a4d0d5e71160d603", "0x1123c39bb52f7e");
     ADD_CHECKPOINT2(491200,  "cedba73ad35ce7f51aaca2beb36dc32d79ecc716d146eb8211e6a815f3666c4a", "0x11334734abbd17");
     ADD_CHECKPOINT2(497100,  "2c4c70ac1ada94151f19d67ccf1aa4e846e6067f49f67c85cc03f78e768ea42b", "0x116906bc97a751");
-    ADD_CHECKPOINT2(691500,  "ed8e2507c0938b7eab7b02eccfb3506aeb591e51fbf6cf145fcc60ea2d351025", "0x163a280f2ce8e3");
+    ADD_CHECKPOINT2(760300,  "50ce41518bb4bea392194c13d0a5ef4cbf01ffb84ba393131e910adb63e2d360", "0x18ef58d8abb8b3");
+    ADD_CHECKPOINT2(771100,  "03e834788e1e33dbba9bc3431a81189cd655f9da80323a728fa0dae56a95145e", "0x192cdb615ada62");
     return true;
   }
 

src/common/dns_utils.cpp

@@ -47,11 +47,6 @@ using namespace epee;
 
 static const char *DEFAULT_DNS_PUBLIC_ADDR[] =
 {
-  "194.150.168.168",    // CCC (Germany)
-  "80.67.169.40",       // FDN (France)
-  "89.233.43.71",       // http://censurfridns.dk (Denmark)
-  "109.69.8.51",        // punCAT (Spain)
-  "193.58.251.251",     // SkyDNS (Russia)
 };
 
 static boost::mutex instance_lock;
@@ -104,8 +99,6 @@ get_builtin_ds(void)
 {
   static const char * const ds[] =
   {
-    ". IN DS 19036 8 2 49AAC11D7B6F6446702E54A1607371607A1A41855200FD2CE1CDDE32F24E8FB5\n",
-    ". IN DS 20326 8 2 E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC683457104237C7F8EC8D\n",
     NULL
   };
   return ds;

src/common/download.cpp

@@ -30,7 +30,6 @@
 #include <atomic>
 #include <boost/filesystem.hpp>
 #include <boost/thread/thread.hpp>
-#include "file_io_utils.h"
 #include "net/http_client.h"
 #include "download.h"
 
@@ -73,8 +72,11 @@ namespace tools
     {
       boost::unique_lock<boost::mutex> lock(control->mutex);
       std::ios_base::openmode mode = std::ios_base::out | std::ios_base::binary;
-      uint64_t existing_size = 0;
-      if (epee::file_io_utils::get_file_size(control->path, existing_size) && existing_size > 0)
+      boost::system::error_code ec{};
+      uint64_t existing_size = static_cast<uint64_t>(boost::filesystem::file_size(control->path, ec));
+      if (ec)
+        existing_size = 0;
+      if (existing_size > 0)
       {
         MINFO("Resuming downloading " << control->uri << " to " << control->path << " from " << existing_size);
         mode |= std::ios_base::app;

src/common/password.cpp

@@ -185,7 +185,7 @@ namespace
         return false;
       if (verify)
       {
-        std::cout << "Confirm password: ";
+        std::cout << "Confirm password: " << std::flush;
         if (!read_from_tty(pass2, hide_input))
           return false;
         if(pass1!=pass2)

src/common/perf_timer.h

@@ -28,6 +28,7 @@
 
 #pragma once
 
+#include <cstdint>
 #include <string>
 #include <stdio.h>
 #include <memory>

src/common/stack_trace.cpp

@@ -34,6 +34,7 @@
 #include "easylogging++/easylogging++.h"
 
 #include <stdexcept>
+#include <iomanip>
 #ifdef USE_UNWIND
 #define UNW_LOCAL_ONLY
 #include <libunwind.h>

src/common/util.cpp

@@ -936,7 +936,7 @@ std::string get_nix_version_display_string()
     }
 
     boost::system::error_code ec;
-    const auto parsed_ip = boost::asio::ip::address::from_string(u_c.host, ec);
+    const auto parsed_ip = boost::asio::ip::make_address(u_c.host, ec);
     if (ec) {
       MDEBUG("Failed to parse '" << address << "' as IP address: " << ec.message() << ". Considering it not local");
       return false;
@@ -1360,30 +1360,13 @@ std::string get_nix_version_display_string()
     // importance.
     static const uint32_t average_block_sizes[] =
     {
-      442, 1211, 1445, 1763, 2272, 8217, 5603, 9999, 16358, 10805, 5290, 4362,
-      4325, 5584, 4515, 5008, 4789, 5196, 7660, 3829, 6034, 2925, 3762, 2545,
-      2437, 2553, 2167, 2761, 2015, 1969, 2350, 1731, 2367, 2078, 2026, 3518,
-      2214, 1908, 1780, 1640, 1976, 1647, 1921, 1716, 1895, 2150, 2419, 2451,
-      2147, 2327, 2251, 1644, 1750, 1481, 1570, 1524, 1562, 1668, 1386, 1494,
-      1637, 1880, 1431, 1472, 1637, 1363, 1762, 1597, 1999, 1564, 1341, 1388,
-      1530, 1476, 1617, 1488, 1368, 1906, 1403, 1695, 1535, 1598, 1318, 1234,
-      1358, 1406, 1698, 1554, 1591, 1758, 1426, 2389, 1946, 1533, 1308, 2701,
-      1525, 1653, 3580, 1889, 2913, 8164, 5154, 3762, 3356, 4360, 3589, 4844,
-      4232, 3781, 3882, 5924, 10790, 7185, 7442, 8214, 8509, 7484, 6939, 7391,
-      8210, 15572, 39680, 44810, 53873, 54639, 68227, 63428, 62386, 68504,
-      83073, 103858, 117573, 98089, 96793, 102337, 94714, 129568, 251584,
-      132026, 94579, 94516, 95722, 106495, 121824, 153983, 162338, 136608,
-      137104, 109872, 91114, 84757, 96339, 74251, 94314, 143216, 155837,
-      129968, 120201, 109913, 101588, 97332, 104611, 95310, 93419, 113345,
-      100743, 92152, 57565, 22533, 37564, 21823, 19980, 18277, 18402, 14344,
-      12142, 15842, 13677, 17631, 18294, 22270, 41422, 39296, 36688, 33512,
-      33831, 27582, 22276, 27516, 27317, 25505, 24426, 20566, 23045, 26766,
-      28185, 26169, 27011, 28642, 34994, 34442, 30682, 34357, 31640, 41167,
-      41301, 48616, 51075, 55061, 49909, 44606, 47091, 53828, 42520, 39023,
-      55245, 56145, 51119, 60398, 71821, 48142, 60310, 56041, 54176, 66220,
-      56336, 55248, 56656, 63305, 54029, 77136, 71902, 71618, 83587, 81068,
-      69062, 54848, 53681, 53555,
-      50616    // Blocks 2,400,000 to 2,409,999 in July 2021
+      23956, 8606, 6156, 5778, 6549, 10028, 10207, 5681, 12915, 13680, 11402,
+      9194, 12082, 10496, 11879, 10304, 10356, 8433, 8252, 12264, 7143, 8484,
+      9226, 8726, 6961, 6082, 4234, 3712, 4307, 4324, 6758, 13616, 11376, 4091,
+      3157, 4450, 2966, 2264, 2290, 2046, 2156, 1902, 1966, 2354, 2295, 2126, 1977,
+      1728, 1591, 1968, 1426, 1280, 1372, 1266, 1399, 2044, 1320, 1100, 1084, 1384,
+      1183, 2458, 1285, 1501, 1270, 1222, 1284, 1246, 1552, 1431, 1325, 1774, 2771,
+      2855, 1394, 1388, 2882    // Blocks 0 to 760,486 in August 2025
     };
     const uint64_t block_range_size = 10000;
 

src/crypto/generic-ops.h

@@ -33,6 +33,7 @@
 #include <cstddef>
 #include <cstring>
 #include <functional>
+#include <memory>
 #include <sodium/crypto_verify_32.h>
 
 #define CRYPTO_MAKE_COMPARABLE(type) \
@@ -60,14 +61,18 @@ namespace crypto { \
 namespace crypto { \
   static_assert(sizeof(std::size_t) <= sizeof(type), "Size of " #type " must be at least that of size_t"); \
   inline std::size_t hash_value(const type &_v) { \
-    return reinterpret_cast<const std::size_t &>(_v); \
+    std::size_t h; \
+    memcpy(&h, std::addressof(_v), sizeof(h)); \
+    return h; \
   } \
 } \
 namespace std { \
   template<> \
   struct hash<crypto::type> { \
     std::size_t operator()(const crypto::type &_v) const { \
-      return reinterpret_cast<const std::size_t &>(_v); \
+      std::size_t h; \
+      memcpy(&h, std::addressof(_v), sizeof(h)); \
+      return h; \
     } \
   }; \
 }

src/crypto/hash.h

@@ -30,8 +30,9 @@
 
 #pragma once
 
-#include <stddef.h>
 #include <iostream>
+#include <stddef.h>
+#include <stdexcept>
 
 #include "common/pod-class.h"
 #include "generic-ops.h"
@@ -70,11 +71,20 @@ namespace crypto {
     return h;
   }
 
+  static constexpr void cn_variant1_check(const std::size_t length, const int variant)
+  {
+    // see VARIANT1_CHECK in slow-hash.c
+    if (variant == 1 && length < 43)
+      throw std::logic_error("Cryptonight variant 1 is undefined for inputs of less than 43 bytes");
+  }
+
   inline void cn_slow_hash(const void *data, std::size_t length, hash &hash, int variant = 0, uint64_t height = 0) {
+    cn_variant1_check(length, variant);
     cn_slow_hash(data, length, reinterpret_cast<char *>(&hash), variant, 0/*prehashed*/, height);
   }
 
   inline void cn_slow_hash_prehashed(const void *data, std::size_t length, hash &hash, int variant = 0, uint64_t height = 0) {
+    cn_variant1_check(length, variant);
     cn_slow_hash(data, length, reinterpret_cast<char *>(&hash), variant, 1/*prehashed*/, height);
   }
 

src/cryptonote_basic/CMakeLists.txt

@@ -71,6 +71,7 @@ target_link_libraries(cryptonote_basic
     checkpoints
     cryptonote_format_utils_basic
     device
+    polyseed_wrapper
     ${Boost_DATE_TIME_LIBRARY}
     ${Boost_PROGRAM_OPTIONS_LIBRARY}
     ${Boost_SERIALIZATION_LIBRARY}

src/cryptonote_basic/account.cpp

@@ -87,12 +87,16 @@ DISABLE_VS_WARNINGS(4244 4345)
   void account_keys::xor_with_key_stream(const crypto::chacha_key &key)
   {
     // encrypt a large enough byte stream with chacha20
-    epee::wipeable_string key_stream = get_key_stream(key, m_encryption_iv, sizeof(crypto::secret_key) * (2 + m_multisig_keys.size()));
+    epee::wipeable_string key_stream = get_key_stream(key, m_encryption_iv, sizeof(crypto::secret_key) * (3 + m_multisig_keys.size()) + m_passphrase.size());
     const char *ptr = key_stream.data();
     for (size_t i = 0; i < sizeof(crypto::secret_key); ++i)
       m_spend_secret_key.data[i] ^= *ptr++;
     for (size_t i = 0; i < sizeof(crypto::secret_key); ++i)
       m_view_secret_key.data[i] ^= *ptr++;
+    for (size_t i = 0; i < sizeof(crypto::secret_key); ++i)
+      m_polyseed.data[i] ^= *ptr++;
+    for (size_t i = 0; i < m_passphrase.size(); ++i)
+      m_passphrase.data()[i] ^= *ptr++;
     for (crypto::secret_key &k: m_multisig_keys)
     {
       for (size_t i = 0; i < sizeof(crypto::secret_key); ++i)
@@ -150,6 +154,19 @@ DISABLE_VS_WARNINGS(4244 4345)
   {
     m_keys.m_spend_secret_key = crypto::secret_key();
     m_keys.m_multisig_keys.clear();
+    m_keys.m_polyseed = crypto::secret_key();
+    m_keys.m_passphrase.wipe();
+  }
+  //-----------------------------------------------------------------
+  void account_base::set_spend_key(const crypto::secret_key& spend_secret_key)
+  {
+    // make sure derived spend public key matches saved public spend key
+    crypto::public_key spend_public_key;
+    crypto::secret_key_to_public_key(spend_secret_key, spend_public_key);
+    CHECK_AND_ASSERT_THROW_MES(m_keys.m_account_address.m_spend_public_key == spend_public_key,
+        "Unexpected derived public spend key");
+
+    m_keys.m_spend_secret_key = spend_secret_key;
   }
   //-----------------------------------------------------------------
   crypto::secret_key account_base::generate(const crypto::secret_key& recovery_key, bool recover, bool two_random)
@@ -244,6 +261,21 @@ DISABLE_VS_WARNINGS(4244 4345)
     create_from_keys(address, fake, viewkey);
   }
   //-----------------------------------------------------------------
+  void account_base::create_from_polyseed(const polyseed::data& seed, const epee::wipeable_string &passphrase)
+  {
+    crypto::secret_key secret_key;
+    seed.keygen(&secret_key, sizeof(secret_key));
+
+    if (!passphrase.empty()) {
+      secret_key = cryptonote::decrypt_key(secret_key, passphrase);
+    }
+
+    generate(secret_key, true, false);
+
+    seed.save(m_keys.m_polyseed.data);
+    m_keys.m_passphrase = passphrase;
+  }
+  //-----------------------------------------------------------------
   bool account_base::make_multisig(const crypto::secret_key &view_secret_key, const crypto::secret_key &spend_secret_key, const crypto::public_key &spend_public_key, const std::vector<crypto::secret_key> &multisig_keys)
   {
     m_keys.m_account_address.m_spend_public_key = spend_public_key;

src/cryptonote_basic/account.h

@@ -33,6 +33,7 @@
 #include "cryptonote_basic.h"
 #include "crypto/crypto.h"
 #include "serialization/keyvalue_serialization.h"
+#include "polyseed/polyseed.hpp"
 
 namespace cryptonote
 {
@@ -45,6 +46,8 @@ namespace cryptonote
     std::vector<crypto::secret_key> m_multisig_keys;
     hw::device *m_device = &hw::get_device("default");
     crypto::chacha_iv m_encryption_iv;
+    crypto::secret_key m_polyseed;
+    epee::wipeable_string m_passphrase; // Only used with polyseed
 
     BEGIN_KV_SERIALIZE_MAP()
       KV_SERIALIZE(m_account_address)
@@ -53,6 +56,8 @@ namespace cryptonote
       KV_SERIALIZE_CONTAINER_POD_AS_BLOB(m_multisig_keys)
       const crypto::chacha_iv default_iv{{0, 0, 0, 0, 0, 0, 0, 0}};
       KV_SERIALIZE_VAL_POD_AS_BLOB_OPT(m_encryption_iv, default_iv)
+      KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE(m_polyseed)
+      KV_SERIALIZE(m_passphrase)
     END_KV_SERIALIZE_MAP()
 
     void encrypt(const crypto::chacha_key &key);
@@ -79,6 +84,7 @@ namespace cryptonote
     void create_from_device(hw::device &hwdev);
     void create_from_keys(const cryptonote::account_public_address& address, const crypto::secret_key& spendkey, const crypto::secret_key& viewkey);
     void create_from_viewkey(const cryptonote::account_public_address& address, const crypto::secret_key& viewkey);
+    void create_from_polyseed(const polyseed::data &polyseed, const epee::wipeable_string &passphrase);
     bool make_multisig(const crypto::secret_key &view_secret_key, const crypto::secret_key &spend_secret_key, const crypto::public_key &spend_public_key, const std::vector<crypto::secret_key> &multisig_keys);
     const account_keys& get_keys() const;
     std::string get_public_address_str(network_type nettype) const;
@@ -95,6 +101,7 @@ namespace cryptonote
     bool store(const std::string& file_path);
 
     void forget_spend_key();
+    void set_spend_key(const crypto::secret_key& spend_secret_key);
     const std::vector<crypto::secret_key> &get_multisig_keys() const { return m_keys.m_multisig_keys; }
 
     void encrypt_keys(const crypto::chacha_key &key) { m_keys.encrypt(key); }

src/cryptonote_basic/connection_context.cpp

@@ -29,6 +29,7 @@
 
 #include "connection_context.h"
 
+#include <boost/optional/optional.hpp>
 #include "cryptonote_protocol/cryptonote_protocol_defs.h"
 #include "p2p/p2p_protocol_defs.h"
 
@@ -69,4 +70,23 @@ namespace cryptonote
     };
     return std::numeric_limits<size_t>::max();
   }
+
+  void cryptonote_connection_context::set_state_normal()
+  {
+    m_state = state_normal;
+    m_expected_heights_start = 0;
+    m_needed_objects.clear();
+    m_needed_objects.shrink_to_fit();
+    m_expected_heights.clear();
+    m_expected_heights.shrink_to_fit();
+    m_requested_objects.clear();
+  }
+
+  boost::optional<crypto::hash> cryptonote_connection_context::get_expected_hash(const uint64_t height) const
+  {
+    const auto difference = height - m_expected_heights_start;
+    if (height < m_expected_heights_start || m_expected_heights.size() <= difference)
+      return boost::none;
+    return m_expected_heights[difference];
+  }
 } // cryptonote

src/cryptonote_basic/connection_context.h

@@ -34,6 +34,7 @@
 #include <atomic>
 #include <algorithm>
 #include <boost/date_time/posix_time/posix_time.hpp>
+#include <boost/optional/optional_fwd.hpp>
 #include "net/net_utils_base.h"
 #include "crypto/hash.h"
 
@@ -42,7 +43,7 @@ namespace cryptonote
   struct cryptonote_connection_context: public epee::net_utils::connection_context_base
   {
     cryptonote_connection_context(): m_state(state_before_handshake), m_remote_blockchain_height(0), m_last_response_height(0),
-        m_last_request_time(boost::date_time::not_a_date_time), m_callback_request_count(0),
+        m_expected_heights_start(0), m_last_request_time(boost::date_time::not_a_date_time), m_callback_request_count(0),
         m_last_known_hash(crypto::null_hash), m_pruning_seed(0), m_rpc_port(0), m_rpc_credits_per_hash(0), m_anchor(false), m_score(0),
         m_expect_response(0), m_expect_height(0), m_num_requested(0) {}
 
@@ -92,11 +93,18 @@ namespace cryptonote
     //! \return Maximum number of bytes permissible for `command`.
     static size_t get_max_bytes(int command) noexcept;
 
+    //! Use this instead of `m_state = state_normal`.
+    void set_state_normal();
+
+    boost::optional<crypto::hash> get_expected_hash(uint64_t height) const;
+
     state m_state;
     std::vector<std::pair<crypto::hash, uint64_t>> m_needed_objects;
+    std::vector<crypto::hash> m_expected_heights;
     std::unordered_set<crypto::hash> m_requested_objects;
     uint64_t m_remote_blockchain_height;
     uint64_t m_last_response_height;
+    uint64_t m_expected_heights_start;
     boost::posix_time::ptime m_last_request_time;
     copyable_atomic m_callback_request_count; //in debug purpose: problem with double callback rise
     crypto::hash m_last_known_hash;

src/cryptonote_basic/cryptonote_basic_impl.h

@@ -39,15 +39,6 @@ namespace cryptonote {
   /************************************************************************/
   /*                                                                      */
   /************************************************************************/
-  template<class t_array>
-  struct array_hasher: std::unary_function<t_array&, std::size_t>
-  {
-    std::size_t operator()(const t_array& val) const
-    {
-      return boost::hash_range(&val.data[0], &val.data[sizeof(val.data)]);
-    }
-  };
-
 
 #pragma pack(push, 1)
   struct public_address_outer_blob

src/cryptonote_basic/cryptonote_boost_serialization.h

@@ -335,7 +335,7 @@ namespace boost
     a & x.type;
     if (x.type == rct::RCTTypeNull)
       return;
-    if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof && x.type != rct::RCTTypeBulletproof2 && x.type != rct::RCTTypeFullBulletproof && x.type != rct::RCTTypeSimpleBulletproof && x.type != rct::RCTTypeCLSAG && x.type != rct::RCTTypeBulletproofPlus)
+    if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof && x.type != rct::RCTTypeBulletproof2 && x.type != rct::RCTTypeFullBulletproof && x.type != rct::RCTTypeSimpleBulletproof && x.type != rct::RCTTypeCLSAG && x.type != rct::RCTTypeBulletproofPlus && x.type != rct::RCTTypeBulletproofPlus_FullCommit)
       throw boost::archive::archive_exception(boost::archive::archive_exception::other_exception, "Unsupported rct type");
     // a & x.message; message is not serialized, as it can be reconstructed from the tx data
     // a & x.mixRing; mixRing is not serialized, as it can be reconstructed from the offsets
@@ -369,7 +369,7 @@ namespace boost
     a & x.type;
     if (x.type == rct::RCTTypeNull)
       return;
-    if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof && x.type != rct::RCTTypeBulletproof2 && x.type != rct::RCTTypeFullBulletproof && x.type != rct::RCTTypeSimpleBulletproof && x.type != rct::RCTTypeCLSAG && x.type != rct::RCTTypeBulletproofPlus)
+    if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof && x.type != rct::RCTTypeBulletproof2 && x.type != rct::RCTTypeFullBulletproof && x.type != rct::RCTTypeSimpleBulletproof && x.type != rct::RCTTypeCLSAG && x.type != rct::RCTTypeBulletproofPlus && x.type != rct::RCTTypeBulletproofPlus_FullCommit)
       throw boost::archive::archive_exception(boost::archive::archive_exception::other_exception, "Unsupported rct type");
     // a & x.message; message is not serialized, as it can be reconstructed from the tx data
     // a & x.mixRing; mixRing is not serialized, as it can be reconstructed from the offsets
@@ -389,7 +389,7 @@ namespace boost
     a & x.p.MGs;
     if (ver >= 1u)
       a & x.p.CLSAGs;
-    if (x.type == rct::RCTTypeBulletproof || x.type == rct::RCTTypeBulletproof2 || x.type == rct::RCTTypeSimpleBulletproof || x.type == rct::RCTTypeCLSAG || x.type == rct::RCTTypeBulletproofPlus)
+    if (x.type == rct::RCTTypeBulletproof || x.type == rct::RCTTypeBulletproof2 || x.type == rct::RCTTypeSimpleBulletproof || x.type == rct::RCTTypeCLSAG || x.type == rct::RCTTypeBulletproofPlus || x.type == rct::RCTTypeBulletproofPlus_FullCommit)
       a & x.p.pseudoOuts;
   }
 

src/cryptonote_basic/cryptonote_format_utils.cpp

@@ -106,7 +106,7 @@ namespace cryptonote
   uint64_t get_transaction_weight_clawback(const transaction &tx, size_t n_padded_outputs)
   {
     const rct::rctSig &rv = tx.rct_signatures;
-    const bool plus = rv.type == rct::RCTTypeBulletproofPlus;
+    const bool plus = rv.type == rct::RCTTypeBulletproofPlus || rv.type == rct::RCTTypeBulletproofPlus_FullCommit;
     const uint64_t bp_base = (32 * ((plus ? 6 : 9) + 7 * 2)) / 2; // notional size of a 2 output proof, normalized to 1 proof (ie, divided by 2)
     const size_t n_outputs = tx.vout.size();
     if (n_padded_outputs <= 2)
@@ -167,7 +167,7 @@ namespace cryptonote
       if (!base_only)
       {
         const bool bulletproof = rct::is_rct_bulletproof(rv.type);
-        const bool bulletproof_plus = rct::is_rct_bulletproof_plus(rv.type);
+        const bool bulletproof_plus = rct::is_rct_bulletproof_plus_any(rv.type);
         if (bulletproof_plus)
         {
           if (rv.p.bulletproofs_plus.size() != 1)
@@ -188,9 +188,11 @@ namespace cryptonote
           }
           const size_t n_amounts = tx.vout.size();
           CHECK_AND_ASSERT_MES(n_amounts == rv.outPk.size(), false, "Internal error filling out V");
-          rv.p.bulletproofs_plus[0].V.resize(n_amounts);
-          for (size_t i = 0; i < n_amounts; ++i)
-            rv.p.bulletproofs_plus[0].V[i] = rv.outPk[i].mask;
+          rv.p.bulletproofs_plus[0].V.resize(n_amounts); 
+          const bool bulletproof_plus_legacy = rct::is_rct_bp_plus_legacy(rv.type);
+          for (size_t i = 0; i < n_amounts; ++i) {
+            rv.p.bulletproofs_plus[0].V[i] = bulletproof_plus_legacy ? rv.outPk[i].mask : rct::scalarmultKey(rv.outPk[i].mask, rct::INV_EIGHT);
+          }
         }
         if (rct::is_rct_new_bulletproof(rv.type))
         {
@@ -226,9 +228,7 @@ namespace cryptonote
           size_t idx = 0;
           for (size_t n = 0; n < rv.outPk.size(); ++n)
           {
-            //rv.p.bulletproofs[n].V.resize(1);
-            //rv.p.bulletproofs[n].V[0] = rv.outPk[n].mask;
-            CHECK_AND_ASSERT_MES(rv.p.bulletproofs[n].L.size() >= 6, false, "Bad bulletproofs L size"); // at least 64 bits
+            CHECK_AND_ASSERT_MES(rv.p.bulletproofs[n].L.size() >= 6, false, "Bad bulletproofs L size");
             const size_t n_amounts = rct::n_bulletproof_v1_amounts(rv.p.bulletproofs[n]);
             CHECK_AND_ASSERT_MES(idx + n_amounts <= rv.outPk.size(), false, "Internal error filling out V");
             rv.p.bulletproofs[n].V.resize(n_amounts);
@@ -277,6 +277,7 @@ namespace cryptonote
     CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob");
     CHECK_AND_ASSERT_MES(expand_transaction_1(tx, false), false, "Failed to expand transaction data");
     tx.invalidate_hashes();
+    tx.set_blob_size(tx_blob.size());
     //TODO: validate tx
 
     return get_transaction_hash(tx, tx_hash);
@@ -468,7 +469,7 @@ namespace cryptonote
       return blob_size;
     const rct::rctSig &rv = tx.rct_signatures;
     const bool bulletproof = rct::is_rct_bulletproof(rv.type);
-    const bool bulletproof_plus = rct::is_rct_bulletproof_plus(rv.type);
+    const bool bulletproof_plus = rct::is_rct_bulletproof_plus_any(rv.type);
     if (!bulletproof && !bulletproof_plus)
       return blob_size;
     const size_t n_outputs = tx.vout.size();
@@ -486,7 +487,7 @@ namespace cryptonote
   {
     CHECK_AND_ASSERT_MES(tx.pruned, std::numeric_limits<uint64_t>::max(), "get_pruned_transaction_weight does not support non pruned txes");
     CHECK_AND_ASSERT_MES(tx.version >= 2, std::numeric_limits<uint64_t>::max(), "get_pruned_transaction_weight does not support v1 txes");
-    CHECK_AND_ASSERT_MES(tx.rct_signatures.type == rct::RCTTypeBulletproof2 || tx.rct_signatures.type == rct::RCTTypeCLSAG || tx.rct_signatures.type == rct::RCTTypeBulletproofPlus,
+    CHECK_AND_ASSERT_MES(tx.rct_signatures.type == rct::RCTTypeBulletproof2 || tx.rct_signatures.type == rct::RCTTypeCLSAG || tx.rct_signatures.type == rct::RCTTypeBulletproofPlus || tx.rct_signatures.type == rct::RCTTypeBulletproofPlus_FullCommit,
         std::numeric_limits<uint64_t>::max(), "Unsupported rct_signatures type in get_pruned_transaction_weight");
     CHECK_AND_ASSERT_MES(!tx.vin.empty(), std::numeric_limits<uint64_t>::max(), "empty vin");
     CHECK_AND_ASSERT_MES(tx.vin[0].type() == typeid(cryptonote::txin_to_key), std::numeric_limits<uint64_t>::max(), "empty vin");
@@ -505,7 +506,7 @@ namespace cryptonote
     while ((n_padded_outputs = (1u << nrl)) < tx.vout.size())
       ++nrl;
     nrl += 6;
-    extra = 32 * ((rct::is_rct_bulletproof_plus(tx.rct_signatures.type) ? 6 : 9) + 2 * nrl) + 2;
+    extra = 32 * ((rct::is_rct_bulletproof_plus_any(tx.rct_signatures.type) ? 6 : 9) + 2 * nrl) + 2;
     weight += extra;
 
     // calculate deterministic CLSAG/MLSAG data size
@@ -545,6 +546,19 @@ namespace cryptonote
     return get_transaction_weight(tx, blob_size);
   }
   //---------------------------------------------------------------
+  uint64_t get_transaction_blob_size(const transaction& tx)
+  {
+    if (!tx.is_blob_size_valid())
+    {
+      const cryptonote::blobdata tx_blob = tx_to_blob(tx);
+      tx.set_blob_size(tx_blob.size());
+    }
+
+    CHECK_AND_ASSERT_THROW_MES(tx.is_blob_size_valid(), "BUG: blob size valid not set");
+
+    return tx.blob_size;
+  }
+  //---------------------------------------------------------------
   bool get_tx_fee(const transaction& tx, uint64_t & fee)
   {
     if (tx.version > 1)
@@ -1284,7 +1298,6 @@ namespace cryptonote
   crypto::hash get_transaction_hash(const transaction& t)
   {
     crypto::hash h = null_hash;
-    get_transaction_hash(t, h, NULL);
     CHECK_AND_ASSERT_THROW_MES(get_transaction_hash(t, h, NULL), "Failed to calculate transaction hash");
     return h;
   }

src/cryptonote_basic/cryptonote_format_utils.h

@@ -139,6 +139,7 @@ namespace cryptonote
   uint64_t get_transaction_weight(const transaction &tx);
   uint64_t get_transaction_weight(const transaction &tx, size_t blob_size);
   uint64_t get_pruned_transaction_weight(const transaction &tx);
+  uint64_t get_transaction_blob_size(const transaction& tx);
 
   bool check_money_overflow(const transaction& tx);
   bool check_outs_overflow(const transaction& tx);

src/cryptonote_basic/difficulty.cpp

@@ -201,7 +201,10 @@ namespace cryptonote {
       return check_hash_128(hash, difficulty);
   }
 
-  difficulty_type next_difficulty(std::vector<uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds, uint64_t HEIGHT) {
+  difficulty_type next_difficulty(std::vector<uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds, uint64_t HEIGHT, network_type nettype) {
+    if ((nettype == TESTNET || nettype == STAGENET) && HEIGHT <= DIFFICULTY_WINDOW) {
+      return 100;
+    }
     //cutoff DIFFICULTY_LAG
     if(timestamps.size() > DIFFICULTY_WINDOW)
     {
@@ -216,7 +219,8 @@ namespace cryptonote {
       return 1;
     }
     // reset difficulty for solo mining to 100 million
-    if (HEIGHT <= 331170 + DIFFICULTY_WINDOW && HEIGHT >= 331170) { return 100000000; }
+    if (nettype == MAINNET && HEIGHT <= 331170 + DIFFICULTY_WINDOW && HEIGHT >= 331170) { return 100000000; }
+
     static_assert(DIFFICULTY_WINDOW >= 2, "Window is too small");
     assert(length <= DIFFICULTY_WINDOW);
     sort(timestamps.begin(), timestamps.end());
@@ -260,9 +264,12 @@ namespace cryptonote {
   // LWMA difficulty algorithm
   // Background:  https://github.com/zawy12/difficulty-algorithms/issues/3
   // Copyright (c) 2017-2018 Zawy
-  difficulty_type next_difficulty_v2(std::vector<uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds, uint64_t HEIGHT) {
+  difficulty_type next_difficulty_v2(std::vector<uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds, uint64_t HEIGHT, network_type nettype) {
     const int64_t T = static_cast<int64_t>(target_seconds);
     size_t N = DIFFICULTY_WINDOW_V2;
+    if ((nettype == TESTNET || nettype == STAGENET) && HEIGHT <= DIFFICULTY_WINDOW) {
+      return 100;
+    }
     if (timestamps.size() < 4) {
         return 1;
     } else if ( timestamps.size() < N+1 ) {
@@ -293,10 +300,13 @@ namespace cryptonote {
   }
 
   // LWMA-2
-  difficulty_type next_difficulty_v3(std::vector<uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, uint64_t HEIGHT) {
+  difficulty_type next_difficulty_v3(std::vector<uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, uint64_t HEIGHT, network_type nettype) {
     int64_t T = DIFFICULTY_TARGET_V2;
     int64_t N = DIFFICULTY_WINDOW_V2;
     int64_t  L(0), ST, sum_3_ST(0), next_D, prev_D;
+    if ((nettype == TESTNET || nettype == STAGENET) && HEIGHT <= DIFFICULTY_WINDOW) {
+      return 100;
+    }
     assert(timestamps.size() == cumulative_difficulties.size() && timestamps.size() <= static_cast<uint64_t>(N+1) );
     for ( int64_t i = 1; i <= N; i++ ) {
       ST = static_cast<int64_t>(timestamps[i]) - static_cast<int64_t>(timestamps[i-1]);
@@ -316,12 +326,15 @@ namespace cryptonote {
   }
 
   // LWMA-4
-  difficulty_type next_difficulty_v4(std::vector<uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, uint64_t HEIGHT) {
+  difficulty_type next_difficulty_v4(std::vector<uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, uint64_t HEIGHT, network_type nettype) {
     uint64_t T = DIFFICULTY_TARGET_V2;
     uint64_t N = DIFFICULTY_WINDOW_V2;
     uint64_t L(0), ST(0), next_D, prev_D, avg_D, i;
+    if ((nettype == TESTNET || nettype == STAGENET) && HEIGHT <= DIFFICULTY_WINDOW) {
+      return 100;
+    }
     assert(timestamps.size() == cumulative_difficulties.size() && timestamps.size() <= N+1 );
-    if (HEIGHT <= 63469 + 1) { return 100000069; }
+    if (nettype == MAINNET && HEIGHT <= 63469 + 1) { return 100000069; }
     std::vector<uint64_t>TS(N+1);
     TS[0] = timestamps[0];
     for ( i = 1; i <= N; i++) {
@@ -363,21 +376,24 @@ namespace cryptonote {
   // LWMA-1 difficulty algorithm
   // Copyright (c) 2017-2019 Zawy, MIT License
   // https://github.com/zawy12/difficulty-algorithms/issues/3
-  difficulty_type next_difficulty_v5(std::vector<uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, uint64_t HEIGHT) {
+  difficulty_type next_difficulty_v5(std::vector<uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, uint64_t HEIGHT, network_type nettype) {
     uint64_t T = DIFFICULTY_TARGET_V2;
     uint64_t N = DIFFICULTY_WINDOW_V3;
+    if ((nettype == TESTNET || nettype == STAGENET) && HEIGHT <= DIFFICULTY_WINDOW) {
+      return 100;
+    }
     assert(timestamps.size() == cumulative_difficulties.size() && timestamps.size() <= N+1 );
 
-    if (HEIGHT >= 81769 && HEIGHT < 81769 + N) { return 10000000; }
+    if (nettype == MAINNET && HEIGHT >= 81769 && HEIGHT < 81769 + N) { return 10000000; }
     assert(timestamps.size() == N+1);
 
     // hardcoding previously erroneously calculated difficulty entries
-    if(HEIGHT == 307686) return 25800000;
-    if(HEIGHT == 307692) return 1890000;
-    if(HEIGHT == 307735) return 17900000;
-    if(HEIGHT == 307742) return 21300000;
-    if(HEIGHT == 307750) return 10900000;
-    if(HEIGHT == 307766) return 2960000;
+    if(nettype == MAINNET && HEIGHT == 307686) return 25800000;
+    if(nettype == MAINNET && HEIGHT == 307692) return 1890000;
+    if(nettype == MAINNET && HEIGHT == 307735) return 17900000;
+    if(nettype == MAINNET && HEIGHT == 307742) return 21300000;
+    if(nettype == MAINNET && HEIGHT == 307750) return 10900000;
+    if(nettype == MAINNET && HEIGHT == 307766) return 2960000;
 
     uint64_t  i, this_timestamp(0), previous_timestamp(0);
     difficulty_type L(0), next_D, avg_D;
@@ -411,7 +427,10 @@ namespace cryptonote {
     return  next_D;
   }
 
-  difficulty_type next_difficulty_v6(std::vector<uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds) {
+  difficulty_type next_difficulty_v6(std::vector<uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds, uint64_t HEIGHT, network_type nettype) {
+    if ((nettype == TESTNET || nettype == STAGENET) && HEIGHT <= DIFFICULTY_WINDOW) {
+      return 100;
+    }
     if(timestamps.size() > DIFFICULTY_WINDOW_V3)
     {
         timestamps.resize(DIFFICULTY_WINDOW_V3);

src/cryptonote_basic/difficulty.h

@@ -35,6 +35,7 @@
 #include <string>
 #include <boost/multiprecision/cpp_int.hpp>
 #include "crypto/hash.h"
+#include "cryptonote_config.h"
 
 namespace cryptonote
 {
@@ -57,12 +58,12 @@ namespace cryptonote
 
     bool check_hash_128(const crypto::hash &hash, difficulty_type difficulty);
     bool check_hash(const crypto::hash &hash, difficulty_type difficulty);
-    difficulty_type next_difficulty(std::vector<std::uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds, uint64_t HEIGHT);
-    difficulty_type next_difficulty_v2(std::vector<std::uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds, uint64_t HEIGHT);
-    difficulty_type next_difficulty_v3(std::vector<std::uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, uint64_t HEIGHT);
-    difficulty_type next_difficulty_v4(std::vector<std::uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, uint64_t HEIGHT);
-    difficulty_type next_difficulty_v5(std::vector<std::uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, uint64_t HEIGHT);
-    difficulty_type next_difficulty_v6(std::vector<std::uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds);
+    difficulty_type next_difficulty(std::vector<std::uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds, uint64_t HEIGHT, cryptonote::network_type nettype);
+    difficulty_type next_difficulty_v2(std::vector<std::uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds, uint64_t HEIGHT, cryptonote::network_type nettype);
+    difficulty_type next_difficulty_v3(std::vector<std::uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, uint64_t HEIGHT, cryptonote::network_type nettype);
+    difficulty_type next_difficulty_v4(std::vector<std::uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, uint64_t HEIGHT, cryptonote::network_type nettype);
+    difficulty_type next_difficulty_v5(std::vector<std::uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, uint64_t HEIGHT, cryptonote::network_type nettype);
+    difficulty_type next_difficulty_v6(std::vector<std::uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds, uint64_t HEIGHT, cryptonote::network_type nettype);
 
     std::string hex(difficulty_type v);
 }

src/cryptonote_basic/miner.cpp

@@ -557,11 +557,8 @@ namespace cryptonote
   bool miner::worker_thread()
   {
     const uint32_t th_local_index = m_thread_index++; // atomically increment, getting value before increment
-    block b;
-    if (b.major_version >= RX_BLOCK_VERSION)
-    {
-      crypto::rx_set_miner_thread(th_local_index, tools::get_max_concurrency());
-    }
+    bool rx_set = false;
+    bool cn_allocated = false;
 
     MLOG_SET_THREAD_NAME(std::string("[miner ") + std::to_string(th_local_index) + "]");
     MGINFO_GREEN("*Spins roulette wheel*... Mining started. Good luck!");
@@ -569,7 +566,7 @@ namespace cryptonote
     uint64_t height = 0;
     difficulty_type local_diff = 0;
     uint32_t local_template_ver = 0;
-    slow_hash_allocate_state();
+    block b;
     ++m_threads_active;
     while(!m_stop)
     {
@@ -635,6 +632,19 @@ namespace cryptonote
       }
 
       crypto::hash h;
+
+      if ((b.major_version >= RX_BLOCK_VERSION) && !rx_set)
+      {
+        crypto::rx_set_miner_thread(th_local_index, tools::get_max_concurrency());
+        rx_set = true;
+      }
+
+      if ((b.major_version < RX_BLOCK_VERSION) && !cn_allocated)
+      {
+        slow_hash_allocate_state();
+        cn_allocated = true;
+      }
+
       m_gbh(b, height, NULL, tools::get_max_concurrency(), h);
 
       if(check_hash(h, local_diff))
@@ -669,14 +679,6 @@ namespace cryptonote
         "                           //@@@@@@@@@@@@@@@@@//                 \n"
         << ENDL);
         MGINFO_GREEN("Awesome, you won a block reward!\n" << get_block_hash(b) << " at height " << height);
-        if (b.vote == 1)
-        {
-            MGINFO_GREEN("Your \"YES\" vote has been cast.");
-        }
-        if (b.vote == 2)
-        {
-            MGINFO_GREEN("Your \"NO\" vote has been cast.");
-        }
         cryptonote::block_verification_context bvc;
         if(!m_phandler->handle_block_found(b, bvc) || !bvc.m_added_to_main_chain)
         {
@@ -692,7 +694,7 @@ namespace cryptonote
       ++m_hashes;
       ++m_total_hashes;
     }
-    slow_hash_free_state();
+    if (cn_allocated) slow_hash_free_state();
     MGINFO("Miner thread stopped ["<< th_local_index << "]");
     --m_threads_active;
     return true;

src/cryptonote_basic/verification_context.h

@@ -69,6 +69,7 @@ namespace cryptonote
     bool m_marked_as_orphaned;
     bool m_already_exists;
     bool m_partial_block_reward;
-    bool m_bad_pow; // if bad pow, bad peer outright for DoS protection
+    bool m_bad_pow; // if bad pow, ban peer outright for DoS protection
+    bool m_missing_txs; // set if, during verif, we don't have all the necessary txs available
   };
 }

src/cryptonote_config.h

@@ -137,6 +137,10 @@
 
 #define COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT     1000
 #define COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT        20000
+#define DEFAULT_RPC_MAX_CONNECTIONS_PER_PUBLIC_IP       3
+#define DEFAULT_RPC_MAX_CONNECTIONS_PER_PRIVATE_IP      25
+#define DEFAULT_RPC_MAX_CONNECTIONS                     100
+#define DEFAULT_RPC_SOFT_LIMIT_SIZE                     25 * 1024 * 1024 // 25 MiB
 #define MAX_RPC_CONTENT_LENGTH                          1048576 // 1 MB
 
 #define P2P_LOCAL_WHITE_PEERLIST_LIMIT                  1000
@@ -155,8 +159,8 @@
 #define P2P_DEFAULT_WHITELIST_CONNECTIONS_PERCENT       70
 #define P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT            2
 #define P2P_DEFAULT_SYNC_SEARCH_CONNECTIONS_COUNT       2
-#define P2P_DEFAULT_LIMIT_RATE_UP                       2048       // kB/s
-#define P2P_DEFAULT_LIMIT_RATE_DOWN                     8192       // kB/s
+#define P2P_DEFAULT_LIMIT_RATE_UP                       8192       // kB/s
+#define P2P_DEFAULT_LIMIT_RATE_DOWN                     32768       // kB/s
 
 #define P2P_FAILED_ADDR_FORGET_SECONDS                  (60*60)     //1 hour
 #define P2P_IP_BLOCKTIME                                (60*60*24)  //24 hour
@@ -199,7 +203,7 @@
 #define HF_VERSION_BLOCK_HEADER_MINER_SIG       18
 #define HF_VERSION_VIEW_TAGS                    20
 #define HF_VERSION_2021_SCALING                 20
-#define HF_VERSION_CAP_TX_EXTRA_SIZE            20
+#define HF_VERSION_BP_PLUS_FULL_COMMIT          21
 
 #define PER_KB_FEE_QUANTIZATION_DECIMALS        8
 #define CRYPTONOTE_SCALING_2021_FEE_ROUNDING_PLACES 2
@@ -219,6 +223,8 @@
 
 #define DNS_BLOCKLIST_LIFETIME (86400 * 8)
 
+#define POLYSEED_COIN POLYSEED_MONERO
+
 //The limit is enough for the mandatory transaction content with 16 outputs (547 bytes),
 //a custom tag (1 byte) and up to 32 bytes of custom data for each recipient.
 // (1+32) + (1+1+16*32) + (1+16*32) = 1060
@@ -253,6 +259,8 @@ namespace config
   const unsigned char HASH_KEY_ENCRYPTED_PAYMENT_ID = 0x8d;
   const unsigned char HASH_KEY_WALLET = 0x8c;
   const unsigned char HASH_KEY_WALLET_CACHE = 0x8d;
+  const unsigned char HASH_KEY_BACKGROUND_CACHE = 0x8e;
+  const unsigned char HASH_KEY_BACKGROUND_KEYS_FILE = 0x8f;
   const unsigned char HASH_KEY_RPC_PAYMENT_NONCE = 0x58;
   const unsigned char HASH_KEY_MEMORY = 'k';
   const unsigned char HASH_KEY_MULTISIG[] = {'M', 'u', 'l', 't' , 'i', 's', 'i', 'g', 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };

src/cryptonote_core/blockchain.cpp

@@ -30,6 +30,7 @@
 
 #include <algorithm>
 #include <cstdio>
+#include <boost/asio/dispatch.hpp>
 #include <boost/filesystem.hpp>
 #include <boost/range/adaptor/reversed.hpp>
 #include <boost/format.hpp>
@@ -40,6 +41,7 @@
 #include "blockchain.h"
 #include "blockchain_db/blockchain_db.h"
 #include "cryptonote_basic/cryptonote_boost_serialization.h"
+#include "cryptonote_basic/events.h"
 #include "cryptonote_config.h"
 #include "cryptonote_basic/miner.h"
 #include "hardforks/hardforks.h"
@@ -375,16 +377,16 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline
 
   // create general purpose async service queue
 
-  m_async_work_idle = std::unique_ptr < boost::asio::io_service::work > (new boost::asio::io_service::work(m_async_service));
+  m_async_work_idle = std::make_unique<boost::asio::executor_work_guard<boost::asio::io_context::executor_type>>(m_async_service.get_executor());
   // we only need 1
-  m_async_pool.create_thread(boost::bind(&boost::asio::io_service::run, &m_async_service));
+  m_async_pool.create_thread(boost::bind(&boost::asio::io_context::run, &m_async_service));
 
 #if defined(PER_BLOCK_CHECKPOINT)
   if (m_nettype != FAKECHAIN)
     load_compiled_in_block_hashes(get_checkpoints);
 #endif
 
-  MINFO("Blockchain initialized. last block: " << m_db->height() - 1 << ", " << epee::misc_utils::get_time_interval_string(timestamp_diff) << " time ago, current difficulty: " << get_difficulty_for_next_block());
+  MINFO("Blockchain initialized. last block: " << m_db->height() - 1 << ", " << epee::misc_utils::get_time_interval_string(timestamp_diff) << " time ago, current difficulty: " << get_difficulty_for_next_block(m_nettype));
 
   rtxn_guard.stop();
 
@@ -643,12 +645,14 @@ block Blockchain::pop_block_from_blockchain()
       // in hf_versions.
       uint8_t version = get_ideal_hard_fork_version(m_db->height());
 
-      // We assume that if they were in a block, the transactions are already
-      // known to the network as a whole. However, if we had mined that block,
-      // that might not be always true. Unlikely though, and always relaying
-      // these again might cause a spike of traffic as many nodes re-relay
-      // all the transactions in a popped block when a reorg happens.
-      bool r = m_tx_pool.add_tx(tx, tvc, relay_method::block, true, version);
+      // We assume that if they were in a block, the transactions are already known to the network
+      // as a whole. However, if we had mined that block, that might not be always true. Unlikely
+      // though, and always relaying these again might cause a spike of traffic as many nodes
+      // re-relay all the transactions in a popped block when a reorg happens. You might notice that
+      // we also set the "nic_verified_hf_version" paramater. Since we know we took this transaction
+      // from the mempool earlier in this function call, when the mempool has the same current fork
+      // version, we can return it without re-verifying the consensus rules on it.
+      const bool r = m_tx_pool.add_tx(tx, tvc, relay_method::block, true, version, version);
       if (!r)
       {
         LOG_ERROR("Error returning transaction to tx_pool");
@@ -660,7 +664,6 @@ block Blockchain::pop_block_from_blockchain()
 
   m_blocks_longhash_table.clear();
   m_scan_table.clear();
-  m_blocks_txs_check.clear();
 
   uint64_t top_block_height;
   crypto::hash top_block_hash = get_tail_id(top_block_height);
@@ -849,22 +852,14 @@ bool Blockchain::get_block_by_hash(const crypto::hash &h, block &blk, bool *orph
 // last DIFFICULTY_BLOCKS_COUNT blocks and passes them to next_difficulty,
 // returning the result of that call.  Ignores the genesis block, and can use
 // less blocks than desired if there aren't enough.
-difficulty_type Blockchain::get_difficulty_for_next_block()
+difficulty_type Blockchain::get_difficulty_for_next_block(const network_type nettype)
 {
-  LOG_PRINT_L3("Blockchain::" << __func__);
-
-  std::stringstream ss;
-  bool print = false;
-
-  int done = 0;
-  ss << "get_difficulty_for_next_block: height " << m_db->height() << std::endl;
   if (m_fixed_difficulty)
   {
     return m_db->height() ? m_fixed_difficulty : 1;
   }
 
-start:
-  difficulty_type D = 0;
+  LOG_PRINT_L3("Blockchain::" << __func__);
 
   crypto::hash top_hash = get_tail_id();
   {
@@ -873,24 +868,16 @@ start:
     // something a bit out of date, but that's fine since anything which
     // requires the blockchain lock will have acquired it in the first place,
     // and it will be unlocked only when called from the getinfo RPC
-    ss << "Locked, tail id " << top_hash << ", cached is " << m_difficulty_for_next_block_top_hash << std::endl;
     if (top_hash == m_difficulty_for_next_block_top_hash)
-    {
-      ss << "Same, using cached diff " << m_difficulty_for_next_block << std::endl;
-      D = m_difficulty_for_next_block;
-    }
+      return m_difficulty_for_next_block;
   }
 
   CRITICAL_REGION_LOCAL(m_blockchain_lock);
   std::vector<uint64_t> timestamps;
   std::vector<difficulty_type> difficulties;
   uint64_t height;
-  auto new_top_hash = get_tail_id(height); // get it again now that we have the lock
-  ++height;
-  if (!(new_top_hash == top_hash)) D=0;
-  ss << "Re-locked, height " << height << ", tail id " << new_top_hash << (new_top_hash == top_hash ? "" : " (different)") << std::endl;
-  top_hash = new_top_hash;
-
+  top_hash = get_tail_id(height); // get it again now that we have the lock
+  ++height; // top block height to blockchain height
   // ND: Speedup
   // 1. Keep a list of the last 735 (or less) blocks that is used to compute difficulty,
   //    then when the next block difficulty is queried, push the latest height data and
@@ -915,12 +902,8 @@ start:
     m_timestamps_and_difficulties_height = height;
     timestamps = m_timestamps;
     difficulties = m_difficulties;
-    check = true;
   }
-  //else
-  std::vector<uint64_t> timestamps_from_cache = timestamps;
-  std::vector<difficulty_type> difficulties_from_cache = difficulties;
-
+  else
   {
     uint64_t offset = height - std::min <uint64_t> (height, static_cast<uint64_t>(difficulty_blocks_count));
     if (offset == 0)
@@ -933,82 +916,36 @@ start:
       timestamps.reserve(height - offset);
       difficulties.reserve(height - offset);
     }
-    ss << "Looking up " << (height - offset) << " from " << offset << std::endl;
     for (; offset < height; offset++)
     {
       timestamps.push_back(m_db->get_block_timestamp(offset));
       difficulties.push_back(m_db->get_block_cumulative_difficulty(offset));
     }
 
-    if (check) if (timestamps != timestamps_from_cache || difficulties !=difficulties_from_cache)
-    {
-      ss << "Inconsistency XXX:" << std::endl;
-      ss << "top hash: "<<top_hash << std::endl;
-      ss << "timestamps: " << timestamps_from_cache.size() << " from cache, but " << timestamps.size() << " without" << std::endl;
-      ss << "difficulties: " << difficulties_from_cache.size() << " from cache, but " << difficulties.size() << " without" << std::endl;
-      ss << "timestamps_from_cache:" << std::endl; for (const auto &v :timestamps_from_cache) ss << "  " << v << std::endl;
-      ss << "timestamps:" << std::endl; for (const auto &v :timestamps) ss << "  " << v << std::endl;
-      ss << "difficulties_from_cache:" << std::endl; for (const auto &v :difficulties_from_cache) ss << "  " << v << std::endl;
-      ss << "difficulties:" << std::endl; for (const auto &v :difficulties) ss << "  " << v << std::endl;
-
-      uint64_t dbh = m_db->height();
-      uint64_t sh = dbh < 10000 ? 0 : dbh - 10000;
-      ss << "History from -10k at :" << dbh << ", from " << sh << std::endl;
-      for (uint64_t h = sh; h < dbh; ++h)
-      {
-        uint64_t ts = m_db->get_block_timestamp(h);
-        difficulty_type d = m_db->get_block_cumulative_difficulty(h);
-        ss << "  " << h << " " << ts << " " << d << std::endl;
-      }
-      print = true;
-    }
     m_timestamps_and_difficulties_height = height;
     m_timestamps = timestamps;
     m_difficulties = difficulties;
   }
-
   size_t target = get_difficulty_target();
   uint64_t HEIGHT = m_db->height();
   difficulty_type diff;
   if (version >= 20) {
-    diff = next_difficulty_v6(timestamps, difficulties, target);
+    diff = next_difficulty_v6(timestamps, difficulties, target, HEIGHT, m_nettype);
   } else if (version <= 17 && version >= 11) {
-    diff = next_difficulty_v5(timestamps, difficulties, HEIGHT);
+    diff = next_difficulty_v5(timestamps, difficulties, HEIGHT, m_nettype);
   } else if (version == 10) {
-    diff = next_difficulty_v4(timestamps, difficulties, HEIGHT);
+    diff = next_difficulty_v4(timestamps, difficulties, HEIGHT, m_nettype);
   } else if (version == 9) {
-    diff = next_difficulty_v3(timestamps, difficulties, HEIGHT);
+    diff = next_difficulty_v3(timestamps, difficulties, HEIGHT, m_nettype);
   } else if (version == 8) {
-    diff = next_difficulty_v2(timestamps, difficulties, target, HEIGHT);
+    diff = next_difficulty_v2(timestamps, difficulties, target, HEIGHT, m_nettype);
   } else {
-    diff = next_difficulty(timestamps, difficulties, target, HEIGHT);
+    diff = next_difficulty(timestamps, difficulties, target, HEIGHT, m_nettype);
   }
 
   CRITICAL_REGION_LOCAL1(m_difficulty_lock);
   m_difficulty_for_next_block_top_hash = top_hash;
   m_difficulty_for_next_block = diff;
-  if (D && D != diff)
-  {
-    ss << "XXX Mismatch at " << height << "/" << top_hash << "/" << get_tail_id() << ": cached " << D << ", real " << diff << std::endl;
-    print = true;
-  }
-
-  ++done;
-  if (done == 1 && D && D != diff)
-  {
-    print = true;
-    ss << "Might be a race. Let's see what happens if we try again..." << std::endl;
-    epee::misc_utils::sleep_no_w(100);
-    goto start;
-  }
-  ss << "Diff for " << top_hash << ": " << diff << std::endl;
-  if (print)
-  {
-    MGINFO("START DUMP");
-    MGINFO(ss.str());
-    MGINFO("END DUMP");
-    MGINFO("Please send wowario on IRC OTFC #wownero-dev the contents of this log, from a couple dozen lines before START DUMP to END DUMP");
-  }
   return diff;
 }
 //------------------------------------------------------------------
@@ -1065,17 +1002,17 @@ size_t Blockchain::recalculate_difficulties(boost::optional<uint64_t> start_heig
     uint64_t HEIGHT = m_db->height();
     difficulty_type recalculated_diff;
     if (version >= 20) {
-      recalculated_diff = next_difficulty_v6(timestamps, difficulties, target);
+      recalculated_diff = next_difficulty_v6(timestamps, difficulties, target, HEIGHT, m_nettype);
     } else if (version <= 17 && version >= 11) {
-      recalculated_diff = next_difficulty_v5(timestamps, difficulties, HEIGHT);
+      recalculated_diff = next_difficulty_v5(timestamps, difficulties, HEIGHT, m_nettype);
     } else if (version == 10) {
-      recalculated_diff = next_difficulty_v4(timestamps, difficulties, HEIGHT);
+      recalculated_diff = next_difficulty_v4(timestamps, difficulties, HEIGHT, m_nettype);
     } else if (version == 9) {
-      recalculated_diff = next_difficulty_v3(timestamps, difficulties, HEIGHT);
+      recalculated_diff = next_difficulty_v3(timestamps, difficulties, HEIGHT, m_nettype);
     } else if (version == 8) {
-      recalculated_diff = next_difficulty_v2(timestamps, difficulties, target, HEIGHT);
+      recalculated_diff = next_difficulty_v2(timestamps, difficulties, target, HEIGHT, m_nettype);
     } else {
-      recalculated_diff = next_difficulty(timestamps, difficulties, target, HEIGHT);
+      recalculated_diff = next_difficulty(timestamps, difficulties, target, HEIGHT, m_nettype);
     }
 
     boost::multiprecision::uint256_t recalculated_cum_diff_256 = boost::multiprecision::uint256_t(recalculated_diff) + last_cum_diff;
@@ -1176,7 +1113,7 @@ bool Blockchain::rollback_blockchain_switching(std::list<block>& original_chain,
   for (auto& bl : original_chain)
   {
     block_verification_context bvc = {};
-    bool r = handle_block_to_main_chain(bl, bvc, false);
+    bool r = handle_block_to_main_chain(bl, bvc);
     CHECK_AND_ASSERT_MES(r && bvc.m_added_to_main_chain, false, "PANIC! failed to add (again) block while chain switching during the rollback!");
   }
 
@@ -1229,7 +1166,7 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info>
     block_verification_context bvc = {};
 
     // add block to main chain
-    bool r = handle_block_to_main_chain(bei.bl, bvc, false);
+    bool r = handle_block_to_main_chain(bei.bl, bvc);
 
     // if adding block to main chain failed, rollback to previous state and
     // return false
@@ -1264,7 +1201,8 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info>
     for (auto& old_ch_ent : disconnected_chain)
     {
       block_verification_context bvc = {};
-      bool r = handle_alternative_block(old_ch_ent, get_block_hash(old_ch_ent), bvc);
+      pool_supplement ps{};
+      bool r = handle_alternative_block(old_ch_ent, get_block_hash(old_ch_ent), bvc, ps);
       if(!r)
       {
         MERROR("Failed to push ex-main chain blocks to alternative chain ");
@@ -1385,17 +1323,17 @@ difficulty_type Blockchain::get_next_difficulty_for_alternative_chain(const std:
   uint64_t HEIGHT = m_db->height();
   difficulty_type next_diff;
   if (version >= 20) {
-    next_diff = next_difficulty_v6(timestamps, cumulative_difficulties, target);
+    next_diff = next_difficulty_v6(timestamps, cumulative_difficulties, target, HEIGHT, m_nettype);
   } else if (version <= 17 && version >= 11) {
-    next_diff = next_difficulty_v5(timestamps, cumulative_difficulties, HEIGHT);
+    next_diff = next_difficulty_v5(timestamps, cumulative_difficulties, HEIGHT, m_nettype);
   } else if (version == 10) {
-    next_diff = next_difficulty_v4(timestamps, cumulative_difficulties, HEIGHT);
+    next_diff = next_difficulty_v4(timestamps, cumulative_difficulties, HEIGHT, m_nettype);
   } else if (version == 9) {
-    next_diff = next_difficulty_v3(timestamps, cumulative_difficulties, HEIGHT);
+    next_diff = next_difficulty_v3(timestamps, cumulative_difficulties, HEIGHT, m_nettype);
   } else if (version == 8) {
-    next_diff = next_difficulty_v2(timestamps, cumulative_difficulties, target, HEIGHT);
+    next_diff = next_difficulty_v2(timestamps, cumulative_difficulties, target, HEIGHT, m_nettype);
   } else {
-    next_diff = next_difficulty(timestamps, cumulative_difficulties, target, HEIGHT);
+    next_diff = next_difficulty(timestamps, cumulative_difficulties, target, HEIGHT, m_nettype);
   }
   return next_diff;
 }
@@ -1443,11 +1381,6 @@ bool Blockchain::prevalidate_miner_transaction(const block& b, uint64_t height,
       }
   }
 
-  if (hf_version >= HF_VERSION_CAP_TX_EXTRA_SIZE && b.miner_tx.extra.size() > MAX_TX_EXTRA_SIZE)
-  {
-    MWARNING("coinbase transaction tx-extra is too big: " << b.miner_tx.extra.size() << " bytes, the limit is: " << MAX_TX_EXTRA_SIZE);
-    return false;
-  }
   LOG_PRINT_L3("Blockchain::" << __func__);
   CHECK_AND_ASSERT_MES(b.miner_tx.vin.size() == 1, false, "coinbase transaction in the block has no inputs");
   CHECK_AND_ASSERT_MES(b.miner_tx.vin[0].type() == typeid(txin_gen), false, "coinbase transaction in the block has the wrong type");
@@ -1761,7 +1694,7 @@ bool Blockchain::create_block_template(block& b, const crypto::hash *from_block,
     b.minor_version = m_hardfork->get_ideal_version();
     b.prev_id = get_tail_id();
     median_weight = m_current_block_cumul_weight_limit / 2;
-    diffic = get_difficulty_for_next_block();
+    diffic = get_difficulty_for_next_block(m_nettype);
     already_generated_coins = m_db->get_block_already_generated_coins(height - 1);
     if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION)
     {
@@ -1928,7 +1861,7 @@ bool Blockchain::get_miner_data(uint8_t& major_version, uint64_t& height, crypto
     seed_hash = get_block_id_by_height(seed_height);
   }
 
-  difficulty = get_difficulty_for_next_block();
+  difficulty = get_difficulty_for_next_block(m_nettype);
   median_weight = m_current_block_cumul_weight_median;
   already_generated_coins = m_db->get_block_already_generated_coins(height - 1);
 
@@ -2022,7 +1955,8 @@ bool Blockchain::build_alt_chain(const crypto::hash &prev_id, std::list<block_ex
 // if that chain is long enough to become the main chain and re-org accordingly
 // if so.  If not, we need to hang on to the block in case it becomes part of
 // a long forked chain eventually.
-bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc)
+bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id,
+  block_verification_context& bvc, pool_supplement& extra_block_txs)
 {
   LOG_PRINT_L3("Blockchain::" << __func__);
   CRITICAL_REGION_LOCAL(m_blockchain_lock);
@@ -2154,6 +2088,47 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id
     }
     bei.cumulative_difficulty += current_diff;
 
+    // Now that we have the PoW verification out of the way, verify all pool supplement txs
+    tx_verification_context tvc{};
+    if (!ver_non_input_consensus(extra_block_txs, tvc, hf_version))
+    {
+      MERROR_VER("Transaction pool supplement verification failure for alt block " << id);
+      bvc.m_verifivation_failed = true;
+      return false;
+    }
+
+    // Add pool supplement txs to the main mempool with relay_method::block
+    CRITICAL_REGION_LOCAL(m_tx_pool);
+    for (auto& extra_block_tx : extra_block_txs.txs_by_txid)
+    {
+      const crypto::hash& txid = extra_block_tx.first;
+      transaction& tx = extra_block_tx.second.first;
+      const blobdata &tx_blob = extra_block_tx.second.second;
+
+      tx_verification_context tvc{};
+      if ((!m_tx_pool.have_tx(txid, relay_category::legacy) &&
+          !m_db->tx_exists(txid) &&
+          !m_tx_pool.add_tx(tx, tvc, relay_method::block, /*relayed=*/true, hf_version, hf_version))
+          || tvc.m_verifivation_failed)
+      {
+        MERROR_VER("Transaction " << txid <<
+          " in pool supplement failed to enter main pool for alt block " << id);
+        bvc.m_verifivation_failed = true;
+        return false;
+      }
+
+      // If new incoming tx in alt block passed verification and entered the pool, notify ZMQ
+      if (tvc.m_added_to_pool)
+        notify_txpool_event({txpool_event{
+          .tx = tx,
+          .hash = txid,
+          .blob_size = tx_blob.size(),
+          .weight = get_transaction_weight(tx),
+          .res = true}});
+    }
+    extra_block_txs.txs_by_txid.clear();
+    extra_block_txs.nic_verified_hf_version = 0;
+
     bei.block_cumulative_weight = cryptonote::get_transaction_weight(b.miner_tx);
     for (const crypto::hash &txid: b.tx_hashes)
     {
@@ -2459,17 +2434,8 @@ void Blockchain::get_output_key_mask_unlocked(const uint64_t& amount, const uint
 bool Blockchain::get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const
 {
   // rct outputs don't exist before v4
-  if (amount == 0)
-  {
-    switch (m_nettype)
-    {
-      case STAGENET: start_height = stagenet_hard_forks[3].height; break;
-      case TESTNET: start_height = testnet_hard_forks[3].height; break;
-      case MAINNET: start_height = 0; break;
-      case FAKECHAIN: start_height = 0; break;
-      default: return false;
-    }
-  }
+  if (amount == 0 && m_nettype != network_type::FAKECHAIN)
+    start_height = m_hardfork->get_earliest_ideal_height_for_version(HF_VERSION_DYNAMIC_FEE);
   else
     start_height = 0;
   base = 0;
@@ -2957,11 +2923,12 @@ bool Blockchain::have_block(const crypto::hash& id, int *where) const
   return have_block_unlocked(id, where);
 }
 //------------------------------------------------------------------
-bool Blockchain::handle_block_to_main_chain(const block& bl, block_verification_context& bvc, bool notify/* = true*/)
+bool Blockchain::handle_block_to_main_chain(const block& bl, block_verification_context& bvc)
 {
     LOG_PRINT_L3("Blockchain::" << __func__);
     crypto::hash id = get_block_hash(bl);
-    return handle_block_to_main_chain(bl, id, bvc, notify);
+    pool_supplement ps{};
+    return handle_block_to_main_chain(bl, id, bvc, ps);
 }
 //------------------------------------------------------------------
 size_t Blockchain::get_total_transactions() const
@@ -3072,24 +3039,6 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<u
   return true;
 }
 //------------------------------------------------------------------
-void Blockchain::on_new_tx_from_block(const cryptonote::transaction &tx)
-{
-#if defined(PER_BLOCK_CHECKPOINT)
-  // check if we're doing per-block checkpointing
-  if (m_db->height() < m_blocks_hash_check.size())
-  {
-    TIME_MEASURE_START(a);
-    m_blocks_txs_check.push_back(get_transaction_hash(tx));
-    TIME_MEASURE_FINISH(a);
-    if(m_show_time_stats)
-    {
-      size_t ring_size = !tx.vin.empty() && tx.vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(tx.vin[0]).key_offsets.size() : 0;
-      MINFO("HASH: " << "-" << " I/M/O: " << tx.vin.size() << "/" << ring_size << "/" << tx.vout.size() << " H: " << 0 << " chcktx: " << a);
-    }
-  }
-#endif
-}
-//------------------------------------------------------------------
 //FIXME: it seems this function is meant to be merely a wrapper around
 //       another function of the same name, this one adding one bit of
 //       functionality.  Should probably move anything more than that
@@ -3129,12 +3078,9 @@ bool Blockchain::check_tx_inputs(transaction& tx, uint64_t& max_used_block_heigh
   return true;
 }
 //------------------------------------------------------------------
-bool Blockchain::check_tx_outputs(const transaction& tx, tx_verification_context &tvc) const
+bool Blockchain::check_tx_outputs(const transaction& tx, tx_verification_context &tvc, std::uint8_t hf_version)
 {
   LOG_PRINT_L3("Blockchain::" << __func__);
-  CRITICAL_REGION_LOCAL(m_blockchain_lock);
-
-  const uint8_t hf_version = m_hardfork->get_current_version();
 
   // from hard fork 2, we forbid dust and compound outputs
   if (hf_version >= 2) {
@@ -3161,21 +3107,6 @@ bool Blockchain::check_tx_outputs(const transaction& tx, tx_verification_context
     }
   }
 
-  // from v4, forbid invalid pubkeys
-  if (hf_version >= 4) {
-    for (const auto &o: tx.vout) {
-      crypto::public_key output_public_key;
-      if (!get_output_public_key(o, output_public_key)) {
-        tvc.m_invalid_output = true;
-        return false;
-      }
-      if (!crypto::check_key(output_public_key)) {
-        tvc.m_invalid_output = true;
-        return false;
-      }
-    }
-  }
-
   // from v8, allow bulletproofs
   if (hf_version < 8) {
     if (tx.version >= 2) {
@@ -3267,8 +3198,8 @@ bool Blockchain::check_tx_outputs(const transaction& tx, tx_verification_context
   // from v18, allow bulletproofs plus
   if (hf_version < HF_VERSION_BULLETPROOF_PLUS) {
     if (tx.version >= 2) {
-      const bool bulletproof_plus = rct::is_rct_bulletproof_plus(tx.rct_signatures.type);
-      if (bulletproof_plus || !tx.rct_signatures.p.bulletproofs_plus.empty())
+      const bool bulletproof_plus_legacy = rct::is_rct_bp_plus_legacy(tx.rct_signatures.type);
+      if (bulletproof_plus_legacy || !tx.rct_signatures.p.bulletproofs_plus.empty())
       {
         MERROR_VER("Bulletproofs plus are not allowed before v" << std::to_string(HF_VERSION_BULLETPROOF_PLUS));
         tvc.m_invalid_output = true;
@@ -3297,13 +3228,26 @@ bool Blockchain::check_tx_outputs(const transaction& tx, tx_verification_context
     return false;
   }
 
-  // from v20, limit tx extra size
-  if (hf_version >= HF_VERSION_CAP_TX_EXTRA_SIZE && tx.extra.size() > MAX_TX_EXTRA_SIZE)
-  {
-    MERROR_VER("transaction tx-extra is too big: " << tx.extra.size() << " bytes, the limit is: " << MAX_TX_EXTRA_SIZE);
-    tvc.m_tx_extra_too_big = true;
+  // from v21, allow bulletproofs plus full commit
+  if (hf_version < HF_VERSION_BP_PLUS_FULL_COMMIT) {
+    if (tx.version >= 2) {
+      const bool bulletproof_plus_full_commit = rct::is_rct_bp_plus_full(tx.rct_signatures.type);
+      if (bulletproof_plus_full_commit)
+      {
+        MERROR_VER("Bulletproofs plus full commit are not allowed before v" << std::to_string(HF_VERSION_BP_PLUS_FULL_COMMIT));
+        tvc.m_invalid_output = true;
+        return false;
+      }
+    }
+  }
+
+  // from v22, forbid bulletproof plus legacy
+  if (hf_version > HF_VERSION_BP_PLUS_FULL_COMMIT && rct::is_rct_bp_plus_legacy(tx.rct_signatures.type)) {
+    MERROR_VER("Bulletproof Plus legacy range proofs are not allowed after v" << (HF_VERSION_BP_PLUS_FULL_COMMIT + 1));
+    tvc.m_invalid_output = true;
     return false;
   }
+
   return true;
 }
 //------------------------------------------------------------------
@@ -3344,7 +3288,7 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr
       }
     }
   }
-  else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof || rv.type == rct::RCTTypeBulletproof2 || rv.type == rct::RCTTypeSimpleBulletproof || rv.type == rct::RCTTypeCLSAG || rv.type == rct::RCTTypeBulletproofPlus)
+  else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof || rv.type == rct::RCTTypeBulletproof2 || rv.type == rct::RCTTypeSimpleBulletproof || rv.type == rct::RCTTypeCLSAG || rv.type == rct::RCTTypeBulletproofPlus || rv.type == rct::RCTTypeBulletproofPlus_FullCommit)
   {
     CHECK_AND_ASSERT_MES(!pubkeys.empty() && !pubkeys[0].empty(), false, "empty pubkeys");
     rv.mixRing.resize(pubkeys.size());
@@ -3385,7 +3329,7 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr
       }
     }
   }
-  else if (rv.type == rct::RCTTypeCLSAG || rv.type == rct::RCTTypeBulletproofPlus)
+  else if (rv.type == rct::RCTTypeCLSAG || rv.type == rct::RCTTypeBulletproofPlus || rv.type == rct::RCTTypeBulletproofPlus_FullCommit)
   {
     if (!tx.pruned)
     {
@@ -3419,10 +3363,6 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
   if(pmax_used_block_height)
     *pmax_used_block_height = 0;
 
-  // pruned txes are skipped, as they're only allowed in sync-pruned-blocks mode, which is within the builtin hashes
-  if (tx.pruned)
-    return true;
-
   crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx);
 
   const uint8_t hf_version = m_hardfork->get_current_version();
@@ -3642,11 +3582,9 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
   }
 
   // Warn that new RCT types are present, and thus the cache is not being used effectively
-  static constexpr const std::uint8_t RCT_CACHE_TYPE = rct::RCTTypeBulletproofPlus;
-  if (tx.rct_signatures.type > RCT_CACHE_TYPE)
-  {
-    MWARNING("RCT cache is not caching new verification results. Please update RCT_CACHE_TYPE!");
-  }
+  const std::uint8_t rct_cache_type = (hf_version >= HF_VERSION_BP_PLUS_FULL_COMMIT) ? static_cast<std::uint8_t>(rct::RCTTypeBulletproofPlus_FullCommit) : static_cast<std::uint8_t>(rct::RCTTypeBulletproofPlus);
+  if (static_cast<std::uint8_t>(tx.rct_signatures.type) > rct_cache_type)
+    MWARNING("RCT cache is not caching new verification results. Please update rct_cache_type.");
 
   if (tx.version == 1)
   {
@@ -3686,8 +3624,9 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
     case rct::RCTTypeBulletproof2:
     case rct::RCTTypeCLSAG:
     case rct::RCTTypeBulletproofPlus:
+    case rct::RCTTypeBulletproofPlus_FullCommit:
     {
-      if (!ver_rct_non_semantics_simple_cached(tx, pubkeys, m_rct_ver_cache, RCT_CACHE_TYPE))
+      if (!ver_rct_non_semantics_simple_cached(tx, pubkeys, m_rct_ver_cache, rct_cache_type))
       {
         MERROR_VER("Failed to check ringct signatures!");
         return false;
@@ -4203,26 +4142,6 @@ bool Blockchain::check_block_timestamp(const block& b, uint64_t& median_ts) cons
   return check_block_timestamp(timestamps, b, median_ts);
 }
 //------------------------------------------------------------------
-void Blockchain::return_tx_to_pool(std::vector<std::pair<transaction, blobdata>> &txs)
-{
-  uint8_t version = get_current_hard_fork_version();
-  for (auto& tx : txs)
-  {
-    cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc);
-    // We assume that if they were in a block, the transactions are already
-    // known to the network as a whole. However, if we had mined that block,
-    // that might not be always true. Unlikely though, and always relaying
-    // these again might cause a spike of traffic as many nodes re-relay
-    // all the transactions in a popped block when a reorg happens.
-    const size_t weight = get_transaction_weight(tx.first, tx.second.size());
-    const crypto::hash tx_hash = get_transaction_hash(tx.first);
-    if (!m_tx_pool.add_tx(tx.first, tx_hash, tx.second, weight, tvc, relay_method::block, true, version))
-    {
-      MERROR("Failed to return taken transaction with hash: " << get_transaction_hash(tx.first) << " to tx_pool");
-    }
-  }
-}
-//------------------------------------------------------------------
 bool Blockchain::flush_txes_from_pool(const std::vector<crypto::hash> &txids)
 {
   CRITICAL_REGION_LOCAL(m_tx_pool);
@@ -4248,7 +4167,8 @@ bool Blockchain::flush_txes_from_pool(const std::vector<crypto::hash> &txids)
 //      Needs to validate the block and acquire each transaction from the
 //      transaction mem_pool, then pass the block and transactions to
 //      m_db->add_block()
-bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc, bool notify/* = true*/)
+bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& id,
+  block_verification_context& bvc, pool_supplement& extra_block_txs)
 {
   LOG_PRINT_L3("Blockchain::" << __func__);
 
@@ -4312,7 +4232,7 @@ leave:
   // so we need to check the return type.
   // FIXME: get_difficulty_for_next_block can also assert, look into
   // changing this to throwing exceptions instead so we can clean up.
-  difficulty_type current_diffic = get_difficulty_for_next_block();
+  difficulty_type current_diffic = get_difficulty_for_next_block(m_nettype);
   CHECK_AND_ASSERT_MES(current_diffic, false, "!!!!!!!!! difficulty overhead !!!!!!!!!");
 
   TIME_MEASURE_FINISH(target_calculating_time);
@@ -4400,10 +4320,66 @@ leave:
     goto leave;
   }
 
+  // verify all non-input consensus rules for txs inside the pool supplement (if not inside checkpoint zone)
+#if defined(PER_BLOCK_CHECKPOINT)
+  if (!fast_check)
+#endif
+  {
+    tx_verification_context tvc{};
+    // If fail non-input consensus rule checking...
+    if (!ver_non_input_consensus(extra_block_txs, tvc, hf_version))
+    {
+      MERROR_VER("Pool supplement provided for block with id: " << id << " failed to pass validation");
+      bvc.m_verifivation_failed = true;
+      goto leave;
+    }
+  }
+
   size_t coinbase_weight = get_transaction_weight(bl.miner_tx);
   size_t cumulative_block_weight = coinbase_weight;
 
   std::vector<std::pair<transaction, blobdata>> txs;
+  //                          txid     weight mempool?
+  std::vector<std::tuple<crypto::hash, size_t, bool>> txs_meta;
+
+  // This will be the data sent to the ZMQ pool listeners for txs which skipped the mempool
+  std::vector<txpool_event> txpool_events;
+
+  // this lambda returns relevant txs back to the mempool
+  auto return_txs_to_pool = [this, &txs, &txs_meta, &hf_version]()
+  {
+    if (txs_meta.size() != txs.size())
+    {
+      MERROR("BUG: txs_meta and txs not matching size!!!");
+      return;
+    }
+
+    for (size_t i = 0; i < txs.size(); ++i)
+    {
+      // if this transaction wasn't ever in the pool, don't return it back to the pool
+      const bool found_in_pool = std::get<2>(txs_meta[i]);
+      if (!found_in_pool)
+        continue;
+
+      transaction &tx = txs[i].first;
+      const crypto::hash &txid = std::get<0>(txs_meta[i]);
+      const blobdata &tx_blob = txs[i].second;
+      const size_t tx_weight = std::get<1>(txs_meta[i]);
+
+      // We assume that if they were in a block, the transactions are already known to the network
+      // as a whole. However, if we had mined that block, that might not be always true. Unlikely
+      // though, and always relaying these again might cause a spike of traffic as many nodes
+      // re-relay all the transactions in a popped block when a reorg happens. You might notice that
+      // we also set the "nic_verified_hf_version" paramater. Since we know we took this transaction
+      // from the mempool earlier in this function call, when the mempool has the same current fork
+      // version, we can return it without re-verifying the consensus rules on it.
+      cryptonote::tx_verification_context tvc{};
+      if (!m_tx_pool.add_tx(tx, txid, tx_blob, tx_weight, tvc, relay_method::block, true,
+          hf_version, hf_version))
+        MERROR("Failed to return taken transaction with hash: " << txid << " to tx_pool");
+    }
+  };
+
   key_images_container keys;
 
   uint64_t fee_summary = 0;
@@ -4416,18 +4392,14 @@ leave:
 
 // XXX old code adds miner tx here
 
-  size_t tx_index = 0;
   // Iterate over the block's transaction hashes, grabbing each
-  // from the tx_pool and validating them.  Each is then added
+  // from the tx_pool (or from extra_block_txs) and validating them.  Each is then added
   // to txs.  Keys spent in each are added to <keys> by the double spend check.
   txs.reserve(bl.tx_hashes.size());
+  txs_meta.reserve(bl.tx_hashes.size());
+  txpool_events.reserve(bl.tx_hashes.size());
   for (const crypto::hash& tx_id : bl.tx_hashes)
   {
-    transaction tx_tmp;
-    blobdata txblob;
-    size_t tx_weight = 0;
-    uint64_t fee = 0;
-    bool relayed = false, do_not_relay = false, double_spend_seen = false, pruned = false;
     TIME_MEASURE_START(aa);
 
 // XXX old code does not check whether tx exists
@@ -4435,21 +4407,73 @@ leave:
     {
       MERROR("Block with id: " << id << " attempting to add transaction already in blockchain with id: " << tx_id);
       bvc.m_verifivation_failed = true;
-      return_tx_to_pool(txs);
-      goto leave;
+      return_txs_to_pool();
+      return false;
     }
 
     TIME_MEASURE_FINISH(aa);
     t_exists += aa;
     TIME_MEASURE_START(bb);
 
-    // get transaction with hash <tx_id> from tx_pool
-    if(!m_tx_pool.take_tx(tx_id, tx_tmp, txblob, tx_weight, fee, relayed, do_not_relay, double_spend_seen, pruned))
+    // get transaction with hash <tx_id> from m_tx_pool or extra_block_txs
+    // tx info we want:
+    //   * tx as `cryptonote::transaction`
+    //   * blob
+    //   * weight
+    //   * fee
+    //   * is pruned?
+    txs.emplace_back();
+    transaction &tx = txs.back().first;
+    blobdata &txblob = txs.back().second;
+    size_t tx_weight{};
+    uint64_t fee{};
+    bool pruned{};
+
+    /* 
+     * Try pulling transaction data from the mempool proper first. If that fails, then try pulling
+     * from the block supplement. We add txs pulled from the block to the txpool events for future
+     * notifications, since if the tx skipped the mempool, then listeners have not yet received a
+     * notification for this tx.
+     */
+    bool _unused1, _unused2, _unused3;
+    const bool found_tx_in_pool{
+        m_tx_pool.take_tx(tx_id, tx, txblob, tx_weight, fee,
+          _unused1, _unused2, _unused3, pruned, /*suppress_missing_msgs=*/true)
+      };
+    bool find_tx_failure{!found_tx_in_pool};
+    if (!found_tx_in_pool) // if not in mempool:
     {
-      MERROR_VER("Block with id: " << id  << " has at least one unknown transaction with id: " << tx_id);
+      const auto extra_txs_it{extra_block_txs.txs_by_txid.find(tx_id)};
+      if (extra_txs_it != extra_block_txs.txs_by_txid.end()) // if in block supplement:
+      {
+        tx = std::move(extra_txs_it->second.first);
+        txblob = std::move(extra_txs_it->second.second);
+        tx_weight = tx.pruned ? get_pruned_transaction_weight(tx) : get_transaction_weight(tx, txblob.size());
+        fee = get_tx_fee(tx);
+        pruned = tx.pruned;
+        extra_block_txs.txs_by_txid.erase(extra_txs_it);
+        txpool_events.emplace_back(txpool_event{tx, tx_id, txblob.size(), tx_weight, true});
+        find_tx_failure = false;
+      }
+    }
+
+    // @TODO: We should move this section (checking if the daemon has all txs from the block) to
+    // right after the PoW check. Since it's now expected the node will sometimes not have all txs
+    // in its pool at this point nor the txs included as fluffy txs (and will need to re-request
+    // missing fluffy txs), then the node will sometimes waste cycles doing verification for some
+    // txs twice.
+    if (find_tx_failure) // did not find txid in mempool or provided extra block txs
+    {
+      const bool fully_supplemented_block = extra_block_txs.txs_by_txid.size() >= bl.tx_hashes.size();
+      if (fully_supplemented_block)
+        MERROR_VER("Block with id: " << id  << " has at least one unknown transaction with id: " << tx_id);
+      else
+        LOG_PRINT_L2("Block with id: " << id  << " has at least one unknown transaction with id: " << tx_id);
+      txs.pop_back(); // We push to the back preemptively. On fail, we need txs & txs_meta to match size
       bvc.m_verifivation_failed = true;
-      return_tx_to_pool(txs);
-      goto leave;
+      bvc.m_missing_txs = true;
+      return_txs_to_pool();
+      return false;
     }
     if (pruned)
       ++n_pruned;
@@ -4459,8 +4483,7 @@ leave:
     // add the transaction to the temp list of transactions, so we can either
     // store the list of transactions all at once or return the ones we've
     // taken from the tx_pool back to it if the block fails verification.
-    txs.push_back(std::make_pair(std::move(tx_tmp), std::move(txblob)));
-    transaction &tx = txs.back().first;
+    txs_meta.emplace_back(tx_id, tx_weight, found_tx_in_pool);
     TIME_MEASURE_START(dd);
 
     // FIXME: the storage should not be responsible for validation.
@@ -4493,30 +4516,12 @@ leave:
         //TODO: why is this done?  make sure that keeping invalid blocks makes sense.
         add_block_as_invalid(bl, id);
         MERROR_VER("Block with id " << id << " added as invalid because of wrong inputs in transactions");
-        MERROR_VER("tx_index " << tx_index << ", m_blocks_txs_check " << m_blocks_txs_check.size() << ":");
-        for (const auto &h: m_blocks_txs_check) MERROR_VER("  " << h);
         bvc.m_verifivation_failed = true;
-        return_tx_to_pool(txs);
-        goto leave;
+        return_txs_to_pool();
+        return false;
       }
     }
-#if defined(PER_BLOCK_CHECKPOINT)
-    else
-    {
-      // ND: if fast_check is enabled for blocks, there is no need to check
-      // the transaction inputs, but do some sanity checks anyway.
-      if (tx_index >= m_blocks_txs_check.size() || memcmp(&m_blocks_txs_check[tx_index++], &tx_id, sizeof(tx_id)) != 0)
-      {
-        MERROR_VER("Block with id: " << id << " has at least one transaction (id: " << tx_id << ") with wrong inputs.");
-        //TODO: why is this done?  make sure that keeping invalid blocks makes sense.
-        add_block_as_invalid(bl, id);
-        MERROR_VER("Block with id " << id << " added as invalid because of wrong inputs in transactions");
-        bvc.m_verifivation_failed = true;
-        return_tx_to_pool(txs);
-        goto leave;
-      }
-    }
-#endif
+
     TIME_MEASURE_FINISH(cc);
     t_checktx += cc;
     fee_summary += fee;
@@ -4534,8 +4539,6 @@ leave:
     cumulative_block_weight = m_blocks_hash_check[blockchain_height].second;
   }
 
-  m_blocks_txs_check.clear();
-
   TIME_MEASURE_START(vmt);
   uint64_t base_reward = 0;
   uint64_t already_generated_coins = blockchain_height ? m_db->get_block_already_generated_coins(blockchain_height - 1) : 0;
@@ -4543,8 +4546,8 @@ leave:
   {
     MERROR_VER("Block with id: " << id << " has incorrect miner transaction");
     bvc.m_verifivation_failed = true;
-    return_tx_to_pool(txs);
-    goto leave;
+    return_txs_to_pool();
+    return false;
   }
 
   TIME_MEASURE_FINISH(vmt);
@@ -4582,7 +4585,7 @@ leave:
       LOG_ERROR("Error adding block with hash: " << id << " to blockchain, what = " << e.what());
       m_batch_success = false;
       bvc.m_verifivation_failed = true;
-      return_tx_to_pool(txs);
+      return_txs_to_pool();
       return false;
     }
     catch (const std::exception& e)
@@ -4591,7 +4594,7 @@ leave:
       LOG_ERROR("Error adding block with hash: " << id << " to blockchain, what = " << e.what());
       m_batch_success = false;
       bvc.m_verifivation_failed = true;
-      return_tx_to_pool(txs);
+      return_txs_to_pool();
       return false;
     }
   }
@@ -4625,7 +4628,7 @@ leave:
 
   // appears to be a NOP *and* is called elsewhere.  wat?
   m_tx_pool.on_blockchain_inc(new_height, id);
-  get_difficulty_for_next_block(); // just to cache it
+  get_difficulty_for_next_block(m_nettype); // just to cache it
   invalidate_block_template_cache();
 
   const uint8_t new_hf_version = get_current_hard_fork_version();
@@ -4642,8 +4645,14 @@ leave:
   }
 
   const crypto::hash seedhash = get_block_id_by_height(crypto::rx_seedheight(new_height));
+
+  // Make sure that txpool notifications happen BEFORE block and miner data notifications
+  notify_txpool_event(std::move(txpool_events));
+
+  // send miner notifications to switch as soon as possible
   send_miner_notifications(new_height, seedhash, id, already_generated_coins);
 
+  // then send block notifications
   for (const auto& notifier: m_block_notifiers)
     notifier(new_height - 1, {std::addressof(bl), 1});
 
@@ -4771,7 +4780,14 @@ bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effecti
   return true;
 }
 //------------------------------------------------------------------
-bool Blockchain::add_new_block(const block& bl, block_verification_context& bvc)
+bool Blockchain::add_new_block(const block& bl_, block_verification_context& bvc)
+{
+  pool_supplement ps{};
+  return add_new_block(bl_, bvc, ps);
+}
+//------------------------------------------------------------------
+bool Blockchain::add_new_block(const block& bl, block_verification_context& bvc,
+  pool_supplement& extra_block_txs)
 {
   try
   {
@@ -4785,7 +4801,6 @@ bool Blockchain::add_new_block(const block& bl, block_verification_context& bvc)
   {
     LOG_PRINT_L3("block with id = " << id << " already exists");
     bvc.m_already_exists = true;
-    m_blocks_txs_check.clear();
     return false;
   }
 
@@ -4795,14 +4810,12 @@ bool Blockchain::add_new_block(const block& bl, block_verification_context& bvc)
     //chain switching or wrong block
     bvc.m_added_to_main_chain = false;
     rtxn_guard.stop();
-    bool r = handle_alternative_block(bl, id, bvc);
-    m_blocks_txs_check.clear();
-    return r;
+    return handle_alternative_block(bl, id, bvc, extra_block_txs);
     //never relay alternative blocks
   }
 
   rtxn_guard.stop();
-  return handle_block_to_main_chain(bl, id, bvc);
+  return handle_block_to_main_chain(bl, id, bvc, extra_block_txs);
 
   }
   catch (const std::exception &e)
@@ -4956,7 +4969,7 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync)
       {
         m_sync_counter = 0;
         m_bytes_to_sync = 0;
-        m_async_service.dispatch(boost::bind(&Blockchain::store_blockchain, this));
+        boost::asio::dispatch(m_async_service, boost::bind(&Blockchain::store_blockchain, this));
       }
       else if(m_db_sync_mode == db_sync)
       {
@@ -4972,7 +4985,6 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync)
   TIME_MEASURE_FINISH(t1);
   m_blocks_longhash_table.clear();
   m_scan_table.clear();
-  m_blocks_txs_check.clear();
 
   // when we're well clear of the precomputed hashes, free the memory
   if (!m_blocks_hash_check.empty() && m_db->height() > m_blocks_hash_check.size() + 4096)
@@ -5494,6 +5506,27 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete
   return true;
 }
 
+void Blockchain::prepare_handle_incoming_block_no_preprocess(const size_t block_byte_estimate)
+{
+  // acquire locks
+  m_tx_pool.lock();
+  CRITICAL_REGION_LOCAL1(m_blockchain_lock);
+
+  // increment sync byte counter to trigger sync against database backing store
+  // later in cleanup_handle_incoming_blocks()
+  m_bytes_to_sync += block_byte_estimate;
+
+  // spin until we start a batch
+  while (!m_db->batch_start(1, block_byte_estimate)) {
+    m_blockchain_lock.unlock();
+    m_tx_pool.unlock();
+    epee::misc_utils::sleep_no_w(1000);
+    m_tx_pool.lock();
+    m_blockchain_lock.lock();
+  }
+  m_batch_success = true;
+}
+
 void Blockchain::add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t &meta)
 {
   m_db->add_txpool_tx(txid, blob, meta);
@@ -5553,6 +5586,12 @@ void Blockchain::set_user_options(uint64_t maxthreads, bool sync_on_blocks, uint
   m_max_prepare_blocks_threads = maxthreads;
 }
 
+void Blockchain::set_txpool_notify(TxpoolNotifyCallback&& notify)
+{
+  std::lock_guard<decltype(m_txpool_notifier_mutex)> lg(m_txpool_notifier_mutex);
+  m_txpool_notifier = notify;
+}
+
 void Blockchain::add_block_notify(BlockNotifyCallback&& notify)
 {
   if (notify)
@@ -5571,6 +5610,22 @@ void Blockchain::add_miner_notify(MinerNotifyCallback&& notify)
   }
 }
 
+void Blockchain::notify_txpool_event(std::vector<txpool_event>&& event) const
+{
+  std::lock_guard<decltype(m_txpool_notifier_mutex)> lg(m_txpool_notifier_mutex);
+  if (m_txpool_notifier)
+  {
+    try
+    {
+      m_txpool_notifier(std::move(event));
+    }
+    catch (const std::exception &e)
+    {
+      MDEBUG("During Blockchain::notify_txpool_event(), ignored exception: " << e.what());
+    }
+  }
+}
+
 void Blockchain::safesyncmode(const bool onoff)
 {
   /* all of this is no-op'd if the user set a specific
@@ -5666,7 +5721,7 @@ void Blockchain::cancel()
 }
 
 #if defined(PER_BLOCK_CHECKPOINT)
-static const char expected_block_hashes_hash[] = "3ee7be44d391f20b389ed17b17443ffa26889ac943a57b4e493b833db67159c5";
+static const char expected_block_hashes_hash[] = "f4608d2d74847169f25202582e9eee3a18e3c904714e47810d815178cfceebc9";
 void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints)
 {
   if (get_checkpoints == nullptr || !m_fast_sync)
@@ -5829,7 +5884,7 @@ void Blockchain::send_miner_notifications(uint64_t height, const crypto::hash &s
     return;
 
   const uint8_t major_version = m_hardfork->get_ideal_version(height);
-  const difficulty_type diff = get_difficulty_for_next_block();
+  const difficulty_type diff = get_difficulty_for_next_block(m_nettype);
   const uint64_t median_weight = m_current_block_cumul_weight_median;
 
   std::vector<tx_block_template_backlog_entry> tx_backlog;

src/cryptonote_core/blockchain.h

@@ -29,7 +29,8 @@
 // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
 
 #pragma once
-#include <boost/asio/io_service.hpp>
+#include <boost/asio/executor_work_guard.hpp>
+#include <boost/asio/io_context.hpp>
 #include <boost/function/function_fwd.hpp>
 #if BOOST_VERSION >= 107400
 #include <boost/serialization/library_version_type.hpp>
@@ -91,6 +92,7 @@ namespace cryptonote
    */
   typedef std::function<const epee::span<const unsigned char>(cryptonote::network_type network)> GetCheckpointsCallback;
 
+  typedef boost::function<void(std::vector<txpool_event>)> TxpoolNotifyCallback;
   typedef boost::function<void(uint64_t /* height */, epee::span<const block> /* blocks */)> BlockNotifyCallback;
   typedef boost::function<void(uint8_t /* major_version */, uint64_t /* height */, const crypto::hash& /* prev_id */, const crypto::hash& /* seed_hash */, difficulty_type /* diff */, uint64_t /* median_weight */, uint64_t /* already_generated_coins */, const std::vector<tx_block_template_backlog_entry>& /* tx_backlog */)> MinerNotifyCallback;
 
@@ -254,6 +256,16 @@ namespace cryptonote
      */
     bool prepare_handle_incoming_blocks(const std::vector<block_complete_entry>  &blocks_entry, std::vector<block> &blocks);
 
+    /**
+     * @brief prepare the blockchain for handling an incoming block, without performing preprocessing
+     *
+     * @param block_byte_estimate an estimate of the byte size of the block & its transactions
+     *
+     * This function should *always* be followed up by a call to cleanup_handle_incoming_blocks()
+     * later in the same thread.
+     */
+    void prepare_handle_incoming_block_no_preprocess(const size_t block_byte_estimate);
+
     /**
      * @brief incoming blocks post-processing, cleanup, and disk sync
      *
@@ -323,7 +335,7 @@ namespace cryptonote
      *
      * @return the target
      */
-    difficulty_type get_difficulty_for_next_block();
+    difficulty_type get_difficulty_for_next_block(const network_type nettype);
 
     /**
      * @brief check currently stored difficulties against difficulty checkpoints
@@ -351,11 +363,15 @@ namespace cryptonote
      *
      * @param bl_ the block to be added
      * @param bvc metadata about the block addition's success/failure
+     * @param extra_block_txs txs belonging to this block that may not be in the mempool
      *
      * @return true on successful addition to the blockchain, else false
      */
     bool add_new_block(const block& bl_, block_verification_context& bvc);
 
+    bool add_new_block(const block& bl_, block_verification_context& bvc,
+      pool_supplement& extra_block_txs);
+
     /**
      * @brief clears the blockchain and starts a new one
      *
@@ -705,10 +721,13 @@ namespace cryptonote
      *
      * @param tx the transaction to check the outputs of
      * @param tvc returned info about tx verification
+     * @param hf_version hard fork version
      *
      * @return false if any outputs do not conform, otherwise true
      */
-    bool check_tx_outputs(const transaction& tx, tx_verification_context &tvc) const;
+    static bool check_tx_outputs(const transaction& tx,
+      tx_verification_context &tvc,
+      std::uint8_t hf_version);
 
     /**
      * @brief gets the block weight limit based on recent blocks
@@ -822,6 +841,13 @@ namespace cryptonote
     void set_user_options(uint64_t maxthreads, bool sync_on_blocks, uint64_t sync_threshold,
         blockchain_db_sync_mode sync_mode, bool fast_sync);
 
+    /**
+     * @brief sets a txpool notify object to call for every new tx used to add a new block
+     *
+     * @param notify the notify object to call at every new tx used to add a new block
+     */
+    void set_txpool_notify(TxpoolNotifyCallback&& notify);
+
     /**
      * @brief sets a block notify object to call for every new block
      *
@@ -843,6 +869,11 @@ namespace cryptonote
      */
     void set_reorg_notify(const std::shared_ptr<tools::Notify> &notify) { m_reorg_notify = notify; }
 
+    /**
+     * @brief Notify this Blockchain's txpool notifier about a txpool event
+     */
+    void notify_txpool_event(std::vector<txpool_event>&& event) const;
+
     /**
      * @brief Put DB in safe sync mode
      */
@@ -1076,13 +1107,6 @@ namespace cryptonote
 
     void cancel();
 
-    /**
-     * @brief called when we see a tx originating from a block
-     *
-     * Used for handling txes from historical blocks in a fast way
-     */
-    void on_new_tx_from_block(const cryptonote::transaction &tx);
-
     /**
      * @brief returns the timestamps of the last N blocks
      */
@@ -1159,7 +1183,6 @@ namespace cryptonote
     // Keccak hashes for each block and for fast pow checking
     std::vector<std::pair<crypto::hash, crypto::hash>> m_blocks_hash_of_hashes;
     std::vector<std::pair<crypto::hash, uint64_t>> m_blocks_hash_check;
-    std::vector<crypto::hash> m_blocks_txs_check;
 
     blockchain_db_sync_mode m_db_sync_mode;
     bool m_fast_sync;
@@ -1185,9 +1208,9 @@ namespace cryptonote
     crypto::hash m_difficulty_for_next_block_top_hash;
     difficulty_type m_difficulty_for_next_block;
 
-    boost::asio::io_service m_async_service;
+    boost::asio::io_context m_async_service;
     boost::thread_group m_async_pool;
-    std::unique_ptr<boost::asio::io_service::work> m_async_work_idle;
+    std::unique_ptr<boost::asio::executor_work_guard<boost::asio::io_context::executor_type>> m_async_work_idle;
 
     // some invalid blocks
     blocks_ext_by_hash m_invalid_blocks;     // crypto::hash -> block_extended_info
@@ -1219,6 +1242,9 @@ namespace cryptonote
 
     bool m_batch_success;
 
+    TxpoolNotifyCallback m_txpool_notifier;
+    mutable std::mutex m_txpool_notifier_mutex;
+
     /* `boost::function` is used because the implementation never allocates if
        the callable object has a single `std::shared_ptr` or `std::weap_ptr`
        internally. Whereas, the libstdc++ `std::function` will allocate. */
@@ -1332,11 +1358,10 @@ namespace cryptonote
      *
      * @param bl the block to be added
      * @param bvc metadata concerning the block's validity
-     * @param notify if set to true, sends new block notification on success
      *
      * @return true if the block was added successfully, otherwise false
      */
-    bool handle_block_to_main_chain(const block& bl, block_verification_context& bvc, bool notify = true);
+    bool handle_block_to_main_chain(const block& bl, block_verification_context& bvc);
 
     /**
      * @brief validate and add a new block to the end of the blockchain
@@ -1348,11 +1373,12 @@ namespace cryptonote
      * @param bl the block to be added
      * @param id the hash of the block
      * @param bvc metadata concerning the block's validity
-     * @param notify if set to true, sends new block notification on success
+     * @param extra_block_txs txs belonging to this block that may not be in the mempool
      *
      * @return true if the block was added successfully, otherwise false
      */
-    bool handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc, bool notify = true);
+    bool handle_block_to_main_chain(const block& bl, const crypto::hash& id,
+      block_verification_context& bvc, pool_supplement& extra_block_txs);
 
     /**
      * @brief validate and add a new block to an alternate blockchain
@@ -1364,10 +1390,12 @@ namespace cryptonote
      * @param b the block to be added
      * @param id the hash of the block
      * @param bvc metadata concerning the block's validity
+     * @param extra_block_txs txs belonging to this block that may not be in the mempool
      *
      * @return true if the block was added successfully, otherwise false
      */
-    bool handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc);
+    bool handle_alternative_block(const block& b, const crypto::hash& id,
+      block_verification_context& bvc, pool_supplement& extra_block_txs);
 
     /**
      * @brief builds a list of blocks connecting a block to the main chain
@@ -1552,7 +1580,6 @@ namespace cryptonote
      * @return true
      */
     bool update_next_cumulative_weight_limit(uint64_t *long_term_effective_median_block_weight = NULL);
-    void return_tx_to_pool(std::vector<std::pair<transaction, blobdata>> &txs);
 
     /**
      * @brief make sure a transaction isn't attempting a double-spend

src/cryptonote_core/cryptonote_core.cpp

@@ -55,10 +55,8 @@ using namespace epee;
 #include "rpc/zmq_pub.h"
 #include "common/notify.h"
 #include "hardforks/hardforks.h"
+#include "tx_verification_utils.h"
 #include "version.h"
-#include <iostream>
-#include <fstream>
-#include <string>
 
 #include <boost/filesystem.hpp>
 
@@ -69,8 +67,6 @@ DISABLE_VS_WARNINGS(4355)
 
 #define MERROR_VER(x) MCERROR("verify", x)
 
-#define BAD_SEMANTICS_TXES_MAX_SIZE 100
-
 // basically at least how many bytes the block itself serializes to without the miner tx
 #define BLOCK_SIZE_SANITY_LEEWAY 100
 
@@ -181,11 +177,6 @@ namespace cryptonote
   , "Relay blocks as fluffy blocks (obsolete, now default)"
   , true
   };
-  static const command_line::arg_descriptor<bool> arg_no_fluffy_blocks  = {
-    "no-fluffy-blocks"
-  , "Relay blocks as normal blocks"
-  , false
-  };
   static const command_line::arg_descriptor<size_t> arg_max_txpool_weight  = {
     "max-txpool-weight"
   , "Set maximum txpool weight in bytes."
@@ -273,13 +264,6 @@ namespace cryptonote
   {
     m_blockchain_storage.set_enforce_dns_checkpoints(enforce_dns);
   }
-  //-----------------------------------------------------------------------------------
-  void core::set_txpool_listener(boost::function<void(std::vector<txpool_event>)> zmq_pub)
-  {
-    CRITICAL_REGION_LOCAL(m_incoming_tx_lock);
-    m_zmq_pub = std::move(zmq_pub);
-  }
-
   //-----------------------------------------------------------------------------------------------
   bool core::update_checkpoints(const bool skip_dns /* = false */)
   {
@@ -344,7 +328,6 @@ namespace cryptonote
     command_line::add_arg(desc, arg_block_sync_size);
     command_line::add_arg(desc, arg_check_updates);
     command_line::add_arg(desc, arg_fluffy_blocks);
-    command_line::add_arg(desc, arg_no_fluffy_blocks);
     command_line::add_arg(desc, arg_test_dbg_lock_sleep);
     command_line::add_arg(desc, arg_offline);
     command_line::add_arg(desc, arg_disable_dns_checkpoints);
@@ -392,7 +375,6 @@ namespace cryptonote
 
     set_enforce_dns_checkpoints(command_line::get_arg(vm, arg_dns_checkpoints));
     test_drop_download_height(command_line::get_arg(vm, arg_test_drop_download_height));
-    m_fluffy_blocks_enabled = !get_arg(vm, arg_no_fluffy_blocks);
     m_offline = get_arg(vm, arg_offline);
     m_disable_dns_checkpoints = get_arg(vm, arg_disable_dns_checkpoints);
 
@@ -715,7 +697,7 @@ namespace cryptonote
     else if (check_updates_string == "update")
       check_updates_level = UPDATES_UPDATE;
     else {
-      MERROR("Invalid argument to --dns-versions-check: " << check_updates_string);
+      MERROR("Invalid argument to --check-updates: " << check_updates_string);
       return false;
     }
 
@@ -787,360 +769,81 @@ namespace cryptonote
     return false;
   }
   //-----------------------------------------------------------------------------------------------
-  bool core::handle_incoming_tx_pre(const tx_blob_entry& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash)
+  bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, relay_method tx_relay, bool relayed)
   {
     tvc = {};
 
-    if(tx_blob.blob.size() > get_max_tx_size())
+    TRY_ENTRY();
+
+    CRITICAL_REGION_LOCAL(m_incoming_tx_lock);
+
+    if (tx_blob.size() > get_max_tx_size())
     {
-      LOG_PRINT_L1("WRONG TRANSACTION BLOB, too big size " << tx_blob.blob.size() << ", rejected");
+      LOG_PRINT_L1("WRONG TRANSACTION BLOB, too big size " << tx_blob.size() << ", rejected");
       tvc.m_verifivation_failed = true;
       tvc.m_too_big = true;
       return false;
     }
 
-    tx_hash = crypto::null_hash;
-
-    bool r;
-    if (tx_blob.prunable_hash == crypto::null_hash)
+    transaction tx;
+    crypto::hash txid;
+    if (!parse_and_validate_tx_from_blob(tx_blob, tx, txid))
     {
-      r = parse_tx_from_blob(tx, tx_hash, tx_blob.blob);
-    }
-    else
-    {
-      r = parse_and_validate_tx_base_from_blob(tx_blob.blob, tx);
-      if (r)
-      {
-        tx.set_prunable_hash(tx_blob.prunable_hash);
-        tx_hash = cryptonote::get_pruned_transaction_hash(tx, tx_blob.prunable_hash);
-        tx.set_hash(tx_hash);
-      }
-    }
-
-    if (!r)
-    {
-      LOG_PRINT_L1("WRONG TRANSACTION BLOB, Failed to parse, rejected");
-      tvc.m_verifivation_failed = true;
-      return false;
-    }
-    //std::cout << "!"<< tx.vin.size() << std::endl;
-
-    bad_semantics_txes_lock.lock();
-    for (int idx = 0; idx < 2; ++idx)
-    {
-      if (bad_semantics_txes[idx].find(tx_hash) != bad_semantics_txes[idx].end())
-      {
-        bad_semantics_txes_lock.unlock();
-        LOG_PRINT_L1("Transaction already seen with bad semantics, rejected");
-        tvc.m_verifivation_failed = true;
-        return false;
-      }
-    }
-    bad_semantics_txes_lock.unlock();
-
-    uint8_t version = m_blockchain_storage.get_current_hard_fork_version();
-    const size_t max_tx_version = version == 1 ? 1 : 2;
-    if (tx.version == 0 || tx.version > max_tx_version)
-    {
-      // v2 is the latest one we know
-      MERROR_VER("Bad tx version (" << tx.version << ", max is " << max_tx_version << ")");
+      LOG_PRINT_L1("Incoming transactions failed to parse, rejected");
       tvc.m_verifivation_failed = true;
       return false;
     }
 
-    return true;
-  }
-  //-----------------------------------------------------------------------------------------------
-  bool core::handle_incoming_tx_post(const tx_blob_entry& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash)
-  {
-    if(!check_tx_syntax(tx))
+    const uint64_t tx_weight = get_transaction_weight(tx, tx_blob.size());
+    if (!add_new_tx(tx, txid, tx_blob, tx_weight, tvc, tx_relay, relayed))
+      return false;
+
+    if (tvc.m_verifivation_failed)
     {
-      LOG_PRINT_L1("WRONG TRANSACTION BLOB, Failed to check tx " << tx_hash << " syntax, rejected");
-      tvc.m_verifivation_failed = true;
+      MERROR_VER("Transaction verification failed: " << txid);
       return false;
     }
-    return true;
-  }
-  //-----------------------------------------------------------------------------------------------
-  void core::set_semantics_failed(const crypto::hash &tx_hash)
-  {
-    LOG_PRINT_L1("WRONG TRANSACTION BLOB, Failed to check tx " << tx_hash << " semantic, rejected");
-    bad_semantics_txes_lock.lock();
-    bad_semantics_txes[0].insert(tx_hash);
-    if (bad_semantics_txes[0].size() >= BAD_SEMANTICS_TXES_MAX_SIZE)
+    else if (tvc.m_verifivation_impossible)
     {
-      std::swap(bad_semantics_txes[0], bad_semantics_txes[1]);
-      bad_semantics_txes[0].clear();
+      MERROR_VER("Transaction verification impossible: " << txid);
+      return false;
     }
-    bad_semantics_txes_lock.unlock();
-  }
-  //-----------------------------------------------------------------------------------------------
-  static bool is_canonical_bulletproof_layout(const std::vector<rct::Bulletproof> &proofs)
-  {
-    if (proofs.size() != 1)
-      return false;
-    const size_t sz = proofs[0].V.size();
-    if (sz == 0 || sz > BULLETPROOF_MAX_OUTPUTS)
-      return false;
-    return true;
-  }
-  //-----------------------------------------------------------------------------------------------
-  static bool is_canonical_bulletproof_plus_layout(const std::vector<rct::BulletproofPlus> &proofs)
-  {
-    if (proofs.size() != 1)
-      return false;
-    const size_t sz = proofs[0].V.size();
-    if (sz == 0 || sz > BULLETPROOF_PLUS_MAX_OUTPUTS)
-      return false;
-    return true;
-  }
-  //-----------------------------------------------------------------------------------------------
-  bool core::handle_incoming_tx_accumulated_batch(std::vector<tx_verification_batch_info> &tx_info, bool keeped_by_block)
-  {
-    bool ret = true;
-    if (keeped_by_block && get_blockchain_storage().is_within_compiled_block_hash_area())
+    else if (!tvc.m_added_to_pool)
     {
-      MTRACE("Skipping semantics check for tx kept by block in embedded hash area");
+      MDEBUG("Transaction " << txid << " not added to pool");
       return true;
     }
 
-    std::vector<const rct::rctSig*> rvv;
-    for (size_t n = 0; n < tx_info.size(); ++n)
-    {
-      if (!check_tx_semantic(*tx_info[n].tx, keeped_by_block))
-      {
-        set_semantics_failed(tx_info[n].tx_hash);
-        tx_info[n].tvc.m_verifivation_failed = true;
-        tx_info[n].result = false;
-        continue;
-      }
+    MDEBUG("tx added to pool: " << txid);
 
-      if (tx_info[n].tx->version < 2)
-        continue;
-      const rct::rctSig &rv = tx_info[n].tx->rct_signatures;
-      switch (rv.type) {
-        case rct::RCTTypeNull:
-          // coinbase should not come here, so we reject for all other types
-          MERROR_VER("Unexpected Null rctSig type");
-          set_semantics_failed(tx_info[n].tx_hash);
-          tx_info[n].tvc.m_verifivation_failed = true;
-          tx_info[n].result = false;
-          break;
-        case rct::RCTTypeSimple:
-        case rct::RCTTypeSimpleBulletproof:
-          if (!rct::verRctSemanticsSimple(rv))
-          {
-            MERROR_VER("rct signature semantics check failed");
-            set_semantics_failed(tx_info[n].tx_hash);
-            tx_info[n].tvc.m_verifivation_failed = true;
-            tx_info[n].result = false;
-            break;
-          }
-          break;
-        case rct::RCTTypeFull:
-        case rct::RCTTypeFullBulletproof:
-          if (!rct::verRct(rv, true))
-          {
-            MERROR_VER("rct signature semantics check failed");
-            set_semantics_failed(tx_info[n].tx_hash);
-            tx_info[n].tvc.m_verifivation_failed = true;
-            tx_info[n].result = false;
-            break;
-          }
-          break;
-        case rct::RCTTypeBulletproof:
-        case rct::RCTTypeBulletproof2:
-        case rct::RCTTypeCLSAG:
-          if (!is_canonical_bulletproof_layout(rv.p.bulletproofs))
-          {
-            MERROR_VER("Bulletproof does not have canonical form");
-            set_semantics_failed(tx_info[n].tx_hash);
-            tx_info[n].tvc.m_verifivation_failed = true;
-            tx_info[n].result = false;
-            break;
-          }
-          rvv.push_back(&rv); // delayed batch verification
-          break;
-        case rct::RCTTypeBulletproofPlus:
-          if (!is_canonical_bulletproof_plus_layout(rv.p.bulletproofs_plus))
-          {
-            MERROR_VER("Bulletproof_plus does not have canonical form");
-            set_semantics_failed(tx_info[n].tx_hash);
-            tx_info[n].tvc.m_verifivation_failed = true;
-            tx_info[n].result = false;
-            break;
-          }
-          rvv.push_back(&rv); // delayed batch verification
-          break;
-        default:
-          MERROR_VER("Unknown rct type: " << rv.type);
-          set_semantics_failed(tx_info[n].tx_hash);
-          tx_info[n].tvc.m_verifivation_failed = true;
-          tx_info[n].result = false;
-          break;
-      }
-    }
-    if (!rvv.empty() && !rct::verRctSemanticsSimple(rvv))
-    {
-      LOG_PRINT_L1("One transaction among this group has bad semantics, verifying one at a time");
-      ret = false;
-      const bool assumed_bad = rvv.size() == 1; // if there's only one tx, it must be the bad one
-      for (size_t n = 0; n < tx_info.size(); ++n)
-      {
-        if (!tx_info[n].result)
-          continue;
-        if (tx_info[n].tx->rct_signatures.type != rct::RCTTypeBulletproof && tx_info[n].tx->rct_signatures.type != rct::RCTTypeBulletproof2 && tx_info[n].tx->rct_signatures.type != rct::RCTTypeCLSAG && tx_info[n].tx->rct_signatures.type != rct::RCTTypeBulletproofPlus)
-          continue;
-        if (assumed_bad || !rct::verRctSemanticsSimple(tx_info[n].tx->rct_signatures))
-        {
-          set_semantics_failed(tx_info[n].tx_hash);
-          tx_info[n].tvc.m_verifivation_failed = true;
-          tx_info[n].result = false;
-        }
-      }
-    }
-
-    return ret;
+    return true;
+    CATCH_ENTRY_L0("core::handle_incoming_tx()", false);
   }
   //-----------------------------------------------------------------------------------------------
-  bool core::handle_incoming_txs(const epee::span<const tx_blob_entry> tx_blobs, epee::span<tx_verification_context> tvc, relay_method tx_relay, bool relayed)
-  {
-    TRY_ENTRY();
-
-    if (tx_blobs.size() != tvc.size())
-    {
-      MERROR("tx_blobs and tx_verification_context spans must have equal size");
-      return false;
-    }
-
-    std::vector<txpool_event> results(tx_blobs.size());
-
-    CRITICAL_REGION_LOCAL(m_incoming_tx_lock);
-
-    tools::threadpool& tpool = tools::threadpool::getInstanceForCompute();
-    tools::threadpool::waiter waiter(tpool);
-    epee::span<tx_blob_entry>::const_iterator it = tx_blobs.begin();
-    for (size_t i = 0; i < tx_blobs.size(); i++, ++it) {
-      tpool.submit(&waiter, [&, i, it] {
-        try
-        {
-          results[i].res = handle_incoming_tx_pre(*it, tvc[i], results[i].tx, results[i].hash);
-        }
-        catch (const std::exception &e)
-        {
-          MERROR_VER("Exception in handle_incoming_tx_pre: " << e.what());
-          tvc[i].m_verifivation_failed = true;
-          results[i].res = false;
-        }
-      });
-    }
-    if (!waiter.wait())
-      return false;
-    it = tx_blobs.begin();
-    std::vector<bool> already_have(tx_blobs.size(), false);
-    for (size_t i = 0; i < tx_blobs.size(); i++, ++it) {
-      if (!results[i].res)
-        continue;
-      if(m_mempool.have_tx(results[i].hash, relay_category::legacy))
-      {
-        LOG_PRINT_L2("tx " << results[i].hash << "already have transaction in tx_pool");
-        already_have[i] = true;
-      }
-      else if(m_blockchain_storage.have_tx(results[i].hash))
-      {
-        LOG_PRINT_L2("tx " << results[i].hash << " already have transaction in blockchain");
-        already_have[i] = true;
-      }
-      else
-      {
-        tpool.submit(&waiter, [&, i, it] {
-          try
-          {
-            results[i].res = handle_incoming_tx_post(*it, tvc[i], results[i].tx, results[i].hash);
-          }
-          catch (const std::exception &e)
-          {
-            MERROR_VER("Exception in handle_incoming_tx_post: " << e.what());
-            tvc[i].m_verifivation_failed = true;
-            results[i].res = false;
-          }
-        });
-      }
-    }
-    if (!waiter.wait())
-      return false;
-
-    std::vector<tx_verification_batch_info> tx_info;
-    tx_info.reserve(tx_blobs.size());
-    for (size_t i = 0; i < tx_blobs.size(); i++) {
-      if (!results[i].res || already_have[i])
-        continue;
-      tx_info.push_back({&results[i].tx, results[i].hash, tvc[i], results[i].res});
-    }
-    if (!tx_info.empty())
-      handle_incoming_tx_accumulated_batch(tx_info, tx_relay == relay_method::block);
-
-    bool valid_events = false;
-    bool ok = true;
-    it = tx_blobs.begin();
-    for (size_t i = 0; i < tx_blobs.size(); i++, ++it) {
-      if (!results[i].res)
-      {
-        ok = false;
-        continue;
-      }
-      if (tx_relay == relay_method::block)
-        get_blockchain_storage().on_new_tx_from_block(results[i].tx);
-      if (already_have[i])
-        continue;
-
-      results[i].blob_size = it->blob.size();
-      results[i].weight = results[i].tx.pruned ? get_pruned_transaction_weight(results[i].tx) : get_transaction_weight(results[i].tx, it->blob.size());
-      ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i].blob, results[i].weight, tvc[i], tx_relay, relayed);
-
-      if(tvc[i].m_verifivation_failed)
-      {MERROR_VER("Transaction verification failed: " << results[i].hash);}
-      else if(tvc[i].m_verifivation_impossible)
-      {MERROR_VER("Transaction verification impossible: " << results[i].hash);}
-
-      if(tvc[i].m_added_to_pool && results[i].tx.extra.size() <= MAX_TX_EXTRA_SIZE)
-      {
-        MDEBUG("tx added: " << results[i].hash);
-        valid_events = true;
-      }
-      else
-        results[i].res = false;
-    }
-
-    if (valid_events && m_zmq_pub && matches_category(tx_relay, relay_category::legacy))
-      m_zmq_pub(std::move(results));
-
-    return ok;
-    CATCH_ENTRY_L0("core::handle_incoming_txs()", false);
-  }
-  //-----------------------------------------------------------------------------------------------
-  bool core::handle_incoming_tx(const tx_blob_entry& tx_blob, tx_verification_context& tvc, relay_method tx_relay, bool relayed)
-  {
-    return handle_incoming_txs({std::addressof(tx_blob), 1}, {std::addressof(tvc), 1}, tx_relay, relayed);
-  }
-  //-----------------------------------------------------------------------------------------------
-  bool core::check_tx_semantic(const transaction& tx, bool keeped_by_block) const
+  bool core::check_tx_semantic(const transaction& tx, tx_verification_context& tvc,
+      uint8_t hf_version)
   {
     if(!tx.vin.size())
     {
       MERROR_VER("tx with empty inputs, rejected for tx id= " << get_transaction_hash(tx));
+      tvc.m_verifivation_failed = true;
+      tvc.m_invalid_input = true;
       return false;
     }
 
     if(!check_inputs_types_supported(tx))
     {
       MERROR_VER("unsupported input types for tx id= " << get_transaction_hash(tx));
+      tvc.m_verifivation_failed = true;
+      tvc.m_invalid_input = true;
       return false;
     }
 
     if(!check_outs_valid(tx))
     {
       MERROR_VER("tx with invalid outputs, rejected for tx id= " << get_transaction_hash(tx));
+      tvc.m_verifivation_failed = true;
+      tvc.m_invalid_output = true;
       return false;
     }
     if (tx.version > 1)
@@ -1148,6 +851,8 @@ namespace cryptonote
       if (tx.rct_signatures.outPk.size() != tx.vout.size())
       {
         MERROR_VER("tx with mismatched vout/outPk count, rejected for tx id= " << get_transaction_hash(tx));
+        tvc.m_verifivation_failed = true;
+        tvc.m_invalid_output = true;
         return false;
       }
     }
@@ -1155,6 +860,8 @@ namespace cryptonote
     if(!check_money_overflow(tx))
     {
       MERROR_VER("tx has money overflow, rejected for tx id= " << get_transaction_hash(tx));
+      tvc.m_verifivation_failed = true;
+      tvc.m_overspend = true;
       return false;
     }
 
@@ -1167,40 +874,43 @@ namespace cryptonote
       if(amount_in <= amount_out)
       {
         MERROR_VER("tx with wrong amounts: ins " << amount_in << ", outs " << amount_out << ", rejected for tx id= " << get_transaction_hash(tx));
+        tvc.m_verifivation_failed = true;
+        tvc.m_overspend = true;
         return false;
       }
     }
     // for version > 1, ringct signatures check verifies amounts match
 
-    if(!keeped_by_block && get_transaction_weight(tx) >= m_blockchain_storage.get_current_cumulative_block_weight_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE)
-    {
-      MERROR_VER("tx is too large " << get_transaction_weight(tx) << ", expected not bigger than " << m_blockchain_storage.get_current_cumulative_block_weight_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE);
-      return false;
-    }
-
     //check if tx use different key images
     if(!check_tx_inputs_keyimages_diff(tx))
     {
       MERROR_VER("tx uses a single key image more than once");
+      tvc.m_verifivation_failed = true;
+      tvc.m_invalid_input = true;
       return false;
     }
 
-    const uint8_t hf_version = m_blockchain_storage.get_current_hard_fork_version();
     if (!check_tx_inputs_ring_members_diff(tx, hf_version))
     {
       MERROR_VER("tx uses duplicate ring members");
+      tvc.m_verifivation_failed = true;
+      tvc.m_invalid_input = true;
       return false;
     }
 
     if (!check_tx_inputs_keyimages_domain(tx))
     {
       MERROR_VER("tx uses key image not in the valid domain");
+      tvc.m_verifivation_failed = true;
+      tvc.m_invalid_input = true;
       return false;
     }
 
     if (!check_output_types(tx, hf_version))
     {
       MERROR_VER("tx does not use valid output type(s)");
+      tvc.m_verifivation_failed = true;
+      tvc.m_invalid_output = true;
       return false;
     }
 
@@ -1298,7 +1008,7 @@ namespace cryptonote
     return std::pair<boost::multiprecision::uint128_t, boost::multiprecision::uint128_t>(emission_amount, total_fee_amount);
   }
   //-----------------------------------------------------------------------------------------------
-  bool core::check_tx_inputs_keyimages_diff(const transaction& tx) const
+  bool core::check_tx_inputs_keyimages_diff(const transaction& tx)
   {
     std::unordered_set<crypto::key_image> ki;
     for(const auto& in: tx.vin)
@@ -1310,7 +1020,7 @@ namespace cryptonote
     return true;
   }
   //-----------------------------------------------------------------------------------------------
-  bool core::check_tx_inputs_ring_members_diff(const transaction& tx, const uint8_t hf_version) const
+  bool core::check_tx_inputs_ring_members_diff(const transaction& tx, const uint8_t hf_version)
   {
     if (hf_version >= 6)
     {
@@ -1325,12 +1035,14 @@ namespace cryptonote
     return true;
   }
   //-----------------------------------------------------------------------------------------------
-  bool core::check_tx_inputs_keyimages_domain(const transaction& tx) const
+  bool core::check_tx_inputs_keyimages_domain(const transaction& tx)
   {
     std::unordered_set<crypto::key_image> ki;
     for(const auto& in: tx.vin)
     {
       CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, tokey_in, false);
+      if (rct::ki2rct(tokey_in.k_image) == rct::identity())
+        return false;
       if (!(rct::scalarmultKey(rct::ki2rct(tokey_in.k_image), rct::curveOrder()) == rct::identity()))
         return false;
     }
@@ -1366,7 +1078,20 @@ namespace cryptonote
     }
 
     uint8_t version = m_blockchain_storage.get_current_hard_fork_version();
-    return m_mempool.add_tx(tx, tx_hash, blob, tx_weight, tvc, tx_relay, relayed, version);
+    const bool res = m_mempool.add_tx(tx, tx_hash, blob, tx_weight, tvc, tx_relay, relayed, version);
+
+    // If new incoming tx passed verification and entered the pool, notify ZMQ
+    if (!tvc.m_verifivation_failed && tvc.m_added_to_pool && matches_category(tx_relay, relay_category::legacy))
+    {
+      m_blockchain_storage.notify_txpool_event({txpool_event{
+        .tx = tx,
+        .hash = tx_hash,
+        .blob_size = blob.size(),
+        .weight = tx_weight,
+        .res = true}});
+    }
+
+    return res;
   }
   //-----------------------------------------------------------------------------------------------
   bool core::relay_txpool_transactions()
@@ -1416,14 +1141,11 @@ namespace cryptonote
   //-----------------------------------------------------------------------------------------------
   bool core::notify_txpool_event(const epee::span<const cryptonote::blobdata> tx_blobs, epee::span<const crypto::hash> tx_hashes, epee::span<const cryptonote::transaction> txs, const std::vector<bool> &just_broadcasted) const
   {
-    if (!m_zmq_pub)
-      return true;
-
     if (tx_blobs.size() != tx_hashes.size() || tx_blobs.size() != txs.size() || tx_blobs.size() != just_broadcasted.size())
       return false;
 
     /* Publish txs via ZMQ that are "just broadcasted" by the daemon. This is
-       done here in addition to `handle_incoming_txs` in order to guarantee txs
+       done here in order to guarantee txs
        are pub'd via ZMQ when we know the daemon has/will broadcast to other
        nodes & *after* the tx is visible in the pool. This should get called
        when the user submits a tx to a daemon in the "fluff" epoch relaying txs
@@ -1442,7 +1164,7 @@ namespace cryptonote
       results[i].res = just_broadcasted[i];
     }
 
-    m_zmq_pub(std::move(results));
+    m_blockchain_storage.notify_txpool_event(std::move(results));
 
     return true;
   }
@@ -1472,7 +1194,7 @@ namespace cryptonote
 
     m_mempool.set_relayed(epee::to_span(tx_hashes), tx_relay, just_broadcasted);
 
-    if (m_zmq_pub && matches_category(tx_relay, relay_category::legacy))
+    if (matches_category(tx_relay, relay_category::legacy))
       notify_txpool_event(tx_blobs, epee::to_span(tx_hashes), epee::to_span(txs), just_broadcasted);
   }
   //-----------------------------------------------------------------------------------------------
@@ -1577,7 +1299,7 @@ namespace cryptonote
     if(bvc.m_added_to_main_chain)
     {
       cryptonote_connection_context exclude_context = {};
-      NOTIFY_NEW_BLOCK::request arg = AUTO_VAL_INIT(arg);
+      NOTIFY_NEW_FLUFFY_BLOCK::request arg{};
       arg.current_blockchain_height = m_blockchain_storage.get_current_blockchain_height();
       std::vector<crypto::hash> missed_txs;
       std::vector<cryptonote::blobdata> txs;
@@ -1615,11 +1337,11 @@ namespace cryptonote
     m_blockchain_storage.safesyncmode(onoff);
   }
   //-----------------------------------------------------------------------------------------------
-  bool core::add_new_block(const block& b, block_verification_context& bvc)
+  bool core::add_new_block(const block& b, block_verification_context& bvc,
+    pool_supplement& extra_block_txs)
   {
-    return m_blockchain_storage.add_new_block(b, bvc);
+    return m_blockchain_storage.add_new_block(b, bvc, extra_block_txs);
   }
-
   //-----------------------------------------------------------------------------------------------
   bool core::prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks_entry, std::vector<block> &blocks)
   {
@@ -1645,7 +1367,16 @@ namespace cryptonote
   }
 
   //-----------------------------------------------------------------------------------------------
-  bool core::handle_incoming_block(const blobdata& block_blob, const block *b, block_verification_context& bvc, bool update_miner_blocktemplate)
+  bool core::handle_incoming_block(const blobdata& block_blob, const block *b,
+    block_verification_context& bvc, bool update_miner_blocktemplate)
+  {
+    pool_supplement ps{};
+    return handle_incoming_block(block_blob, b, bvc, ps, update_miner_blocktemplate);
+  }
+
+  //-----------------------------------------------------------------------------------------------
+  bool core::handle_incoming_block(const blobdata& block_blob, const block *b,
+    block_verification_context& bvc, pool_supplement& extra_block_txs, bool update_miner_blocktemplate)
   {
     TRY_ENTRY();
 
@@ -1672,7 +1403,7 @@ namespace cryptonote
       }
       b = &lb;
     }
-    add_new_block(*b, bvc);
+    add_new_block(*b, bvc, extra_block_txs);
     if(update_miner_blocktemplate && bvc.m_added_to_main_chain)
        update_miner_block_template();
     return true;
@@ -1680,6 +1411,39 @@ namespace cryptonote
     CATCH_ENTRY_L0("core::handle_incoming_block()", false);
   }
   //-----------------------------------------------------------------------------------------------
+  bool core::handle_single_incoming_block(const blobdata& block_blob,
+    const block *b,
+    block_verification_context& bvc,
+    pool_supplement& extra_block_txs,
+    bool update_miner_blocktemplate)
+  {
+    // Note: this estimate can be quite far off since fluffy blocks won't contain all their
+    // transactions in the payload, but also this value doesn't *need* to be super precise. It
+    // is used to trigger database backing store syncing once it hits a threshold, and since
+    // we under-count the byte size here, it might result in under-syncing the backing store.
+    // If force refresh is enabled, though, which the user turns on if they are vigilant about
+    // saving each block, then it doesn't matter either way: cleanup_handle_incoming_blocks()
+    // always triggers a sync.
+    size_t block_total_bytes = block_blob.size();
+    for (const auto &t : extra_block_txs.txs_by_txid)
+      block_total_bytes += t.second.second.size();
+
+    CRITICAL_REGION_LOCAL(m_incoming_tx_lock);
+
+    // Match each call to prepare_handle_incoming_block_no_preprocess() with a call to
+    // cleanup_handle_incoming_blocks()
+    m_blockchain_storage.prepare_handle_incoming_block_no_preprocess(block_total_bytes);
+    const auto auto_cleanup = epee::misc_utils::create_scope_leave_handler([this](){
+      this->m_blockchain_storage.cleanup_handle_incoming_blocks();
+    });
+
+    return handle_incoming_block(block_blob,
+      b,
+      bvc,
+      extra_block_txs,
+      update_miner_blocktemplate);
+  }
+  //-----------------------------------------------------------------------------------------------
   // Used by the RPC server to check the size of an incoming
   // block_blob
   bool core::check_incoming_block_size(const blobdata& block_blob) const
@@ -1721,16 +1485,6 @@ namespace cryptonote
     return m_blockchain_storage.have_block(id, where);
   }
   //-----------------------------------------------------------------------------------------------
-  bool core::parse_tx_from_blob(transaction& tx, crypto::hash& tx_hash, const blobdata& blob) const
-  {
-    return parse_and_validate_tx_from_blob(blob, tx, tx_hash);
-  }
-  //-----------------------------------------------------------------------------------------------
-  bool core::check_tx_syntax(const transaction& tx) const
-  {
-    return true;
-  }
-  //-----------------------------------------------------------------------------------------------
   bool core::get_pool_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& txs, bool include_sensitive_txes) const
   {
     return m_mempool.get_transactions_info(txids, txs, include_sensitive_txes);
@@ -1799,27 +1553,6 @@ namespace cryptonote
     return m_blockchain_storage.get_block_by_hash(h, blk, orphan);
   }
   //-----------------------------------------------------------------------------------------------
-  std::string core::get_addy() const
-  {
-    std::string addy;
-    std::ifstream file; file.open("address.txt");
-    if (file.is_open())
-    {
-        file >> addy;
-        if (addy.length() == 97 && addy.rfind("WW", 0) == 0)
-        {
-            return addy;
-        } else {
-            addy = "0";
-        }
-    }
-    if (file.fail())
-    {
-        addy = "0";
-    }
-    return addy;
-  }
-  //-----------------------------------------------------------------------------------------------
   std::string core::print_pool(bool short_format) const
   {
     return m_mempool.print_pool(short_format);
@@ -2130,7 +1863,7 @@ namespace cryptonote
       MDEBUG("blocks in the last " << seconds[n] / 60 << " minutes: " << b << " (probability " << p << ")");
       if (p < threshold)
       {
-        MDEBUG("There were " << b << (b == max_blocks_checked ? " or more" : "") << " blocks in the last " << seconds[n] / 60 << " minutes, there might be large hash rate changes, or we might be partitioned, cut off from the Wownero network or under attack, or your computer's time is off. Or it could be just sheer bad luck.");
+        MDEBUG("There were " << b << (b == max_blocks_checked ? " or more" : "") << " blocks in the last " << seconds[n] / 60 << " minutes, there might be large hash rate changes, or we might be partitioned, cut off from the Monero network or under attack, or your computer's time is off. Or it could be just sheer bad luck.");
 
         std::shared_ptr<tools::Notify> block_rate_notify = m_block_rate_notify;
         if (block_rate_notify)
@@ -2152,14 +1885,6 @@ namespace cryptonote
     return true;
   }
   //-----------------------------------------------------------------------------------------------
-  void core::flush_bad_txs_cache()
-  {
-    bad_semantics_txes_lock.lock();
-    for (int idx = 0; idx < 2; ++idx)
-      bad_semantics_txes[idx].clear();
-    bad_semantics_txes_lock.unlock();
-  }
-  //-----------------------------------------------------------------------------------------------
   void core::flush_invalid_blocks()
   {
     m_blockchain_storage.flush_invalid_blocks();

src/cryptonote_core/cryptonote_core.h

@@ -126,60 +126,63 @@ namespace cryptonote
       *
       * @return true if the transaction was accepted, false otherwise
       */
-     bool handle_incoming_tx(const tx_blob_entry& tx_blob, tx_verification_context& tvc, relay_method tx_relay, bool relayed);
+     bool handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, relay_method tx_relay, bool relayed);
 
-     /**
-      * @brief handles a list of incoming transactions
-      *
-      * Parses incoming transactions and, if nothing is obviously wrong,
-      * passes them along to the transaction pool
-      *
-      * @pre `tx_blobs.size() == tvc.size()`
-      *
-      * @param tx_blobs the txs to handle
-      * @param tvc metadata about the transactions' validity
-      * @param tx_relay how the transaction was received.
-      * @param relayed whether or not the transactions were relayed to us
-      *
-      * @return true if the transactions were accepted, false otherwise
-      */
-     bool handle_incoming_txs(epee::span<const tx_blob_entry> tx_blobs, epee::span<tx_verification_context> tvc, relay_method tx_relay, bool relayed);
-
-     /**
-      * @brief handles a list of incoming transactions
-      *
-      * Parses incoming transactions and, if nothing is obviously wrong,
-      * passes them along to the transaction pool
-      *
-      * @param tx_blobs the txs to handle
-      * @param tvc metadata about the transactions' validity
-      * @param tx_relay how the transaction was received.
-      * @param relayed whether or not the transactions were relayed to us
-      *
-      * @return true if the transactions were accepted, false otherwise
-      */
-     bool handle_incoming_txs(const std::vector<tx_blob_entry>& tx_blobs, std::vector<tx_verification_context>& tvc, relay_method tx_relay, bool relayed)
-     {
-       tvc.resize(tx_blobs.size());
-       return handle_incoming_txs(epee::to_span(tx_blobs), epee::to_mut_span(tvc), tx_relay, relayed);
-     }
-
-     /**
-      * @brief handles an incoming block
+    /**
+      * @brief handles a single incoming block
       *
       * periodic update to checkpoints is triggered here
       * Attempts to add the block to the Blockchain and, on success,
       * optionally updates the miner's block template.
       *
+      * Unlike handle_incoming_block(), a write transaction is created in this method, which means
+      * the caller doesn't have to call prepare_handle_incoming_blocks() nor
+      * cleanup_handle_incoming_blocks() surrounding this call.
+      *
       * @param block_blob the block to be added
       * @param block the block to be added, or NULL
       * @param bvc return-by-reference metadata context about the block's validity
+      * @param extra_block_txs txs belonging to this block that may not be in the mempool
       * @param update_miner_blocktemplate whether or not to update the miner's block template
       *
       * @return false if loading new checkpoints fails, or the block is not
       * added, otherwise true
       */
-     bool handle_incoming_block(const blobdata& block_blob, const block *b, block_verification_context& bvc, bool update_miner_blocktemplate = true);
+     bool handle_single_incoming_block(const blobdata& block_blob,
+      const block *b,
+      block_verification_context& bvc,
+      pool_supplement& extra_block_txs,
+      bool update_miner_blocktemplate = true);
+
+     /**
+      * @brief handles an incoming block as part of a batch
+      *
+      * periodic update to checkpoints is triggered here
+      * Attempts to add the block to the Blockchain and, on success,
+      * optionally updates the miner's block template.
+      *
+      * Prerequisite: There must be an active write transaction for the blockchain storage on this
+      *               thread. Typically, this is done by calling prepare_handle_incoming_blocks() on
+      *               this thread before calls to handle_incoming_block(). Then, after calls to
+      *               handle_incoming_block(), a call to cleanup_handle_incoming_blocks() is made
+      *               on this thread to either abort or commit the write transaction.
+      *
+      * @param block_blob the block to be added
+      * @param block the block to be added, or NULL
+      * @param bvc return-by-reference metadata context about the block's validity
+      * @param extra_block_txs txs belonging to this block that may not be in the mempool
+      * @param update_miner_blocktemplate whether or not to update the miner's block template
+      *
+      * @return false if loading new checkpoints fails, or the block is not
+      * added, otherwise true
+      */
+     bool handle_incoming_block(const blobdata& block_blob, const block *b,
+       block_verification_context& bvc,
+       bool update_miner_blocktemplate = true);
+
+     bool handle_incoming_block(const blobdata& block_blob, const block *b,
+       block_verification_context& bvc, pool_supplement& extra_block_txs,
+       bool update_miner_blocktemplate = true);
 
      /**
       * @copydoc Blockchain::prepare_handle_incoming_blocks
@@ -464,13 +467,6 @@ namespace cryptonote
       */
      void set_enforce_dns_checkpoints(bool enforce_dns);
 
-     /**
-      * @brief set a listener for txes being added to the txpool
-      *
-      * @param callable to notify, or empty function to disable.
-      */
-     void set_txpool_listener(boost::function<void(std::vector<txpool_event>)> zmq_pub);
-
      /**
       * @brief set whether or not to enable or disable DNS checkpoints
       *
@@ -664,13 +660,6 @@ namespace cryptonote
       */
      const Blockchain& get_blockchain_storage()const{return m_blockchain_storage;}
 
-     /**
-      * @brief gets addy
-      *
-      * @note get addy
-      */
-     std::string get_addy() const;
-
      /**
       * @copydoc tx_memory_pool::print_pool
       *
@@ -834,13 +823,6 @@ namespace cryptonote
       */
      bool is_update_available() const { return m_update_available; }
 
-     /**
-      * @brief get whether fluffy blocks are enabled
-      *
-      * @return whether fluffy blocks are enabled
-      */
-     bool fluffy_blocks_enabled() const { return m_fluffy_blocks_enabled; }
-
      /**
       * @brief check a set of hashes against the precompiled hash set
       *
@@ -904,11 +886,6 @@ namespace cryptonote
       */
      bool has_block_weights(uint64_t height, uint64_t nblocks) const;
 
-     /**
-      * @brief flushes the bad txs cache
-      */
-     void flush_bad_txs_cache();
-
      /**
       * @brief flushes the invalid block cache
       */
@@ -923,6 +900,55 @@ namespace cryptonote
       */
      bool get_txpool_complement(const std::vector<crypto::hash> &hashes, std::vector<cryptonote::blobdata> &txes);
 
+     /**
+      * @brief validates some simple properties of a transaction
+      *
+      * Currently checks: tx has inputs,
+      *                   tx inputs all of supported type(s),
+      *                   tx outputs valid (type, key, amount),
+      *                   input and output total amounts don't overflow,
+      *                   output amount <= input amount,
+      *                   tx not too large,
+      *                   each input has a different key image.
+      *
+      * @param tx the transaction to check
+      * @param tvc tx verification context where extra fail flags are stored
+      * @param hf_version hard fork version
+      *
+      * @return true if all the checks pass, otherwise false
+      */
+     static bool check_tx_semantic(const transaction& tx, tx_verification_context& tvc,
+      uint8_t hf_version);
+
+     /**
+      * @brief verify that each input key image in a transaction is unique
+      *
+      * @param tx the transaction to check
+      *
+      * @return false if any key image is repeated, otherwise true
+      */
+     static bool check_tx_inputs_keyimages_diff(const transaction& tx);
+
+     /**
+      * @brief verify that each ring uses distinct members
+      *
+      * @param tx the transaction to check
+      * @param hf_version the hard fork version rules to use
+      *
+      * @return false if any ring uses duplicate members, true otherwise
+      */
+     static bool check_tx_inputs_ring_members_diff(const transaction& tx, const uint8_t hf_version);
+
+     /**
+      * @brief verify that each input key image in a transaction is in
+      * the valid domain
+      *
+      * @param tx the transaction to check
+      *
+      * @return false if any key image is not in the valid domain, otherwise true
+      */
+     static bool check_tx_inputs_keyimages_domain(const transaction& tx);
+
    private:
 
      /**
@@ -958,7 +984,8 @@ namespace cryptonote
       *
       * @note see Blockchain::add_new_block
       */
-     bool add_new_block(const block& b, block_verification_context& bvc);
+     bool add_new_block(const block& b, block_verification_context& bvc,
+       pool_supplement& extra_block_txs);
 
      /**
       * @brief load any core state stored on disk
@@ -969,49 +996,6 @@ namespace cryptonote
       */
      bool load_state_data();
 
-     /**
-      * @copydoc parse_tx_from_blob(transaction&, crypto::hash&, crypto::hash&, const blobdata&) const
-      *
-      * @note see parse_tx_from_blob(transaction&, crypto::hash&, crypto::hash&, const blobdata&) const
-      */
-     bool parse_tx_from_blob(transaction& tx, crypto::hash& tx_hash, const blobdata& blob) const;
-
-     /**
-      * @brief check a transaction's syntax
-      *
-      * For now this does nothing, but it may check something about the tx
-      * in the future.
-      *
-      * @param tx the transaction to check
-      *
-      * @return true
-      */
-     bool check_tx_syntax(const transaction& tx) const;
-
-     /**
-      * @brief validates some simple properties of a transaction
-      *
-      * Currently checks: tx has inputs,
-      *                   tx inputs all of supported type(s),
-      *                   tx outputs valid (type, key, amount),
-      *                   input and output total amounts don't overflow,
-      *                   output amount <= input amount,
-      *                   tx not too large,
-      *                   each input has a different key image.
-      *
-      * @param tx the transaction to check
-      * @param keeped_by_block if the transaction has been in a block
-      *
-      * @return true if all the checks pass, otherwise false
-      */
-     bool check_tx_semantic(const transaction& tx, bool keeped_by_block) const;
-     void set_semantics_failed(const crypto::hash &tx_hash);
-
-     bool handle_incoming_tx_pre(const tx_blob_entry& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash);
-     bool handle_incoming_tx_post(const tx_blob_entry& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash);
-     struct tx_verification_batch_info { const cryptonote::transaction *tx; crypto::hash tx_hash; tx_verification_context &tvc; bool &result; };
-     bool handle_incoming_tx_accumulated_batch(std::vector<tx_verification_batch_info> &tx_info, bool keeped_by_block);
-
      /**
       * @copydoc miner::on_block_chain_update
       *
@@ -1030,35 +1014,6 @@ namespace cryptonote
       */
      bool handle_command_line(const boost::program_options::variables_map& vm);
 
-     /**
-      * @brief verify that each input key image in a transaction is unique
-      *
-      * @param tx the transaction to check
-      *
-      * @return false if any key image is repeated, otherwise true
-      */
-     bool check_tx_inputs_keyimages_diff(const transaction& tx) const;
-
-     /**
-      * @brief verify that each ring uses distinct members
-      *
-      * @param tx the transaction to check
-      * @param hf_version the hard fork version rules to use
-      *
-      * @return false if any ring uses duplicate members, true otherwise
-      */
-     bool check_tx_inputs_ring_members_diff(const transaction& tx, const uint8_t hf_version) const;
-
-     /**
-      * @brief verify that each input key image in a transaction is in
-      * the valid domain
-      *
-      * @param tx the transaction to check
-      *
-      * @return false if any key image is not in the valid domain, otherwise true
-      */
-     bool check_tx_inputs_keyimages_domain(const transaction& tx) const;
-
      /**
       * @brief attempts to relay any transactions in the mempool which need it
       *
@@ -1146,9 +1101,6 @@ namespace cryptonote
 
      time_t start_time;
 
-     std::unordered_set<crypto::hash> bad_semantics_txes[2];
-     boost::mutex bad_semantics_txes_lock;
-
      enum {
        UPDATES_DISABLED,
        UPDATES_NOTIFY,
@@ -1160,15 +1112,9 @@ namespace cryptonote
      size_t m_last_update_length;
      boost::mutex m_update_mutex;
 
-     bool m_fluffy_blocks_enabled;
      bool m_offline;
 
-    /* `boost::function` is used because the implementation never allocates if
-       the callable object has a single `std::shared_ptr` or `std::weap_ptr`
-       internally. Whereas, the libstdc++ `std::function` will allocate. */
-
      std::shared_ptr<tools::Notify> m_block_rate_notify;
-     boost::function<void(std::vector<txpool_event>)> m_zmq_pub;
    };
 }
 

src/cryptonote_core/tx_pool.cpp

@@ -36,6 +36,7 @@
 #include "tx_pool.h"
 #include "cryptonote_tx_utils.h"
 #include "cryptonote_basic/cryptonote_boost_serialization.h"
+#include "cryptonote_basic/events.h"
 #include "cryptonote_config.h"
 #include "blockchain.h"
 #include "blockchain_db/locked_txn.h"
@@ -43,6 +44,8 @@
 #include "common/boost_serialization_helper.h"
 #include "int-util.h"
 #include "misc_language.h"
+#include "misc_log_ex.h"
+#include "tx_verification_utils.h"
 #include "warnings.h"
 #include "common/perf_timer.h"
 #include "crypto/hash.h"
@@ -110,15 +113,6 @@ namespace cryptonote
       return amount * ACCEPT_THRESHOLD;
     }
 
-    uint64_t get_transaction_weight_limit(uint8_t version)
-    {
-      // from v12, limit a tx to 50% of the minimum block weight
-      if (version >= 12)
-        return get_min_block_weight(version) / 2 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE;
-      else
-        return get_min_block_weight(version) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE;
-    }
-
     // external lock must be held for the comparison+set to work properly
     void set_if_less(std::atomic<time_t>& next_check, const time_t candidate) noexcept
     {
@@ -141,7 +135,10 @@ namespace cryptonote
     // corresponding lists.
   }
   //---------------------------------------------------------------------------------
-  bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, relay_method tx_relay, bool relayed, uint8_t version)
+  bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/
+    const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight,
+    tx_verification_context& tvc, relay_method tx_relay, bool relayed,
+    uint8_t version, uint8_t nic_verified_hf_version)
   {
     const bool kept_by_block = (tx_relay == relay_method::block);
 
@@ -149,13 +146,6 @@ namespace cryptonote
     CRITICAL_REGION_LOCAL(m_transactions_lock);
 
     PERF_TIMER(add_tx);
-    if (tx.version == 0)
-    {
-      // v0 never accepted
-      LOG_PRINT_L1("transaction version 0 is invalid");
-      tvc.m_verifivation_failed = true;
-      return false;
-    }
 
     // we do not accept transactions that timed out before, unless they're
     // kept_by_block
@@ -167,49 +157,24 @@ namespace cryptonote
       return false;
     }
 
-    if(!check_inputs_types_supported(tx))
+    if (version != nic_verified_hf_version && !cryptonote::ver_non_input_consensus(tx, tvc, version))
     {
-      tvc.m_verifivation_failed = true;
-      tvc.m_invalid_input = true;
+      LOG_PRINT_L1("transaction " << id << " failed non-input consensus rule checks");
+      tvc.m_verifivation_failed = true; // should already be set, but just in case
       return false;
     }
 
-    // fee per kilobyte, size rounded up.
     uint64_t fee;
-
-    if (tx.version == 1)
+    bool fee_good = false;
+    try
     {
-      uint64_t inputs_amount = 0;
-      if(!get_inputs_money_amount(tx, inputs_amount))
-      {
-        tvc.m_verifivation_failed = true;
-        return false;
-      }
-
-      uint64_t outputs_amount = get_outs_money_amount(tx);
-      if(outputs_amount > inputs_amount)
-      {
-        LOG_PRINT_L1("transaction use more money than it has: use " << print_money(outputs_amount) << ", have " << print_money(inputs_amount));
-        tvc.m_verifivation_failed = true;
-        tvc.m_overspend = true;
-        return false;
-      }
-      else if(outputs_amount == inputs_amount)
-      {
-        LOG_PRINT_L1("transaction fee is zero: outputs_amount == inputs_amount, rejecting.");
-        tvc.m_verifivation_failed = true;
-        tvc.m_fee_too_low = true;
-        return false;
-      }
-
-      fee = inputs_amount - outputs_amount;
+      // get_tx_fee() can throw. It shouldn't throw because we check preconditions in
+      // ver_non_input_consensus(), but let's put it in a try block just in case.
+      fee = get_tx_fee(tx);
+      fee_good = kept_by_block || m_blockchain.check_fee(tx_weight, fee);
     }
-    else
-    {
-      fee = tx.rct_signatures.txnFee;
-    }
-
-    if (!kept_by_block && !m_blockchain.check_fee(tx_weight, fee))
+    catch(...) {}
+    if (!fee_good) // if fee calculation failed or fee in relayed tx is too low...
     {
       tvc.m_verifivation_failed = true;
       tvc.m_fee_too_low = true;
@@ -217,15 +182,6 @@ namespace cryptonote
       return false;
     }
 
-    size_t tx_weight_limit = get_transaction_weight_limit(version);
-    if ((!kept_by_block || version >= HF_VERSION_PER_BYTE_FEE) && tx_weight > tx_weight_limit)
-    {
-      LOG_PRINT_L1("transaction is too heavy: " << tx_weight << " bytes, maximum weight: " << tx_weight_limit);
-      tvc.m_verifivation_failed = true;
-      tvc.m_too_big = true;
-      return false;
-    }
-
     size_t tx_extra_size = tx.extra.size();
     if (!kept_by_block && tx_extra_size > MAX_TX_EXTRA_SIZE)
     {
@@ -261,14 +217,6 @@ namespace cryptonote
       }
     }
 
-    if (!m_blockchain.check_tx_outputs(tx, tvc))
-    {
-      LOG_PRINT_L1("Transaction with id= "<< id << " has at least one invalid output");
-      tvc.m_verifivation_failed = true;
-      tvc.m_invalid_output = true;
-      return false;
-    }
-
     // assume failure during verification steps until success is certain
     tvc.m_verifivation_failed = true;
 
@@ -382,13 +330,13 @@ namespace cryptonote
           add_tx_to_transient_lists(id, meta.fee / (double)(tx_weight ? tx_weight : 1), receive_time);
         }
         lock.commit();
+        tvc.m_added_to_pool = !existing_tx;
       }
       catch (const std::exception &e)
       {
         MERROR("internal error: error adding transaction to txpool: " << e.what());
         return false;
       }
-      tvc.m_added_to_pool = true;
 
       static_assert(unsigned(relay_method::none) == 0, "expected relay_method::none value to be zero");
       if(meta.fee > 0 && tx_relay != relay_method::forward)
@@ -407,14 +355,16 @@ namespace cryptonote
     return true;
   }
   //---------------------------------------------------------------------------------
-  bool tx_memory_pool::add_tx(transaction &tx, tx_verification_context& tvc, relay_method tx_relay, bool relayed, uint8_t version)
+  bool tx_memory_pool::add_tx(transaction &tx, tx_verification_context& tvc, relay_method tx_relay,
+    bool relayed, uint8_t version, uint8_t nic_verified_hf_version)
   {
     crypto::hash h = null_hash;
     cryptonote::blobdata bl;
     t_serializable_object_to_blob(tx, bl);
     if (bl.size() == 0 || !get_transaction_hash(tx, h))
       return false;
-    return add_tx(tx, h, bl, get_transaction_weight(tx, bl.size()), tvc, tx_relay, relayed, version);
+    return add_tx(tx, h, bl, get_transaction_weight(tx, bl.size()), tvc, tx_relay, relayed, version,
+      nic_verified_hf_version);
   }
   //---------------------------------------------------------------------------------
   size_t tx_memory_pool::get_txpool_weight() const
@@ -580,7 +530,7 @@ namespace cryptonote
     return true;
   }
   //---------------------------------------------------------------------------------
-  bool tx_memory_pool::take_tx(const crypto::hash &id, transaction &tx, cryptonote::blobdata &txblob, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen, bool &pruned)
+  bool tx_memory_pool::take_tx(const crypto::hash &id, transaction &tx, cryptonote::blobdata &txblob, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen, bool &pruned, const bool suppress_missing_msgs)
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
     CRITICAL_REGION_LOCAL1(m_blockchain);
@@ -592,7 +542,10 @@ namespace cryptonote
       txpool_tx_meta_t meta;
       if (!m_blockchain.get_txpool_tx_meta(id, meta))
       {
-        MERROR("Failed to find tx_meta in txpool");
+        if (!suppress_missing_msgs)
+        {
+          MERROR("Failed to find tx_meta in txpool");
+        }
         return false;
       }
       txblob = m_blockchain.get_txpool_tx_blob(id, relay_category::all);
@@ -1466,44 +1419,21 @@ namespace cryptonote
       bool parsed;
     } lazy_tx(txblob, txid, tx);
 
-    //not the best implementation at this time, sorry :(
-    //check is ring_signature already checked ?
-    if(txd.max_used_block_id == null_hash)
-    {//not checked, lets try to check
+    const std::uint64_t top_block_height{m_blockchain.get_current_blockchain_height() - 1};
+    const crypto::hash top_block_hash{m_blockchain.get_block_id_by_height(top_block_height)};
 
-      if(txd.last_failed_id != null_hash && m_blockchain.get_current_blockchain_height() > txd.last_failed_height && txd.last_failed_id == m_blockchain.get_block_id_by_height(txd.last_failed_height))
-        return false;//we already sure that this tx is broken for this height
+    if (txd.last_failed_id == top_block_hash)
+      return false; // we are already sure that this tx isn't passing for this exact chain
 
-      tx_verification_context tvc;
-      if(!check_tx_inputs([&lazy_tx]()->cryptonote::transaction&{ return lazy_tx(); }, txid, txd.max_used_block_height, txd.max_used_block_id, tvc))
-      {
-        txd.last_failed_height = m_blockchain.get_current_blockchain_height()-1;
-        txd.last_failed_id = m_blockchain.get_block_id_by_height(txd.last_failed_height);
-        return false;
-      }
-    }else
+    tx_verification_context tvc{};
+    if (!check_tx_inputs([&lazy_tx]()->cryptonote::transaction&{ return lazy_tx(); },
+      txid,
+      txd.max_used_block_height,
+      txd.max_used_block_id,
+      tvc))
     {
-      if(txd.max_used_block_height >= m_blockchain.get_current_blockchain_height())
-        return false;
-      if(true)
-      {
-        //if we already failed on this height and id, skip actual ring signature check
-        if(txd.last_failed_id == m_blockchain.get_block_id_by_height(txd.last_failed_height))
-          return false;
-        //check ring signature again, it is possible (with very small chance) that this transaction become again valid
-        tx_verification_context tvc;
-        if(!check_tx_inputs([&lazy_tx]()->cryptonote::transaction&{ return lazy_tx(); }, txid, txd.max_used_block_height, txd.max_used_block_id, tvc))
-        {
-          txd.last_failed_height = m_blockchain.get_current_blockchain_height()-1;
-          txd.last_failed_id = m_blockchain.get_block_id_by_height(txd.last_failed_height);
-          return false;
-        }
-      }
-    }
-    //if we here, transaction seems valid, but, anyway, check for key_images collisions with blockchain, just to be sure
-    if(m_blockchain.have_tx_keyimges_as_spent(lazy_tx()))
-    {
-      txd.double_spend_seen = true;
+      txd.last_failed_height = top_block_height;
+      txd.last_failed_id = top_block_hash;
       return false;
     }
 

src/cryptonote_core/tx_pool.h

@@ -113,7 +113,9 @@ namespace cryptonote
      * @tx_relay how the transaction was received
      * @param tx_weight the transaction's weight
      */
-    bool add_tx(transaction &tx, const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, relay_method tx_relay, bool relayed, uint8_t version);
+    bool add_tx(transaction &tx, const crypto::hash &id, const cryptonote::blobdata &blob,
+      size_t tx_weight, tx_verification_context& tvc, relay_method tx_relay, bool relayed,
+      uint8_t version, uint8_t nic_verified_hf_version = 0);
 
     /**
      * @brief add a transaction to the transaction pool
@@ -128,10 +130,18 @@ namespace cryptonote
      * @tx_relay how the transaction was received
      * @param relayed was this transaction from the network or a local client?
      * @param version the version used to create the transaction
+     * @param nic_verified_hf_version hard fork which "tx" is known to pass non-input consensus test
+     *
+     * If "nic_verified_hf_version" parameter is equal to "version" parameter, then we skip the
+     * asserting `ver_non_input_consensus(tx)`, which greatly speeds up block popping and returning
+     * txs to mempool for txs which we know will pass the test. If nothing is known about how "tx"
+     * passes the non-input consensus tests (e.g. for newly received relayed txs), then leave
+     * "nic_verified_hf_version" as its default value of 0 (there is no v0 fork).
      *
      * @return true if the transaction passes validations, otherwise false
      */
-    bool add_tx(transaction &tx, tx_verification_context& tvc, relay_method tx_relay, bool relayed, uint8_t version);
+    bool add_tx(transaction &tx, tx_verification_context& tvc, relay_method tx_relay, bool relayed,
+      uint8_t version, uint8_t nic_verified_hf_version = 0);
 
     /**
      * @brief takes a transaction with the given hash from the pool
@@ -145,10 +155,11 @@ namespace cryptonote
      * @param do_not_relay return-by-reference is transaction not to be relayed to the network?
      * @param double_spend_seen return-by-reference was a double spend seen for that transaction?
      * @param pruned return-by-reference is the tx pruned
+     * @param suppress_missing_msgs suppress warning msgs when txid is missing (optional, defaults to `false`)
      *
      * @return true unless the transaction cannot be found in the pool
      */
-    bool take_tx(const crypto::hash &id, transaction &tx, cryptonote::blobdata &txblob, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen, bool &pruned);
+    bool take_tx(const crypto::hash &id, transaction &tx, cryptonote::blobdata &txblob, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen, bool &pruned, bool suppress_missing_msgs = false);
 
     /**
      * @brief checks if the pool has a transaction with the given hash

src/cryptonote_core/tx_verification_utils.cpp

@@ -26,8 +26,12 @@
 // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+#include <boost/iterator/transform_iterator.hpp>
+
 #include "cryptonote_core/blockchain.h"
+#include "cryptonote_core/cryptonote_core.h"
 #include "cryptonote_core/tx_verification_utils.h"
+#include "hardforks/hardforks.h"
 #include "ringct/rctSigs.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
@@ -105,11 +109,104 @@ static crypto::hash calc_tx_mixring_hash(const transaction& tx, const rct::ctkey
     return tx_and_mixring_hash;
 }
 
+static bool is_canonical_bulletproof_layout(const std::vector<rct::Bulletproof> &proofs)
+{
+    if (proofs.size() != 1)
+        return false;
+    const size_t sz = proofs[0].V.size();
+    if (sz == 0 || sz > BULLETPROOF_MAX_OUTPUTS)
+        return false;
+    return true;
+}
+
+static bool is_canonical_bulletproof_plus_layout(const std::vector<rct::BulletproofPlus> &proofs)
+{
+    if (proofs.size() != 1)
+        return false;
+    const size_t sz = proofs[0].V.size();
+    if (sz == 0 || sz > BULLETPROOF_PLUS_MAX_OUTPUTS)
+        return false;
+    return true;
+}
+
+template <class TxForwardIt>
+static bool ver_non_input_consensus_templated(TxForwardIt tx_begin, TxForwardIt tx_end,
+        tx_verification_context& tvc, std::uint8_t hf_version)
+{
+    std::vector<const rct::rctSig*> rvv;
+    rvv.reserve(static_cast<size_t>(std::distance(tx_begin, tx_end)));
+
+    const size_t max_tx_version = hf_version < HF_VERSION_DYNAMIC_FEE ? 1 : 2;
+
+    const size_t tx_weight_limit = get_transaction_weight_limit(hf_version);
+
+    for (; tx_begin != tx_end; ++tx_begin)
+    {
+        const transaction& tx = *tx_begin;
+        const uint64_t blob_size = get_transaction_blob_size(tx);
+
+        // Rule 1
+        if (blob_size > get_max_tx_size())
+        {
+            tvc.m_verifivation_failed = true;
+            tvc.m_too_big = true;
+            return false;
+        }
+
+        // Rule 2 & 3
+        if (tx.version == 0 || tx.version > max_tx_version)
+        {
+            tvc.m_verifivation_failed = true;
+            return false;
+        }
+
+        // Rule 4
+        const size_t tx_weight = get_transaction_weight(tx, blob_size);
+        if (hf_version >= HF_VERSION_PER_BYTE_FEE && tx_weight > tx_weight_limit)
+        {
+            tvc.m_verifivation_failed = true;
+            tvc.m_too_big = true;
+            return false;
+        }
+
+        // Rule 5
+        if (!core::check_tx_semantic(tx, tvc, hf_version))
+            return false;
+
+        // Rule 6
+        if (!Blockchain::check_tx_outputs(tx, tvc, hf_version) || tvc.m_verifivation_failed)
+            return false;
+
+        // We only want to check RingCT semantics if this is actually a RingCT transaction
+        if (tx.version >= 2)
+            rvv.push_back(&tx.rct_signatures);
+    }
+
+    // Rule 7
+    if (!ver_mixed_rct_semantics(std::move(rvv)))
+    {
+        tvc.m_verifivation_failed = true;
+        tvc.m_invalid_input = true;
+        return false;
+    }
+
+    return true;
+}
+
 ////////////////////////////////////////////////////////////////////////////////////////////////////
 
 namespace cryptonote
 {
 
+uint64_t get_transaction_weight_limit(const uint8_t hf_version)
+{
+    // from v8, limit a tx to 50% of the minimum block weight
+    if (hf_version >= HF_VERSION_PER_BYTE_FEE)
+        return get_min_block_weight(hf_version) / 2 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE;
+    else
+        return get_min_block_weight(hf_version) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE;
+}
+
 bool ver_rct_non_semantics_simple_cached
 (
     transaction& tx,
@@ -129,7 +226,7 @@ bool ver_rct_non_semantics_simple_cached
     // mixring. Future versions of the protocol may differ in this regard, but if this assumptions
     // holds true in the future, enable the verification hash by modifying the `untested_tx`
     // condition below.
-    const bool untested_tx = tx.version > 2 || tx.rct_signatures.type > rct::RCTTypeBulletproofPlus;
+    const bool untested_tx = (tx.version > 2) || (static_cast<std::uint8_t>(tx.rct_signatures.type) > rct_type_to_cache);
     VER_ASSERT(!untested_tx, "Unknown TX type. Make sure RCT cache works correctly with this type and then enable it in the code here.");
 
     // Don't cache older (or newer) rctSig types
@@ -164,4 +261,105 @@ bool ver_rct_non_semantics_simple_cached
     return true;
 }
 
+bool ver_mixed_rct_semantics(std::vector<const rct::rctSig*> rvv)
+{
+    size_t batch_rv_size = 0; // this acts as an "end" iterator to the last simple batchable sig ptr
+    for (size_t i = 0; i < rvv.size(); ++i)
+    {
+        const rct::rctSig& rv = *rvv[i];
+
+        bool is_batchable_rv = false;
+
+        switch (rv.type)
+        {
+        case rct::RCTTypeNull:
+            // coinbase should not come here, so we reject for all other types
+            MERROR("Unexpected Null rctSig type");
+            return false;
+            break;
+        case rct::RCTTypeSimple:
+            if (!rct::verRctSemanticsSimple(rv))
+            {
+                MERROR("rct signature semantics check failed: type simple");
+                return false;
+            }
+            break;
+        case rct::RCTTypeFull:
+            if (!rct::verRct(rv, /*semantics=*/true))
+            {
+                MERROR("rct signature semantics check failed: type full");
+                return false;
+            }
+            break;
+        case rct::RCTTypeBulletproof:
+        case rct::RCTTypeBulletproof2:
+        case rct::RCTTypeCLSAG:
+            if (!is_canonical_bulletproof_layout(rv.p.bulletproofs))
+            {
+                MERROR("Bulletproof does not have canonical form");
+                return false;
+            }
+            is_batchable_rv = true;
+            break;
+        case rct::RCTTypeBulletproofPlus:
+        case rct::RCTTypeBulletproofPlus_FullCommit:
+            if (!is_canonical_bulletproof_plus_layout(rv.p.bulletproofs_plus))
+            {
+                MERROR("Bulletproof_plus does not have canonical form");
+                return false;
+            }
+            is_batchable_rv = true;
+            break;
+        default:
+            MERROR("Unknown rct type: " << rv.type);
+            return false;
+            break;
+        }
+
+        // Save this ring sig for later, as we will attempt simple RCT semantics batch verification
+        if (is_batchable_rv)
+            rvv[batch_rv_size++] = rvv[i];
+    }
+
+    if (batch_rv_size) // if any simple, batchable ring sigs...
+    {
+        rvv.resize(batch_rv_size);
+        if (!rct::verRctSemanticsSimple(rvv))
+        {
+            MERROR("rct signature semantics check failed: simple-style batch verification failed");
+            return false;
+        }
+    }
+
+    return true;
+}
+
+bool ver_non_input_consensus(const transaction& tx, tx_verification_context& tvc,
+    std::uint8_t hf_version)
+{
+    return ver_non_input_consensus_templated(&tx, &tx + 1, tvc, hf_version);
+}
+
+bool ver_non_input_consensus(const pool_supplement& ps, tx_verification_context& tvc,
+    const std::uint8_t hf_version)
+{
+    // We already verified the pool supplement for this hard fork version! Yippee!
+    if (ps.nic_verified_hf_version == hf_version)
+        return true;
+
+    const auto it_transform = [] (const decltype(ps.txs_by_txid)::value_type& in)
+        -> const transaction& { return in.second.first; };
+    const auto tx_begin = boost::make_transform_iterator(ps.txs_by_txid.cbegin(), it_transform);
+    const auto tx_end = boost::make_transform_iterator(ps.txs_by_txid.cend(), it_transform);
+
+    // Perform the checks...
+    const bool verified = ver_non_input_consensus_templated(tx_begin, tx_end, tvc, hf_version);
+
+    // Cache the hard fork version on success
+    if (verified)
+        ps.nic_verified_hf_version = hf_version;
+
+    return verified;
+}
+
 } // namespace cryptonote

src/cryptonote_core/tx_verification_utils.h

@@ -30,10 +30,19 @@
 
 #include "common/data_cache.h"
 #include "cryptonote_basic/cryptonote_basic.h"
+#include "cryptonote_basic/verification_context.h"
 
 namespace cryptonote
 {
 
+/**
+ * @brief Get the maximum transaction weight for a given hardfork
+ *
+ * @param hf_version hard fork version
+ * @return the maximum unconditional transaction weight
+ */
+uint64_t get_transaction_weight_limit(uint8_t hf_version);
+
 // Modifying this value should not affect consensus. You can adjust it for performance needs
 static constexpr const size_t RCT_VER_CACHE_SIZE = 8192;
 
@@ -75,4 +84,57 @@ bool ver_rct_non_semantics_simple_cached
     std::uint8_t rct_type_to_cache
 );
 
+/**
+ * @brief Verify the semantics of a group of RingCT signatures as a batch (if applicable)
+ *
+ * Coinbase txs or other transaction with a RingCT type of RCTTypeNull will fail to verify.
+ *
+ * @param rvv list of signatures to verify
+ * @return true if all signatures verified semantics successfully, false otherwise
+ */
+bool ver_mixed_rct_semantics(std::vector<const rct::rctSig*> rvv);
+
+/**
+ * @brief Used to provide transaction info that skips the mempool to block handling code 
+ */
+struct pool_supplement
+{
+    // Map of supplemental tx info that we might need to validate a block
+    // Maps TXID -> transaction and blob
+    std::unordered_map<crypto::hash, std::pair<transaction, blobdata>> txs_by_txid;
+    // If non-zero, then consider all the txs' non-input consensus (NIC) rules verified for this
+    // hard fork. User: If you add an unverified transaction to txs_by_txid, set this field to zero!
+    mutable std::uint8_t nic_verified_hf_version = 0;
+};
+
+/**
+ * @brief Verify every non-input consensus rule for a group of non-coinbase transactions
+ *
+ * List of checks that we do for each transaction:
+ *     1. Check tx blob size < get_max_tx_size()
+ *     2. Check tx version != 0
+ *     3. Check tx version is less than maximum for given hard fork version
+ *     4. Check tx weight < get_transaction_weight_limit()
+ *     5. Passes core::check_tx_semantic()
+ *     6. Passes Blockchain::check_tx_outputs()
+ *     7. Passes ver_mixed_rct_semantics() [Uses batch RingCT verification when applicable]
+ *
+ * For pool_supplement input:
+ * We assume the structure of the pool supplement is already correct: for each value entry, the
+ * cryptonote::transaction matches its corresponding blobdata and the TXID map key is correctly
+ * calculated for that transaction. We use the .nic_verified_hf_version field to skip verification
+ * for the pool supplement if hf_version matches, and we cache that version on success.
+ *
+ * @param tx single transaction to verify
+ * @param pool_supplement pool supplement to verify
+ * @param tvc relevant flags will be set for if/why verification failed
+ * @param hf_version Hard fork version to run rules against
+ * @return true if all relevant transactions verify, false otherwise
+ */
+bool ver_non_input_consensus(const transaction& tx, tx_verification_context& tvc,
+    std::uint8_t hf_version);
+
+bool ver_non_input_consensus(const pool_supplement& ps, tx_verification_context& tvc,
+    std::uint8_t hf_version);
+
 } // namespace cryptonote

src/cryptonote_protocol/CMakeLists.txt

@@ -26,9 +26,6 @@
 # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
 # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-cmake_minimum_required (VERSION 3.5)
-project (monero CXX)
-
 file(GLOB CRYPTONOTE_PROTOCOL *)
 source_group(cryptonote_protocol FILES ${CRYPTONOTE_PROTOCOL})
 

src/cryptonote_protocol/block_queue.cpp

@@ -51,10 +51,10 @@ void block_queue::add_blocks(uint64_t height, std::vector<cryptonote::block_comp
   blocks.insert(span(height, std::move(bcel), connection_id, addr, rate, size));
   if (has_hashes)
   {
-    for (const crypto::hash &h: hashes)
+    for (std::size_t i = 0; i < hashes.size(); ++i)
     {
-      requested_hashes.insert(h);
-      have_blocks.insert(h);
+      requested_hashes.insert(hashes[i]);
+      have_blocks.emplace(hashes[i], height + i);
     }
     set_span_hashes(height, connection_id, hashes);
   }
@@ -219,6 +219,16 @@ bool block_queue::have(const crypto::hash &hash) const
   return have_blocks.find(hash) != have_blocks.end();
 }
 
+std::uint64_t block_queue::have_height(const crypto::hash &hash) const
+{
+  boost::unique_lock<boost::recursive_mutex> lock(mutex);
+  const auto elem = have_blocks.find(hash);
+  if (elem == have_blocks.end())
+    return std::numeric_limits<std::uint64_t>::max();
+  return elem->second;
+}
+
+
 std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const epee::net_utils::network_address &addr, bool sync_pruned_blocks, uint32_t local_pruning_seed, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<std::pair<crypto::hash, uint64_t>> &block_hashes, boost::posix_time::ptime time)
 {
   boost::unique_lock<boost::recursive_mutex> lock(mutex);

src/cryptonote_protocol/block_queue.h

@@ -98,6 +98,7 @@ namespace cryptonote
     bool foreach(std::function<bool(const span&)> f) const;
     bool requested(const crypto::hash &hash) const;
     bool have(const crypto::hash &hash) const;
+    std::uint64_t have_height(const crypto::hash &hash) const;
 
   private:
     void erase_block(block_map::iterator j);
@@ -107,6 +108,6 @@ namespace cryptonote
     block_map blocks;
     mutable boost::recursive_mutex mutex;
     std::unordered_set<crypto::hash> requested_hashes;
-    std::unordered_set<crypto::hash> have_blocks;
+    std::unordered_map<crypto::hash, std::uint64_t> have_blocks;
   };
 }

src/cryptonote_protocol/cryptonote_protocol_handler.h

@@ -145,7 +145,7 @@ namespace cryptonote
     int handle_notify_get_txpool_complement(int command, NOTIFY_GET_TXPOOL_COMPLEMENT::request& arg, cryptonote_connection_context& context);
 		
     //----------------- i_bc_protocol_layout ---------------------------------------
-    virtual bool relay_block(NOTIFY_NEW_BLOCK::request& arg, cryptonote_connection_context& exclude_context);
+    virtual bool relay_block(NOTIFY_NEW_FLUFFY_BLOCK::request& arg, cryptonote_connection_context& exclude_context);
     virtual bool relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, const boost::uuids::uuid& source, epee::net_utils::zone zone, relay_method tx_relay);
     //----------------------------------------------------------------------------------
     //bool get_payload_sync_data(HANDSHAKE_DATA::request& hshd, cryptonote_connection_context& context);
@@ -157,6 +157,7 @@ namespace cryptonote
     bool should_ask_for_pruned_data(cryptonote_connection_context& context, uint64_t first_block_height, uint64_t nblocks, bool check_block_weights) const;
     void drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans);
     void drop_connection_with_score(cryptonote_connection_context &context, unsigned int score, bool flush_all_spans);
+    void drop_connection(const boost::uuids::uuid&);
     void drop_connections(const epee::net_utils::network_address address);
     bool kick_idle_peers();
     bool check_standby_peers();

src/cryptonote_protocol/cryptonote_protocol_handler_common.h

@@ -41,7 +41,7 @@ namespace cryptonote
   struct i_cryptonote_protocol
   {
     virtual bool is_synchronized() const = 0;
-    virtual bool relay_block(NOTIFY_NEW_BLOCK::request& arg, cryptonote_connection_context& exclude_context)=0;
+    virtual bool relay_block(NOTIFY_NEW_FLUFFY_BLOCK::request& arg, cryptonote_connection_context& exclude_context)=0;
     virtual bool relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, const boost::uuids::uuid& source, epee::net_utils::zone zone, relay_method tx_relay)=0;
     //virtual bool request_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, cryptonote_connection_context& context)=0;
   };
@@ -55,7 +55,7 @@ namespace cryptonote
     {
       return false;
     }
-    virtual bool relay_block(NOTIFY_NEW_BLOCK::request& arg, cryptonote_connection_context& exclude_context)
+    virtual bool relay_block(NOTIFY_NEW_FLUFFY_BLOCK::request& arg, cryptonote_connection_context& exclude_context)
     {
       return false;
     }

src/cryptonote_protocol/levin_notify.cpp

@@ -28,6 +28,8 @@
 
 #include "levin_notify.h"
 
+#include <boost/asio/dispatch.hpp>
+#include <boost/asio/post.hpp>
 #include <boost/asio/steady_timer.hpp>
 #include <boost/system/system_error.hpp>
 #include <boost/uuid/uuid_io.hpp>
@@ -221,7 +223,7 @@ namespace levin
        `dispatch` is used heavily, which means "execute immediately in _this_
        thread if the strand is not in use, otherwise queue the callback to be
        executed immediately after the strand completes its current task".
-       `post` is used where deferred execution to an `asio::io_service::run`
+       `post` is used where deferred execution to an `asio::io_context::run`
        thread is preferred.
 
        The strand per "zone" is useful because the levin
@@ -238,7 +240,7 @@ namespace levin
     //! A queue of levin messages for a noise i2p/tor link
     struct noise_channel
     {
-      explicit noise_channel(boost::asio::io_service& io_service)
+      explicit noise_channel(boost::asio::io_context& io_service)
         : active(nullptr),
           queue(),
           strand(io_service),
@@ -246,7 +248,7 @@ namespace levin
           connection(boost::uuids::nil_uuid())
       {}
 
-      // `asio::io_service::strand` cannot be copied or moved
+      // `asio::io_context::strand` cannot be copied or moved
       noise_channel(const noise_channel&) = delete;
       noise_channel& operator=(const noise_channel&) = delete;
 
@@ -254,7 +256,7 @@ namespace levin
 
       epee::byte_slice active;
       std::deque<epee::byte_slice> queue;
-      boost::asio::io_service::strand strand;
+      boost::asio::io_context::strand strand;
       boost::asio::steady_timer next_noise;
       boost::uuids::uuid connection;
     };
@@ -264,7 +266,7 @@ namespace levin
   {
     struct zone
     {
-      explicit zone(boost::asio::io_service& io_service, std::shared_ptr<connections> p2p, epee::byte_slice noise_in, epee::net_utils::zone zone, bool pad_txs)
+      explicit zone(boost::asio::io_context& io_service, std::shared_ptr<connections> p2p, epee::byte_slice noise_in, epee::net_utils::zone zone, bool pad_txs)
         : p2p(std::move(p2p)),
           noise(std::move(noise_in)),
           next_epoch(io_service),
@@ -286,7 +288,7 @@ namespace levin
       const epee::byte_slice noise; //!< `!empty()` means zone is using noise channels
       boost::asio::steady_timer next_epoch;
       boost::asio::steady_timer flush_txs;
-      boost::asio::io_service::strand strand;
+      boost::asio::io_context::strand strand;
       struct context_t {
         std::vector<cryptonote::blobdata> fluff_txs;
         std::chrono::steady_clock::time_point flush_time;
@@ -454,7 +456,7 @@ namespace levin
 
         if (next_flush == std::chrono::steady_clock::time_point::max())
           MWARNING("Unable to send transaction(s), no available connections");
-        else if (!zone->flush_callbacks || next_flush < zone->flush_txs.expires_at())
+        else if (!zone->flush_callbacks || next_flush < zone->flush_txs.expiry())
           fluff_flush::queue(std::move(zone), next_flush);
       }
     };
@@ -515,7 +517,7 @@ namespace levin
         for (auto id = zone->map.begin(); id != zone->map.end(); ++id)
         {
           const std::size_t i = id - zone->map.begin();
-          zone->channels[i].strand.post(update_channel{zone, i, *id});
+          boost::asio::post(zone->channels[i].strand, update_channel{zone, i, *id});
         }
       }
 
@@ -674,7 +676,7 @@ namespace levin
               MWARNING("Unable to send transaction(s) to " << epee::net_utils::zone_to_string(zone_->nzone) <<
 			" - no suitable outbound connections at height " << height);
 
-            zone_->strand.post(update_channels{zone_, std::move(connections)});
+            boost::asio::post(zone_->strand, update_channels{zone_, std::move(connections)});
           }
         }
 
@@ -704,7 +706,8 @@ namespace levin
         const bool fluffing = crypto::rand_idx(unsigned(100)) < CRYPTONOTE_DANDELIONPP_FLUFF_PROBABILITY;
         const auto start = std::chrono::steady_clock::now();
         auto connections = get_out_connections(*(zone_->p2p), core_);
-        zone_->strand.dispatch(
+        boost::asio::dispatch(
+          zone_->strand,
           change_channels{zone_, net::dandelionpp::connection_map{std::move(connections), count_}, fluffing}
         );
 
@@ -715,7 +718,7 @@ namespace levin
     };
   } // anonymous
 
-  notify::notify(boost::asio::io_service& service, std::shared_ptr<connections> p2p, epee::byte_slice noise, epee::net_utils::zone zone, const bool pad_txs, i_core_events& core)
+  notify::notify(boost::asio::io_context& service, std::shared_ptr<connections> p2p, epee::byte_slice noise, epee::net_utils::zone zone, const bool pad_txs, i_core_events& core)
     : zone_(std::make_shared<detail::zone>(service, std::move(p2p), std::move(noise), zone, pad_txs))
     , core_(std::addressof(core))
   {
@@ -758,7 +761,8 @@ namespace levin
     if (!zone_ || zone_->noise.empty() || CRYPTONOTE_NOISE_CHANNELS <= zone_->connection_count)
       return;
 
-    zone_->strand.dispatch(
+    boost::asio::dispatch(
+      zone_->strand,
       update_channels{zone_, get_out_connections(*(zone_->p2p), core_)}
     );
   }
@@ -769,7 +773,7 @@ namespace levin
       return;
 
     auto& zone = zone_;
-    zone_->strand.dispatch([zone, id, is_income]{
+    boost::asio::dispatch(zone_->strand, [zone, id, is_income] {
       zone->contexts[id] = {
         .fluff_txs = {},
         .flush_time = std::chrono::steady_clock::time_point::max(),
@@ -784,7 +788,7 @@ namespace levin
       return;
 
     auto& zone = zone_;
-    zone_->strand.dispatch([zone, id]{
+    boost::asio::dispatch(zone_->strand, [zone, id]{
       zone->contexts.erase(id);
     });
   }
@@ -859,7 +863,8 @@ namespace levin
 
       for (std::size_t channel = 0; channel < zone_->channels.size(); ++channel)
       {
-        zone_->channels[channel].strand.dispatch(
+        boost::asio::dispatch(
+          zone_->channels[channel].strand,
           queue_covert_notify{zone_, message.clone(), channel}
         );
       }
@@ -878,7 +883,8 @@ namespace levin
           if (zone_->nzone == epee::net_utils::zone::public_)
           {
             // this will change a local/forward tx to stem or fluff ...
-            zone_->strand.dispatch(
+            boost::asio::dispatch(
+              zone_->strand,
               dandelionpp_notify{zone_, core_, std::move(txs), source, tx_relay}
             );
             break;
@@ -891,7 +897,7 @@ namespace levin
              ipv4/6. Marking it as "fluff" here will make the tx immediately
              visible externally from this node, which is not desired. */
           core_->on_transactions_relayed(epee::to_span(txs), tx_relay);
-          zone_->strand.dispatch(fluff_notify{zone_, std::move(txs), source});
+          boost::asio::dispatch(zone_->strand, fluff_notify{zone_, std::move(txs), source});
           break;
       }
     }

src/cryptonote_protocol/levin_notify.h

@@ -28,7 +28,7 @@
 
 #pragma once
 
-#include <boost/asio/io_service.hpp>
+#include <boost/asio/io_context.hpp>
 #include <boost/uuid/uuid.hpp>
 #include <memory>
 #include <vector>
@@ -86,7 +86,7 @@ namespace levin
     {}
 
     //! Construct an instance with available notification `zones`.
-    explicit notify(boost::asio::io_service& service, std::shared_ptr<connections> p2p, epee::byte_slice noise, epee::net_utils::zone zone, bool pad_txs, i_core_events& core);
+    explicit notify(boost::asio::io_context& service, std::shared_ptr<connections> p2p, epee::byte_slice noise, epee::net_utils::zone zone, bool pad_txs, i_core_events& core);
 
     notify(const notify&) = delete;
     notify(notify&&) = default;

src/daemon/command_parser_executor.cpp

@@ -1063,7 +1063,7 @@ bool t_command_parser_executor::set_bootstrap_daemon(const std::vector<std::stri
 
 bool t_command_parser_executor::flush_cache(const std::vector<std::string>& args)
 {
-  bool bad_txs = false, bad_blocks = false;
+  bool bad_blocks = false;
   std::string arg;
 
   if (args.empty())
@@ -1072,18 +1072,16 @@ bool t_command_parser_executor::flush_cache(const std::vector<std::string>& args
   for (size_t i = 0; i < args.size(); ++i)
   {
     arg = args[i];
-    if (arg == "bad-txs")
-      bad_txs = true;
-    else if (arg == "bad-blocks")
+    if (arg == "bad-blocks")
       bad_blocks = true;
     else
       goto show_list;
   }
-  return m_executor.flush_cache(bad_txs, bad_blocks);
+  return m_executor.flush_cache(bad_blocks);
 
 show_list:
   std::cout << "Invalid cache type: " << arg << std::endl;
-  std::cout << "Cache types: bad-txs bad-blocks" << std::endl;
+  std::cout << "Cache types: bad-blocks" << std::endl;
   return true;
 }
 

src/daemon/daemon.cpp

@@ -119,9 +119,9 @@ public:
 
       if (shared)
       {
+        core.get().get_blockchain_storage().set_txpool_notify(cryptonote::listener::zmq_pub::txpool_add{shared});
         core.get().get_blockchain_storage().add_block_notify(cryptonote::listener::zmq_pub::chain_main{shared});
         core.get().get_blockchain_storage().add_miner_notify(cryptonote::listener::zmq_pub::miner_data{shared});
-        core.get().set_txpool_listener(cryptonote::listener::zmq_pub::txpool_add{shared});
       }
     }
   }

src/daemon/main.cpp

@@ -83,7 +83,7 @@ uint16_t parse_public_rpc_port(const po::variables_map &vm)
   }
 
   uint16_t rpc_port;
-  if (!string_tools::get_xtype_from_string(rpc_port, rpc_port_str))
+  if (!epee::string_tools::get_xtype_from_string(rpc_port, rpc_port_str))
   {
     throw std::runtime_error("invalid RPC port " + rpc_port_str);
   }

src/daemon/rpc_command_executor.cpp

@@ -1064,7 +1064,7 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash,
       cryptonote::blobdata blob;
       std::string source = as_hex.empty() ? pruned_as_hex + prunable_as_hex : as_hex;
       bool pruned = !pruned_as_hex.empty() && prunable_as_hex.empty();
-      if (!string_tools::parse_hexstr_to_binbuff(source, blob))
+      if (!epee::string_tools::parse_hexstr_to_binbuff(source, blob))
       {
         tools::fail_msg_writer() << "Failed to parse tx to get json format";
       }
@@ -2441,14 +2441,13 @@ bool t_rpc_command_executor::set_bootstrap_daemon(
     return true;
 }
 
-bool t_rpc_command_executor::flush_cache(bool bad_txs, bool bad_blocks)
+bool t_rpc_command_executor::flush_cache(bool bad_blocks)
 {
     cryptonote::COMMAND_RPC_FLUSH_CACHE::request req;
     cryptonote::COMMAND_RPC_FLUSH_CACHE::response res;
     std::string fail_message = "Unsuccessful";
     epee::json_rpc::error error_resp;
 
-    req.bad_txs = bad_txs;
     req.bad_blocks = bad_blocks;
 
     if (m_is_rpc)