Merge pull request 'update readme and checkpoints [CI SKIP]' (#358) from wowario/wownero:checkpoint into master

Reviewed-on: https://git.wownero.com/wownero/wownero/pulls/358

.drone.yml

@@ -7,4 +7,4 @@ steps:
 - name: linux-build
   image: wownero/wow-dependencies:v1.0
   commands:
-  - make -j2 release-static-linux-x86_64
+  - make release-static-linux-x86_64

.github/FUNDING.yml

@@ -1 +1 @@
-custom: https://dev-funding.webui.wowkira.co
+custom: https://dev-funding.webui.wowkira.com

.github/workflows/build.yml

@@ -38,10 +38,15 @@ jobs:
         submodules: recursive
     - name: remove bundled boost
       run: sudo rm -rf /usr/local/share/boost
+    - name: set apt conf
+      run: |
+        echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
+        echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
+        echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
     - name: update apt
       run: sudo apt update
     - name: install monero dependencies
-      run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev
+      run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler
     - name: build
       run: make -j3
 
@@ -53,10 +58,39 @@ jobs:
         submodules: recursive
     - name: remove bundled boost
       run: sudo rm -rf /usr/local/share/boost
+    - name: set apt conf
+      run: |
+        echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
+        echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
+        echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
     - name: update apt
       run: sudo apt update
     - name: install monero dependencies
-      run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev
+      run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler
     - name: build
       run: cmake -DBUILD_GUI_DEPS=ON && make -j3
 
+  test-ubuntu:
+    needs: build-ubuntu
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+      with:
+        submodules: recursive
+    - name: remove bundled boost
+      run: sudo rm -rf /usr/local/share/boost
+    - name: set apt conf
+      run: |
+        echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
+        echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
+        echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
+    - name: update apt
+      run: sudo apt update
+    - name: install monero dependencies
+      run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler
+    - name: install requests
+      run: pip install requests
+    - name: tests
+      env:
+        CTEST_OUTPUT_ON_FAILURE: ON
+      run: make release-test -j3

.gitignore

@@ -23,6 +23,7 @@ cscope.po.out
 external/miniupnpc/Makefile
 miniupnpcstrings.h
 version/
+ClangBuildAnalyzerSession.txt
 # Created by https://www.gitignore.io
 
 ### C++ ###

.gitmodules

@@ -13,3 +13,7 @@
 	path = external/RandomWOW
 	url = https://git.wownero.com/wownero/RandomWOW
         branch = 1.1.8-wow
+[submodule "external/supercop"]
+	path = external/supercop
+	url = https://git.wownero.com/wownero/supercop
+	branch = monero

.travis.yml

@@ -25,7 +25,7 @@ env:
 # ARM v8
     - HOST=aarch64-linux-gnu PACKAGES="python3 gperf g++-aarch64-linux-gnu"
 # i686 Win
-    - HOST=i686-w64-mingw32 DEP_OPTS="NO_QT=1" PACKAGES="python3 g++-mingw-w64-i686 qttools5-dev-tools"
+    - HOST=i686-w64-mingw32 DEP_OPTS="NO_QT=1" PACKAGES="python3 g++-mingw-w64-i686 qttools5-dev-tools" MAKEJOBS=-j2
 # i686 Linux
     - HOST=i686-pc-linux-gnu PACKAGES="gperf cmake g++-multilib python3-zmq"
 # Win64

CMakeLists.txt

@@ -54,6 +54,13 @@ if (USE_CCACHE)
 else()
 	message(STATUS "ccache deselected")
 endif()
+option (USE_COMPILATION_TIME_PROFILER "Use compilation time profiler (for CLang >= 9 only)" OFF)
+if (USE_COMPILATION_TIME_PROFILER)
+	if (NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
+		message(FATAL_ERROR "The flag USE_COMPILATION_TIME_PROFILER is meant to be set only for CLang compiler!")
+	endif()
+	add_compile_options("-ftime-trace")
+endif()
 
 # Job pool feature requires Ninja.
 if (${CMAKE_VERSION} VERSION_GREATER "3.0.0")
@@ -68,6 +75,20 @@ if (${CMAKE_VERSION} VERSION_GREATER "3.0.0")
     set_property(GLOBAL APPEND PROPERTY JOB_POOLS link_job_pool=${WOWNERO_PARALLEL_LINK_JOBS})
     set(CMAKE_JOB_POOL_LINK link_job_pool)
   endif ()
+endif ()
+  
+option (USE_CLANG_TIDY_C   "Lint the code with clang-tidy - variant C"   OFF)
+option (USE_CLANG_TIDY_CXX "Lint the code with clang-tidy - variant C++" OFF)
+if (USE_CLANG_TIDY_C AND USE_CLANG_TIDY_CXX)
+	message(FATAL_ERROR "Enabling both USE_CLANG_TIDY_C and USE_CLANG_TIDY_CXX simultaneously crashes clang-tidy.")
+endif()
+if (USE_CLANG_TIDY_C OR USE_CLANG_TIDY_CXX)
+	include(SetClangTidy)
+endif()
+if (USE_CLANG_TIDY_C)
+	monero_clang_tidy("C")
+elseif (USE_CLANG_TIDY_CXX)
+	monero_clang_tidy("CXX")
 endif()
 
 enable_language(C ASM)
@@ -126,6 +147,24 @@ function (add_definition_if_library_exists library function header var)
   endif()
 endfunction()
 
+option(RELINK_TARGETS "Relink targets, when just a dependant .so changed, but not its header?" OFF)
+function (monero_set_target_no_relink target)
+  if (RELINK_TARGETS MATCHES OFF)
+    # Will not relink the target, when just its dependant .so has changed, but not it's interface
+    set_target_properties("${target}" PROPERTIES LINK_DEPENDS_NO_SHARED true)
+  endif()
+endfunction()
+
+function (monero_add_minimal_executable name)
+  source_group("${name}"
+    FILES
+      ${ARGN})
+
+  add_executable("${name}"
+    ${ARGN})
+    monero_set_target_no_relink( ${name} )
+endfunction()
+
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
   message(STATUS "Setting default build type: ${CMAKE_BUILD_TYPE}")
@@ -233,6 +272,7 @@ if(NOT MANUAL_SUBMODULES)
     check_submodule(external/unbound)
     check_submodule(external/rapidjson)
     check_submodule(external/RandomWOW)
+    check_submodule(external/supercop)
   endif()
 endif()
 
@@ -290,6 +330,12 @@ else()
 endif()
 option(BUILD_DEBUG_UTILITIES "Build debug utilities." DEFAULT_BUILD_DEBUG_UTILITIES)
 
+if(OSSFUZZ)
+  message(STATUS "Using OSS-Fuzz fuzzing system")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DOSSFUZZ")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DOSSFUZZ")
+endif()
+
 # Check whether we're on a 32-bit or 64-bit system
 if(CMAKE_SIZEOF_VOID_P EQUAL "8")
   set(DEFAULT_BUILD_64 ON)
@@ -328,7 +374,7 @@ endif()
 # elseif(CMAKE_SYSTEM_NAME MATCHES ".*BSDI.*")
 #   set(BSDI TRUE)
 
-include_directories(external/rapidjson/include external/easylogging++ src contrib/epee/include external)
+include_directories(external/rapidjson/include external/easylogging++ src contrib/epee/include external external/supercop/include)
 
 if(APPLE)
   include_directories(SYSTEM /usr/include/malloc)
@@ -411,7 +457,7 @@ elseif(CMAKE_C_COMPILER_ID STREQUAL "GNU" AND NOT MINGW)
   set(DEFAULT_STACK_TRACE ON)
   set(STACK_TRACE_LIB "easylogging++") # for diag output only
   set(LIBUNWIND_LIBRARIES "")
-elseif (ARM AND STATIC)
+elseif (ARM)
   set(DEFAULT_STACK_TRACE OFF)
   set(LIBUNWIND_LIBRARIES "")
 else()
@@ -448,7 +494,7 @@ endif ()
 
 if (APPLE AND NOT IOS)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=x86-64 -fvisibility=default -std=c++11")
-  if (NOT OpenSSL_DIR)
+  if (NOT OPENSSL_ROOT_DIR)
       EXECUTE_PROCESS(COMMAND brew --prefix openssl
         OUTPUT_VARIABLE OPENSSL_ROOT_DIR
         OUTPUT_STRIP_TRAILING_WHITESPACE)
@@ -473,6 +519,9 @@ add_definition_if_function_found(strptime HAVE_STRPTIME)
 
 add_definitions(-DAUTO_INITIALIZE_EASYLOGGINGPP)
 
+set(MONERO_GENERATED_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated_include")
+include_directories(${MONERO_GENERATED_HEADERS_DIR})
+
 # Generate header for embedded translations
 # Generate header for embedded translations, use target toolchain if depends, otherwise use the
 # lrelease and lupdate binaries from the host
@@ -486,14 +535,6 @@ ExternalProject_Add(generate_translations_header
 include_directories("${CMAKE_CURRENT_BINARY_DIR}/translations")
 add_subdirectory(external)
 
-# Final setup for miniupnpc
-if(UPNP_STATIC OR IOS)
-  add_definitions("-DUPNP_STATIC")
-else()
-  add_definitions("-DUPNP_DYNAMIC")
-  include_directories(${UPNP_INCLUDE})
-endif()
-
 # Final setup for libunbound
 include_directories(${UNBOUND_INCLUDE})
 link_directories(${UNBOUND_LIBRARY_DIRS})
@@ -682,7 +723,8 @@ else()
   endif()
 
   # linker
-  if (NOT (WIN32 AND (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS 9.1)))
+  if (NOT SANITIZE AND NOT OSSFUZZ AND NOT (WIN32 AND (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS 9.1)))
+    # PIE executables randomly crash at startup with ASAN
     # Windows binaries die on startup with PIE when compiled with GCC <9.x
     add_linker_flag_if_supported(-pie LD_SECURITY_FLAGS)
   endif()
@@ -804,7 +846,8 @@ else()
 
   endif(ARM)
 
-  if(ANDROID AND NOT BUILD_GUI_DEPS STREQUAL "ON" OR IOS)
+  # random crash on startup when asan is on if pie is enabled
+  if(NOT SANITIZE AND ANDROID AND NOT BUILD_GUI_DEPS STREQUAL "ON" OR IOS)
     #From Android 5: "only position independent executables (PIE) are supported" 
     message(STATUS "Enabling PIE executable")
     set(PIC_FLAG "")
@@ -939,10 +982,12 @@ list(APPEND EXTRA_LIBRARIES ${CMAKE_DL_LIBS})
 if (HIDAPI_FOUND OR LibUSB_COMPILE_TEST_PASSED)
   if (APPLE)
     if(DEPENDS)
-      list(APPEND EXTRA_LIBRARIES "-framework Foundation -framework IOKit")
+      list(APPEND EXTRA_LIBRARIES "-framework Foundation -framework AppKit -framework IOKit")
     else()
       find_library(COREFOUNDATION CoreFoundation)
+      find_library(APPKIT AppKit)
       find_library(IOKIT IOKit)
+      list(APPEND EXTRA_LIBRARIES ${APPKIT})
       list(APPEND EXTRA_LIBRARIES ${IOKIT})
       list(APPEND EXTRA_LIBRARIES ${COREFOUNDATION})
     endif()
@@ -988,12 +1033,10 @@ find_path(ZMQ_INCLUDE_PATH zmq.h)
 find_library(ZMQ_LIB zmq)
 find_library(PGM_LIBRARY pgm)
 find_library(NORM_LIBRARY norm)
+find_library(GSSAPI_LIBRARY gssapi_krb5)
 find_library(PROTOLIB_LIBRARY protolib)
 find_library(SODIUM_LIBRARY sodium)
 
-set(ZMQ_INCLUDE_PATH zmq_dummy_include_path)
-set(ZMQ_LIB zmq_dummy_lib_2)
-
 if(NOT ZMQ_INCLUDE_PATH)
   message(FATAL_ERROR "Could not find required header zmq.h")
 endif()
@@ -1006,6 +1049,9 @@ endif()
 if(NORM_LIBRARY)
   set(ZMQ_LIB "${ZMQ_LIB};${NORM_LIBRARY}")
 endif()
+if(GSSAPI_LIBRARY)
+  set(ZMQ_LIB "${ZMQ_LIB};${GSSAPI_LIBRARY}")
+endif()
 if(PROTOLIB_LIBRARY)
   set(ZMQ_LIB "${ZMQ_LIB};${PROTOLIB_LIBRARY}")
 endif()
@@ -1013,6 +1059,7 @@ if(SODIUM_LIBRARY)
   set(ZMQ_LIB "${ZMQ_LIB};${SODIUM_LIBRARY}")
 endif()
 
+include(external/supercop/functions.cmake) # place after setting flags and before src directory inclusion
 add_subdirectory(contrib)
 add_subdirectory(src)
 

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2014-2020, The Monero Project
+Copyright (c) 2014-2021, The Monero Project
 
 All rights reserved.
 

README.md

@@ -1,55 +1,34 @@
 [![Build Status](https://ci.wownero.com/api/badges/wownero/wownero/status.svg)](https://ci.wownero.com/wownero/wownero)
 # ~~Mo~~Wownero -  Such privacy! Many coins! Wow! 🐕
 
-Copyright (c) 2014-2020 The Monero Project.   
+Copyright (c) 2014-2021 The Monero Project.   
 Portions Copyright (c) 2012-2013 The Cryptonote developers.
 
-# CyberWOW
-An Android pruned full node for Wownero
-
-[<img src="https://git.wownero.com/wownero/meta/raw/branch/master/images/fdroid_badge.png"
-      alt="Get it on F-Droid"
-      height="80">](https://f-droid.org/en/packages/org.wownero.cyberwow/)
-<a href='https://play.google.com/store/apps/details?id=org.wownero.cyberwow'><img alt='Get it on Google Play' src='https://git.wownero.com/wownero/meta/raw/branch/master/images/google_badge.png' height='80'/></a>
-
-# Wownerujo
-An Android Wallet for Wownero
-
-[<img src="https://git.wownero.com/wownero/meta/raw/branch/master/images/fdroid_badge.png"
-      alt="Get it on F-Droid"
-      height="80">](https://f-droid.org/packages/com.wownero.wownerujo/)
-<a href='https://play.google.com/store/apps/details?id=com.wownero.wownerujo'><img alt='Get it on Google Play' src='https://git.wownero.com/wownero/meta/raw/branch/master/images/google_badge.png' height='80'/></a>
-
 ## Resources
 
-- Web: [wownero.org](http://wownero.org)
+- Web: [wownero.org](https://wownero.org)
 - Twitter: [@w0wn3r0](https://twitter.com/w0wn3r0)
 - Reddit: [/r/wownero](https://www.reddit.com/r/wownero)
 - Mail: [wownero@protonmail.com](mailto:wownero@protonmail.com)
-- GitHub: [git.wownero.com/wownero/wownero](https://git.wownero.com/wownero/wownero)
-- IRC: [#wownero on Freenode](https://kiwiirc.com/client/irc.freenode.net/?nick=suchchatter|?#wownero)
-- Bitmessage Chan: wownero (`BM-2cSzWtrj2pzLva9GF1Jp2TYsnLjrnJpvba`)
+- Git: [git.wownero.com/wownero/wownero](https://git.wownero.com/wownero/wownero)
 - Wownero Funding System: [funding.wownero.com](https://funding.wownero.com)
 - Wownero Forum: [forum.wownero.com](https://forum.wownero.com)
-- Discord: [discord.gg/abgaJbN](https://discord.gg/abgaJbN)
+- Discord: [discord.gg/ENbgme4bWq](https://discord.com/invite/ENbgme4bWq)
 - Telegram: [t.me/wownero](https://t.me/wownero)
+- Feather-WOW Desktop Wallet: [featherwallet.org/wownero](https://featherwallet.org/wownero)
+- WOW Stash Web Wallet: [wowstash.app](https://wowstash.app)
+- Public Node Status: [monero.fail](https://monero.fail/?crypto=wownero)
+- Map of Nodes: [wownero.fyi](https://wownero.fyi)
+- Wownero Memes: [suchwow.xyz]https://suchwow.xyz/posts/top)
+- XMR/WOW Swap: [nero Swap](https://neroswap.com)
+- Mining Pools: [miningpoolstats.stream](https://miningpoolstats.stream/wownero)
+- Market Info: [coinmarketcap.com](https://coinmarketcap.com/currencies/wownero), [coingecko.com](https://www.coingecko.com/en/coins/wownero/usd)
 
-Blockchain Explorers
+### Blockchain Explorers
 - https://explore.wownero.com
-- https://explorer.wowkira.com
 - http://wow5eqtzqvsg5jctqzg5g7uk3u62sfqiacj5x6lo4by7bvnj6jkvubyd.onion
-- https://wownero.xmrauctions.com
-
-Free Public Nodes
-- global.wownodes.com:34568 (Global)
-- node.suchwow.xyz:34568    (US)
-- wow.pwned.systems:34568   (NL)
-- wowbux.org:34568          (CA)
-- wowbuxx535x4exuexja2xfezpwcyznxkofui4ndjiectj4yuh2xheiid.onion:34568
-
-Tor Peers
-- wowp2p5gelm6vhl2d5tvfqills63jilgy6hkvlrqljooov5ktaxgqdad.onion
-- f3moshycuklu3mb3wnlfjwn26nsgzreqtfzfuyjjk46u7jqxlhf7d5id.onion
+- https://wownero.club
+- https://explorer.wownero.fyi
 
 ## Introduction
 
@@ -65,7 +44,7 @@ Wownero is a 100% community-sponsored endeavor. Supporting services are also gra
 
 Developers are volunteers doing this mostly for shits and giggles. If you would like to support our shenanigans and stimulant addictions, please consider donating to [WFS proposals](https://funding.wownero.com/proposals) or the dev slush fund.
 
-Donation Addresses
+### Donation Addresses
 
 WOW: `Wo3MWeKwtA918DU4c69hVSNgejdWFCRCuWjShRY66mJkU2Hv58eygJWDJS1MNa2Ge5M1WjUkGHuLqHkweDxwZZU42d16v94mP`
 
@@ -102,7 +81,7 @@ Dates are provided in the format YYYY-MM-DD.
 | 114,969                          | 2019-06-14 | F For Fappening    | v0.6.1.0               | v0.6.1.2                  | RandomWOW, new block weight algorithm, slightly more efficient RingCT format
 | 160,777                          | 2019-11-20 | Gaping Goatse    | v0.7.0.0               | v0.7.1.0                  | Only allow >= 2 outputs, change to the block median used to calculate penalty, rct sigs in coinbase forbidden, 4 unlock time as protocol rule
 | -                         | 2020-06-28 | Hallucinogenic Hypnotoad  | v0.8.0.0               | v0.8.0.2                  | Dandelion++ support
-| XXXX                         | 2020-09-XX | XXXXX  | v0.9.0.0               | v0.9.0.0                  | RandomWOW v2 (WOWv2), increase coinbase unlock to 8,640 (1 mo.), CLSAG
+| 253,999                         | 2020-10-09 | Illiterate Illuminati  | v0.9.0.0               | v0.9.2.2                  | Dynamic coinbase unlock (up to 1 mo.), Deterministic unlock times, Enforce maximum coinbase amount, show_qr_code wallet command, CLSAG
 
 X's indicate that these details have not been determined as of commit date.
 
@@ -116,7 +95,7 @@ Packages are available for
 
         yay -S wownero-git
 
-* Gentoo
+* Gentoo - Russian hacking tool
 
         emerge --noreplace eselect-repository
         eselect repository enable monero
@@ -154,7 +133,6 @@ Packaging for your favorite distribution would be a welcome contribution!
         git clone https://git.wownero.com/wownero/wownero && cd wownero
         make -j2
 
-
 * Debian/Ubuntu
 
         sudo apt update && sudo apt install build-essential cmake pkg-config libboost-all-dev libssl-dev libzmq3-dev libunbound-dev libsodium-dev libunwind8-dev liblzma-dev libreadline6-dev libldns-dev libexpat1-dev libpgm-dev libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler libudev-dev git -y
@@ -165,7 +143,7 @@ Packaging for your favorite distribution would be a welcome contribution!
 ## Running Binaries
 
 The build places the binary in `bin/` sub-directory within the build directory
-from which cmake was invoked (repository root by default). To run in
+from which cmake was invoked (repository root by default). To run in the
 foreground:
 
     ./bin/wownerod
@@ -174,7 +152,7 @@ To list all available options, run `./bin/wownerod --help`.  Options can be
 specified either on the command line or in a configuration file passed by the
 `--config-file` argument.  To specify an option in the configuration file, add
 a line with the syntax `argumentname=value`, where `argumentname` is the name
-of the argument without the leading dashes, for example `log-level=1`.
+of the argument without the leading dashes, for example, `log-level=1`.
 
 To run in background:
 

ZMQ.md

@@ -0,0 +1,61 @@
+# The Current/Future Status of ZMQ in Monero
+
+## ZMQ Pub/Sub
+Client `ZMQ_SUB` sockets must "subscribe" to topics before it receives any data.
+This allows filtering on the server side, so network traffic is reduced. Monero
+allows for filtering on: (1) format, (2) context, and (3) event.
+
+ * **format** refers to the _wire_ format (i.e. JSON) used to send event
+   information.
+ * **context** allows for a reduction in fields for the event, so the
+   daemon doesn't waste cycles serializing fields that get ignored.
+ * **event** refers to status changes occurring within the daemon (i.e. new
+   block to main chain).
+
+ * Formats:
+   * `json`
+ * Contexts:
+   * `full` - the entire block or transaction is transmitted (the hash can be
+     computed remotely).
+   * `minimal` - the bare minimum for a remote client to react to an event is
+     sent.
+ * Events:
+   * `chain_main` - changes to the primary/main blockchain.
+   * `txpool_add` - new _publicly visible_ transactions in the mempool.
+     Includes previously unseen transactions in a block but _not_ the
+     `miner_tx`. Does not "re-publish" after a reorg. Includes `do_not_relay`
+     transactions.
+
+The subscription topics are formatted as `format-context-event`, with prefix
+matching supported by both Monero and ZMQ. The `format`, `context` and `event`
+will _never_ have hyphens or colons in their name. For example, subscribing to
+`json-minimal-chain_main` will send minimal information in JSON when changes
+to the main/primary blockchain occur. Whereas, subscribing to `json-minimal`
+will send minimal information in JSON on all available events supported by the
+daemon.
+
+The Monero daemon will ensure that events prefixed by `chain` will be sent in
+"chain-order" - the `prev_id` (hash) field will _always_ refer to a previous
+block. On rollbacks/reorgs, the event will reference an earlier block in the
+chain instead of the last block. The Monero daemon also ensures that
+`txpool_add` events are sent before `chain_*` events - the `chain_*` messages
+will only serialize miner transactions since the other transactions were
+previously published via `txpool_add`. This prevents transactions from being
+serialized twice, even when the transaction was first observed in a block.
+
+ZMQ Pub/Sub will drop messages if the network is congested, so the above rules
+for send order are used for detecting lost messages. A missing gap in `height`
+or `prev_id` for `chain_*` events indicates a lost pub message. Missing
+`txpool_add` messages can only be detected at the next `chain_` message.
+
+Since blockchain events can be dropped, clients will likely want to have a
+timeout against `chain_main` events. The `GetLastBlockHeader` RPC is useful
+for checking the current chain state. Dropped messages should be rare in most
+conditions.
+
+The Monero daemon will send a `txpool_add` pub exactly once for each
+transaction, even after a reorg or restarts. Clients should use the
+`GetTransactionPool` after a reorg to get all transactions that have been put
+back into the tx pool or been invalidated due to a double-spend.
+
+

cmake/GitVersion.cmake

@@ -48,29 +48,21 @@ function (get_version_tag_from_git GIT)
         message(STATUS "You are currently on commit ${COMMIT}")
 
         # Get all the tags
-        execute_process(COMMAND "${GIT}" rev-list --tags --max-count=1 --abbrev-commit 
+        execute_process(COMMAND "${GIT}" tag -l --points-at HEAD
                         WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
                         RESULT_VARIABLE RET
-                        OUTPUT_VARIABLE TAGGEDCOMMIT
+                        OUTPUT_VARIABLE TAG
                         OUTPUT_STRIP_TRAILING_WHITESPACE)
 
-        if(NOT TAGGEDCOMMIT)
-            message(WARNING "Cannot determine most recent tag. Make sure that you are building either from a Git working tree or from a source archive.")
+        # Check if we're building that tagged commit or a different one
+        if(TAG)
+            message(STATUS "You are building a tagged release")
+            set(VERSIONTAG "release")
+            set(VERSION_IS_RELEASE "true")
+        else()
+            message(STATUS "You are ahead of or behind a tagged release")
             set(VERSIONTAG "${COMMIT}")
             set(VERSION_IS_RELEASE "false")
-        else()
-            message(STATUS "The most recent tag was at ${TAGGEDCOMMIT}")
-
-            # Check if we're building that tagged commit or a different one
-            if(COMMIT STREQUAL TAGGEDCOMMIT)
-                message(STATUS "You are building a tagged release")
-                set(VERSIONTAG "release")
-                set(VERSION_IS_RELEASE "true")
-            else()
-                message(STATUS "You are ahead of or behind a tagged release")
-                set(VERSIONTAG "${COMMIT}")
-                set(VERSION_IS_RELEASE "false")
-            endif()
         endif()	    
     endif()
 

cmake/SetClangTidy.cmake

@@ -0,0 +1,72 @@
+# Copyright (c) 2014-2020, The Monero Project
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# https://cmake.org/cmake/help/latest/variable/CMAKE_LANG_CLANG_TIDY.html
+# This module sets the following variables:
+# CMAKE_C_CLANG_TIDY
+# CMAKE_CXX_CLANG_TIDY
+# when clang-tidy is found in PATH. Afterwards, the code is being linted by the tool.
+# The checks to be enabled can be manipulated with the variable MONERO_CLANG_TIDY_CHECKS
+
+macro (monero_clang_tidy LANGUAGE)
+	set(TOOL_NAME "clang-tidy")
+	set(MONERO_CLANG_TIDY_MIN_VERSION "3.6")
+	if(${CMAKE_VERSION} VERSION_LESS "${MONERO_CLANG_TIDY_MIN_VERSION}")
+		message(FATAL_ERROR "Sorry, ${TOOL_NAME} is available for CMake from version ${MONERO_CLANG_TIDY_MIN_VERSION}")
+	else()
+		message(STATUS "Trying to enable ${TOOL_NAME}")
+		find_program(MONERO_CLANG_BIN ${TOOL_NAME})
+		if(NOT MONERO_CLANG_BIN)
+			message(FATAL_ERROR "${TOOL_NAME} not found! Try running: sudo apt install ${TOOL_NAME}")
+		else()
+			message(STATUS "Found ${MONERO_CLANG_BIN}")
+			set(MONERO_CLANG_TIDY_CHECKS
+			    -header-filter=.; 	# By default the headers are excluded. This line enables them.
+			    -checks=*; 		# Currently enabling all checks
+			    # An example of selectively enabling checks:
+			    #-checks=bugprone-*,cppcoreguidelines-avoid-goto # Have to be in one line :(
+			)
+			# Current list of checks is avaibale under:
+			# https://clang.llvm.org/extra/clang-tidy/
+			if (${LANGUAGE} STREQUAL "C")
+				set(CMAKE_C_CLANG_TIDY
+					${MONERO_CLANG_BIN}; # Mind the semicolon
+					${MONERO_CLANG_TIDY_CHECKS}
+				)
+			elseif (${LANGUAGE} STREQUAL "CXX")
+				set(CMAKE_CXX_CLANG_TIDY
+					${MONERO_CLANG_BIN}; # Mind the semicolon
+					${MONERO_CLANG_TIDY_CHECKS}
+				)
+			else()
+				message(FATAL_ERROR "${TOOL_NAME}: Unsupported language: ${LANGUAGE}")
+			endif()
+		endif()
+	endif()
+endmacro()
+

cmake/Version.cmake

@@ -28,7 +28,7 @@
 
 function (write_version tag)
   set(VERSIONTAG "${tag}" CACHE STRING "The tag portion of the Monero software version" FORCE)
-  configure_file("${CMAKE_SOURCE_DIR}/src/version.cpp.in" "${CMAKE_BINARY_DIR}/version.cpp")
+  configure_file("${CMAKE_CURRENT_LIST_DIR}/../src/version.cpp.in" "${CMAKE_BINARY_DIR}/version.cpp")
 endfunction ()
 
 find_package(Git QUIET)

contrib/depends/packages/expat.mk

@@ -1,6 +1,6 @@
 package=expat
 $(package)_version=2.2.4
-$(package)_download_path=https://downloads.sourceforge.net/project/expat/expat/$($(package)_version)
+$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_2_2_4
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=03ad85db965f8ab2d27328abcf0bc5571af6ec0a414874b2066ee3fdd372019e
 

contrib/depends/packages/openssl.mk

@@ -1,36 +1,31 @@
 package=openssl
-$(package)_version=1.0.2r
-$(package)_download_path=https://ftp.openssl.org/source/old/1.0.2
+$(package)_version=1.1.1i
+$(package)_download_path=https://www.openssl.org/source
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
-$(package)_sha256_hash=ae51d08bba8a83958e894946f15303ff894d75c2b8bbd44a852b64e3fe11d0d6
-$(package)_patches=fix_arflags.patch
+$(package)_sha256_hash=e8be6a35fe41d10603c3cc635e93289ed00bf34b79671a3a4de64fcee00d5242
 
 define $(package)_set_vars
 $(package)_config_env=AR="$($(package)_ar)" ARFLAGS=$($(package)_arflags) RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)"
+$(package)_config_env_arm_android=ANDROID_NDK_HOME="$(host_prefix)/native" PATH="$(host_prefix)/native/bin" CC=clang AR=ar RANLIB=ranlib
+$(package)_config_env_aarch64_android=ANDROID_NDK_HOME="$(host_prefix)/native" PATH="$(host_prefix)/native/bin" CC=clang AR=ar RANLIB=ranlib
+$(package)_build_env_arm_android=ANDROID_NDK_HOME="$(host_prefix)/native"
+$(package)_build_env_aarch64_android=ANDROID_NDK_HOME="$(host_prefix)/native"
 $(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/openssl
 $(package)_config_opts+=no-capieng
 $(package)_config_opts+=no-dso
 $(package)_config_opts+=no-dtls1
 $(package)_config_opts+=no-ec_nistp_64_gcc_128
 $(package)_config_opts+=no-gost
-$(package)_config_opts+=no-gmp
 $(package)_config_opts+=no-heartbeats
-$(package)_config_opts+=no-jpake
-$(package)_config_opts+=no-krb5
-$(package)_config_opts+=no-libunbound
 $(package)_config_opts+=no-md2
 $(package)_config_opts+=no-rc5
 $(package)_config_opts+=no-rdrand
 $(package)_config_opts+=no-rfc3779
-$(package)_config_opts+=no-rsax
 $(package)_config_opts+=no-sctp
-$(package)_config_opts+=no-sha0
 $(package)_config_opts+=no-shared
 $(package)_config_opts+=no-ssl-trace
 $(package)_config_opts+=no-ssl2
 $(package)_config_opts+=no-ssl3
-$(package)_config_opts+=no-static_engine
-$(package)_config_opts+=no-store
 $(package)_config_opts+=no-unit-test
 $(package)_config_opts+=no-weak-ssl-ciphers
 $(package)_config_opts+=no-zlib
@@ -42,8 +37,8 @@ $(package)_config_opts_x86_64_linux=linux-x86_64
 $(package)_config_opts_i686_linux=linux-generic32
 $(package)_config_opts_arm_linux=linux-generic32
 $(package)_config_opts_aarch64_linux=linux-generic64
-$(package)_config_opts_arm_android=--static android-armv7 no-asm
-$(package)_config_opts_aarch64_android=--static android no-asm
+$(package)_config_opts_arm_android=--static android-arm
+$(package)_config_opts_aarch64_android=--static android-arm64
 $(package)_config_opts_riscv64_linux=linux-generic64
 $(package)_config_opts_mipsel_linux=linux-generic32
 $(package)_config_opts_mips_linux=linux-generic32
@@ -55,10 +50,8 @@ $(package)_config_opts_x86_64_freebsd=BSD-x86_64
 endef
 
 define $(package)_preprocess_cmds
-  sed -i.old "/define DATE/d" util/mkbuildinf.pl && \
-  sed -i.old "s|engines apps test|engines|" Makefile.org && \
-  sed -i -e "s/-mandroid //" Configure && \
-  patch < $($(package)_patch_dir)/fix_arflags.patch
+  sed -i.old 's|"engines", "apps", "test", "util", "tools", "fuzz"|"engines", "tools"|' Configure && \
+  sed -i -e 's|cflags --sysroot.*",|cflags",|' Configurations/15-android.conf
 endef
 
 define $(package)_config_cmds
@@ -70,7 +63,7 @@ define $(package)_build_cmds
 endef
 
 define $(package)_stage_cmds
-  $(MAKE) INSTALL_PREFIX=$($(package)_staging_dir) -j1 install_sw
+  $(MAKE) DESTDIR=$($(package)_staging_dir) -j1 install_sw
 endef
 
 define $(package)_postprocess_cmds

contrib/depends/packages/qt.mk

@@ -1,18 +1,18 @@
 PACKAGE=qt
-$(package)_version=5.7.1
-$(package)_download_path=http://linorg.usp.br/Qt/archive/qt/5.7/5.7.1/submodules
-$(package)_suffix=opensource-src-$($(package)_version).tar.gz
+$(package)_version=5.15.1
+$(package)_download_path=https://download.qt.io/official_releases/qt/5.15/$($(package)_version)/submodules
+$(package)_suffix=everywhere-src-$($(package)_version).tar.xz
 $(package)_file_name=qtbase-$($(package)_suffix)
-$(package)_sha256_hash=95f83e532d23b3ddbde7973f380ecae1bac13230340557276f75f2e37984e410
+$(package)_sha256_hash=33960404d579675b7210de103ed06a72613bfc4305443e278e2d32a3eb1f3d8c
 $(package)_build_subdir=qtbase
 $(package)_qt_libs=corelib
-$(package)_patches=pidlist_absolute.patch fix_qt_pkgconfig.patch qfixed-coretext.patch
+$(package)_patches=fix_qt_pkgconfig.patch fix_no_printer.patch fix_rcc_determinism.patch no-xlib.patch
 
 $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix)
-$(package)_qttranslations_sha256_hash=3a15aebd523c6d89fb97b2d3df866c94149653a26d27a00aac9b6d3020bc5a1d
+$(package)_qttranslations_sha256_hash=46e0c0e3a511fbcc803a4146204062e47f6ed43b34d98a3c27372a03b8746bd8
 
 $(package)_qttools_file_name=qttools-$($(package)_suffix)
-$(package)_qttools_sha256_hash=22d67de915cb8cd93e16fdd38fa006224ad9170bd217c2be1e53045a8dd02f0f
+$(package)_qttools_sha256_hash=c98ee5f0f980bf68cbf0c94d62434816a92441733de50bd9adbe9b9055f03498
 
 $(package)_extra_sources  = $($(package)_qttranslations_file_name)
 $(package)_extra_sources += $($(package)_qttools_file_name)
@@ -24,28 +24,26 @@ $(package)_config_opts += -bindir $(build_prefix)/bin
 $(package)_config_opts += -c++std c++11
 $(package)_config_opts += -confirm-license
 $(package)_config_opts += -dbus-runtime
-$(package)_config_opts += -no-alsa
-$(package)_config_opts += -no-audio-backend
+$(package)_config_opts += -hostprefix $(build_prefix)
+$(package)_config_opts += -no-compile-examples
 $(package)_config_opts += -no-cups
 $(package)_config_opts += -no-egl
 $(package)_config_opts += -no-eglfs
-$(package)_config_opts += -no-feature-style-windowsmobile
-$(package)_config_opts += -no-feature-style-windowsce
+$(package)_config_opts += -no-evdev
+$(package)_config_opts += -no-gui
 $(package)_config_opts += -no-freetype
 $(package)_config_opts += -no-gif
 $(package)_config_opts += -no-glib
-$(package)_config_opts += -no-gstreamer
 $(package)_config_opts += -no-icu
+$(package)_config_opts += -no-ico
 $(package)_config_opts += -no-iconv
 $(package)_config_opts += -no-kms
 $(package)_config_opts += -no-linuxfb
+$(package)_config_opts += -no-libjpeg
 $(package)_config_opts += -no-libudev
-$(package)_config_opts += -no-mitshm
 $(package)_config_opts += -no-mtdev
-$(package)_config_opts += -no-pulseaudio
 $(package)_config_opts += -no-openvg
 $(package)_config_opts += -no-reduce-relocations
-$(package)_config_opts += -no-qml-debug
 $(package)_config_opts += -no-sql-db2
 $(package)_config_opts += -no-sql-ibase
 $(package)_config_opts += -no-sql-oci
@@ -56,8 +54,6 @@ $(package)_config_opts += -no-sql-psql
 $(package)_config_opts += -no-sql-sqlite
 $(package)_config_opts += -no-sql-sqlite2
 $(package)_config_opts += -no-use-gold-linker
-$(package)_config_opts += -no-xinput2
-$(package)_config_opts += -no-xrender
 $(package)_config_opts += -nomake examples
 $(package)_config_opts += -nomake tests
 $(package)_config_opts += -opensource
@@ -65,25 +61,46 @@ $(package)_config_opts += -no-openssl
 $(package)_config_opts += -optimized-qmake
 $(package)_config_opts += -pch
 $(package)_config_opts += -pkg-config
+$(package)_config_opts += -prefix $(host_prefix)
 $(package)_config_opts += -no-libpng
-$(package)_config_opts += -no-libjpeg
 $(package)_config_opts += -qt-pcre
+$(package)_config_opts += -qt-harfbuzz
 $(package)_config_opts += -no-zlib
-$(package)_config_opts += -reduce-exports
 $(package)_config_opts += -static
 $(package)_config_opts += -silent
 $(package)_config_opts += -v
-$(package)_config_opts += -no-feature-printer
+$(package)_config_opts += -no-feature-bearermanagement
+$(package)_config_opts += -no-feature-colordialog
+$(package)_config_opts += -no-feature-dial
+$(package)_config_opts += -no-feature-filesystemwatcher
+$(package)_config_opts += -no-feature-fontcombobox
+$(package)_config_opts += -no-feature-ftp
+$(package)_config_opts += -no-feature-image_heuristic_mask
+$(package)_config_opts += -no-feature-keysequenceedit
+$(package)_config_opts += -no-feature-lcdnumber
+$(package)_config_opts += -no-feature-pdf
 $(package)_config_opts += -no-feature-printdialog
-$(package)_config_opts += -no-gui
-$(package)_config_opts += -no-freetype
-$(package)_config_opts += -no-sm
-$(package)_config_opts += -no-fontconfig
-$(package)_config_opts += -no-opengl
-$(package)_config_opts += -no-xkb
-$(package)_config_opts += -no-xcb
-$(package)_config_opts += -no-xshape
-$(package)_build_env  = QT_RCC_TEST=1
+$(package)_config_opts += -no-feature-printer
+$(package)_config_opts += -no-feature-printpreviewdialog
+$(package)_config_opts += -no-feature-printpreviewwidget
+$(package)_config_opts += -no-feature-sessionmanager
+$(package)_config_opts += -no-feature-sql
+$(package)_config_opts += -no-feature-statemachine
+$(package)_config_opts += -no-feature-syntaxhighlighter
+$(package)_config_opts += -no-feature-textbrowser
+$(package)_config_opts += -no-feature-textodfwriter
+$(package)_config_opts += -no-feature-topleveldomain
+$(package)_config_opts += -no-feature-udpsocket
+$(package)_config_opts += -no-feature-undocommand
+$(package)_config_opts += -no-feature-undogroup
+$(package)_config_opts += -no-feature-undostack
+$(package)_config_opts += -no-feature-undoview
+$(package)_config_opts += -no-feature-vnc
+$(package)_config_opts += -no-feature-wizard
+$(package)_config_opts_linux = -no-fontconfig
+$(package)_config_opts_linux += -no-opengl
+$(package)_config_opts_linux += -no-xcb
+$(package)_config_opts_linux += -no-feature-xlib
 endef
 
 define $(package)_fetch_cmds
@@ -108,14 +125,24 @@ endef
 
 
 define $(package)_preprocess_cmds
+  sed -i.old "s|FT_Get_Font_Format|FT_Get_X11_Font_Format|" qtbase/src/platformsupport/fontdatabases/freetype/qfontengine_ft.cpp && \
   sed -i.old "s|updateqm.commands = \$$$$\$$$$LRELEASE|updateqm.commands = $($(package)_extract_dir)/qttools/bin/lrelease|" qttranslations/translations/translations.pro && \
   sed -i.old "/updateqm.depends =/d" qttranslations/translations/translations.pro && \
-  patch -p1 < $($(package)_patch_dir)/pidlist_absolute.patch && \
-  patch -p1 < $($(package)_patch_dir)/fix_qt_pkgconfig.patch && \
-  patch -p1 < $($(package)_patch_dir)/qfixed-coretext.patch && \
+  sed -i.old "s/src_plugins.depends = src_sql src_network/src_plugins.depends = src_network/" qtbase/src/src.pro && \
+  cp -r qtbase/mkspecs/linux-arm-gnueabi-g++ qtbase/mkspecs/bitcoin-linux-g++ && \
+  sed -i.old "s/arm-linux-gnueabi-/$(host)-/g" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \
+  patch -p1 -i $($(package)_patch_dir)/fix_qt_pkgconfig.patch && \
+  patch -p1 -i $($(package)_patch_dir)/fix_no_printer.patch && \
   echo "!host_build: QMAKE_CFLAGS     += $($(package)_cflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \
   echo "!host_build: QMAKE_CXXFLAGS   += $($(package)_cxxflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \
-  echo "!host_build: QMAKE_LFLAGS     += $($(package)_ldflags)" >> qtbase/mkspecs/common/gcc-base.conf
+  echo "!host_build: QMAKE_LFLAGS     += $($(package)_ldflags)" >> qtbase/mkspecs/common/gcc-base.conf && \
+  patch -p1 -i $($(package)_patch_dir)/no-xlib.patch && \
+  echo "QMAKE_LINK_OBJECT_MAX = 10" >> qtbase/mkspecs/win32-g++/qmake.conf && \
+  echo "QMAKE_LINK_OBJECT_SCRIPT = object_script" >> qtbase/mkspecs/win32-g++/qmake.conf && \
+  sed -i.old "s|QMAKE_CFLAGS           += |!host_build: QMAKE_CFLAGS            = $($(package)_cflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \
+  sed -i.old "s|QMAKE_CXXFLAGS         += |!host_build: QMAKE_CXXFLAGS            = $($(package)_cxxflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \
+  sed -i.old "0,/^QMAKE_LFLAGS_/s|^QMAKE_LFLAGS_|!host_build: QMAKE_LFLAGS            = $($(package)_ldflags)\n&|" qtbase/mkspecs/win32-g++/qmake.conf && \
+  sed -i.old "s/LIBRARY_PATH/(CROSS_)?\0/g" qtbase/mkspecs/features/toolchain.prf
 endef
 
 define $(package)_config_cmds

contrib/depends/patches/openssl/fix_arflags.patch

@@ -1,24 +0,0 @@
---- Makefile.org.O	2019-02-26 14:20:20.000000000 +0000
-+++ Makefile.org	2019-11-15 13:05:54.370086856 +0000
-@@ -63,8 +63,8 @@
- PEX_LIBS= 
- EX_LIBS= 
- EXE_EXT= 
--ARFLAGS=
--AR=ar $(ARFLAGS) r
-+ARFLAGS= r
-+AR=ar $(ARFLAGS)
- RANLIB= ranlib
- RC= windres
- NM= nm
---- Configure.O	2019-02-26 14:20:20.000000000 +0000
-+++ Configure	2019-11-16 07:43:14.933990774 +0000
-@@ -1251,7 +1251,7 @@
- my $shared_extension = $fields[$idx_shared_extension];
- my $ranlib = $ENV{'RANLIB'} || $fields[$idx_ranlib];
- my $ar = $ENV{'AR'} || "ar";
--my $arflags = $fields[$idx_arflags];
-+my $arflags = $ENV{'ARFLAGS'} || $fields[$idx_arflags];
- my $windres = $ENV{'RC'} || $ENV{'WINDRES'} || "windres";
- my $multilib = $fields[$idx_multilib];
- 

contrib/depends/patches/qt/fix_no_printer.patch

@@ -0,0 +1,19 @@
+--- x/qtbase/src/plugins/platforms/cocoa/qprintengine_mac_p.h
++++ y/qtbase/src/plugins/platforms/cocoa/qprintengine_mac_p.h
+@@ -52,6 +52,7 @@
+ //
+
+ #include <QtCore/qglobal.h>
++#include <qpa/qplatformprintdevice.h>
+
+ #ifndef QT_NO_PRINTER
+
+--- x/qtbase/src/plugins/plugins.pro
++++ y/qtbase/src/plugins/plugins.pro
+@@ -9,6 +9,3 @@ qtHaveModule(gui) {
+     !android:qtConfig(library): SUBDIRS *= generic
+ }
+ qtHaveModule(widgets): SUBDIRS += styles
+-
+-!winrt:qtHaveModule(printsupport): \
+-    SUBDIRS += printsupport

contrib/depends/patches/qt/fix_qt_pkgconfig.patch

@@ -1,11 +1,11 @@
 --- old/qtbase/mkspecs/features/qt_module.prf
 +++ new/qtbase/mkspecs/features/qt_module.prf
-@@ -245,7 +245,7 @@
+@@ -269,7 +269,7 @@ load(qt_installs)
  load(qt_targets)
  
  # this builds on top of qt_common
--!internal_module:!lib_bundle:if(unix|mingw) {
-+unix|mingw {
+-!internal_module:if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) {
++if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) {
      CONFIG += create_pc
      QMAKE_PKGCONFIG_DESTDIR = pkgconfig
      host_build: \

contrib/depends/patches/qt/fix_rcc_determinism.patch

@@ -0,0 +1,15 @@
+--- old/qtbase/src/tools/rcc/rcc.cpp
++++ new/qtbase/src/tools/rcc/rcc.cpp
+@@ -207,7 +207,11 @@ void RCCFileInfo::writeDataInfo(RCCResourceLibrary &lib)
+     if (lib.formatVersion() >= 2) {
+         // last modified time stamp
+         const QDateTime lastModified = m_fileInfo.lastModified();
+-        lib.writeNumber8(quint64(lastModified.isValid() ? lastModified.toMSecsSinceEpoch() : 0));
++        quint64 lastmod = quint64(lastModified.isValid() ? lastModified.toMSecsSinceEpoch() : 0);
++        static const quint64 sourceDate = 1000 * qgetenv("QT_RCC_SOURCE_DATE_OVERRIDE").toULongLong();
++        if (sourceDate != 0)
++            lastmod = sourceDate;
++        lib.writeNumber8(lastmod);
+         if (text || pass1)
+             lib.writeChar('\n');
+     }

contrib/depends/patches/qt/no-xlib.patch

@@ -0,0 +1,69 @@
+From 9563cef873ae82e06f60708d706d054717e801ce Mon Sep 17 00:00:00 2001
+From: Carl Dong <contact@carldong.me>
+Date: Thu, 18 Jul 2019 17:22:05 -0400
+Subject: [PATCH] Wrap xlib related code blocks in #if's
+
+They are not necessary to compile QT.
+---
+ qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp | 8 ++++++++
+ 1 file changed, 8 insertions(+)
+
+diff --git a/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp b/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp
+index 7c62c2e2b3..c05c6c0a07 100644
+--- a/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp
++++ b/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp
+@@ -49,7 +49,9 @@
+ #include <QtGui/QWindow>
+ #include <QtGui/QBitmap>
+ #include <QtGui/private/qguiapplication_p.h>
++#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library)
+ #include <X11/cursorfont.h>
++#endif
+ #include <xcb/xfixes.h>
+ #include <xcb/xcb_image.h>
+ 
+@@ -391,6 +393,7 @@ void QXcbCursor::changeCursor(QCursor *cursor, QWindow *window)
+     xcb_flush(xcb_connection());
+ }
+ 
++#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library)
+ static int cursorIdForShape(int cshape)
+ {
+     int cursorId = 0;
+@@ -444,6 +447,7 @@ static int cursorIdForShape(int cshape)
+     }
+     return cursorId;
+ }
++#endif
+ 
+ xcb_cursor_t QXcbCursor::createNonStandardCursor(int cshape)
+ {
+@@ -556,7 +560,9 @@ static xcb_cursor_t loadCursor(void *dpy, int cshape)
+ xcb_cursor_t QXcbCursor::createFontCursor(int cshape)
+ {
+     xcb_connection_t *conn = xcb_connection();
++#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library)
+     int cursorId = cursorIdForShape(cshape);
++#endif
+     xcb_cursor_t cursor = XCB_NONE;
+ 
+     // Try Xcursor first
+@@ -586,6 +592,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape)
+     // Non-standard X11 cursors are created from bitmaps
+     cursor = createNonStandardCursor(cshape);
+ 
++#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library)
+     // Create a glpyh cursor if everything else failed
+     if (!cursor && cursorId) {
+         cursor = xcb_generate_id(conn);
+@@ -593,6 +600,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape)
+                                 cursorId, cursorId + 1,
+                                 0xFFFF, 0xFFFF, 0xFFFF, 0, 0, 0);
+     }
++#endif
+ 
+     if (cursor && cshape >= 0 && cshape < Qt::LastCursor && connection()->hasXFixes()) {
+         const char *name = cursorNames[cshape].front();
+---
+2.22.0
+

contrib/depends/patches/qt/pidlist_absolute.patch

@@ -1,37 +0,0 @@
-diff -dur old/qtbase/src/plugins/platforms/windows/qwindowscontext.h new/qtbase/src/plugins/platforms/windows/qwindowscontext.h
---- old/qtbase/src/plugins/platforms/windows/qwindowscontext.h
-+++ new/qtbase/src/plugins/platforms/windows/qwindowscontext.h
-@@ -136,10 +136,18 @@
-     inline void init();
- 
-     typedef HRESULT (WINAPI *SHCreateItemFromParsingName)(PCWSTR, IBindCtx *, const GUID&, void **);
-+#if defined(Q_CC_MINGW) && (!defined(__MINGW64_VERSION_MAJOR) || __MINGW64_VERSION_MAJOR < 3)
-+    typedef HRESULT (WINAPI *SHGetKnownFolderIDList)(const GUID &, DWORD, HANDLE, ITEMIDLIST **);
-+#else
-     typedef HRESULT (WINAPI *SHGetKnownFolderIDList)(const GUID &, DWORD, HANDLE, PIDLIST_ABSOLUTE *);
-+#endif
-     typedef HRESULT (WINAPI *SHGetStockIconInfo)(int , int , _SHSTOCKICONINFO *);
-     typedef HRESULT (WINAPI *SHGetImageList)(int, REFIID , void **);
-+#if defined(Q_CC_MINGW) && (!defined(__MINGW64_VERSION_MAJOR) || __MINGW64_VERSION_MAJOR < 3)
-+    typedef HRESULT (WINAPI *SHCreateItemFromIDList)(const ITEMIDLIST *, REFIID, void **);
-+#else
-     typedef HRESULT (WINAPI *SHCreateItemFromIDList)(PCIDLIST_ABSOLUTE, REFIID, void **);
-+#endif
- 
-     SHCreateItemFromParsingName sHCreateItemFromParsingName;
-     SHGetKnownFolderIDList sHGetKnownFolderIDList;
-diff -dur old/qtbase/src/plugins/platforms/windows/qwindowsdialoghelpers.cpp new/qtbase/src/plugins/platforms/windows/qwindowsdialoghelpers.cpp
---- old/qtbase/src/plugins/platforms/windows/qwindowsdialoghelpers.cpp
-+++ new/qtbase/src/plugins/platforms/windows/qwindowsdialoghelpers.cpp
-@@ -1016,7 +1016,11 @@
-             qWarning() << __FUNCTION__ << ": Invalid CLSID: " << url.path();
-             return Q_NULLPTR;
-         }
-+#if defined(Q_CC_MINGW) && (!defined(__MINGW64_VERSION_MAJOR) || __MINGW64_VERSION_MAJOR < 3)
-+        ITEMIDLIST *idList;
-+#else
-         PIDLIST_ABSOLUTE idList;
-+#endif
-         HRESULT hr = QWindowsContext::shell32dll.sHGetKnownFolderIDList(uuid, 0, 0, &idList);
-         if (FAILED(hr)) {
-             qErrnoWarning("%s: SHGetKnownFolderIDList(%s)) failed", __FUNCTION__, qPrintable(url.toString()));

contrib/depends/patches/qt/qfixed-coretext.patch

@@ -1,34 +0,0 @@
-From dbdd5f0ffbce52c8b789ed09f1aa3f1da6c02e23 Mon Sep 17 00:00:00 2001
-From: Gabriel de Dietrich <gabriel.dedietrich@qt.io>
-Date: Fri, 30 Mar 2018 11:58:16 -0700
-Subject: [PATCH] QCoreTextFontEngine: Fix build with Xcode 9.3
-
-Apple LLVM version 9.1.0 (clang-902.0.39.1)
-
-Error message:
-
-.../qfontengine_coretext.mm:827:20: error: qualified reference to
-      'QFixed' is a constructor name rather than a type in this context
-    return QFixed::QFixed(int(CTFontGetUnitsPerEm(ctfont)));
-
-Change-Id: Iebe26b3b087a16b10664208fc8851cbddb47f043
-Reviewed-by: Konstantin Ritt <ritt.ks@gmail.com>
----
- src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git old/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm new/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm
-index 25ff69d877d..98b753eff96 100644
---- old/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm
-+++ new/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm
-@@ -824,7 +824,7 @@ void QCoreTextFontEngine::getUnscaledGlyph(glyph_t glyph, QPainterPath *path, gl
- 
- QFixed QCoreTextFontEngine::emSquareSize() const
- {
--    return QFixed::QFixed(int(CTFontGetUnitsPerEm(ctfont)));
-+    return QFixed(int(CTFontGetUnitsPerEm(ctfont)));
- }
- 
- QFontEngine *QCoreTextFontEngine::cloneWithSize(qreal pixelSize) const
--- 
-2.16.3

contrib/depends/toolchain.cmake.in

@@ -1,5 +1,6 @@
 # Set the system name to one of Android, Darwin, FreeBSD, Linux, or Windows
 SET(CMAKE_SYSTEM_NAME @depends@)
+SET(CMAKE_SYSTEM_PROCESSOR @arch@)
 SET(CMAKE_BUILD_TYPE @release_type@)
 
 OPTION(STATIC "Link libraries statically" ON)
@@ -55,7 +56,7 @@ SET(Boost_NO_SYSTEM_PATHS ON)
 SET(Boost_USE_STATIC_LIBS ON)
 SET(Boost_USE_STATIC_RUNTIME ON)
 
-SET(OpenSSL_DIR @prefix@/lib)
+SET(OPENSSL_ROOT_DIR @prefix@)
 SET(ARCHITECTURE @arch@)
 
 # for libraries and headers in the target directories
@@ -63,14 +64,14 @@ set (CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) # Find programs on host
 set (CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) # Find libs in target
 set (CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) # Find includes in target
 
-set(CMAKE_SYSTEM_PROCESSOR ${CMAKE_HOST_SYSTEM_PROCESSOR} CACHE STRING "" FORCE)
-
 # specify the cross compiler to be used. Darwin uses clang provided by the SDK.
 if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
   SET(CMAKE_C_COMPILER @prefix@/native/bin/clang)
   SET(CMAKE_C_COMPILER_TARGET x86_64-apple-darwin11)
   SET(CMAKE_CXX_COMPILER @prefix@/native/bin/clang++ -stdlib=libc++)
   SET(CMAKE_CXX_COMPILER_TARGET x86_64-apple-darwin11)
+  SET(CMAKE_ASM_COMPILER_TARGET x86_64-apple-darwin11)
+  SET(CMAKE_ASM-ATT_COMPILER_TARGET x86_64-apple-darwin11)
   SET(_CMAKE_TOOLCHAIN_PREFIX x86_64-apple-darwin11-)
   SET(APPLE True)
   SET(BUILD_TAG "mac-x64")

contrib/epee/include/byte_stream.h

@@ -58,7 +58,6 @@ namespace epee
     byte_buffer buffer_;        //! Beginning of buffer
     std::uint8_t* next_write_;  //! Current write position
     const std::uint8_t* end_;   //! End of buffer
-    std::size_t increase_size_; //! Minimum buffer size increase
 
     //! \post `requested <= available()`
     void overflow(const std::size_t requested);
@@ -75,29 +74,17 @@ namespace epee
     using char_type = std::uint8_t;
     using Ch = char_type;
 
-    //! \return Default minimum size increase on buffer overflow
-    static constexpr std::size_t default_increase() noexcept { return 4096; }
-
     //! Increase internal buffer by at least `byte_stream_increase` bytes.
     byte_stream() noexcept
-      : byte_stream(default_increase())
-    {}
-
-    //! Increase internal buffer by at least `increase` bytes.
-    explicit byte_stream(const std::size_t increase) noexcept
       : buffer_(nullptr),
         next_write_(nullptr),
-        end_(nullptr),
-        increase_size_(increase)
+        end_(nullptr)
     {}
 
     byte_stream(byte_stream&& rhs) noexcept;
     ~byte_stream() noexcept = default;
     byte_stream& operator=(byte_stream&& rhs) noexcept;
 
-    //! \return The minimum increase size on buffer overflow
-    std::size_t increase_size() const noexcept { return increase_size_; }
-
     const std::uint8_t* data() const noexcept { return buffer_.get(); }
     std::uint8_t* tellp() const noexcept { return next_write_; }
     std::size_t available() const noexcept { return end_ - next_write_; }
@@ -117,6 +104,9 @@ namespace epee
       check(more);
     }
 
+    //! Reset write position, but do not release internal memory. \post `size() == 0`.
+    void clear() noexcept { next_write_ = buffer_.get(); }
+
     /*! Copy `length` bytes starting at `ptr` to end of stream.
         \throw std::range_error If exceeding max size_t value.
         \throw std::bad_alloc If allocation fails. */
@@ -185,7 +175,7 @@ namespace epee
     void put_n(const std::uint8_t ch, const std::size_t count)
     {
       check(count);
-      std::memset(tellp(), count, ch);
+      std::memset(tellp(), ch, count);
       next_write_ += count;
     }
 

contrib/epee/include/console_handler.h

@@ -543,6 +543,31 @@ eof:
       return it->second.second;
     }
 
+    std::vector<std::string> get_command_list(const std::vector<std::string>& keywords = std::vector<std::string>())
+    {
+      std::vector<std::string> list;
+      list.reserve(m_command_handlers.size());
+      for(auto const& x:m_command_handlers)
+      {
+        bool take = true;
+        for(auto const& y:keywords)
+        {
+          bool in_usage = x.second.second.first.find(y) != std::string::npos;
+          bool in_description = x.second.second.second.find(y) != std::string::npos;
+          if (!(in_usage || in_description))
+          {
+            take = false;
+            break;
+          }
+        }
+        if (take)
+        {
+          list.push_back(x.first);
+        }
+      }
+      return list;
+    }
+
     void set_handler(const std::string& cmd, const callback& hndlr, const std::string& usage = "", const std::string& description = "")
     {
       lookup::mapped_type & vt = m_command_handlers[cmd];

contrib/epee/include/misc_language.h

@@ -106,6 +106,14 @@ namespace misc_utils
 		return true;
 	}
 
+  template <typename T>
+  T get_mid(const T &a, const T &b)
+  {
+    //returns the average of two numbers; overflow safe and works with at least all integral and floating point types
+    //(a+b)/2 = (a/2) + (b/2) + ((a - 2*(a/2)) + (b - 2*(b/2)))/2
+    return (a/2) + (b/2) + ((a - 2*(a/2)) + (b - 2*(b/2)))/2;
+  }
+
   template<class type_vec_type>
   type_vec_type median(std::vector<type_vec_type> &v)
   {
@@ -122,7 +130,7 @@ namespace misc_utils
       return v[n];
     }else 
     {//2, 4, 6...
-      return (v[n-1] + v[n])/2;
+      return get_mid<type_vec_type>(v[n-1],v[n]);
     }
 
   }

contrib/epee/include/net/abstract_http_client.h

@@ -64,12 +64,13 @@ namespace http
     abstract_http_client() {}
     virtual ~abstract_http_client() {}
     bool set_server(const std::string& address, boost::optional<login> user, ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect);
+    virtual bool set_proxy(const std::string& address);
     virtual void set_server(std::string host, std::string port, boost::optional<login> user, ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect) = 0;
     virtual void set_auto_connect(bool auto_connect) = 0;
     virtual bool connect(std::chrono::milliseconds timeout) = 0;
     virtual bool disconnect() = 0;
     virtual bool is_connected(bool *ssl = NULL) = 0;
-    virtual bool invoke(const boost::string_ref uri, const boost::string_ref method, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0;
+    virtual bool invoke(const boost::string_ref uri, const boost::string_ref method, const boost::string_ref body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0;
     virtual bool invoke_get(const boost::string_ref uri, std::chrono::milliseconds timeout, const std::string& body = std::string(), const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0;
     virtual bool invoke_post(const boost::string_ref uri, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0;
     virtual uint64_t get_bytes_sent() const = 0;

contrib/epee/include/net/abstract_tcp_server2.inl

@@ -207,7 +207,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     buffer_ssl_init_fill = 0;
     if (is_income && m_ssl_support != epee::net_utils::ssl_support_t::e_ssl_support_disabled)
       socket().async_receive(boost::asio::buffer(buffer_),
-        boost::asio::socket_base::message_peek,
         strand_.wrap(
           std::bind(&connection<t_protocol_handler>::handle_receive, self,
             std::placeholders::_1,
@@ -334,6 +333,9 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     TRY_ENTRY();
     //_info("[sock " << socket().native_handle() << "] Async read calledback.");
     
+    if (m_was_shutdown)
+        return;
+
     if (!e)
     {
         double current_speed_down;
@@ -360,6 +362,9 @@ PRAGMA_WARNING_DISABLE_VS(4355)
 					CRITICAL_REGION_LOCAL(	epee::net_utils::network_throttle_manager::m_lock_get_global_throttle_in );
 					delay = epee::net_utils::network_throttle_manager::get_global_throttle_in().get_sleep_time_after_tick( bytes_transferred );
 				}
+
+				if (m_was_shutdown)
+					return;
 				
 				delay *= 0.5;
 				long int ms = (long int)(delay * 100);
@@ -431,6 +436,9 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     std::size_t bytes_transferred)
   {
     TRY_ENTRY();
+
+    if (m_was_shutdown) return;
+
     if (e)
     {
       // offload the error case
@@ -438,13 +446,11 @@ PRAGMA_WARNING_DISABLE_VS(4355)
       return;
     }
 
-    reset_timer(get_timeout_from_bytes_read(bytes_transferred), false);
-
     buffer_ssl_init_fill += bytes_transferred;
-    if (buffer_ssl_init_fill <= get_ssl_magic_size())
+    MTRACE("we now have " << buffer_ssl_init_fill << "/" << get_ssl_magic_size() << " bytes needed to detect SSL");
+    if (buffer_ssl_init_fill < get_ssl_magic_size())
     {
       socket().async_receive(boost::asio::buffer(buffer_.data() + buffer_ssl_init_fill, buffer_.size() - buffer_ssl_init_fill),
-        boost::asio::socket_base::message_peek,
         strand_.wrap(
           boost::bind(&connection<t_protocol_handler>::handle_receive, connection<t_protocol_handler>::shared_from_this(),
             boost::asio::placeholders::error,
@@ -470,7 +476,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     if (m_ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_enabled)
     {
       // Handshake
-      if (!handshake(boost::asio::ssl::stream_base::server))
+      if (!handshake(boost::asio::ssl::stream_base::server, boost::asio::const_buffer(buffer_.data(), buffer_ssl_init_fill)))
       {
         MERROR("SSL handshake failed");
         boost::interprocess::ipcdetail::atomic_write32(&m_want_close_connection, 1);
@@ -485,6 +491,11 @@ PRAGMA_WARNING_DISABLE_VS(4355)
         return;
       }
     }
+    else
+    {
+      handle_read(e, buffer_ssl_init_fill);
+      return;
+    }
 
     async_read_some(boost::asio::buffer(buffer_),
       strand_.wrap(
@@ -651,6 +662,8 @@ PRAGMA_WARNING_DISABLE_VS(4355)
         boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) );
         m_send_que_lock.lock();
         _dbg1("sleep for queue: " << ms);
+	if (m_was_shutdown)
+		return false;
 
         if (retry > retry_limit) {
             MWARNING("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection");
@@ -748,7 +761,8 @@ PRAGMA_WARNING_DISABLE_VS(4355)
   template<class t_protocol_handler>
   void connection<t_protocol_handler>::reset_timer(boost::posix_time::milliseconds ms, bool add)
   {
-    if (ms.total_milliseconds() < 0)
+    const auto tms = ms.total_milliseconds();
+    if (tms < 0 || (add && tms == 0))
     {
       MWARNING("Ignoring negative timeout " << ms);
       return;

contrib/epee/include/net/connection_basic.hpp

@@ -132,10 +132,10 @@ class connection_basic { // not-templated base class for rapid developmet of som
 		ssl_support_t get_ssl_support() const { return m_ssl_support; }
 		void disable_ssl() { m_ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_disabled; }
 
-		bool handshake(boost::asio::ssl::stream_base::handshake_type type)
+		bool handshake(boost::asio::ssl::stream_base::handshake_type type, boost::asio::const_buffer buffer = {})
 		{
 			//m_state != nullptr verified in constructor
-			return m_state->ssl_options().handshake(socket_, type);
+			return m_state->ssl_options().handshake(socket_, type, buffer);
 		}
 
 		template<typename MutableBufferSequence, typename ReadHandler>

contrib/epee/include/net/http_client.h

@@ -233,7 +233,7 @@ namespace net_utils
 			}
 
 			//---------------------------------------------------------------------------
-			inline bool invoke(const boost::string_ref uri, const boost::string_ref method, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) override
+			inline bool invoke(const boost::string_ref uri, const boost::string_ref method, const boost::string_ref body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) override
 			{
 				CRITICAL_REGION_LOCAL(m_lock);
 				if(!is_connected())
@@ -885,14 +885,6 @@ namespace net_utils
 			}
 		};
 		typedef http_simple_client_template<blocked_mode_client> http_simple_client;
-
-    class http_simple_client_factory : public http_client_factory
-    {
-    public:
-      std::unique_ptr<abstract_http_client> create() override {
-        return std::unique_ptr<epee::net_utils::http::abstract_http_client>(new epee::net_utils::http::http_simple_client());
-      }
-    };
 	}
 }
 }

contrib/epee/include/net/http_server_handlers_map2.h

@@ -42,8 +42,17 @@
   MINFO("HTTP [" << m_conn_context.m_remote_address.host_str() << "] " << query_info.m_http_method_str << " " << query_info.m_URI); \
   response.m_response_code = 200; \
   response.m_response_comment = "Ok"; \
-  if(!handle_http_request_map(query_info, response, m_conn_context)) \
-  {response.m_response_code = 404;response.m_response_comment = "Not found";} \
+  try \
+  { \
+    if(!handle_http_request_map(query_info, response, m_conn_context)) \
+    {response.m_response_code = 404;response.m_response_comment = "Not found";} \
+  } \
+  catch (const std::exception &e) \
+  { \
+    MERROR(m_conn_context << "Exception in handle_http_request_map: " << e.what()); \
+    response.m_response_code = 500; \
+    response.m_response_comment = "Internal Server Error"; \
+  } \
   return true; \
 }
 
@@ -69,9 +78,11 @@
       uint64_t ticks1 = epee::misc_utils::get_tick_count(); \
       boost::value_initialized<command_type::response> resp;\
       MINFO(m_conn_context << "calling " << s_pattern); \
-      if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context)) \
+      bool res = false; \
+      try { res = callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context); } \
+      catch (const std::exception &e) { MERROR(m_conn_context << "Failed to " << #callback_f << "(): " << e.what()); } \
+      if (!res) \
       { \
-        MERROR(m_conn_context << "Failed to " << #callback_f << "()"); \
         response_info.m_response_code = 500; \
         response_info.m_response_comment = "Internal Server Error"; \
         return true; \
@@ -97,16 +108,20 @@
       uint64_t ticks1 = misc_utils::get_tick_count(); \
       boost::value_initialized<command_type::response> resp;\
       MINFO(m_conn_context << "calling " << s_pattern); \
-      if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context)) \
+      bool res = false; \
+      try { res = callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context); } \
+      catch (const std::exception &e) { MERROR(m_conn_context << "Failed to " << #callback_f << "()"); } \
+      if (!res) \
       { \
-        MERROR(m_conn_context << "Failed to " << #callback_f << "()"); \
         response_info.m_response_code = 500; \
         response_info.m_response_comment = "Internal Server Error"; \
         return true; \
       } \
       uint64_t ticks2 = misc_utils::get_tick_count(); \
-      epee::serialization::store_t_to_binary(static_cast<command_type::response&>(resp), response_info.m_body); \
+      epee::byte_slice buffer; \
+      epee::serialization::store_t_to_binary(static_cast<command_type::response&>(resp), buffer, 64 * 1024); \
       uint64_t ticks3 = epee::misc_utils::get_tick_count(); \
+      response_info.m_body.assign(reinterpret_cast<const char*>(buffer.data()), buffer.size()); \
       response_info.m_mime_tipe = " application/octet-stream"; \
       response_info.m_header_info.m_content_type = " application/octet-stream"; \
       MDEBUG( s_pattern << "() processed with " << ticks1-ticks << "/"<< ticks2-ticks1 << "/" << ticks3-ticks2 << "ms"); \
@@ -184,7 +199,10 @@
   fail_resp.jsonrpc = "2.0"; \
   fail_resp.id = req.id; \
   MINFO(m_conn_context << "Calling RPC method " << method_name); \
-  if(!callback_f(req.params, resp.result, fail_resp.error, &m_conn_context)) \
+  bool res = false; \
+  try { res = callback_f(req.params, resp.result, fail_resp.error, &m_conn_context); } \
+  catch (const std::exception &e) { MERROR(m_conn_context << "Failed to " << #callback_f << "(): " << e.what()); } \
+  if (!res) \
   { \
     epee::serialization::store_t_to_json(static_cast<epee::json_rpc::error_response&>(fail_resp), response_info.m_body); \
     return true; \
@@ -203,7 +221,10 @@
   fail_resp.jsonrpc = "2.0"; \
   fail_resp.id = req.id; \
   MINFO(m_conn_context << "calling RPC method " << method_name); \
-  if(!callback_f(req.params, resp.result, fail_resp.error, response_info, &m_conn_context)) \
+  bool res = false; \
+  try { res = callback_f(req.params, resp.result, fail_resp.error, response_info, &m_conn_context); } \
+  catch (const std::exception &e) { MERROR(m_conn_context << "Failed to " << #callback_f << "(): " << e.what()); } \
+  if (!res) \
   { \
     epee::serialization::store_t_to_json(static_cast<epee::json_rpc::error_response&>(fail_resp), response_info.m_body); \
     return true; \
@@ -217,7 +238,10 @@
 { \
   PREPARE_OBJECTS_FROM_JSON(command_type) \
   MINFO(m_conn_context << "calling RPC method " << method_name); \
-  if(!callback_f(req.params, resp.result, &m_conn_context)) \
+  bool res = false; \
+  try { res = callback_f(req.params, resp.result, &m_conn_context); } \
+  catch (const std::exception &e) { MERROR(m_conn_context << "Failed to " << #callback_f << "(): " << e.what()); } \
+  if (!res) \
   { \
     epee::json_rpc::error_response fail_resp = AUTO_VAL_INIT(fail_resp); \
     fail_resp.jsonrpc = "2.0"; \

contrib/epee/include/net/levin_base.h

@@ -31,7 +31,6 @@
 
 #include <cstdint>
 
-#include "byte_slice.h"
 #include "net_utils_base.h"
 #include "span.h"
 
@@ -39,6 +38,7 @@
 
 namespace epee
 {
+class byte_slice;
 namespace levin
 {
 #pragma pack(push)
@@ -72,7 +72,8 @@ namespace levin
 
 
 #define LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED 0
-#define LEVIN_DEFAULT_MAX_PACKET_SIZE 100000000      //100MB by default
+#define LEVIN_INITIAL_MAX_PACKET_SIZE  256*1024      // 256 KiB before handshake
+#define LEVIN_DEFAULT_MAX_PACKET_SIZE 100000000      //100MB by default after handshake
 
 #define LEVIN_PACKET_REQUEST			0x00000001
 #define LEVIN_PACKET_RESPONSE		0x00000002
@@ -86,7 +87,7 @@ namespace levin
   template<class t_connection_context = net_utils::connection_context_base>
   struct levin_commands_handler
   {
-    virtual int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, t_connection_context& context)=0;
+    virtual int invoke(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, t_connection_context& context)=0;
     virtual int notify(int command, const epee::span<const uint8_t> in_buff, t_connection_context& context)=0;
     virtual void callback(t_connection_context& context){};
 

contrib/epee/include/net/levin_protocol_handler_async.h

@@ -84,7 +84,8 @@ class async_protocol_handler_config
 
 public:
   typedef t_connection_context connection_context;
-  uint64_t m_max_packet_size; 
+  uint64_t m_initial_max_packet_size;
+  uint64_t m_max_packet_size;
   uint64_t m_invoke_timeout;
 
   int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, boost::uuids::uuid connection_id);
@@ -105,7 +106,7 @@ public:
   size_t get_in_connections_count();
   void set_handler(levin_commands_handler<t_connection_context>* handler, void (*destroy)(levin_commands_handler<t_connection_context>*) = NULL);
 
-  async_protocol_handler_config():m_pcommands_handler(NULL), m_pcommands_handler_destroy(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE), m_invoke_timeout(LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED)
+  async_protocol_handler_config():m_pcommands_handler(NULL), m_pcommands_handler_destroy(NULL), m_initial_max_packet_size(LEVIN_INITIAL_MAX_PACKET_SIZE), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE), m_invoke_timeout(LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED)
   {}
   ~async_protocol_handler_config() { set_handler(NULL, NULL); }
   void del_out_connections(size_t count);
@@ -162,6 +163,7 @@ public:
   net_utils::i_service_endpoint* m_pservice_endpoint; 
   config_type& m_config;
   t_connection_context& m_connection_context;
+  std::atomic<uint64_t> m_max_packet_size;
 
   net_utils::buffer m_cache_in_buffer;
   stream_state m_state;
@@ -289,7 +291,8 @@ public:
             m_current_head(bucket_head2()),
             m_pservice_endpoint(psnd_hndlr), 
             m_config(config), 
-            m_connection_context(conn_context), 
+            m_connection_context(conn_context),
+            m_max_packet_size(config.m_initial_max_packet_size),
             m_cache_in_buffer(4 * 1024),
             m_state(stream_state_head)
   {
@@ -399,13 +402,14 @@ public:
     }
 
     // these should never fail, but do runtime check for safety
-    CHECK_AND_ASSERT_MES(m_config.m_max_packet_size >= m_cache_in_buffer.size(), false, "Bad m_cache_in_buffer.size()");
-    CHECK_AND_ASSERT_MES(m_config.m_max_packet_size - m_cache_in_buffer.size() >= m_fragment_buffer.size(), false, "Bad m_cache_in_buffer.size() + m_fragment_buffer.size()");
+    const uint64_t max_packet_size = m_max_packet_size;
+    CHECK_AND_ASSERT_MES(max_packet_size >= m_cache_in_buffer.size(), false, "Bad m_cache_in_buffer.size()");
+    CHECK_AND_ASSERT_MES(max_packet_size - m_cache_in_buffer.size() >= m_fragment_buffer.size(), false, "Bad m_cache_in_buffer.size() + m_fragment_buffer.size()");
 
     // flipped to subtraction; prevent overflow since m_max_packet_size is variable and public
-    if(cb > m_config.m_max_packet_size - m_cache_in_buffer.size() - m_fragment_buffer.size())
+    if(cb > max_packet_size - m_cache_in_buffer.size() - m_fragment_buffer.size())
     {
-      MWARNING(m_connection_context << "Maximum packet size exceed!, m_max_packet_size = " << m_config.m_max_packet_size
+      MWARNING(m_connection_context << "Maximum packet size exceed!, m_max_packet_size = " << max_packet_size
                           << ", packet received " << m_cache_in_buffer.size() +  cb 
                           << ", connection will be closed.");
       return false;
@@ -430,7 +434,7 @@ public:
               //async call scenario
               boost::shared_ptr<invoke_response_handler_base> response_handler = m_invoke_response_handlers.front();
               response_handler->reset_timer();
-              MDEBUG(m_connection_context << "LEVIN_PACKET partial msg received. len=" << cb);
+              MDEBUG(m_connection_context << "LEVIN_PACKET partial msg received. len=" << cb << ", current total " << m_cache_in_buffer.size() << "/" << m_current_head.m_cb << " (" << (100.0f * m_cache_in_buffer.size() / (m_current_head.m_cb ? m_current_head.m_cb : 1)) << "%)");
             }
           }
           break;
@@ -465,6 +469,14 @@ public:
             temp = std::move(m_fragment_buffer);
             m_fragment_buffer.clear();
             std::memcpy(std::addressof(m_current_head), std::addressof(temp[0]), sizeof(bucket_head2));
+            const size_t max_bytes = m_connection_context.get_max_bytes(m_current_head.m_command);
+            if(m_current_head.m_cb > std::min<size_t>(max_packet_size, max_bytes))
+            {
+              MERROR(m_connection_context << "Maximum packet size exceed!, m_max_packet_size = " << std::min<size_t>(max_packet_size, max_bytes)
+                << ", packet header received " << m_current_head.m_cb << ", command " << m_current_head.m_command
+                << ", connection will be closed.");
+              return false;
+            }
             buff_to_invoke = {reinterpret_cast<const uint8_t*>(temp.data()) + sizeof(bucket_head2), temp.size() - sizeof(bucket_head2)};
           }
 
@@ -514,16 +526,19 @@ public:
           {
             if(m_current_head.m_have_to_return_data)
             {
-              std::string return_buff;
+              byte_slice return_buff;
               const uint32_t return_code = m_config.m_pcommands_handler->invoke(
                 m_current_head.m_command, buff_to_invoke, return_buff, m_connection_context
               );
 
+              // peer_id remains unset if dropped
+              if (m_current_head.m_command == m_connection_context.handshake_command() && m_connection_context.handshake_complete())
+                m_max_packet_size = m_config.m_max_packet_size;
+
               bucket_head2 head = make_header(m_current_head.m_command, return_buff.size(), LEVIN_PACKET_RESPONSE, false);
               head.m_return_code = SWAP32LE(return_code);
-              return_buff.insert(0, reinterpret_cast<const char*>(&head), sizeof(head));
 
-              if(!m_pservice_endpoint->do_send(byte_slice{std::move(return_buff)}))
+              if(!m_pservice_endpoint->do_send(byte_slice{{epee::as_byte_span(head), epee::to_span(return_buff)}}))
                 return false;
 
               MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb
@@ -577,10 +592,11 @@ public:
           m_cache_in_buffer.erase(sizeof(bucket_head2));
           m_state = stream_state_body;
           m_oponent_protocol_ver = m_current_head.m_protocol_version;
-          if(m_current_head.m_cb > m_config.m_max_packet_size)
+          const size_t max_bytes = m_connection_context.get_max_bytes(m_current_head.m_command);
+          if(m_current_head.m_cb > std::min<size_t>(max_packet_size, max_bytes))
           {
-            LOG_ERROR_CC(m_connection_context, "Maximum packet size exceed!, m_max_packet_size = " << m_config.m_max_packet_size 
-              << ", packet header received " << m_current_head.m_cb 
+            LOG_ERROR_CC(m_connection_context, "Maximum packet size exceed!, m_max_packet_size = " << std::min<size_t>(max_packet_size, max_bytes)
+              << ", packet header received " << m_current_head.m_cb << ", command " << m_current_head.m_command
               << ", connection will be closed.");
             return false;
           }
@@ -634,6 +650,9 @@ public:
       boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0);
       CRITICAL_REGION_BEGIN(m_invoke_response_handlers_lock);
 
+      if (command == m_connection_context.handshake_command())
+        m_max_packet_size = m_config.m_max_packet_size;
+
       if(!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true))
       {
         LOG_ERROR_CC(m_connection_context, "Failed to do_send");
@@ -675,6 +694,9 @@ public:
 
     boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0);
 
+    if (command == m_connection_context.handshake_command())
+      m_max_packet_size = m_config.m_max_packet_size;
+
     if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true))
     {
       LOG_ERROR_CC(m_connection_context, "Failed to send request");

contrib/epee/include/net/local_ip.h

@@ -28,8 +28,6 @@
 #pragma once
 
 #include <string>
-#include <boost/algorithm/string/predicate.hpp>
-#include <boost/asio/ip/address_v6.hpp>
 #include "int-util.h"
 
 // IP addresses are kept in network byte order
@@ -40,30 +38,6 @@ namespace epee
 {
   namespace net_utils
   {
-
-    inline
-    bool is_ipv6_local(const std::string& ip)
-    {
-      auto addr = boost::asio::ip::address_v6::from_string(ip);
-
-      // ipv6 link-local unicast addresses are fe80::/10
-      bool is_link_local = addr.is_link_local();
-
-      auto addr_bytes = addr.to_bytes();
-
-      // ipv6 unique local unicast addresses start with fc00::/7 -- (fcXX or fdXX)
-      bool is_unique_local_unicast = (addr_bytes[0] == 0xfc || addr_bytes[0] == 0xfd);
-
-      return is_link_local || is_unique_local_unicast;
-    }
-
-    inline
-    bool is_ipv6_loopback(const std::string& ip)
-    {
-      // ipv6 loopback is ::1
-      return boost::asio::ip::address_v6::from_string(ip).is_loopback();
-    }
-
     inline
     bool is_ip_local(uint32_t ip)
     {

contrib/epee/include/net/net_helper.h

@@ -44,6 +44,7 @@
 #include <boost/lambda/lambda.hpp>
 #include <boost/interprocess/detail/atomic.hpp>
 #include <boost/system/error_code.hpp>
+#include <boost/utility/string_ref.hpp>
 #include <functional>
 #include "net/net_utils_base.h"
 #include "net/net_ssl.h"
@@ -178,7 +179,7 @@ namespace net_utils
 					// SSL Options
 					if (m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_enabled || m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect)
 					{
-						if (!m_ssl_options.handshake(*m_ssl_socket, boost::asio::ssl::stream_base::client, addr, timeout))
+						if (!m_ssl_options.handshake(*m_ssl_socket, boost::asio::ssl::stream_base::client, {}, addr, timeout))
 						{
 							if (m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect)
 							{
@@ -280,7 +281,7 @@ namespace net_utils
 
 
 		inline 
-		bool send(const std::string& buff, std::chrono::milliseconds timeout)
+		bool send(const boost::string_ref buff, std::chrono::milliseconds timeout)
 		{
 
 			try
@@ -298,7 +299,7 @@ namespace net_utils
 				// object is used as a callback and will update the ec variable when the
 				// operation completes. The blocking_udp_client.cpp example shows how you
 				// can use boost::bind rather than boost::lambda.
-				async_write(buff.c_str(), buff.size(), ec);
+				async_write(buff.data(), buff.size(), ec);
 
 				// Block until the asynchronous operation has completed.
 				while (ec == boost::asio::error::would_block)

contrib/epee/include/net/net_parse_helpers.h

@@ -131,8 +131,8 @@ namespace net_utils
   inline
     bool parse_url_ipv6(const std::string url_str, http::url_content& content)
   {
-    STATIC_REGEXP_EXPR_1(rexp_match_uri, "^((.*?)://)?(\\[(.*)\\](:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal);
-    //                                     12         3   4      5 6        7
+    STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(\\[(.*)\\](:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal);
+    //                                     12            3   4      5 6        7
 
     content.port = 0;
     boost::smatch result;
@@ -175,8 +175,8 @@ namespace net_utils
 
     ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash=
     //STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal);
-    STATIC_REGEXP_EXPR_1(rexp_match_uri, "^((.*?)://)?(([^/:]*)(:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal);
-    //                                     12         34      5 6        7
+    STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(([^/:]*)(:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal);
+    //                                     12            34       5 6        7
     content.port = 0;
     boost::smatch result;	
     if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched))

contrib/epee/include/net/net_ssl.h

@@ -132,6 +132,7 @@ namespace net_utils
     bool handshake(
       boost::asio::ssl::stream<boost::asio::ip::tcp::socket> &socket,
       boost::asio::ssl::stream_base::handshake_type type,
+      boost::asio::const_buffer buffer = {},
       const std::string& host = {},
       std::chrono::milliseconds timeout = std::chrono::seconds(15)) const;
   };

contrib/epee/include/rolling_median.h

@@ -34,6 +34,8 @@
 
 #pragma once
 
+#include "misc_language.h"
+
 #include <stdlib.h>
 #include <stdint.h>
 
@@ -139,7 +141,6 @@ public:
 
   rolling_median_t(rolling_median_t &&m)
   {
-    free(data);
     memcpy(this, &m, sizeof(rolling_median_t));
     m.data = NULL;
   }
@@ -226,7 +227,7 @@ public:
     Item v = data[heap[0]];
     if (minCt < maxCt)
     {
-      v = (v + data[heap[-1]]) / 2;
+      v = get_mid<Item>(v, data[heap[-1]]);
     }
     return v;
   }

contrib/epee/include/storages/http_abstract_invoke.h

@@ -29,6 +29,7 @@
 #include <boost/utility/string_ref.hpp>
 #include <chrono>
 #include <string>
+#include "byte_slice.h"
 #include "portable_storage_template_helper.h"
 #include "net/http_base.h"
 #include "net/http_server_handlers_map2.h"
@@ -74,12 +75,12 @@ namespace epee
     template<class t_request, class t_response, class t_transport>
     bool invoke_http_bin(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref method = "POST")
     {
-      std::string req_param;
-      if(!serialization::store_t_to_binary(out_struct, req_param))
+      byte_slice req_param;
+      if(!serialization::store_t_to_binary(out_struct, req_param, 16 * 1024))
         return false;
 
       const http::http_response_info* pri = NULL;
-      if(!transport.invoke(uri, method, req_param, timeout, std::addressof(pri)))
+      if(!transport.invoke(uri, method, boost::string_ref{reinterpret_cast<const char*>(req_param.data()), req_param.size()}, timeout, std::addressof(pri)))
       {
         LOG_PRINT_L1("Failed to invoke http request to  " << uri);
         return false;
@@ -97,7 +98,12 @@ namespace epee
         return false;
       }
 
-      return serialization::load_t_from_binary(result_struct, epee::strspan<uint8_t>(pri->m_body));
+      static const constexpr epee::serialization::portable_storage::limits_t default_http_bin_limits = {
+        65536 * 3, // objects
+        65536 * 3, // fields
+        65536 * 3, // strings
+      };
+      return serialization::load_t_from_binary(result_struct, epee::strspan<uint8_t>(pri->m_body), &default_http_bin_limits);
     }
 
     template<class t_request, class t_response, class t_transport>

contrib/epee/include/storages/levin_abstract_invoke2.h

@@ -27,8 +27,10 @@
 #pragma once
 
 #include "portable_storage_template_helper.h"
+#include <boost/utility/string_ref.hpp>
 #include <boost/utility/value_init.hpp>
 #include <functional>
+#include "byte_slice.h"
 #include "span.h"
 #include "net/levin_base.h"
 
@@ -50,6 +52,11 @@ namespace
     snprintf(buf, sizeof(buf),  "command-%u", command);
     return on_levin_traffic(context, initiator, sent, error, bytes, buf);
   }
+  static const constexpr epee::serialization::portable_storage::limits_t default_levin_limits = {
+    8192, // objects
+    16384, // fields
+    16384, // strings
+  };
 }
 
 namespace epee
@@ -75,7 +82,7 @@ namespace epee
         return false;
       }
       serialization::portable_storage stg_ret;
-      if(!stg_ret.load_from_binary(buff_to_recv))
+      if(!stg_ret.load_from_binary(buff_to_recv, &default_levin_limits))
       {
         LOG_ERROR("Failed to load_from_binary on command " << command);
         return false;
@@ -110,18 +117,19 @@ namespace epee
       const boost::uuids::uuid &conn_id = context.m_connection_id;
       typename serialization::portable_storage stg;
       out_struct.store(stg);
-      std::string buff_to_send, buff_to_recv;
-      stg.store_to_binary(buff_to_send);
+      byte_slice buff_to_send;
+      std::string buff_to_recv;
+      stg.store_to_binary(buff_to_send, 16 * 1024);
 
       on_levin_traffic(context, true, true, false, buff_to_send.size(), command);
-      int res = transport.invoke(command, buff_to_send, buff_to_recv, conn_id);
+      int res = transport.invoke(command, boost::string_ref{reinterpret_cast<const char*>(buff_to_send.data()), buff_to_send.size()}, buff_to_recv, conn_id);
       if( res <=0 )
       {
         LOG_PRINT_L1("Failed to invoke command " << command << " return code " << res);
         return false;
       }
       typename serialization::portable_storage stg_ret;
-      if(!stg_ret.load_from_binary(buff_to_recv))
+      if(!stg_ret.load_from_binary(buff_to_recv, &default_levin_limits))
       {
         on_levin_traffic(context, true, false, true, buff_to_recv.size(), command);
         LOG_ERROR("Failed to load_from_binary on command " << command);
@@ -137,10 +145,10 @@ namespace epee
       const boost::uuids::uuid &conn_id = context.m_connection_id;
       typename serialization::portable_storage stg;
       const_cast<t_arg&>(out_struct).store(stg);//TODO: add true const support to searilzation
-      std::string buff_to_send;
-      stg.store_to_binary(buff_to_send);
+      byte_slice buff_to_send;
+      stg.store_to_binary(buff_to_send, 16 * 1024);
       on_levin_traffic(context, true, true, false, buff_to_send.size(), command);
-      int res = transport.invoke_async(command, epee::strspan<uint8_t>(buff_to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool 
+      int res = transport.invoke_async(command, epee::to_span(buff_to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool
       {
         t_result result_struct = AUTO_VAL_INIT(result_struct);
         if( code <=0 )
@@ -152,7 +160,7 @@ namespace epee
           return false;
         }
         serialization::portable_storage stg_ret;
-        if(!stg_ret.load_from_binary(buff))
+        if(!stg_ret.load_from_binary(buff, &default_levin_limits))
         {
           on_levin_traffic(context, true, false, true, buff.size(), command);
           LOG_ERROR("Failed to load_from_binary on command " << command);
@@ -184,11 +192,11 @@ namespace epee
       const boost::uuids::uuid &conn_id = context.m_connection_id;
       serialization::portable_storage stg;
       out_struct.store(stg);
-      std::string buff_to_send;
+      byte_slice buff_to_send;
       stg.store_to_binary(buff_to_send);
 
       on_levin_traffic(context, true, true, false, buff_to_send.size(), command);
-      int res = transport.notify(command, epee::strspan<uint8_t>(buff_to_send), conn_id);
+      int res = transport.notify(command, epee::to_span(buff_to_send), conn_id);
       if(res <=0 )
       {
         MERROR("Failed to notify command " << command << " return code " << res);
@@ -199,10 +207,10 @@ namespace epee
     //----------------------------------------------------------------------------------------------------
     //----------------------------------------------------------------------------------------------------
     template<class t_owner, class t_in_type, class t_out_type, class t_context, class callback_t>
-    int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, callback_t cb, t_context& context )
+    int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, callback_t cb, t_context& context )
     {
       serialization::portable_storage strg;
-      if(!strg.load_from_binary(in_buff))
+      if(!strg.load_from_binary(in_buff, &default_levin_limits))
       {
         on_levin_traffic(context, false, false, true, in_buff.size(), command);
         LOG_ERROR("Failed to load_from_binary in command " << command);
@@ -222,7 +230,7 @@ namespace epee
       serialization::portable_storage strg_out;
       static_cast<t_out_type&>(out_struct).store(strg_out);
 
-      if(!strg_out.store_to_binary(buff_out))
+      if(!strg_out.store_to_binary(buff_out, 32 * 1024))
       {
         LOG_ERROR("Failed to store_to_binary in command" << command);
         return -1;
@@ -236,7 +244,7 @@ namespace epee
     int buff_to_t_adapter(t_owner* powner, int command, const epee::span<const uint8_t> in_buff, callback_t cb, t_context& context)
     {
       serialization::portable_storage strg;
-      if(!strg.load_from_binary(in_buff))
+      if(!strg.load_from_binary(in_buff, &default_levin_limits))
       {
         on_levin_traffic(context, false, false, true, in_buff.size(), command);
         LOG_ERROR("Failed to load_from_binary in notify " << command);
@@ -254,7 +262,7 @@ namespace epee
     }
 
 #define CHAIN_LEVIN_INVOKE_MAP2(context_type) \
-  int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, context_type& context) \
+  int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, context_type& context) \
   { \
   bool handled = false; \
   return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \
@@ -263,13 +271,13 @@ namespace epee
 #define CHAIN_LEVIN_NOTIFY_MAP2(context_type) \
   int notify(int command, const epee::span<const uint8_t> in_buff, context_type& context) \
   { \
-  bool handled = false; std::string fake_str;\
+  bool handled = false; epee::byte_slice fake_str; \
   return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \
   } 
 
 
 #define CHAIN_LEVIN_INVOKE_MAP() \
-  int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, epee::net_utils::connection_context_base& context) \
+  int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, epee::net_utils::connection_context_base& context) \
   { \
   bool handled = false; \
   return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \
@@ -289,8 +297,9 @@ namespace epee
   } 
 
 #define BEGIN_INVOKE_MAP2(owner_type) \
-  template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, t_context& context, bool& handled) \
+  template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, t_context& context, bool& handled) \
   { \
+  try { \
   typedef owner_type internal_owner_type_name;
 
 #define HANDLE_INVOKE2(command_id, func, type_name_in, typename_out) \
@@ -336,7 +345,13 @@ namespace epee
   LOG_ERROR("Unknown command:" << command); \
   on_levin_traffic(context, false, false, true, in_buff.size(), "invalid-command"); \
   return LEVIN_ERROR_CONNECTION_HANDLER_NOT_DEFINED; \
+  } \
+  catch (const std::exception &e) { \
+    MERROR("Error in handle_invoke_map: " << e.what()); \
+    return LEVIN_ERROR_CONNECTION_TIMEDOUT; /* seems kinda appropriate */ \
+  } \
   }
+
   }
 }
 

contrib/epee/include/storages/portable_storage.h

@@ -32,7 +32,6 @@
 
 #include "misc_language.h"
 #include "portable_storage_base.h"
-#include "portable_storage_to_bin.h"
 #include "portable_storage_from_bin.h"
 #include "portable_storage_to_json.h"
 #include "portable_storage_from_json.h"
@@ -42,6 +41,7 @@
 
 namespace epee
 {
+  class byte_slice;
   namespace serialization
   {
     /************************************************************************/
@@ -54,6 +54,13 @@ namespace epee
       typedef epee::serialization::harray  harray;
       typedef storage_entry meta_entry;
 
+      struct limits_t
+      {
+        size_t n_objects;
+        size_t n_fields;
+        size_t n_strings; // not counting field names
+      };
+
       portable_storage(){}
       virtual ~portable_storage(){}
       hsection   open_section(const std::string& section_name,  hsection hparent_section, bool create_if_notexist = false);
@@ -83,9 +90,9 @@ namespace epee
       bool        delete_entry(const std::string& pentry_name, hsection hparent_section = nullptr);
 
       //-------------------------------------------------------------------------------
-      bool		store_to_binary(binarybuffer& target);
-      bool		load_from_binary(const epee::span<const uint8_t> target);
-      bool		load_from_binary(const std::string& target) { return load_from_binary(epee::strspan<uint8_t>(target)); }
+      bool		store_to_binary(byte_slice& target, std::size_t initial_buffer_size = 8192);
+      bool		load_from_binary(const epee::span<const uint8_t> target, const limits_t *limits = NULL);
+      bool		load_from_binary(const std::string& target, const limits_t *limits = NULL) { return load_from_binary(epee::strspan<uint8_t>(target), limits); }
       template<class trace_policy>
       bool		  dump_as_xml(std::string& targetObj, const std::string& root_name = "");
       bool		  dump_as_json(std::string& targetObj, size_t indent = 0, bool insert_newlines = true);
@@ -133,24 +140,8 @@ namespace epee
     {
       return false;//TODO: don't think i ever again will use xml - ambiguous and "overtagged" format
     }
-
     inline
-    bool portable_storage::store_to_binary(binarybuffer& target)
-    {
-      TRY_ENTRY();
-      std::stringstream ss;
-      storage_block_header sbh = AUTO_VAL_INIT(sbh);
-      sbh.m_signature_a = SWAP32LE(PORTABLE_STORAGE_SIGNATUREA);
-      sbh.m_signature_b = SWAP32LE(PORTABLE_STORAGE_SIGNATUREB);
-      sbh.m_ver = PORTABLE_STORAGE_FORMAT_VER;
-      ss.write((const char*)&sbh, sizeof(storage_block_header));
-      pack_entry_to_buff(ss, m_root);
-      target = ss.str();
-      return true;
-      CATCH_ENTRY("portable_storage::store_to_binary", false)
-    }
-    inline
-    bool portable_storage::load_from_binary(const epee::span<const uint8_t> source)
+    bool portable_storage::load_from_binary(const epee::span<const uint8_t> source, const limits_t *limits)
     {
       m_root.m_entries.clear();
       if(source.size() < sizeof(storage_block_header))
@@ -173,6 +164,8 @@ namespace epee
       }
       TRY_ENTRY();
       throwable_buffer_reader buf_reader(source.data()+sizeof(storage_block_header), source.size()-sizeof(storage_block_header));
+      if (limits)
+        buf_reader.set_limits(limits->n_objects, limits->n_fields, limits->n_strings);
       buf_reader.read(m_root);
       return true;//TODO:
       CATCH_ENTRY("portable_storage::load_from_binary", false);
@@ -282,6 +275,7 @@ namespace epee
       static_assert(std::is_rvalue_reference<entry_type&&>(), "unexpected copy of value");
       TRY_ENTRY();
       CHECK_AND_ASSERT(psection, nullptr);
+      CHECK_AND_ASSERT(!pentry_name.empty(), nullptr);
       auto ins_res = psection->m_entries.emplace(pentry_name, std::forward<entry_type>(entry));
       return &ins_res.first->second;
       CATCH_ENTRY("portable_storage::insert_new_entry_get_storage_entry", nullptr);

contrib/epee/include/storages/portable_storage_from_bin.h

@@ -29,6 +29,7 @@
 #pragma once 
 
 #include "misc_language.h"
+#include "misc_log_ex.h"
 #include "portable_storage_base.h"
 #include "portable_storage_bin_utils.h"
 
@@ -42,6 +43,24 @@ namespace epee
 {
   namespace serialization
   {
+    template<typename T>
+    struct ps_min_bytes {
+      static constexpr const size_t strict = 4096; // actual low bound
+    };
+    template<> struct ps_min_bytes<uint64_t> { static constexpr const size_t strict = 8; };
+    template<> struct ps_min_bytes<int64_t> { static constexpr const size_t strict = 8; };
+    template<> struct ps_min_bytes<uint32_t> { static constexpr const size_t strict = 4; };
+    template<> struct ps_min_bytes<int32_t> { static constexpr const size_t strict = 4; };
+    template<> struct ps_min_bytes<uint16_t> { static constexpr const size_t strict = 2; };
+    template<> struct ps_min_bytes<int16_t> { static constexpr const size_t strict = 2; };
+    template<> struct ps_min_bytes<uint8_t> { static constexpr const size_t strict = 1; };
+    template<> struct ps_min_bytes<int8_t> { static constexpr const size_t strict = 1; };
+    template<> struct ps_min_bytes<double> { static constexpr const size_t strict = 8; };
+    template<> struct ps_min_bytes<bool> { static constexpr const size_t strict = 1; };
+    template<> struct ps_min_bytes<std::string> { static constexpr const size_t strict = 2; };
+    template<> struct ps_min_bytes<section> { static constexpr const size_t strict = 1; };
+    template<> struct ps_min_bytes<array_entry> { static constexpr const size_t strict = 1; };
+
     struct throwable_buffer_reader
     {
       throwable_buffer_reader(const void* ptr, size_t sz);
@@ -61,6 +80,9 @@ namespace epee
       void read(section& sec);
       void read(std::string& str);
       void read(array_entry &ae);
+      template<class t_type>
+      size_t min_bytes() const;
+      void set_limits(size_t objects, size_t fields, size_t strings);
     private:
       struct recursuion_limitation_guard
       {
@@ -81,6 +103,13 @@ namespace epee
       const uint8_t* m_ptr;
       size_t m_count;
       size_t m_recursion_count;
+      size_t m_objects;
+      size_t m_fields;
+      size_t m_strings;
+
+      size_t max_objects;
+      size_t max_fields;
+      size_t max_strings;
     };
 
     inline throwable_buffer_reader::throwable_buffer_reader(const void* ptr, size_t sz)
@@ -92,6 +121,12 @@ namespace epee
       m_ptr = (uint8_t*)ptr;
       m_count = sz;
       m_recursion_count = 0;
+      m_objects = 0;
+      m_fields = 0;
+      m_strings = 0;
+      max_objects = std::numeric_limits<size_t>::max();
+      max_fields = std::numeric_limits<size_t>::max();
+      max_strings = std::numeric_limits<size_t>::max();
     }
     inline 
     void throwable_buffer_reader::read(void* target, size_t count)
@@ -108,6 +143,7 @@ namespace epee
       RECURSION_LIMITATION();
       uint8_t name_len = 0;
       read(name_len);
+      CHECK_AND_ASSERT_THROW_MES(name_len > 0, "Section name is missing");
       sce_name.resize(name_len);
       read((void*)sce_name.data(), name_len);
     }
@@ -138,7 +174,18 @@ namespace epee
       //for pod types
       array_entry_t<type_name> sa;
       size_t size = read_varint();
-      CHECK_AND_ASSERT_THROW_MES(size <= m_count, "Size sanity check failed");
+      CHECK_AND_ASSERT_THROW_MES(size <= m_count / ps_min_bytes<type_name>::strict, "Size sanity check failed");
+      if (std::is_same<type_name, section>())
+      {
+        CHECK_AND_ASSERT_THROW_MES(size <= max_objects - m_objects, "Too many objects");
+        m_objects += size;
+      }
+      else if (std::is_same<type_name, std::string>())
+      {
+        CHECK_AND_ASSERT_THROW_MES(size <= max_strings - m_strings, "Too many strings");
+        m_strings += size;
+      }
+
       sa.reserve(size);
       //TODO: add some optimization here later
       while(size--)
@@ -204,6 +251,8 @@ namespace epee
     inline storage_entry throwable_buffer_reader::read_se<std::string>()
     {
       RECURSION_LIMITATION();
+      CHECK_AND_ASSERT_THROW_MES(m_strings + 1 <= max_strings, "Too many strings");
+      m_strings += 1;
       return storage_entry(read<std::string>());
     }
 
@@ -212,6 +261,8 @@ namespace epee
     inline storage_entry throwable_buffer_reader::read_se<section>()
     {
       RECURSION_LIMITATION();
+      CHECK_AND_ASSERT_THROW_MES(m_objects < max_objects, "Too many objects");
+      ++m_objects;
       section s;//use extra variable due to vs bug, line "storage_entry se(section()); " can't be compiled in visual studio
       storage_entry se(std::move(s));
       section& section_entry = boost::get<section>(se);
@@ -263,12 +314,16 @@ namespace epee
       RECURSION_LIMITATION();
       sec.m_entries.clear();
       size_t count = read_varint();
+      CHECK_AND_ASSERT_THROW_MES(count <= max_fields - m_fields, "Too many object fields");
+      m_fields += count;
       while(count--)
       {
         //read section name string
         std::string sec_name;
         read_sec_name(sec_name);
-        sec.m_entries.emplace(std::move(sec_name), load_storage_entry());
+        const auto insert_loc = sec.m_entries.lower_bound(sec_name);
+        CHECK_AND_ASSERT_THROW_MES(insert_loc == sec.m_entries.end() || insert_loc->first != sec_name, "duplicate key: " << sec_name);
+        sec.m_entries.emplace_hint(insert_loc, std::move(sec_name), load_storage_entry());
       }
     }
     inline 
@@ -289,5 +344,12 @@ namespace epee
       RECURSION_LIMITATION();
       CHECK_AND_ASSERT_THROW_MES(false, "Reading array entry is not supported");
     }
+    inline
+    void throwable_buffer_reader::set_limits(size_t objects, size_t fields, size_t strings)
+    {
+      max_objects = objects;
+      max_fields = fields;
+      max_strings = strings;
+    }
   }
 }

contrib/epee/include/storages/portable_storage_template_helper.h

@@ -28,6 +28,7 @@
 
 #include <string>
 
+#include "byte_slice.h"
 #include "parserse_base_utils.h"
 #include "portable_storage.h"
 #include "file_io_utils.h"
@@ -84,10 +85,10 @@ namespace epee
     }
     //-----------------------------------------------------------------------------------------------------------
     template<class t_struct>
-    bool load_t_from_binary(t_struct& out, const epee::span<const uint8_t> binary_buff)
+    bool load_t_from_binary(t_struct& out, const epee::span<const uint8_t> binary_buff, const epee::serialization::portable_storage::limits_t *limits = NULL)
     {
       portable_storage ps;
-      bool rs = ps.load_from_binary(binary_buff);
+      bool rs = ps.load_from_binary(binary_buff, limits);
       if(!rs)
         return false;
 
@@ -111,18 +112,18 @@ namespace epee
     }
     //-----------------------------------------------------------------------------------------------------------
     template<class t_struct>
-    bool store_t_to_binary(t_struct& str_in, std::string& binary_buff, size_t indent = 0)
+    bool store_t_to_binary(t_struct& str_in, byte_slice& binary_buff, size_t initial_buffer_size = 8192)
     {
       portable_storage ps;
       str_in.store(ps);
-      return ps.store_to_binary(binary_buff);
+      return ps.store_to_binary(binary_buff, initial_buffer_size);
     }
     //-----------------------------------------------------------------------------------------------------------
     template<class t_struct>
-    std::string store_t_to_binary(t_struct& str_in, size_t indent = 0)
+    byte_slice store_t_to_binary(t_struct& str_in, size_t initial_buffer_size = 8192)
     {
-      std::string binary_buff;
-      store_t_to_binary(str_in, binary_buff, indent);
+      byte_slice binary_buff;
+      store_t_to_binary(str_in, binary_buff, initial_buffer_size);
       return binary_buff;
     }
   }

contrib/epee/include/storages/portable_storage_to_bin.h

@@ -211,6 +211,7 @@ namespace epee
       for(const section_pair& se: sec.m_entries)
       {
         CHECK_AND_ASSERT_THROW_MES(se.first.size() < std::numeric_limits<uint8_t>::max(), "storage_entry_name is too long: " << se.first.size() << ", val: " << se.first);
+        CHECK_AND_ASSERT_THROW_MES(!se.first.empty(), "storage_entry_name is empty");
         uint8_t len = static_cast<uint8_t>(se.first.size());
         strm.write((const char*)&len, sizeof(len));
         strm.write(se.first.data(), size_t(len));

contrib/epee/src/CMakeLists.txt

@@ -29,7 +29,7 @@
 
 add_library(epee STATIC byte_slice.cpp byte_stream.cpp hex.cpp abstract_http_client.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp
     wipeable_string.cpp levin_base.cpp memwipe.c connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp
-    int-util.cpp)
+    int-util.cpp portable_storage.cpp)
 
 if (USE_READLINE AND (GNU_READLINE_FOUND OR (DEPENDS AND NOT MINGW)))
   add_library(epee_readline STATIC readline_buffer.cpp)

contrib/epee/src/abstract_http_client.cpp

@@ -137,6 +137,11 @@ namespace http
     set_server(std::move(parsed.host), std::to_string(parsed.port), std::move(user), std::move(ssl_options));
     return true;
   }
+
+  bool epee::net_utils::http::abstract_http_client::set_proxy(const std::string& address)
+  {
+    return false;
+  }
 }
 }
 }

contrib/epee/src/byte_stream.cpp

@@ -34,6 +34,11 @@
 
 #include <iostream>
 
+namespace
+{
+  constexpr const std::size_t minimum_increase = 4096;
+}
+
 namespace epee
 {
   void byte_stream::overflow(const std::size_t requested)
@@ -46,7 +51,7 @@ namespace epee
 
     const std::size_t len = size();
     const std::size_t cap = capacity();
-    const std::size_t increase = std::max(need, increase_size());
+    const std::size_t increase = std::max(std::max(need, cap), minimum_increase);
 
     next_write_ = nullptr;
     end_ = nullptr;
@@ -62,8 +67,7 @@ namespace epee
   byte_stream::byte_stream(byte_stream&& rhs) noexcept
     : buffer_(std::move(rhs.buffer_)),
       next_write_(rhs.next_write_),
-      end_(rhs.end_),
-      increase_size_(rhs.increase_size_)
+      end_(rhs.end_)
   {
     rhs.next_write_ = nullptr;
     rhs.end_ = nullptr;
@@ -76,7 +80,6 @@ namespace epee
       buffer_ = std::move(rhs.buffer_);
       next_write_ = rhs.next_write_;
       end_ = rhs.end_;
-      increase_size_ = rhs.increase_size_;
       rhs.next_write_ = nullptr;
       rhs.end_ = nullptr;
     }

contrib/epee/src/net_ssl.cpp

@@ -473,6 +473,7 @@ bool ssl_options_t::has_fingerprint(boost::asio::ssl::verify_context &ctx) const
 bool ssl_options_t::handshake(
   boost::asio::ssl::stream<boost::asio::ip::tcp::socket> &socket,
   boost::asio::ssl::stream_base::handshake_type type,
+  boost::asio::const_buffer buffer,
   const std::string& host,
   std::chrono::milliseconds timeout) const
 {
@@ -530,7 +531,7 @@ bool ssl_options_t::handshake(
   });
 
   boost::system::error_code ec = boost::asio::error::would_block;
-  socket.async_handshake(type, boost::lambda::var(ec) = boost::lambda::_1);
+  socket.async_handshake(type, boost::asio::buffer(buffer), boost::lambda::var(ec) = boost::lambda::_1);
   if (io_service.stopped())
   {
     io_service.reset();

contrib/epee/src/net_utils_base.cpp

@@ -6,6 +6,17 @@
 #include "string_tools.h"
 #include "net/local_ip.h"
 
+static inline uint32_t make_address_v4_from_v6(const boost::asio::ip::address_v6& a)
+{
+  const auto &bytes = a.to_bytes();
+  uint32_t v4 = 0;
+  v4 = (v4 << 8) | bytes[12];
+  v4 = (v4 << 8) | bytes[13];
+  v4 = (v4 << 8) | bytes[14];
+  v4 = (v4 << 8) | bytes[15];
+  return htonl(v4);
+}
+
 namespace epee { namespace net_utils
 {
 	bool ipv4_network_address::equal(const ipv4_network_address& other) const noexcept
@@ -83,8 +94,28 @@ namespace epee { namespace net_utils
 		network_address::interface const* const other_self = other.self.get();
 		if (self_ == other_self) return true;
 		if (!self_ || !other_self) return false;
-		if (typeid(*self_) != typeid(*other_self)) return false;
-		return self_->is_same_host(*other_self);
+		if (typeid(*self_) == typeid(*other_self))
+			return self_->is_same_host(*other_self);
+		const auto this_id = get_type_id();
+		if (this_id == ipv4_network_address::get_type_id() && other.get_type_id() == ipv6_network_address::get_type_id())
+		{
+			const boost::asio::ip::address_v6 &actual_ip = other.as<const epee::net_utils::ipv6_network_address>().ip();
+			if (actual_ip.is_v4_mapped())
+			{
+				const uint32_t v4ip = make_address_v4_from_v6(actual_ip);
+				return is_same_host(ipv4_network_address(v4ip, 0));
+			}
+		}
+		else if (this_id == ipv6_network_address::get_type_id() && other.get_type_id() == ipv4_network_address::get_type_id())
+		{
+			const boost::asio::ip::address_v6 &actual_ip = this->as<const epee::net_utils::ipv6_network_address>().ip();
+			if (actual_ip.is_v4_mapped())
+			{
+				const uint32_t v4ip = make_address_v4_from_v6(actual_ip);
+				return other.is_same_host(ipv4_network_address(v4ip, 0));
+			}
+		}
+		return false;
 	}
 
   std::string print_connection_context(const connection_context_base& ctx)

contrib/epee/src/portable_storage.cpp

@@ -0,0 +1,29 @@
+
+#include "byte_slice.h"
+#include "byte_stream.h"
+#include "misc_log_ex.h"
+#include "span.h"
+#include "storages/portable_storage.h"
+#include "storages/portable_storage_to_bin.h"
+
+namespace epee
+{
+namespace serialization
+{
+  bool portable_storage::store_to_binary(byte_slice& target, const std::size_t initial_buffer_size)
+  {
+    TRY_ENTRY();
+    byte_stream ss;
+    ss.reserve(initial_buffer_size);
+    storage_block_header sbh{};
+    sbh.m_signature_a = SWAP32LE(PORTABLE_STORAGE_SIGNATUREA);
+    sbh.m_signature_b = SWAP32LE(PORTABLE_STORAGE_SIGNATUREB);
+    sbh.m_ver = PORTABLE_STORAGE_FORMAT_VER;
+    ss.write(epee::as_byte_span(sbh));
+    pack_entry_to_buff(ss, m_root);
+    target = epee::byte_slice{std::move(ss)};
+    return true;
+    CATCH_ENTRY("portable_storage::store_to_binary", false)
+  }
+}
+}

contrib/epee/src/readline_buffer.cpp

@@ -6,6 +6,7 @@
 #include <boost/thread/lock_guard.hpp>
 #include <boost/algorithm/string.hpp>
 
+static bool same_as_last_line(const std::string&);
 static void install_line_handler();
 static void remove_line_handler();
 
@@ -51,6 +52,7 @@ rdln::readline_buffer::readline_buffer()
 
 void rdln::readline_buffer::start()
 {
+  boost::lock_guard<boost::mutex> lock(sync_mutex);
   if(m_cout_buf != NULL)
     return;
   m_cout_buf = std::cout.rdbuf();
@@ -60,6 +62,7 @@ void rdln::readline_buffer::start()
 
 void rdln::readline_buffer::stop()
 {
+  boost::lock_guard<boost::mutex> lock(sync_mutex);
   if(m_cout_buf == NULL)
     return;
   std::cout.rdbuf(m_cout_buf);
@@ -88,9 +91,9 @@ rdln::linestatus rdln::readline_buffer::get_line(std::string& line) const
 
 void rdln::readline_buffer::set_prompt(const std::string& prompt)
 {
+  boost::lock_guard<boost::mutex> lock(sync_mutex);
   if(m_cout_buf == NULL)
     return;
-  boost::lock_guard<boost::mutex> lock(sync_mutex);
   rl_set_prompt(std::string(m_prompt_length, ' ').c_str());
   rl_redisplay();
   rl_set_prompt(prompt.c_str());
@@ -113,6 +116,12 @@ const std::vector<std::string>& rdln::readline_buffer::get_completions()
 int rdln::readline_buffer::sync()
 {
   boost::lock_guard<boost::mutex> lock(sync_mutex);
+
+  if (m_cout_buf == nullptr)
+  {
+    return -1;
+  }
+
 #if RL_READLINE_VERSION < 0x0700
   char lbuf[2] = {0,0};
   char *line = NULL;
@@ -167,8 +176,11 @@ static void handle_line(char* line)
     boost::trim_right(test_line);
     if(!test_line.empty())
     {
-      add_history(test_line.c_str());
-      history_set_pos(history_length);
+      if (!same_as_last_line(test_line))
+      {
+        add_history(test_line.c_str());
+        history_set_pos(history_length);
+      }
       if (test_line == "exit" || test_line == "q")
         exit = true;
     }
@@ -184,6 +196,16 @@ static void handle_line(char* line)
   return;
 }
 
+// same_as_last_line returns true, if the last line in the history is
+// equal to test_line.
+static bool same_as_last_line(const std::string& test_line)
+{
+  // Note that state->offset == state->length, when a new line was entered.
+  HISTORY_STATE* state = history_get_history_state();
+  return state->length > 0
+    && test_line.compare(state->entries[state->length-1]->line) == 0;
+}
+
 static char* completion_matches(const char* text, int state)
 {
   static size_t list_index;

contrib/fuzz_testing/fuzz.sh

@@ -14,8 +14,8 @@ then
   exit 1
 fi
 case "$type" in
-  block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof) ;;
-  *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof"; exit 1 ;;
+  block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof|utf8) ;;
+  *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof|utf8"; exit 1 ;;
 esac
 
 if test -d "fuzz-out/$type"

contrib/gitian/README.md

@@ -126,7 +126,7 @@ Setup for LXC:
 
 ```bash
 GH_USER=fluffypony
-VERSION=v0.15.0.0
+VERSION=v0.17.0.0
 
 ./gitian-build.py --setup $GH_USER $VERSION
 ```
@@ -182,12 +182,13 @@ If you chose to do detached signing using `--detach-sign` above (recommended), y
 
 ```bash
 GH_USER=fluffypony
-VERSION=v0.15.0.0
+VERSION=v0.17.0.0
 
 gpg --detach-sign ${VERSION}-linux/${GH_USER}/monero-linux-*-build.assert
 gpg --detach-sign ${VERSION}-win/${GH_USER}/monero-win-*-build.assert
 gpg --detach-sign ${VERSION}-osx/${GH_USER}/monero-osx-*-build.assert
 gpg --detach-sign ${VERSION}-android/${GH_USER}/monero-android-*-build.assert
+gpg --detach-sign ${VERSION}-freebsd/${GH_USER}/monero-freebsd-*-build.assert
 ```
 
 This will create a `.sig` file for each `.assert` file above (2 files for each platform).

contrib/gitian/gitian-android.yml

@@ -1,5 +1,5 @@
 ---
-name: "wownero-android-0.8"
+name: "wownero-android-0.9"
 enable_cache: true
 suites:
 - "bionic"
@@ -32,7 +32,7 @@ packages:
 - "python3-zmq"
 - "unzip"
 remotes:
-- "url": "https://git.wownero.com/qvqc/wownero.git"
+- "url": "https://git.wownero.com/wownero/wownero.git"
   "dir": "wownero"
 files: []
 script: |

contrib/gitian/gitian-freebsd.yml

@@ -1,5 +1,5 @@
 ---
-name: "wownero-freebsd-0.8"
+name: "wownero-freebsd-0.9"
 enable_cache: true
 suites:
 - "bionic"
@@ -119,7 +119,7 @@ script: |
   for i in ${HOSTS}; do
     export PATH=${WRAP_DIR}:${BASEPREFIX}/${i}/native/bin:${ORIGPATH}
     mkdir build && cd build
-    cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DCMAKE_BUILD_TYPE=Release
+    cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_SKIP_RPATH=ON
     make ${MAKEOPTS}
     chmod 755 bin/*
     cp ../LICENSE bin

contrib/gitian/gitian-linux.yml

@@ -1,5 +1,5 @@
 ---
-name: "wownero-linux-0.8"
+name: "wownero-linux-0.9"
 enable_cache: true
 suites:
 - "bionic"
@@ -111,6 +111,11 @@ script: |
   rm -f $WRAP_DIR/extra_includes/i686-linux-gnu/asm
   ln -s /usr/include/x86_64-linux-gnu/asm $EXTRA_INCLUDES_BASE/i686-linux-gnu/asm
 
+  # glibc 2.23 breaks compatibility with <=2.19 use of lgamma function.
+  # Hack the math header to restore the old behavior.
+  mkdir $EXTRA_INCLUDES_BASE/bits
+  sed -e '/__REDIRFROM .lgamma,/,+3s/_USE_/_DONTUSE_/g' /usr/include/x86_64-linux-gnu/bits/math-finite.h > $EXTRA_INCLUDES_BASE/bits/math-finite.h
+
   # gcc 7+ honors SOURCE_DATE_EPOCH, no faketime needed
   export SOURCE_DATE_EPOCH=`date -d 2000-01-01T12:00:00 +%s`
 
@@ -127,14 +132,14 @@ script: |
   # Build dependencies for each host
   export TAR_OPTIONS=--mtime=2000-01-01T12:00:00
   for i in $HOSTS; do
-    EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i"
-    if [ -d "$EXTRA_INCLUDES" ]; then
-      export C_INCLUDE_PATH="$EXTRA_INCLUDES"
-      export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES"
+    ARCH_INCLUDES="$EXTRA_INCLUDES_BASE/$i"
+    if [ -d "$ARCH_INCLUDES" ]; then
+      EXTRA_INCLUDES="${EXTRA_INCLUDES_BASE}:${ARCH_INCLUDES}"
     else
-      unset C_INCLUDE_PATH
-      unset CPLUS_INCLUDE_PATH
+      EXTRA_INCLUDES="${EXTRA_INCLUDES_BASE}"
     fi
+    export C_INCLUDE_PATH="$EXTRA_INCLUDES"
+    export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES"
     make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" V=1
   done
 
@@ -151,15 +156,15 @@ script: |
   for i in ${HOSTS}; do
     export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH}
     mkdir build && cd build
-    EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i"
-    if [ -d "$EXTRA_INCLUDES" ]; then
-      export C_INCLUDE_PATH="$EXTRA_INCLUDES"
-      export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES"
+    ARCH_INCLUDES="$EXTRA_INCLUDES_BASE/$i"
+    if [ -d "$ARCH_INCLUDES" ]; then
+      EXTRA_INCLUDES="${EXTRA_INCLUDES_BASE}:${ARCH_INCLUDES}"
     else
-      unset C_INCLUDE_PATH
-      unset CPLUS_INCLUDE_PATH
+      EXTRA_INCLUDES="${EXTRA_INCLUDES_BASE}"
     fi
-    cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=ON
+    export C_INCLUDE_PATH="$EXTRA_INCLUDES"
+    export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES"
+    cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=ON -DCMAKE_SKIP_RPATH=ON
     make ${MAKEOPTS}
     chmod 755 bin/*
     cp ../LICENSE bin

contrib/gitian/gitian-osx.yml

@@ -1,5 +1,5 @@
 ---
-name: "wownero-osx-0.8"
+name: "wownero-osx-0.9"
 enable_cache: true
 suites:
 - "bionic"

contrib/gitian/gitian-win.yml

@@ -1,5 +1,5 @@
 ---
-name: "wownero-win-0.8"
+name: "wownero-win-0.9"
 enable_cache: true
 suites:
 - "bionic"

external/CMakeLists.txt

@@ -34,42 +34,22 @@
 # We always compile if we are building statically to reduce static dependency issues...
 # ...except for FreeBSD, because FreeBSD is a special case that doesn't play well with
 # others.
-if(NOT IOS)
-    find_package(Miniupnpc QUIET)
+
+find_package(Miniupnpc REQUIRED)
+
+message(STATUS "Using in-tree miniupnpc")
+add_subdirectory(miniupnp/miniupnpc)
+set_property(TARGET libminiupnpc-static PROPERTY FOLDER "external")
+if(MSVC)
+  set_property(TARGET libminiupnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " -wd4244 -wd4267")
+elseif(NOT MSVC)
+  set_property(TARGET libminiupnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-undef -Wno-unused-result -Wno-unused-value")
+endif()
+if(CMAKE_SYSTEM_NAME MATCHES "NetBSD")
+	set_property(TARGET libminiupnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " -D_NETBSD_SOURCE")
 endif()
 
-# If we have the correct shared version and we're not building static, use it
-if(STATIC OR IOS)
- set(USE_SHARED_MINIUPNPC false)
-elseif(MINIUPNP_FOUND AND MINIUPNPC_VERSION_1_7_OR_HIGHER)
- set(USE_SHARED_MINIUPNPC true)
-endif()
-
-if(USE_SHARED_MINIUPNPC)
-  message(STATUS "Using shared miniupnpc found at ${MINIUPNP_INCLUDE_DIR}")
-
-  set(UPNP_STATIC false PARENT_SCOPE)
-  set(UPNP_INCLUDE ${MINIUPNP_INCLUDE_DIR} PARENT_SCOPE)
-  set(UPNP_LIBRARIES ${MINIUPNP_LIBRARY} PARENT_SCOPE)
-else()
-  if(STATIC)
-    message(STATUS "Using miniupnpc from local source tree for static build")
-  else()
-    message(STATUS "Using miniupnpc from local source tree (/external/miniupnp/miniupnpc)")
-  endif()
-
-  add_subdirectory(miniupnp/miniupnpc)
-
-  set_property(TARGET libminiupnpc-static PROPERTY FOLDER "external")
-  if(MSVC)
-    set_property(TARGET libminiupnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " -wd4244 -wd4267")
-  elseif(NOT MSVC)
-    set_property(TARGET libminiupnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-undef -Wno-unused-result -Wno-unused-value")
-  endif()
-
-  set(UPNP_STATIC true PARENT_SCOPE)
-  set(UPNP_LIBRARIES "libminiupnpc-static" PARENT_SCOPE)
-endif()
+set(UPNP_LIBRARIES "libminiupnpc-static" PARENT_SCOPE)
 
 find_package(Unbound)
 

external/easylogging++/ea_config.h

@@ -11,7 +11,7 @@
 #define ELPP_UTC_DATETIME
 
 #ifdef EASYLOGGING_CC
-#if !(!defined __GLIBC__ || !defined __GNUC__ || defined __MINGW32__ || defined __MINGW64__ || defined __ANDROID__)
+#if !(!defined __GLIBC__ || !defined __GNUC__ || defined __MINGW32__ || defined __MINGW64__ || defined __ANDROID__ || defined __NetBSD__)
 #define ELPP_FEATURE_CRASH_LOG
 #endif
 #endif

external/easylogging++/easylogging++.cc

@@ -2968,6 +2968,16 @@ void Writer::initializeLogger(Logger *logger, bool needLock) {
 }
 
 void Writer::processDispatch() {
+  static std::atomic_flag in_dispatch;
+  if (in_dispatch.test_and_set())
+  {
+    if (m_proceed && m_logger != NULL)
+    {
+      m_logger->stream().str(ELPP_LITERAL(""));
+      m_logger->releaseLock();
+    }
+    return;
+  }
 #if ELPP_LOGGING_ENABLED
   if (ELPP->hasFlag(LoggingFlag::MultiLoggerSupport)) {
     bool firstDispatched = false;
@@ -3006,6 +3016,7 @@ void Writer::processDispatch() {
     m_logger->releaseLock();
   }
 #endif // ELPP_LOGGING_ENABLED
+  in_dispatch.clear();
 }
 
 void Writer::triggerDispatch(void) {

external/qrcodegen/CMakeLists.txt

@@ -1,7 +1,8 @@
 project(libqrcodegen)
 
 add_library(qrcodegen STATIC QrCode.cpp)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
+set_target_properties(qrcodegen PROPERTIES POSITION_INDEPENDENT_CODE ON)
+set_target_properties(qrcodegen PROPERTIES CXX_STANDARD 11)
 
 target_include_directories(qrcodegen PUBLIC 
     ${CMAKE_CURRENT_SOURCE_DIR})

src/CMakeLists.txt

@@ -75,6 +75,8 @@ function (monero_add_executable name)
     PROPERTY
       RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
   enable_stack_trace("${name}")
+  
+  monero_set_target_no_relink("${name}")
 endfunction ()
 
 function (monero_add_library name)
@@ -92,6 +94,7 @@ function (monero_add_library_with_deps)
   set(objlib obj_${MONERO_ADD_LIBRARY_NAME})
   add_library(${objlib} OBJECT ${MONERO_ADD_LIBRARY_SOURCES})
   add_library("${MONERO_ADD_LIBRARY_NAME}" $<TARGET_OBJECTS:${objlib}>)
+  monero_set_target_no_relink("${MONERO_ADD_LIBRARY_NAME}")
   if (MONERO_ADD_LIBRARY_DEPENDS)
     add_dependencies(${objlib} ${MONERO_ADD_LIBRARY_DEPENDS})
   endif()

src/blockchain_db/blockchain_db.cpp

@@ -63,6 +63,7 @@ bool matches_category(relay_method method, relay_category category) noexcept
   {
     default:
     case relay_method::local:
+    case relay_method::forward:
     case relay_method::stem:
       return false;
     case relay_method::block:
@@ -79,6 +80,7 @@ void txpool_tx_meta_t::set_relay_method(relay_method method) noexcept
   kept_by_block = 0;
   do_not_relay = 0;
   is_local = 0;
+  is_forwarding = 0;
   dandelionpp_stem = 0;
 
   switch (method)
@@ -89,8 +91,8 @@ void txpool_tx_meta_t::set_relay_method(relay_method method) noexcept
     case relay_method::local:
       is_local = 1;
       break;
-    default:
-    case relay_method::fluff:
+    case relay_method::forward:
+      is_forwarding = 1;
       break;
     case relay_method::stem:
       dandelionpp_stem = 1;
@@ -98,26 +100,45 @@ void txpool_tx_meta_t::set_relay_method(relay_method method) noexcept
     case relay_method::block:
       kept_by_block = 1;
       break;
+    default:
+    case relay_method::fluff:
+      break;
   }
 }
 
 relay_method txpool_tx_meta_t::get_relay_method() const noexcept
 {
-  if (kept_by_block)
-    return relay_method::block;
-  if (do_not_relay)
-    return relay_method::none;
-  if (is_local)
-    return relay_method::local;
-  if (dandelionpp_stem)
-    return relay_method::stem;
+  const uint8_t state =
+    uint8_t(kept_by_block) +
+    (uint8_t(do_not_relay) << 1) +
+    (uint8_t(is_local) << 2) +
+    (uint8_t(is_forwarding) << 3) +
+    (uint8_t(dandelionpp_stem) << 4);
+
+  switch (state)
+  {
+    default: // error case
+    case 0:
+      break;
+    case 1:
+      return relay_method::block;
+    case 2:
+      return relay_method::none;
+    case 4:
+      return relay_method::local;
+    case 8:
+      return relay_method::forward;
+    case 16:
+      return relay_method::stem;
+  };
   return relay_method::fluff;
 }
 
 bool txpool_tx_meta_t::upgrade_relay_method(relay_method method) noexcept
 {
   static_assert(relay_method::none < relay_method::local, "bad relay_method value");
-  static_assert(relay_method::local < relay_method::stem, "bad relay_method value");
+  static_assert(relay_method::local < relay_method::forward, "bad relay_method value");
+  static_assert(relay_method::forward < relay_method::stem, "bad relay_method value");
   static_assert(relay_method::stem < relay_method::fluff, "bad relay_method value");
   static_assert(relay_method::fluff < relay_method::block, "bad relay_method value");
 
@@ -158,7 +179,7 @@ void BlockchainDB::pop_block()
   pop_block(blk, txs);
 }
 
-void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& txp, const crypto::hash* tx_hash_ptr, const crypto::hash* tx_prunable_hash_ptr)
+void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& txp, const crypto::hash* tx_hash_ptr, const crypto::hash* tx_prunable_hash_ptr)
 {
   const transaction &tx = txp.first;
 
@@ -260,12 +281,13 @@ uint64_t BlockchainDB::add_block( const std::pair<block, blobdata>& blck
   time1 = epee::misc_utils::get_tick_count();
 
   uint64_t num_rct_outs = 0;
-  add_transaction(blk_hash, std::make_pair(blk.miner_tx, tx_to_blob(blk.miner_tx)));
+  blobdata miner_bd = tx_to_blob(blk.miner_tx);
+  add_transaction(blk_hash, std::make_pair(blk.miner_tx, blobdata_ref(miner_bd)));
   if (blk.miner_tx.version == 2)
     num_rct_outs += blk.miner_tx.vout.size();
   int tx_i = 0;
   crypto::hash tx_hash = crypto::null_hash;
-  for (const std::pair<transaction, blobdata>& tx : txs)
+  for (const std::pair<transaction, blobdata_ref>& tx : txs)
   {
     tx_hash = blk.tx_hashes[tx_i];
     add_transaction(blk_hash, tx, &tx_hash);

src/blockchain_db/blockchain_db.h

@@ -160,7 +160,7 @@ struct txpool_tx_meta_t
   uint64_t max_used_block_height;
   uint64_t last_failed_height;
   uint64_t receive_time;
-  uint64_t last_relayed_time; //!< If Dandelion++ stem, randomized embargo timestamp. Otherwise, last relayed timestmap.
+  uint64_t last_relayed_time; //!< If received over i2p/tor, randomized forward time. If Dandelion++stem, randomized embargo time. Otherwise, last relayed timestamp
   // 112 bytes
   uint8_t kept_by_block;
   uint8_t relayed;
@@ -169,7 +169,8 @@ struct txpool_tx_meta_t
   uint8_t pruned: 1;
   uint8_t is_local: 1;
   uint8_t dandelionpp_stem : 1;
-  uint8_t bf_padding: 4;
+  uint8_t is_forwarding: 1;
+  uint8_t bf_padding: 3;
 
   uint8_t padding[76]; // till 192 bytes
 
@@ -439,7 +440,7 @@ private:
    * @param tx_prunable_hash the hash of the prunable part of the transaction
    * @return the transaction ID
    */
-  virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) = 0;
+  virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) = 0;
 
   /**
    * @brief remove data about a transaction
@@ -567,7 +568,7 @@ protected:
    * @param tx_hash_ptr the hash of the transaction, if already calculated
    * @param tx_prunable_hash_ptr the hash of the prunable part of the transaction, if already calculated
    */
-  void add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& tx, const crypto::hash* tx_hash_ptr = NULL, const crypto::hash* tx_prunable_hash_ptr = NULL);
+  void add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& tx, const crypto::hash* tx_hash_ptr = NULL, const crypto::hash* tx_prunable_hash_ptr = NULL);
 
   mutable uint64_t time_tx_exists = 0;  //!< a performance metric
   uint64_t time_commit1 = 0;  //!< a performance metric
@@ -1311,17 +1312,20 @@ public:
    * height. The number of blocks returned is variable, based on the max_size passed.
    *
    * @param start_height the height of the first block
-   * @param min_count the minimum number of blocks to return, if they exist
-   * @param max_count the maximum number of blocks to return
+   * @param min_block_count the minimum number of blocks to return, if they exist
+   * @param max_block_count the maximum number of blocks to return
+   * @param max_tx_count the maximum number of txes to return
    * @param max_size the maximum size of block/transaction data to return (will be exceeded by one blocks's worth at most, if min_count is met)
    * @param blocks the returned block/transaction data
    * @param pruned whether to return full or pruned tx data
    * @param skip_coinbase whether to return or skip coinbase transactions (they're in blocks regardless)
    * @param get_miner_tx_hash whether to calculate and return the miner (coinbase) tx hash
    *
+   * The call will return at least min_block_count if possible, even if this contravenes max_tx_count
+   *
    * @return true iff the blocks and transactions were found
    */
-  virtual bool get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const = 0;
+  virtual bool get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const = 0;
 
   /**
    * @brief fetches the prunable transaction blob with the given hash
@@ -1523,7 +1527,7 @@ public:
    *
    * @param details the details of the transaction to add
    */
-  virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t& details) = 0;
+  virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata_ref &blob, const txpool_tx_meta_t& details) = 0;
 
   /**
    * @brief update a txpool transaction's metadata
@@ -1643,7 +1647,7 @@ public:
    * @param: data: the metadata for the block
    * @param: blob: the block's blob
    */
-  virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata &blob) = 0;
+  virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata_ref &blob) = 0;
 
   /**
    * @brief get an alternative block by hash
@@ -1686,7 +1690,7 @@ public:
    *
    * @return false if the function returns false for any transaction, otherwise true
    */
-  virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false, relay_category category = relay_category::broadcasted) const = 0;
+  virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata_ref*)>, bool include_blob = false, relay_category category = relay_category::broadcasted) const = 0;
 
   /**
    * @brief runs a function over all key images stored
@@ -1778,7 +1782,7 @@ public:
    *
    * @return false if the function returns false for any output, otherwise true
    */
-  virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata *blob)> f, bool include_blob = false) const = 0;
+  virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata_ref *blob)> f, bool include_blob = false) const = 0;
 
 
   //

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -856,7 +856,7 @@ void BlockchainLMDB::remove_block()
       throw1(DB_ERROR(lmdb_error("Failed to add removal of block info to db transaction: ", result).c_str()));
 }
 
-uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& txp, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash)
+uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& txp, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -896,7 +896,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons
   if (result)
     throw0(DB_ERROR(lmdb_error("Failed to add tx data to db transaction: ", result).c_str()));
 
-  const cryptonote::blobdata &blob = txp.second;
+  const cryptonote::blobdata_ref &blob = txp.second;
   MDB_val_sized(blobval, blob);
 
   unsigned int unprunable_size = tx.unprunable_size;
@@ -1756,7 +1756,7 @@ void BlockchainLMDB::unlock()
       auto_txn.commit(); \
   } while(0)
 
-void BlockchainLMDB::add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t &meta)
+void BlockchainLMDB::add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata_ref &blob, const txpool_tx_meta_t &meta)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -2308,7 +2308,7 @@ bool BlockchainLMDB::check_pruning()
   return prune_worker(prune_mode_check, 0);
 }
 
-bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob, relay_category category) const
+bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata_ref*)> f, bool include_blob, relay_category category) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -2334,8 +2334,7 @@ bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&,
     const txpool_tx_meta_t &meta = *(const txpool_tx_meta_t*)v.mv_data;
     if (!meta.matches(category))
       continue;
-    const cryptonote::blobdata *passed_bd = NULL;
-    cryptonote::blobdata bd;
+    cryptonote::blobdata_ref bd;
     if (include_blob)
     {
       MDB_val b;
@@ -2344,11 +2343,10 @@ bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&,
         throw0(DB_ERROR("Failed to find txpool tx blob to match metadata"));
       if (result)
         throw0(DB_ERROR(lmdb_error("Failed to enumerate txpool tx blob: ", result).c_str()));
-      bd.assign(reinterpret_cast<const char*>(b.mv_data), b.mv_size);
-      passed_bd = &bd;
+      bd = {reinterpret_cast<const char*>(b.mv_data), b.mv_size};
     }
 
-    if (!f(txid, meta, passed_bd)) {
+    if (!f(txid, meta, &bd)) {
       ret = false;
       break;
     }
@@ -2359,7 +2357,7 @@ bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&,
   return ret;
 }
 
-bool BlockchainLMDB::for_all_alt_blocks(std::function<bool(const crypto::hash&, const alt_block_data_t&, const cryptonote::blobdata*)> f, bool include_blob) const
+bool BlockchainLMDB::for_all_alt_blocks(std::function<bool(const crypto::hash&, const alt_block_data_t&, const cryptonote::blobdata_ref*)> f, bool include_blob) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -2384,15 +2382,13 @@ bool BlockchainLMDB::for_all_alt_blocks(std::function<bool(const crypto::hash&,
     if (v.mv_size < sizeof(alt_block_data_t))
       throw0(DB_ERROR("alt_blocks record is too small"));
     const alt_block_data_t *data = (const alt_block_data_t*)v.mv_data;
-    const cryptonote::blobdata *passed_bd = NULL;
-    cryptonote::blobdata bd;
+    cryptonote::blobdata_ref bd;
     if (include_blob)
     {
-      bd.assign(reinterpret_cast<const char*>(v.mv_data) + sizeof(alt_block_data_t), v.mv_size - sizeof(alt_block_data_t));
-      passed_bd = &bd;
+      bd = {reinterpret_cast<const char*>(v.mv_data) + sizeof(alt_block_data_t), v.mv_size - sizeof(alt_block_data_t)};
     }
 
-    if (!f(blkid, *data, passed_bd)) {
+    if (!f(blkid, *data, &bd)) {
       ret = false;
       break;
     }
@@ -3005,7 +3001,7 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const
 
   if (! tx_found)
   {
-    LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
+    LOG_PRINT_L3("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
     return false;
   }
 
@@ -3036,7 +3032,7 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h, uint64_t& tx_id) const
   bool ret = false;
   if (get_result == MDB_NOTFOUND)
   {
-    LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
+    LOG_PRINT_L3("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
   }
   else if (get_result)
     throw0(DB_ERROR(lmdb_error("DB error attempting to fetch transaction from hash", get_result).c_str()));
@@ -3175,7 +3171,7 @@ bool BlockchainLMDB::get_pruned_tx_blobs_from(const crypto::hash& h, size_t coun
   return true;
 }
 
-bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const
+bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -3189,14 +3185,15 @@ bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_count, si
     RCURSOR(txs_prunable);
   }
 
-  blocks.reserve(std::min<size_t>(max_count, 10000)); // guard against very large max count if only checking bytes
+  blocks.reserve(std::min<size_t>(max_block_count, 10000)); // guard against very large max count if only checking bytes
   const uint64_t blockchain_height = height();
   uint64_t size = 0;
+  size_t num_txes = 0;
   MDB_val_copy<uint64_t> key(start_height);
   MDB_val k, v, val_tx_id;
   uint64_t tx_id = ~0;
   MDB_cursor_op op = MDB_SET;
-  for (uint64_t h = start_height; h < blockchain_height && blocks.size() < max_count && (size < max_size || blocks.size() < min_count); ++h)
+  for (uint64_t h = start_height; h < blockchain_height && blocks.size() < max_block_count && (size < max_size || blocks.size() < min_block_count); ++h)
   {
     MDB_cursor_op op = h == start_height ? MDB_SET : MDB_NEXT;
     int result = mdb_cursor_get(m_cur_blocks, &key, &v, op);
@@ -3247,6 +3244,7 @@ bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_count, si
     op = MDB_NEXT;
 
     current_block.second.reserve(b.tx_hashes.size());
+    num_txes += b.tx_hashes.size() + (skip_coinbase ? 0 : 1);
     for (const auto &tx_hash: b.tx_hashes)
     {
       // get pruned data
@@ -3266,6 +3264,9 @@ bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_count, si
       current_block.second.push_back(std::make_pair(tx_hash, std::move(tx_blob)));
       size += current_block.second.back().second.size();
     }
+
+    if (blocks.size() >= min_block_count && num_txes >= max_tx_count)
+      break;
   }
 
   TXN_POSTFIX_RDONLY();
@@ -3604,8 +3605,7 @@ bool BlockchainLMDB::for_blocks_range(const uint64_t& h1, const uint64_t& h2, st
     if (ret)
       throw0(DB_ERROR("Failed to enumerate blocks"));
     uint64_t height = *(const uint64_t*)k.mv_data;
-    blobdata bd;
-    bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
+    blobdata_ref bd{reinterpret_cast<char*>(v.mv_data), v.mv_size};
     block b;
     if (!parse_and_validate_block_from_blob(bd, b))
       throw0(DB_ERROR("Failed to parse block from blob retrieved from the db"));
@@ -3660,15 +3660,16 @@ bool BlockchainLMDB::for_all_transactions(std::function<bool(const crypto::hash&
     if (ret)
       throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str()));
     transaction tx;
-    blobdata bd;
-    bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
     if (pruned)
     {
+      blobdata_ref bd{reinterpret_cast<char*>(v.mv_data), v.mv_size};
       if (!parse_and_validate_tx_base_from_blob(bd, tx))
         throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db"));
     }
     else
     {
+      blobdata bd;
+      bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
       ret = mdb_cursor_get(m_cur_txs_prunable, &k, &v, MDB_SET);
       if (ret)
         throw0(DB_ERROR(lmdb_error("Failed to get prunable tx data the db: ", ret).c_str()));
@@ -4402,7 +4403,7 @@ uint8_t BlockchainLMDB::get_hard_fork_version(uint64_t height) const
   return ret;
 }
 
-void BlockchainLMDB::add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata &blob)
+void BlockchainLMDB::add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata_ref &blob)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -4969,7 +4970,7 @@ void BlockchainLMDB::migrate_0_1()
     }
 
     MDB_dbi o_txs;
-    blobdata bd;
+    blobdata_ref bd;
     block b;
     MDB_val hk;
 
@@ -5051,7 +5052,7 @@ void BlockchainLMDB::migrate_0_1()
       } else if (result)
         throw0(DB_ERROR(lmdb_error("Failed to get a record from blocks: ", result).c_str()));
 
-      bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
+      bd = {reinterpret_cast<char*>(v.mv_data), v.mv_size};
       if (!parse_and_validate_block_from_blob(bd, b))
         throw0(DB_ERROR("Failed to parse block from blob retrieved from the db"));
 
@@ -5062,7 +5063,7 @@ void BlockchainLMDB::migrate_0_1()
         result = mdb_cursor_get(c_txs, &hk, &v, MDB_SET);
         if (result)
           throw0(DB_ERROR(lmdb_error("Failed to get record from txs: ", result).c_str()));
-        bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
+        bd = {reinterpret_cast<char*>(v.mv_data), v.mv_size};
         if (!parse_and_validate_tx_from_blob(bd, tx))
           throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db"));
         add_transaction(null_hash, std::make_pair(std::move(tx), bd), &b.tx_hashes[j]);
@@ -5184,8 +5185,7 @@ void BlockchainLMDB::migrate_1_2()
       else if (result)
         throw0(DB_ERROR(lmdb_error("Failed to get a record from txs: ", result).c_str()));
 
-      cryptonote::blobdata bd;
-      bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
+      cryptonote::blobdata bd{reinterpret_cast<char*>(v.mv_data), v.mv_size};
       transaction tx;
       if (!parse_and_validate_tx_from_blob(bd, tx))
         throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db"));

src/blockchain_db/lmdb/db_lmdb.h

@@ -257,7 +257,7 @@ public:
   virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const;
   virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const;
   virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<cryptonote::blobdata> &bd) const;
-  virtual bool get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const;
+  virtual bool get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const;
   virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const;
   virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const;
 
@@ -283,7 +283,7 @@ public:
 
   virtual bool has_key_image(const crypto::key_image& img) const;
 
-  virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t& meta);
+  virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata_ref &blob, const txpool_tx_meta_t& meta);
   virtual void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t& meta);
   virtual uint64_t get_txpool_tx_count(relay_category category = relay_category::broadcasted) const;
   virtual bool txpool_has_tx(const crypto::hash &txid, relay_category tx_category) const;
@@ -296,20 +296,20 @@ public:
   virtual bool update_pruning();
   virtual bool check_pruning();
 
-  virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata &blob);
+  virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata_ref &blob);
   virtual bool get_alt_block(const crypto::hash &blkid, alt_block_data_t *data, cryptonote::blobdata *blob);
   virtual void remove_alt_block(const crypto::hash &blkid);
   virtual uint64_t get_alt_block_count();
   virtual void drop_alt_blocks();
 
-  virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob = false, relay_category category = relay_category::broadcasted) const;
+  virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata_ref*)> f, bool include_blob = false, relay_category category = relay_category::broadcasted) const;
 
   virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const;
   virtual bool for_blocks_range(const uint64_t& h1, const uint64_t& h2, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const;
   virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>, bool pruned) const;
   virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const;
   virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const;
-  virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata *blob)> f, bool include_blob = false) const;
+  virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata_ref *blob)> f, bool include_blob = false) const;
 
   virtual uint64_t add_block( const std::pair<block, blobdata>& blk
                             , size_t block_weight
@@ -376,7 +376,7 @@ private:
 
   virtual void remove_block();
 
-  virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash);
+  virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash);
 
   virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx);
 

src/blockchain_db/testdb.h

@@ -70,7 +70,7 @@ public:
   virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; }
   virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; }
   virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<cryptonote::blobdata> &bd) const override { return false; }
-  virtual bool get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const override { return false; }
+  virtual bool get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const override { return false; }
   virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; }
   virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const override { return false; }
   virtual uint64_t get_block_height(const crypto::hash& h) const override { return 0; }
@@ -111,7 +111,7 @@ public:
   virtual std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(const uint64_t tx_index, size_t n_txes) const override { return std::vector<std::vector<uint64_t>>(); }
   virtual bool has_key_image(const crypto::key_image& img) const override { return false; }
   virtual void remove_block() override { }
-  virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<cryptonote::transaction, cryptonote::blobdata>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) override {return 0;}
+  virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<cryptonote::transaction, cryptonote::blobdata_ref>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) override {return 0;}
   virtual void remove_transaction_data(const crypto::hash& tx_hash, const cryptonote::transaction& tx) override {}
   virtual uint64_t add_output(const crypto::hash& tx_hash, const cryptonote::tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) override {return 0;}
   virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector<uint64_t>& amount_output_indices) override {}
@@ -127,7 +127,7 @@ public:
   virtual std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> get_output_histogram(const std::vector<uint64_t> &amounts, bool unlocked, uint64_t recent_cutoff, uint64_t min_count) const override { return std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>>(); }
   virtual bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, std::vector<uint64_t> &distribution, uint64_t &base) const override { return false; }
 
-  virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const cryptonote::txpool_tx_meta_t& details) override {}
+  virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata_ref &blob, const cryptonote::txpool_tx_meta_t& details) override {}
   virtual void update_txpool_tx(const crypto::hash &txid, const cryptonote::txpool_tx_meta_t& details) override {}
   virtual uint64_t get_txpool_tx_count(relay_category tx_relay = relay_category::broadcasted) const override { return 0; }
   virtual bool txpool_has_tx(const crypto::hash &txid, relay_category tx_category) const override { return  false; }
@@ -136,7 +136,7 @@ public:
   virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd, relay_category tx_category) const override { return false; }
   virtual uint64_t get_database_size() const override { return 0; }
   virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid, relay_category tx_category) const override { return ""; }
-  virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const cryptonote::txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false, relay_category category = relay_category::broadcasted) const override { return false; }
+  virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const cryptonote::txpool_tx_meta_t&, const cryptonote::blobdata_ref*)>, bool include_blob = false, relay_category category = relay_category::broadcasted) const override { return false; }
 
   virtual void add_block( const cryptonote::block& blk
                         , size_t block_weight
@@ -160,12 +160,12 @@ public:
   virtual uint64_t get_max_block_size() override { return 100000000; }
   virtual void add_max_block_size(uint64_t sz) override { }
 
-  virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata &blob) override {}
+  virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata_ref &blob) override {}
   virtual bool get_alt_block(const crypto::hash &blkid, alt_block_data_t *data, cryptonote::blobdata *blob) override { return false; }
   virtual void remove_alt_block(const crypto::hash &blkid) override {}
   virtual uint64_t get_alt_block_count() override { return 0; }
   virtual void drop_alt_blocks() override {}
-  virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata *blob)> f, bool include_blob = false) const override { return true; }
+  virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata_ref *blob)> f, bool include_blob = false) const override { return true; }
 };
 
 }

src/blockchain_utilities/blockchain_blackball.cpp

@@ -28,8 +28,6 @@
 
 #include <boost/range/adaptor/transformed.hpp>
 #include <boost/algorithm/string.hpp>
-#include <boost/archive/portable_binary_iarchive.hpp>
-#include <boost/archive/portable_binary_oarchive.hpp>
 #include "common/unordered_containers_boost_serialization.h"
 #include "common/command_line.h"
 #include "common/varint.h"

src/blockchain_utilities/blockchain_export.cpp

@@ -47,6 +47,7 @@ int main(int argc, char* argv[])
   epee::string_tools::set_module_name_and_folder(argv[0]);
 
   uint32_t log_level = 0;
+  uint64_t block_start = 0;
   uint64_t block_stop = 0;
   bool blocks_dat = false;
 
@@ -58,6 +59,7 @@ int main(int argc, char* argv[])
   po::options_description desc_cmd_sett("Command line options and settings options");
   const command_line::arg_descriptor<std::string> arg_output_file = {"output-file", "Specify output file", "", true};
   const command_line::arg_descriptor<std::string> arg_log_level  = {"log-level",  "0-4 or categories", ""};
+  const command_line::arg_descriptor<uint64_t> arg_block_start = {"block-start", "Start at block number", block_start};
   const command_line::arg_descriptor<uint64_t> arg_block_stop = {"block-stop", "Stop at block number", block_stop};
   const command_line::arg_descriptor<bool> arg_blocks_dat = {"blocksdat", "Output in blocks.dat format", blocks_dat};
 
@@ -67,6 +69,7 @@ int main(int argc, char* argv[])
   command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on);
   command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on);
   command_line::add_arg(desc_cmd_sett, arg_log_level);
+  command_line::add_arg(desc_cmd_sett, arg_block_start);
   command_line::add_arg(desc_cmd_sett, arg_block_stop);
   command_line::add_arg(desc_cmd_sett, arg_blocks_dat);
 
@@ -97,6 +100,7 @@ int main(int argc, char* argv[])
     mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str());
   else
     mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
+  block_start = command_line::get_arg(vm, arg_block_start);
   block_stop = command_line::get_arg(vm, arg_block_stop);
 
   LOG_PRINT_L0("Starting...");
@@ -178,7 +182,7 @@ int main(int argc, char* argv[])
   else
   {
     BootstrapFile bootstrap;
-    r = bootstrap.store_blockchain_raw(core_storage, NULL, output_file_path, block_stop);
+    r = bootstrap.store_blockchain_raw(core_storage, NULL, output_file_path, block_start, block_stop);
   }
   CHECK_AND_ASSERT_MES(r, 1, "Failed to export blockchain raw data");
   LOG_PRINT_L0("Blockchain raw data exported OK");

src/blockchain_utilities/blockchain_import.cpp

@@ -227,6 +227,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
     return false;
   }
 
+  uint64_t block_first, block_last;
   uint64_t start_height = 1, seek_height;
   if (opt_resume)
     start_height = core.get_blockchain_storage().get_current_blockchain_height();
@@ -235,10 +236,10 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
   BootstrapFile bootstrap;
   std::streampos pos;
   // BootstrapFile bootstrap(import_file_path);
-  uint64_t total_source_blocks = bootstrap.count_blocks(import_file_path, pos, seek_height);
-  MINFO("bootstrap file last block number: " << total_source_blocks-1 << " (zero-based height)  total blocks: " << total_source_blocks);
+  uint64_t total_source_blocks = bootstrap.count_blocks(import_file_path, pos, seek_height, block_first);
+  MINFO("bootstrap file last block number: " << total_source_blocks+block_first-1 << " (zero-based height)  total blocks: " << total_source_blocks);
 
-  if (total_source_blocks-1 <= start_height)
+  if (total_source_blocks+block_first-1 <= start_height)
   {
     return false;
   }
@@ -260,7 +261,8 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
 
   // 4 byte magic + (currently) 1024 byte header structures
   uint8_t major_version, minor_version;
-  bootstrap.seek_to_first_chunk(import_file, major_version, minor_version);
+  uint64_t dummy;
+  bootstrap.seek_to_first_chunk(import_file, major_version, minor_version, dummy, dummy);
 
   std::string str1;
   char buffer1[1024];
@@ -275,7 +277,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
 
   if (! block_stop)
   {
-    block_stop = total_source_blocks - 1;
+    block_stop = total_source_blocks+block_first - 1;
   }
 
   // These are what we'll try to use, and they don't have to be a determination

src/blockchain_utilities/blockchain_stats.cpp

@@ -68,6 +68,9 @@ int main(int argc, char* argv[])
   const command_line::arg_descriptor<bool> arg_outputs  = {"with-outputs", "with output stats", false};
   const command_line::arg_descriptor<bool> arg_ringsize  = {"with-ringsize", "with ringsize stats", false};
   const command_line::arg_descriptor<bool> arg_hours  = {"with-hours", "with txns per hour", false};
+  const command_line::arg_descriptor<bool> arg_emission  = {"with-emission", "with coin emission", false};
+  const command_line::arg_descriptor<bool> arg_fees  = {"with-fees", "with txn fees", false};
+  const command_line::arg_descriptor<bool> arg_diff  = {"with-diff", "with difficulty", false};
 
   command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir);
   command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on);
@@ -79,6 +82,9 @@ int main(int argc, char* argv[])
   command_line::add_arg(desc_cmd_sett, arg_outputs);
   command_line::add_arg(desc_cmd_sett, arg_ringsize);
   command_line::add_arg(desc_cmd_sett, arg_hours);
+  command_line::add_arg(desc_cmd_sett, arg_emission);
+  command_line::add_arg(desc_cmd_sett, arg_fees);
+  command_line::add_arg(desc_cmd_sett, arg_diff);
   command_line::add_arg(desc_cmd_only, command_line::arg_help);
 
   po::options_description desc_options("Allowed options");
@@ -120,6 +126,9 @@ int main(int argc, char* argv[])
   bool do_outputs = command_line::get_arg(vm, arg_outputs);
   bool do_ringsize = command_line::get_arg(vm, arg_ringsize);
   bool do_hours = command_line::get_arg(vm, arg_hours);
+  bool do_emission = command_line::get_arg(vm, arg_emission);
+  bool do_fees = command_line::get_arg(vm, arg_fees);
+  bool do_diff = command_line::get_arg(vm, arg_diff);
 
   LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)");
   std::unique_ptr<Blockchain> core_storage;
@@ -177,12 +186,20 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
   // spit out a comment that GnuPlot can use as an index
   std::cout << ENDL << "# DATA" << ENDL;
   std::cout << "Date\tBlocks/day\tBlocks\tTxs/Day\tTxs\tBytes/Day\tBytes";
+  if (do_emission)
+    std::cout << "\tEmission/day\tEmission";
+  if (do_fees)
+    std::cout << "\tFees/day\tFees";
+  if (do_diff)
+    std::cout << "\tDiffMin\tDiffMax\tDiffAvg";
   if (do_inputs)
     std::cout << "\tInMin\tInMax\tInAvg";
   if (do_outputs)
     std::cout << "\tOutMin\tOutMax\tOutAvg";
   if (do_ringsize)
     std::cout << "\tRingMin\tRingMax\tRingAvg";
+  if (do_inputs || do_outputs || do_ringsize)
+    std::cout << std::setprecision(2) << std::fixed;
   if (do_hours) {
     char buf[8];
     unsigned int i;
@@ -193,14 +210,20 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
   }
   std::cout << ENDL;
 
+#define MAX_INOUT	0xffffffff
+#define MAX_RINGS	0xffffffff
+
   struct tm prevtm = {0}, currtm;
   uint64_t prevsz = 0, currsz = 0;
   uint64_t prevtxs = 0, currtxs = 0;
   uint64_t currblks = 0;
   uint64_t totins = 0, totouts = 0, totrings = 0;
-  uint32_t minins = 10, maxins = 0;
-  uint32_t minouts = 10, maxouts = 0;
-  uint32_t minrings = 50, maxrings = 0;
+  boost::multiprecision::uint128_t prevemission = 0, prevfees = 0;
+  boost::multiprecision::uint128_t emission = 0, fees = 0;
+  boost::multiprecision::uint128_t totdiff = 0, mindiff = 0, maxdiff = 0;
+  uint32_t minins = MAX_INOUT, maxins = 0;
+  uint32_t minouts = MAX_INOUT, maxouts = 0;
+  uint32_t minrings = MAX_RINGS, maxrings = 0;
   uint32_t io, tottxs = 0;
   uint32_t txhr[24] = {0};
   unsigned int i;
@@ -230,34 +253,50 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
       std::cout << timebuf << "\t" << currblks << "\t" << h << "\t" << currtxs << "\t" << prevtxs + currtxs << "\t" << currsz << "\t" << prevsz + currsz;
       prevsz += currsz;
       currsz = 0;
-      currblks = 0;
       prevtxs += currtxs;
       currtxs = 0;
       if (!tottxs)
         tottxs = 1;
+      if (do_emission) {
+        std::cout << "\t" << print_money(emission) << "\t" << print_money(prevemission + emission);
+        prevemission += emission;
+        emission = 0;
+      }
+      if (do_fees) {
+        std::cout << "\t" << print_money(fees) << "\t" << print_money(prevfees + fees);
+        prevfees += fees;
+        fees = 0;
+      }
+      if (do_diff) {
+        std::cout << "\t" << (maxdiff ? mindiff : 0) << "\t" << maxdiff << "\t" << totdiff / currblks;
+        mindiff = 0; maxdiff = 0; totdiff = 0;
+      }
       if (do_inputs) {
-        std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins / tottxs;
-        minins = 10; maxins = 0; totins = 0;
+        std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins * 1.0 / tottxs;
+        minins = MAX_INOUT; maxins = 0; totins = 0;
       }
       if (do_outputs) {
-        std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts / tottxs;
-        minouts = 10; maxouts = 0; totouts = 0;
+        std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts * 1.0 / tottxs;
+        minouts = MAX_INOUT; maxouts = 0; totouts = 0;
       }
       if (do_ringsize) {
-        std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings / tottxs;
-        minrings = 50; maxrings = 0; totrings = 0;
+        std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings * 1.0 / tottxs;
+        minrings = MAX_RINGS; maxrings = 0; totrings = 0;
       }
-      tottxs = 0;
       if (do_hours) {
         for (i=0; i<24; i++) {
           std::cout << "\t" << txhr[i];
           txhr[i] = 0;
         }
       }
+      currblks = 0;
+      tottxs = 0;
       std::cout << ENDL;
     }
 skip:
     currsz += bd.size();
+    uint64_t coinbase_amount;
+    uint64_t tx_fee_amount = 0;
     for (const auto& tx_id : blk.tx_hashes)
     {
       if (tx_id == crypto::null_hash)
@@ -275,7 +314,12 @@ skip:
         return 1;
       }
       currsz += bd.size();
+      if (db->get_prunable_tx_blob(tx_id, bd))
+        currsz += bd.size();
       currtxs++;
+      if (do_fees || do_emission) {
+        tx_fee_amount += get_tx_fee(tx);
+      }
       if (do_hours)
         txhr[currtm.tm_hour]++;
       if (do_inputs) {
@@ -306,6 +350,21 @@ skip:
       }
       tottxs++;
     }
+    if (do_diff) {
+      difficulty_type diff = db->get_block_difficulty(h);
+      if (!mindiff || diff < mindiff)
+        mindiff = diff;
+      if (diff > maxdiff)
+        maxdiff = diff;
+      totdiff += diff;
+    }
+    if (do_emission) {
+      coinbase_amount = get_outs_money_amount(blk.miner_tx);
+      emission += coinbase_amount - tx_fee_amount;
+    }
+    if (do_fees) {
+      fees += tx_fee_amount;
+    }
     currblks++;
 
     if (stop_requested)

src/blockchain_utilities/bootstrap_file.cpp

@@ -52,7 +52,7 @@ namespace
 
 
 
-bool BootstrapFile::open_writer(const boost::filesystem::path& file_path)
+bool BootstrapFile::open_writer(const boost::filesystem::path& file_path, uint64_t start_block, uint64_t stop_block)
 {
   const boost::filesystem::path dir_path = file_path.parent_path();
   if (!dir_path.empty())
@@ -78,7 +78,7 @@ bool BootstrapFile::open_writer(const boost::filesystem::path& file_path)
   m_raw_data_file = new std::ofstream();
 
   bool do_initialize_file = false;
-  uint64_t num_blocks = 0;
+  uint64_t num_blocks = 0, block_first = 0;
 
   if (! boost::filesystem::exists(file_path))
   {
@@ -88,10 +88,12 @@ bool BootstrapFile::open_writer(const boost::filesystem::path& file_path)
   }
   else
   {
-    num_blocks = count_blocks(file_path.string());
-    MDEBUG("appending to existing file with height: " << num_blocks-1 << "  total blocks: " << num_blocks);
+    std::streampos dummy_pos;
+    uint64_t dummy_height = 0;
+    num_blocks = count_blocks(file_path.string(), dummy_pos, dummy_height, block_first);
+    MDEBUG("appending to existing file with height: " << num_blocks+block_first-1 << "  total blocks: " << num_blocks);
   }
-  m_height = num_blocks;
+  m_height = num_blocks+block_first;
 
   if (do_initialize_file)
     m_raw_data_file->open(file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::trunc);
@@ -106,13 +108,12 @@ bool BootstrapFile::open_writer(const boost::filesystem::path& file_path)
     return false;
 
   if (do_initialize_file)
-    initialize_file();
+    initialize_file(start_block, stop_block);
 
   return true;
 }
 
-
-bool BootstrapFile::initialize_file()
+bool BootstrapFile::initialize_file(uint64_t first_block, uint64_t last_block)
 {
   const uint32_t file_magic = blockchain_raw_magic;
 
@@ -129,8 +130,8 @@ bool BootstrapFile::initialize_file()
   bfi.header_size = header_size;
 
   bootstrap::blocks_info bbi;
-  bbi.block_first = 0;
-  bbi.block_last = 0;
+  bbi.block_first = first_block;
+  bbi.block_last = last_block;
   bbi.block_last_pos = 0;
 
   buffer_type buffer2;
@@ -261,7 +262,7 @@ bool BootstrapFile::close()
 }
 
 
-bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_file, uint64_t requested_block_stop)
+bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_file, uint64_t start_block, uint64_t requested_block_stop)
 {
   uint64_t num_blocks_written = 0;
   m_max_chunk = 0;
@@ -269,17 +270,11 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
   m_tx_pool = _tx_pool;
   uint64_t progress_interval = 100;
   MINFO("Storing blocks raw data...");
-  if (!BootstrapFile::open_writer(output_file))
-  {
-    MFATAL("failed to open raw file for write");
-    return false;
-  }
   block b;
 
   // block_start, block_stop use 0-based height. m_height uses 1-based height. So to resume export
   // from last exported block, block_start doesn't need to add 1 here, as it's already at the next
   // height.
-  uint64_t block_start = m_height;
   uint64_t block_stop = 0;
   MINFO("source blockchain height: " <<  m_blockchain_storage->get_current_blockchain_height()-1);
   if ((requested_block_stop > 0) && (requested_block_stop < m_blockchain_storage->get_current_blockchain_height()))
@@ -292,6 +287,13 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
     block_stop = m_blockchain_storage->get_current_blockchain_height() - 1;
     MINFO("Using block height of source blockchain: " << block_stop);
   }
+  if (!BootstrapFile::open_writer(output_file, start_block, block_stop))
+  {
+    MFATAL("failed to open raw file for write");
+    return false;
+  }
+  uint64_t block_start = m_height ? m_height : start_block;
+  MINFO("Starting block height: " << block_start);
   for (m_cur_height = block_start; m_cur_height <= block_stop; ++m_cur_height)
   {
     // this method's height refers to 0-based height (genesis block = height 0)
@@ -323,7 +325,8 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
   return BootstrapFile::close();
 }
 
-uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version)
+uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version,
+	uint64_t &block_first, uint64_t &block_last)
 {
   uint32_t file_magic;
 
@@ -368,11 +371,35 @@ uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file, uint8_t
   MINFO("bootstrap magic size: " << sizeof(file_magic));
   MINFO("bootstrap header size: " << bfi.header_size);
 
+  uint32_t buflen_blocks_info;
+
+  import_file.read(buf1, sizeof(buflen_blocks_info));
+  str1.assign(buf1, sizeof(buflen_blocks_info));
+  if (! import_file)
+    throw std::runtime_error("Error reading expected number of bytes");
+  if (! ::serialization::parse_binary(str1, buflen_blocks_info))
+    throw std::runtime_error("Error in deserialization of buflen_blocks_info");
+  MINFO("bootstrap::blocks_info size: " << buflen_blocks_info);
+
+  if (buflen_blocks_info > sizeof(buf1))
+    throw std::runtime_error("Error: bootstrap::blocks_info size exceeds buffer size");
+  import_file.read(buf1, buflen_blocks_info);
+  if (! import_file)
+    throw std::runtime_error("Error reading expected number of bytes");
+  str1.assign(buf1, buflen_blocks_info);
+  bootstrap::blocks_info bbi;
+  if (! ::serialization::parse_binary(str1, bbi))
+    throw std::runtime_error("Error in deserialization of bootstrap::blocks_info");
+  MINFO("bootstrap first block:" << bbi.block_first);
+  MINFO("bootstrap last block:" << bbi.block_last);
+
   uint64_t full_header_size = sizeof(file_magic) + bfi.header_size;
   import_file.seekg(full_header_size);
 
   major_version = bfi.major_version;
   minor_version = bfi.minor_version;
+  block_first = bbi.block_first;
+  block_last = bbi.block_last;
   return full_header_size;
 }
 
@@ -436,13 +463,14 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path)
 {
   std::streampos dummy_pos;
   uint64_t dummy_height = 0;
-  return count_blocks(import_file_path, dummy_pos, dummy_height);
+  return count_blocks(import_file_path, dummy_pos, dummy_height, dummy_height);
 }
 
 // If seek_height is non-zero on entry, return a stream position <= this height when finished.
 // And return the actual height corresponding to this position. Allows the caller to locate its
 // starting position without having to reread the entire file again.
-uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::streampos &start_pos, uint64_t& seek_height)
+uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::streampos &start_pos,
+	uint64_t& seek_height, uint64_t &block_first)
 {
   boost::filesystem::path raw_file_path(import_file_path);
   boost::system::error_code ec;
@@ -464,7 +492,8 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::s
 
   uint64_t full_header_size; // 4 byte magic + length of header structures
   uint8_t major_version, minor_version;
-  full_header_size = seek_to_first_chunk(import_file, major_version, minor_version);
+  uint64_t block_last;
+  full_header_size = seek_to_first_chunk(import_file, major_version, minor_version, block_first, block_last);
 
   MINFO("Scanning blockchain from bootstrap file...");
   bool quit = false;
@@ -473,11 +502,11 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::s
 
   while (! quit)
   {
-    if (start_height && h + progress_interval >= start_height - 1)
+    if (start_height && h + block_first + progress_interval >= start_height - 1)
     {
       start_height = 0;
       start_pos = import_file.tellg();
-      seek_height = h;
+      seek_height = h + block_first;
     }
     bytes_read += count_bytes(import_file, progress_interval, blocks, quit);
     h += blocks;

src/blockchain_utilities/bootstrap_file.h

@@ -57,12 +57,12 @@ class BootstrapFile
 public:
 
   uint64_t count_bytes(std::ifstream& import_file, uint64_t blocks, uint64_t& h, bool& quit);
-  uint64_t count_blocks(const std::string& dir_path, std::streampos& start_pos, uint64_t& seek_height);
+  uint64_t count_blocks(const std::string& dir_path, std::streampos& start_pos, uint64_t& seek_height, uint64_t& block_first);
   uint64_t count_blocks(const std::string& dir_path);
-  uint64_t seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version);
+  uint64_t seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version, uint64_t &block_first, uint64_t &block_last);
 
   bool store_blockchain_raw(cryptonote::Blockchain* cs, cryptonote::tx_memory_pool* txp,
-      boost::filesystem::path& output_file, uint64_t use_block_height=0);
+      boost::filesystem::path& output_file, uint64_t start_block=0, uint64_t stop_block=0);
 
 protected:
 
@@ -75,8 +75,8 @@ protected:
   boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>* m_output_stream;
 
   // open export file for write
-  bool open_writer(const boost::filesystem::path& file_path);
-  bool initialize_file();
+  bool open_writer(const boost::filesystem::path& file_path, uint64_t start_block, uint64_t stop_block);
+  bool initialize_file(uint64_t start_block, uint64_t stop_block);
   bool close();
   void write_block(block& block);
   void flush_chunk();

src/blocks/CMakeLists.txt

@@ -45,4 +45,4 @@ foreach(BLOB_NAME checkpoints)
     )
 endforeach()
 
-add_library(blocks STATIC blocks.cpp ${GENERATED_SOURCES})
+monero_add_library(blocks blocks.cpp ${GENERATED_SOURCES})

src/checkpoints/checkpoints.cpp

@@ -182,12 +182,26 @@ namespace cryptonote
   {
     if (nettype == TESTNET)
     {
+    ADD_CHECKPOINT2(1,   "c47b8effcc84e22ddd1b876d059c8fbd3e26e4bdffb6bed8b8f4fe283104a7af", "0x2");
+    ADD_CHECKPOINT2(5,   "292add330f6cf5f3845dc4cd2f28ce8d211dd487ddcfe1b5d1d545d7b90ff7d1", "0x5df");  //Hard fork to v8
+    ADD_CHECKPOINT2(10,  "fbdf7d812aa6dd4c5915aadcbf023e722eab1d8ad47c24a230634e688d314798", "0xfa3");  //Hard fork to v9
+    ADD_CHECKPOINT2(15,  "9b86e40b959d7cfe75366686dfd2657699e85b8c658aa8c20b23f7bc01b38af0", "0x1967"); //Hard fork to v10
+    ADD_CHECKPOINT2(20,  "8f65c309ba3ed2bf25df2b7d91e1db40c864230aa3caaf823db81bc19e0bd6a3", "0x232b"); //Hard fork to v11
+    ADD_CHECKPOINT2(25,  "35f2957fdba45d5421561ba2f126a61ec0101b5c2eb79b9e296310acbbdbbe58", "0x2cef"); //Hard fork to v12
+    ADD_CHECKPOINT2(30,  "32cc5aacce8bea10ae869313e194ba51a30720810f3665433ffeea2818938429", "0x36b3"); //Hard fork to v13
+    ADD_CHECKPOINT2(35,  "7f1cdd2c4a6002772343ad9de6c5dba743a0be1d326a6f63924511f08b15863a", "0x4077"); //Hard fork to v14
+    ADD_CHECKPOINT2(40,  "63cf4f703489a881baf63bacd6a3af2b5bde79b558ce123e97313890be6f747f", "0x4a3b"); //Hard fork to v15
+    ADD_CHECKPOINT2(45,  "65da4c966ae02983c6c8b7ab9b959df612863c379c5e48669061cbbb1e02a3ab", "0x53ff"); //Hard fork to v16
+    ADD_CHECKPOINT2(50,  "8c68a444b6743a14152f130357be9751e42e82a84f69f25320eda532a830c7b0", "0x5dc3"); //Hard fork to v17
       return true;
     }
     if (nettype == STAGENET)
     {
       return true;
     }
+    // make RPC call to daemon
+    // curl http://127.0.0.1:34568/json_rpc -d '{"jsonrpc":"2.0","id":"0","method":"get_block","params":{"height":282700}}' -H 'Content-Type: application/json'
+    // "wide_cumulative_difficulty": "0x14eb4d0131fe8",
     ADD_CHECKPOINT2(1,       "97f4ce4d7879b3bea54dcec738cd2ebb7952b4e9bb9743262310cd5fec749340", "0x2");
     ADD_CHECKPOINT2(6969,    "aa7b66e8c461065139b55c29538a39c33ceda93e587f84d490ed573d80511c87", "0x118eef693fd");  //Hard fork to v8
     ADD_CHECKPOINT2(53666,   "3f43f56f66ef0c43cf2fd14d0d28fa2aae0ef8f40716773511345750770f1255", "0xb677d6405ae");  //Hard fork to v9
@@ -197,7 +211,12 @@ namespace cryptonote
     ADD_CHECKPOINT2(114969,  "b48245956b87f243048fd61021f4b3e5443e57eee7ff8ba4762d18926e80b80c", "0x1ca552b3ec68"); //Hard fork to v13
     ADD_CHECKPOINT2(115257,  "338e056551087fe23d6c4b4280244bc5362b004716d85ec799a775f190f9fea9", "0x1cb25f5d4628"); //Hard fork to v14
     ADD_CHECKPOINT2(160777,  "9496690579af21f38f00e67e11c2e85a15912fe4f412aad33d1162be1579e755", "0x5376eaa196a8"); //Hard fork to v15
-    ADD_CHECKPOINT2(230200,  "0ebb018c452fa84997f573937db67ef3946a857ee57cd40fa51ffe368896c666", "0xbcc1409c7268");
+    ADD_CHECKPOINT2(253999,  "755a289fe8a68e96a0f69069ba4007b676ec87dce2e47dfb9647fe5691f49883", "0x172d026ef7fe8"); //Hard fork to v16
+    ADD_CHECKPOINT2(254287,  "b37cb55abe73965b424f8028bf71bef98d069645077ffa52f0c134907b7734e3", "0x1746622f56668"); //Hard fork to v17
+    ADD_CHECKPOINT2(256700,  "389a8ab95a80e84ec74639c1078bc67b33af208ef00f53bd9609cfc40efa7059", "0x185ace3c1bd68");
+    ADD_CHECKPOINT2(271600,  "9597cdbdc52ca57d7dbd8f9c0a23a73194ef2ebbcfdc75c21992672706108d43", "0x1e2d2d6a2a9e8");
+    ADD_CHECKPOINT2(278300,  "b10dcdf7a51651f60fbcc0447409773eef1458d2c706d9a61daf467571ac19c9", "0x20a83a16d3968");
+    ADD_CHECKPOINT2(282700,  "79c06cafd7cb5f76bcebbf8f1ae16203bb41fd75b284bcd0eb0b457991ab7d4a", "0x22e3baf142de8");
     return true;
   }
 

src/common/dns_utils.cpp

@@ -519,14 +519,14 @@ bool load_txt_records_from_dns(std::vector<std::string> &good_records, const std
   // send all requests in parallel
   std::deque<bool> avail(dns_urls.size(), false), valid(dns_urls.size(), false);
   tools::threadpool& tpool = tools::threadpool::getInstance();
-  tools::threadpool::waiter waiter;
+  tools::threadpool::waiter waiter(tpool);
   for (size_t n = 0; n < dns_urls.size(); ++n)
   {
     tpool.submit(&waiter,[n, dns_urls, &records, &avail, &valid](){
       records[n] = tools::DNSResolver::instance().get_txt_record(dns_urls[n], avail[n], valid[n]); 
     });
   }
-  waiter.wait(&tpool);
+  waiter.wait();
 
   size_t cur_index = first_index;
   do

src/common/notify.cpp

@@ -62,7 +62,7 @@ static void replace(std::vector<std::string> &v, const char *tag, const char *s)
     boost::replace_all(str, tag, s);
 }
 
-int Notify::notify(const char *tag, const char *s, ...)
+int Notify::notify(const char *tag, const char *s, ...) const
 {
   std::vector<std::string> margs = args;
 

src/common/notify.h

@@ -38,8 +38,12 @@ class Notify
 {
 public:
   Notify(const char *spec);
+  Notify(const Notify&) = default;
+  Notify(Notify&&) = default;
+  Notify& operator=(const Notify&) = default;
+  Notify& operator=(Notify&&) = default;
 
-  int notify(const char *tag, const char *s, ...);
+  int notify(const char *tag, const char *s, ...) const;
 
 private:
   std::string filename;
@@ -47,3 +51,4 @@ private:
 };
 
 }
+

src/common/powerof.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include <stdint.h>
+
+namespace tools
+{
+  template<uint64_t a, uint64_t b>
+  struct PowerOf
+  {
+    enum Data : uint64_t
+    {
+      // a^b = a * a^(b-1)
+      Value = a * PowerOf<a, b - 1>::Value,
+    };
+  };
+
+  template<uint64_t a>
+  struct PowerOf<a, 0>
+  {
+    enum Data : uint64_t
+    {
+      // a^0 = 1
+      Value = 1,
+    };
+  };
+}

src/common/threadpool.cpp

@@ -120,7 +120,7 @@ threadpool::waiter::~waiter()
   catch (...) { /* ignore */ }
   try
   {
-    wait(NULL);
+    wait();
   }
   catch (const std::exception &e)
   {
@@ -128,12 +128,12 @@ threadpool::waiter::~waiter()
   }
 }
 
-void threadpool::waiter::wait(threadpool *tpool) {
-  if (tpool)
-    tpool->run(true);
+bool threadpool::waiter::wait() {
+  pool.run(true);
   boost::unique_lock<boost::mutex> lock(mt);
   while(num)
     cv.wait(lock);
+  return !error();
 }
 
 void threadpool::waiter::inc() {
@@ -166,7 +166,8 @@ void threadpool::run(bool flush) {
     lock.unlock();
     ++depth;
     is_leaf = e.leaf;
-    e.f();
+    try { e.f(); }
+    catch (const std::exception &ex) { e.wo->set_error(); try { MERROR("Exception in threadpool job: " << ex.what()); } catch (...) {} }
     --depth;
     is_leaf = false;
 

src/common/threadpool.h

@@ -55,12 +55,16 @@ public:
   class waiter {
     boost::mutex mt;
     boost::condition_variable cv;
+    threadpool &pool;
     int num;
+    bool error_flag;
     public:
     void inc();
     void dec();
-    void wait(threadpool *tpool);  //! Wait for a set of tasks to finish.
-    waiter() : num(0){}
+    bool wait();  //! Wait for a set of tasks to finish, returns false iff any error
+    void set_error() noexcept { error_flag = true; }
+    bool error() const noexcept { return error_flag; }
+    waiter(threadpool &pool) : pool(pool), num(0), error_flag(false) {}
     ~waiter();
   };
 

src/common/updates.cpp

@@ -98,6 +98,8 @@ namespace tools
     const char *base = user ? "" : "";
 #ifdef _WIN32
     static const char *extension = strncmp(buildtag.c_str(), "source", 6) ? (strncmp(buildtag.c_str(), "install-", 8) ? ".zip" : ".exe") : ".tar.bz2";
+#elif defined(__APPLE__)
+    static const char *extension = strncmp(software.c_str(), "monero-gui", 10) ? ".tar.bz2" : ".dmg";
 #else
     static const char extension[] = ".tar.bz2";
 #endif

src/common/util.cpp

@@ -871,10 +871,19 @@ std::string get_nix_version_display_string()
     return max_concurrency;
   }
 
+  bool is_privacy_preserving_network(const std::string &address)
+  {
+    if (boost::ends_with(address, ".onion"))
+      return true;
+    if (boost::ends_with(address, ".i2p"))
+      return true;
+    return false;
+  }
+
   bool is_local_address(const std::string &address)
   {
     // always assume Tor/I2P addresses to be untrusted by default
-    if (boost::ends_with(address, ".onion") || boost::ends_with(address, ".i2p"))
+    if (is_privacy_preserving_network(address))
     {
       MDEBUG("Address '" << address << "' is Tor/I2P, non local");
       return false;
@@ -1000,13 +1009,13 @@ std::string get_nix_version_display_string()
     for (char c: val)
     {
       if (c == '*')
-        newval += escape ? "*" : ".*";
+        newval += escape ? "*" : ".*", escape = false;
       else if (c == '?')
-        newval += escape ? "?" : ".";
+        newval += escape ? "?" : ".", escape = false;
       else if (c == '\\')
         newval += '\\', escape = !escape;
       else
-        newval += c;
+        newval += c, escape = false;
     }
     return newval;
   }
@@ -1116,7 +1125,7 @@ std::string get_nix_version_display_string()
     static constexpr const byte_map sizes[] =
     {
         {"%.0f B", 1024},
-        {"%.2f KB", 1024 * 1024},
+        {"%.2f kB", 1024 * 1024},
         {"%.2f MB", std::uint64_t(1024) * 1024 * 1024},
         {"%.2f GB", std::uint64_t(1024) * 1024 * 1024 * 1024},
         {"%.2f TB", std::uint64_t(1024) * 1024 * 1024 * 1024 * 1024}

src/common/util.h

@@ -228,6 +228,7 @@ namespace tools
   unsigned get_max_concurrency();
 
   bool is_local_address(const std::string &address);
+  bool is_privacy_preserving_network(const std::string &address);
   int vercmp(const char *v0, const char *v1); // returns < 0, 0, > 0, similar to strcmp, but more human friendly than lexical - does not attempt to validate
 
   bool sha256sum(const uint8_t *data, size_t len, crypto::hash &hash);

src/crypto/CMakeLists.txt

@@ -116,3 +116,6 @@ endif()
 
 # cheat because cmake and ccache hate each other
 set_property(SOURCE CryptonightR_template.S PROPERTY LANGUAGE C)
+
+# Must be done last, because it references libraries in this directory
+add_subdirectory(wallet)

src/crypto/crypto-ops.c

@@ -1234,6 +1234,56 @@ void ge_double_scalarmult_base_vartime(ge_p2 *r, const unsigned char *a, const g
   }
 }
 
+// Computes aG + bB + cC (G is the fixed basepoint)
+void ge_triple_scalarmult_base_vartime(ge_p2 *r, const unsigned char *a, const unsigned char *b, const ge_dsmp Bi, const unsigned char *c, const ge_dsmp Ci) {
+  signed char aslide[256];
+  signed char bslide[256];
+  signed char cslide[256];
+  ge_p1p1 t;
+  ge_p3 u;
+  int i;
+
+  slide(aslide, a);
+  slide(bslide, b);
+  slide(cslide, c);
+
+  ge_p2_0(r);
+
+  for (i = 255; i >= 0; --i) {
+    if (aslide[i] || bslide[i] || cslide[i]) break;
+  }
+
+  for (; i >= 0; --i) {
+    ge_p2_dbl(&t, r);
+
+    if (aslide[i] > 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_madd(&t, &u, &ge_Bi[aslide[i]/2]);
+    } else if (aslide[i] < 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_msub(&t, &u, &ge_Bi[(-aslide[i])/2]);
+    }
+
+    if (bslide[i] > 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_add(&t, &u, &Bi[bslide[i]/2]);
+    } else if (bslide[i] < 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_sub(&t, &u, &Bi[(-bslide[i])/2]);
+    }
+
+    if (cslide[i] > 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_add(&t, &u, &Ci[cslide[i]/2]);
+    } else if (cslide[i] < 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_sub(&t, &u, &Ci[(-cslide[i])/2]);
+    }
+
+    ge_p1p1_to_p2(r, &t);
+  }
+}
+
 void ge_double_scalarmult_base_vartime_p3(ge_p3 *r3, const unsigned char *a, const ge_p3 *A, const unsigned char *b) {
   signed char aslide[256];
   signed char bslide[256];
@@ -2148,6 +2198,56 @@ void ge_double_scalarmult_precomp_vartime2(ge_p2 *r, const unsigned char *a, con
   }
 }
 
+// Computes aA + bB + cC (all points require precomputation)
+void ge_triple_scalarmult_precomp_vartime(ge_p2 *r, const unsigned char *a, const ge_dsmp Ai, const unsigned char *b, const ge_dsmp Bi, const unsigned char *c, const ge_dsmp Ci) {
+  signed char aslide[256];
+  signed char bslide[256];
+  signed char cslide[256];
+  ge_p1p1 t;
+  ge_p3 u;
+  int i;
+
+  slide(aslide, a);
+  slide(bslide, b);
+  slide(cslide, c);
+
+  ge_p2_0(r);
+
+  for (i = 255; i >= 0; --i) {
+    if (aslide[i] || bslide[i] || cslide[i]) break;
+  }
+
+  for (; i >= 0; --i) {
+    ge_p2_dbl(&t, r);
+
+    if (aslide[i] > 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_add(&t, &u, &Ai[aslide[i]/2]);
+    } else if (aslide[i] < 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_sub(&t, &u, &Ai[(-aslide[i])/2]);
+    }
+
+    if (bslide[i] > 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_add(&t, &u, &Bi[bslide[i]/2]);
+    } else if (bslide[i] < 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_sub(&t, &u, &Bi[(-bslide[i])/2]);
+    }
+
+    if (cslide[i] > 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_add(&t, &u, &Ci[cslide[i]/2]);
+    } else if (cslide[i] < 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_sub(&t, &u, &Ci[(-cslide[i])/2]);
+    }
+
+    ge_p1p1_to_p2(r, &t);
+  }
+}
+
 void ge_double_scalarmult_precomp_vartime2_p3(ge_p3 *r3, const unsigned char *a, const ge_dsmp Ai, const unsigned char *b, const ge_dsmp Bi) {
   signed char aslide[256];
   signed char bslide[256];

src/crypto/crypto-ops.h

@@ -79,6 +79,7 @@ typedef ge_cached ge_dsmp[8];
 extern const ge_precomp ge_Bi[8];
 void ge_dsm_precomp(ge_dsmp r, const ge_p3 *s);
 void ge_double_scalarmult_base_vartime(ge_p2 *, const unsigned char *, const ge_p3 *, const unsigned char *);
+void ge_triple_scalarmult_base_vartime(ge_p2 *, const unsigned char *, const unsigned char *, const ge_dsmp, const unsigned char *, const ge_dsmp);
 void ge_double_scalarmult_base_vartime_p3(ge_p3 *, const unsigned char *, const ge_p3 *, const unsigned char *);
 
 /* From ge_frombytes.c, modified */
@@ -130,6 +131,7 @@ void sc_reduce(unsigned char *);
 void ge_scalarmult(ge_p2 *, const unsigned char *, const ge_p3 *);
 void ge_scalarmult_p3(ge_p3 *, const unsigned char *, const ge_p3 *);
 void ge_double_scalarmult_precomp_vartime(ge_p2 *, const unsigned char *, const ge_p3 *, const unsigned char *, const ge_dsmp);
+void ge_triple_scalarmult_precomp_vartime(ge_p2 *, const unsigned char *, const ge_dsmp, const unsigned char *, const ge_dsmp, const unsigned char *, const ge_dsmp);
 void ge_double_scalarmult_precomp_vartime2(ge_p2 *, const unsigned char *, const ge_dsmp, const unsigned char *, const ge_dsmp);
 void ge_double_scalarmult_precomp_vartime2_p3(ge_p3 *, const unsigned char *, const ge_dsmp, const unsigned char *, const ge_dsmp);
 void ge_mul8(ge_p1p1 *, const ge_p2 *);

src/crypto/crypto.cpp

@@ -43,6 +43,8 @@
 #include "crypto.h"
 #include "hash.h"
 
+#include "cryptonote_config.h"
+
 namespace {
   static void local_abort(const char *msg)
   {
@@ -121,13 +123,17 @@ namespace crypto {
   void random32_unbiased(unsigned char *bytes)
   {
     // l = 2^252 + 27742317777372353535851937790883648493.
-    // it fits 15 in 32 bytes
+    // l fits 15 times in 32 bytes (iow, 15 l is the highest multiple of l that fits in 32 bytes)
     static const unsigned char limit[32] = { 0xe3, 0x6a, 0x67, 0x72, 0x8b, 0xce, 0x13, 0x29, 0x8f, 0x30, 0x82, 0x8c, 0x0b, 0xa4, 0x10, 0x39, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0 };
-    do
+    while(1)
     {
       generate_random_bytes_thread_safe(32, bytes);
-    } while (!sc_isnonzero(bytes) && !less32(bytes, limit)); // should be good about 15/16 of the time
-    sc_reduce32(bytes);
+      if (!less32(bytes, limit))
+        continue;
+      sc_reduce32(bytes);
+      if (sc_isnonzero(bytes))
+        break;
+    }
   }
   /* generate a random 32-byte (256-bit) integer and copy it to res */
   static inline void random_scalar(ec_scalar &res) {
@@ -261,11 +267,24 @@ namespace crypto {
     ec_point comm;
   };
 
+  // Used in v1 tx proofs
+  struct s_comm_2_v1 {
+    hash msg;
+    ec_point D;
+    ec_point X;
+    ec_point Y;
+  };
+
+  // Used in v1/v2 tx proofs
   struct s_comm_2 {
     hash msg;
     ec_point D;
     ec_point X;
     ec_point Y;
+    hash sep; // domain separation
+    ec_point R;
+    ec_point A;
+    ec_point B;
   };
 
   void crypto_ops::generate_signature(const hash &prefix_hash, const public_key &pub, const secret_key &sec, signature &sig) {
@@ -321,6 +340,86 @@ namespace crypto {
     return sc_isnonzero(&c) == 0;
   }
 
+  // Generate a proof of knowledge of `r` such that (`R = rG` and `D = rA`) or (`R = rB` and `D = rA`) via a Schnorr proof
+  // This handles use cases for both standard addresses and subaddresses
+  //
+  // NOTE: This generates old v1 proofs, and is for TESTING ONLY
+  void crypto_ops::generate_tx_proof_v1(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const secret_key &r, signature &sig) {
+    // sanity check
+    ge_p3 R_p3;
+    ge_p3 A_p3;
+    ge_p3 B_p3;
+    ge_p3 D_p3;
+    if (ge_frombytes_vartime(&R_p3, &R) != 0) throw std::runtime_error("tx pubkey is invalid");
+    if (ge_frombytes_vartime(&A_p3, &A) != 0) throw std::runtime_error("recipient view pubkey is invalid");
+    if (B && ge_frombytes_vartime(&B_p3, &*B) != 0) throw std::runtime_error("recipient spend pubkey is invalid");
+    if (ge_frombytes_vartime(&D_p3, &D) != 0) throw std::runtime_error("key derivation is invalid");
+#if !defined(NDEBUG)
+    {
+      assert(sc_check(&r) == 0);
+      // check R == r*G or R == r*B
+      public_key dbg_R;
+      if (B)
+      {
+        ge_p2 dbg_R_p2;
+        ge_scalarmult(&dbg_R_p2, &r, &B_p3);
+        ge_tobytes(&dbg_R, &dbg_R_p2);
+      }
+      else
+      {
+        ge_p3 dbg_R_p3;
+        ge_scalarmult_base(&dbg_R_p3, &r);
+        ge_p3_tobytes(&dbg_R, &dbg_R_p3);
+      }
+      assert(R == dbg_R);
+      // check D == r*A
+      ge_p2 dbg_D_p2;
+      ge_scalarmult(&dbg_D_p2, &r, &A_p3);
+      public_key dbg_D;
+      ge_tobytes(&dbg_D, &dbg_D_p2);
+      assert(D == dbg_D);
+    }
+#endif
+
+    // pick random k
+    ec_scalar k;
+    random_scalar(k);
+    
+    s_comm_2_v1 buf;
+    buf.msg = prefix_hash;
+    buf.D = D;
+    
+    if (B)
+    {
+      // compute X = k*B
+      ge_p2 X_p2;
+      ge_scalarmult(&X_p2, &k, &B_p3);
+      ge_tobytes(&buf.X, &X_p2);
+    }
+    else
+    {
+      // compute X = k*G
+      ge_p3 X_p3;
+      ge_scalarmult_base(&X_p3, &k);
+      ge_p3_tobytes(&buf.X, &X_p3);
+    }
+    
+    // compute Y = k*A
+    ge_p2 Y_p2;
+    ge_scalarmult(&Y_p2, &k, &A_p3);
+    ge_tobytes(&buf.Y, &Y_p2);
+
+    // sig.c = Hs(Msg || D || X || Y) 
+    hash_to_scalar(&buf, sizeof(buf), sig.c);
+
+    // sig.r = k - sig.c*r
+    sc_mulsub(&sig.r, &sig.c, &unwrap(r), &k);
+  }
+
+  // Generate a proof of knowledge of `r` such that (`R = rG` and `D = rA`) or (`R = rB` and `D = rA`) via a Schnorr proof
+  // This handles use cases for both standard addresses and subaddresses
+  //
+  // Generates only proofs for InProofV2 and OutProofV2
   void crypto_ops::generate_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const secret_key &r, signature &sig) {
     // sanity check
     ge_p3 R_p3;
@@ -362,10 +461,20 @@ namespace crypto {
     ec_scalar k;
     random_scalar(k);
     
+    // if B is not present
+    static const ec_point zero = {{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }};
+
     s_comm_2 buf;
     buf.msg = prefix_hash;
     buf.D = D;
-
+    buf.R = R;
+    buf.A = A;
+    if (B)
+        buf.B = *B;
+    else
+        buf.B = zero;
+    cn_fast_hash(config::HASH_KEY_TXPROOF_V2, sizeof(config::HASH_KEY_TXPROOF_V2)-1, buf.sep);
+    
     if (B)
     {
       // compute X = k*B
@@ -386,7 +495,7 @@ namespace crypto {
     ge_scalarmult(&Y_p2, &k, &A_p3);
     ge_tobytes(&buf.Y, &Y_p2);
 
-    // sig.c = Hs(Msg || D || X || Y)
+    // sig.c = Hs(Msg || D || X || Y || sep || R || A || B) 
     hash_to_scalar(&buf, sizeof(buf), sig.c);
 
     // sig.r = k - sig.c*r
@@ -395,7 +504,8 @@ namespace crypto {
     memwipe(&k, sizeof(k));
   }
 
-  bool crypto_ops::check_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const signature &sig) {
+  // Verify a proof: either v1 (version == 1) or v2 (version == 2)
+  bool crypto_ops::check_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const signature &sig, const int version) {
     // sanity check
     ge_p3 R_p3;
     ge_p3 A_p3;
@@ -467,14 +577,31 @@ namespace crypto {
     ge_p2 Y_p2;
     ge_p1p1_to_p2(&Y_p2, &Y_p1p1);
 
-    // compute c2 = Hs(Msg || D || X || Y)
+    // Compute hash challenge
+    // for v1, c2 = Hs(Msg || D || X || Y)
+    // for v2, c2 = Hs(Msg || D || X || Y || sep || R || A || B)
+
+    // if B is not present
+    static const ec_point zero = {{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }};
+
     s_comm_2 buf;
     buf.msg = prefix_hash;
     buf.D = D;
+    buf.R = R;
+    buf.A = A;
+    if (B)
+        buf.B = *B;
+    else
+        buf.B = zero;
+    cn_fast_hash(config::HASH_KEY_TXPROOF_V2, sizeof(config::HASH_KEY_TXPROOF_V2)-1, buf.sep);
     ge_tobytes(&buf.X, &X_p2);
     ge_tobytes(&buf.Y, &Y_p2);
     ec_scalar c2;
-    hash_to_scalar(&buf, sizeof(s_comm_2), c2);
+
+    // Hash depends on version
+    if (version == 1) hash_to_scalar(&buf, sizeof(s_comm_2) - 3*sizeof(ec_point) - sizeof(hash), c2);
+    else if (version == 2) hash_to_scalar(&buf, sizeof(s_comm_2), c2);
+    else return false;
 
     // test if c2 == sig.c
     sc_sub(&c2, &c2, &sig.c);

src/crypto/crypto.h

@@ -132,8 +132,10 @@ namespace crypto {
     friend bool check_signature(const hash &, const public_key &, const signature &);
     static void generate_tx_proof(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const secret_key &, signature &);
     friend void generate_tx_proof(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const secret_key &, signature &);
-    static bool check_tx_proof(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const signature &);
-    friend bool check_tx_proof(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const signature &);
+    static void generate_tx_proof_v1(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const secret_key &, signature &);
+    friend void generate_tx_proof_v1(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const secret_key &, signature &);
+    static bool check_tx_proof(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const signature &, const int);
+    friend bool check_tx_proof(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const signature &, const int);
     static void generate_key_image(const public_key &, const secret_key &, key_image &);
     friend void generate_key_image(const public_key &, const secret_key &, key_image &);
     static void generate_ring_signature(const hash &, const key_image &,
@@ -248,8 +250,11 @@ namespace crypto {
   inline void generate_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const secret_key &r, signature &sig) {
     crypto_ops::generate_tx_proof(prefix_hash, R, A, B, D, r, sig);
   }
-  inline bool check_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const signature &sig) {
-    return crypto_ops::check_tx_proof(prefix_hash, R, A, B, D, sig);
+  inline void generate_tx_proof_v1(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const secret_key &r, signature &sig) {
+    crypto_ops::generate_tx_proof_v1(prefix_hash, R, A, B, D, r, sig);
+  }
+  inline bool check_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const signature &sig, const int version) {
+    return crypto_ops::check_tx_proof(prefix_hash, R, A, B, D, sig, version);
   }
 
   /* To send money to a key:

src/crypto/hash.h

@@ -73,7 +73,7 @@ namespace crypto {
   inline void cn_slow_hash(const void *data, std::size_t length, hash &hash, int variant = 0, uint64_t height = 0) {
     cn_slow_hash(data, length, reinterpret_cast<char *>(&hash), variant, 0/*prehashed*/, height);
   }
-  
+
   inline void cn_slow_hash_prehashed(const void *data, std::size_t length, hash &hash, int variant = 0, uint64_t height = 0) {
     cn_slow_hash(data, length, reinterpret_cast<char *>(&hash), variant, 1/*prehashed*/, height);
   }

src/crypto/rx-slow-hash.c

@@ -116,6 +116,46 @@ static inline int enabled_flags(void) {
 #define SEEDHASH_EPOCH_BLOCKS	2048	/* Must be same as BLOCKS_SYNCHRONIZING_MAX_COUNT in cryptonote_config.h */
 #define SEEDHASH_EPOCH_LAG		64
 
+static inline int is_power_of_2(uint64_t n) { return n && (n & (n-1)) == 0; }
+
+static int get_seedhash_epoch_lag(void)
+{
+  static unsigned int lag = (unsigned int)-1;
+  if (lag != (unsigned int)-1)
+    return lag;
+  const char *e = getenv("SEEDHASH_EPOCH_LAG");
+  if (e)
+  {
+    lag = atoi(e);
+    if (lag > SEEDHASH_EPOCH_LAG || !is_power_of_2(lag))
+      lag = SEEDHASH_EPOCH_LAG;
+  }
+  else
+  {
+    lag = SEEDHASH_EPOCH_LAG;
+  }
+  return lag;
+}
+
+static unsigned int get_seedhash_epoch_blocks(void)
+{
+  static unsigned int blocks = (unsigned int)-1;
+  if (blocks != (unsigned int)-1)
+    return blocks;
+  const char *e = getenv("SEEDHASH_EPOCH_BLOCKS");
+  if (e)
+  {
+    blocks = atoi(e);
+    if (blocks < 2 || blocks > SEEDHASH_EPOCH_BLOCKS || !is_power_of_2(blocks))
+      blocks = SEEDHASH_EPOCH_BLOCKS;
+  }
+  else
+  {
+    blocks = SEEDHASH_EPOCH_BLOCKS;
+  }
+  return blocks;
+}
+
 void rx_reorg(const uint64_t split_height) {
   int i;
   CTHR_MUTEX_LOCK(rx_mutex);
@@ -130,14 +170,16 @@ void rx_reorg(const uint64_t split_height) {
 }
 
 uint64_t rx_seedheight(const uint64_t height) {
-  uint64_t s_height =  (height <= SEEDHASH_EPOCH_BLOCKS+SEEDHASH_EPOCH_LAG) ? 0 :
-                       (height - SEEDHASH_EPOCH_LAG - 1) & ~(SEEDHASH_EPOCH_BLOCKS-1);
+  const uint64_t seedhash_epoch_lag = get_seedhash_epoch_lag();
+  const uint64_t seedhash_epoch_blocks = get_seedhash_epoch_blocks();
+  uint64_t s_height =  (height <= seedhash_epoch_blocks+seedhash_epoch_lag) ? 0 :
+                       (height - seedhash_epoch_lag - 1) & ~(seedhash_epoch_blocks-1);
   return s_height;
 }
 
 void rx_seedheights(const uint64_t height, uint64_t *seedheight, uint64_t *nextheight) {
   *seedheight = rx_seedheight(height);
-  *nextheight = rx_seedheight(height + SEEDHASH_EPOCH_LAG);
+  *nextheight = rx_seedheight(height + get_seedhash_epoch_lag());
 }
 
 typedef struct seedinfo {
@@ -161,11 +203,11 @@ static void rx_initdata(randomx_cache *rs_cache, const int miners, const uint64_
     CTHR_THREAD_TYPE *st;
     si = malloc(miners * sizeof(seedinfo));
     if (si == NULL)
-      local_abort("Couldn't allocate RandomWOW mining threadinfo");
+      local_abort("Couldn't allocate RandomX mining threadinfo");
     st = malloc(miners * sizeof(CTHR_THREAD_TYPE));
     if (st == NULL) {
       free(si);
-      local_abort("Couldn't allocate RandomWOW mining threadlist");
+      local_abort("Couldn't allocate RandomX mining threadlist");
     }
     for (i=0; i<miners-1; i++) {
       si[i].si_cache = rs_cache;
@@ -194,7 +236,7 @@ static void rx_initdata(randomx_cache *rs_cache, const int miners, const uint64_
 void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const char *seedhash, const void *data, size_t length,
   char *hash, int miners, int is_alt) {
   uint64_t s_height = rx_seedheight(mainheight);
-  int toggle = (s_height & SEEDHASH_EPOCH_BLOCKS) != 0;
+  int toggle = (s_height & get_seedhash_epoch_blocks()) != 0;
   randomx_flags flags = enabled_flags() & ~disabled_flags();
   rx_state *rx_sp;
   randomx_cache *cache;
@@ -225,11 +267,11 @@ void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const ch
     if (cache == NULL) {
       cache = randomx_alloc_cache(flags | RANDOMX_FLAG_LARGE_PAGES);
       if (cache == NULL) {
-        mdebug(RX_LOGCAT, "Couldn't use largePages for RandomWOW cache");
+        mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX cache");
         cache = randomx_alloc_cache(flags);
       }
       if (cache == NULL)
-        local_abort("Couldn't allocate RandomWOW cache");
+        local_abort("Couldn't allocate RandomX cache");
     }
   }
   if (rx_sp->rs_height != seedheight || rx_sp->rs_cache == NULL || memcmp(seedhash, rx_sp->rs_hash, HASH_SIZE)) {
@@ -251,7 +293,7 @@ void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const ch
         if (rx_dataset == NULL) {
           rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_LARGE_PAGES);
           if (rx_dataset == NULL) {
-            mdebug(RX_LOGCAT, "Couldn't use largePages for RandomWOW dataset");
+            mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX dataset");
             rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_DEFAULT);
           }
           if (rx_dataset != NULL)
@@ -264,14 +306,14 @@ void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const ch
         miners = 0;
         if (!rx_dataset_nomem) {
           rx_dataset_nomem = 1;
-          mwarning(RX_LOGCAT, "Couldn't allocate RandomWOW dataset for miner");
+          mwarning(RX_LOGCAT, "Couldn't allocate RandomX dataset for miner");
         }
       }
       CTHR_MUTEX_UNLOCK(rx_dataset_mutex);
     }
     rx_vm = randomx_create_vm(flags | RANDOMX_FLAG_LARGE_PAGES, rx_sp->rs_cache, rx_dataset);
     if(rx_vm == NULL) { //large pages failed
-      mdebug(RX_LOGCAT, "Couldn't use largePages for RandomWOW VM");
+      mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX VM");
       rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset);
     }
     if(rx_vm == NULL) {//fallback if everything fails
@@ -279,7 +321,7 @@ void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const ch
       rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset);
     }
     if (rx_vm == NULL)
-      local_abort("Couldn't allocate RandomWOW VM");
+      local_abort("Couldn't allocate RandomX VM");
   } else if (miners) {
     CTHR_MUTEX_LOCK(rx_dataset_mutex);
     if (rx_dataset != NULL && rx_dataset_height != seedheight)

src/crypto/wallet/CMakeLists.txt

@@ -0,0 +1,62 @@
+# Copyright (c) 2020, The Monero Project
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#
+# Possibly user defined values.
+#
+set(MONERO_WALLET_CRYPTO_LIBRARY "auto" CACHE STRING "Select a wallet crypto library")
+
+#
+# If the user specified "auto", detect best library defaulting to internal.
+#
+if (${MONERO_WALLET_CRYPTO_LIBRARY} STREQUAL "auto")
+   monero_crypto_autodetect(AVAILABLE BEST)
+  if (DEFINED BEST)
+    message("Wallet crypto is using ${BEST} backend")
+    set(MONERO_WALLET_CRYPTO_LIBRARY ${BEST})
+  else ()
+    message("Defaulting to internal crypto library for wallet")
+    set(MONERO_WALLET_CRYPTO_LIBRARY "cn")
+  endif ()
+endif ()
+
+#
+# Configure library target "wallet-crypto" - clients will use this as a
+# library dependency which in turn will depend on the crypto library selected.
+#
+if (${MONERO_WALLET_CRYPTO_LIBRARY} STREQUAL "cn")
+  configure_file(${CMAKE_CURRENT_SOURCE_DIR}/empty.h.in ${MONERO_GENERATED_HEADERS_DIR}/crypto/wallet/ops.h)
+  add_library(wallet-crypto ALIAS cncrypto)
+else ()
+  monero_crypto_generate_header(${MONERO_WALLET_CRYPTO_LIBRARY} "${MONERO_GENERATED_HEADERS_DIR}/crypto/wallet/ops.h")
+  monero_crypto_get_target(${MONERO_WALLET_CRYPTO_LIBRARY} CRYPTO_TARGET)
+  add_library(wallet-crypto $<TARGET_OBJECTS:${CRYPTO_TARGET}>)
+  target_link_libraries(wallet-crypto cncrypto)
+endif ()
+
+

src/crypto/wallet/crypto.h

@@ -0,0 +1,56 @@
+// Copyright (c) 2020, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <cstddef>
+#include "crypto/wallet/ops.h"
+
+namespace crypto {
+  namespace wallet {
+// if C functions defined from external/supercop - cmake generates crypto/wallet/ops.h
+#if defined(monero_crypto_generate_key_derivation)
+      inline
+      bool generate_key_derivation(const public_key &tx_pub, const secret_key &view_sec, key_derivation &out)
+      {
+        return monero_crypto_generate_key_derivation(out.data, tx_pub.data, view_sec.data) == 0;
+      }
+
+      inline
+      bool derive_subaddress_public_key(const public_key &output_pub, const key_derivation &d, std::size_t index, public_key &out)
+      {
+        ec_scalar scalar;
+        derivation_to_scalar(d, index, scalar);
+        return monero_crypto_generate_subaddress_public_key(out.data, output_pub.data, scalar.data) == 0;
+      }
+#else
+    using ::crypto::generate_key_derivation;
+    using ::crypto::derive_subaddress_public_key;
+#endif
+  }
+}

src/crypto/wallet/empty.h.in

@@ -0,0 +1,31 @@
+// Copyright (c) 2020, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+// Left empty so internal cryptonote crypto library is used.

src/cryptonote_basic/CMakeLists.txt

@@ -38,6 +38,7 @@ endif()
 
 set(cryptonote_basic_sources
   account.cpp
+  connection_context.cpp
   cryptonote_basic_impl.cpp
   cryptonote_format_utils.cpp
   difficulty.cpp

src/cryptonote_basic/blobdatatype.h

@@ -31,10 +31,11 @@
 #pragma once
 
 #include <string>
+#include <boost/utility/string_ref_fwd.hpp>
 #include "span.h"
 
 namespace cryptonote
 {
   typedef std::string blobdata;
-  typedef epee::span<const char> blobdata_ref;
+  typedef boost::string_ref blobdata_ref;
 }