Merge pull request 'diff hotfix 2' (#365) from wowario/wownero:fix into master

Reviewed-on: https://git.wownero.com/wownero/wownero/pulls/365

.drone.yml

@@ -7,4 +7,4 @@ steps:
 - name: linux-build
   image: wownero/wow-dependencies:v1.0
   commands:
-  - make -j2 release-static-linux-x86_64
+  - make release-static-linux-x86_64

.github/FUNDING.yml

@@ -1 +1 @@
-custom: https://dev-funding.webui.wowkira.co
+custom: https://dev-funding.webui.wowkira.com

.github/workflows/build.yml

@@ -38,10 +38,15 @@ jobs:
         submodules: recursive
     - name: remove bundled boost
       run: sudo rm -rf /usr/local/share/boost
+    - name: set apt conf
+      run: |
+        echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
+        echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
+        echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
     - name: update apt
       run: sudo apt update
     - name: install monero dependencies
-      run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev
+      run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler
     - name: build
       run: make -j3
 
@@ -53,10 +58,39 @@ jobs:
         submodules: recursive
     - name: remove bundled boost
       run: sudo rm -rf /usr/local/share/boost
+    - name: set apt conf
+      run: |
+        echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
+        echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
+        echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
     - name: update apt
       run: sudo apt update
     - name: install monero dependencies
-      run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev
+      run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler
     - name: build
       run: cmake -DBUILD_GUI_DEPS=ON && make -j3
 
+  test-ubuntu:
+    needs: build-ubuntu
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+      with:
+        submodules: recursive
+    - name: remove bundled boost
+      run: sudo rm -rf /usr/local/share/boost
+    - name: set apt conf
+      run: |
+        echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
+        echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
+        echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
+    - name: update apt
+      run: sudo apt update
+    - name: install monero dependencies
+      run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler
+    - name: install requests
+      run: pip install requests
+    - name: tests
+      env:
+        CTEST_OUTPUT_ON_FAILURE: ON
+      run: make release-test -j3

.gitignore

@@ -23,6 +23,7 @@ cscope.po.out
 external/miniupnpc/Makefile
 miniupnpcstrings.h
 version/
+ClangBuildAnalyzerSession.txt
 # Created by https://www.gitignore.io
 
 ### C++ ###

.gitmodules

@@ -13,3 +13,7 @@
 	path = external/RandomWOW
 	url = https://git.wownero.com/wownero/RandomWOW
         branch = 1.1.8-wow
+[submodule "external/supercop"]
+	path = external/supercop
+	url = https://git.wownero.com/wownero/supercop
+	branch = monero

.travis.yml

@@ -25,7 +25,7 @@ env:
 # ARM v8
     - HOST=aarch64-linux-gnu PACKAGES="python3 gperf g++-aarch64-linux-gnu"
 # i686 Win
-    - HOST=i686-w64-mingw32 DEP_OPTS="NO_QT=1" PACKAGES="python3 g++-mingw-w64-i686 qttools5-dev-tools"
+    - HOST=i686-w64-mingw32 DEP_OPTS="NO_QT=1" PACKAGES="python3 g++-mingw-w64-i686 qttools5-dev-tools" MAKEJOBS=-j2
 # i686 Linux
     - HOST=i686-pc-linux-gnu PACKAGES="gperf cmake g++-multilib python3-zmq"
 # Win64

CMakeLists.txt

@@ -54,6 +54,42 @@ if (USE_CCACHE)
 else()
 	message(STATUS "ccache deselected")
 endif()
+option (USE_COMPILATION_TIME_PROFILER "Use compilation time profiler (for CLang >= 9 only)" OFF)
+if (USE_COMPILATION_TIME_PROFILER)
+	if (NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
+		message(FATAL_ERROR "The flag USE_COMPILATION_TIME_PROFILER is meant to be set only for CLang compiler!")
+	endif()
+	add_compile_options("-ftime-trace")
+endif()
+
+# Job pool feature requires Ninja.
+if (${CMAKE_VERSION} VERSION_GREATER "3.0.0")
+  set(WOWNERO_PARALLEL_COMPILE_JOBS "" CACHE STRING "The maximum number of concurrent compilation jobs.")
+  if (WOWNERO_PARALLEL_COMPILE_JOBS)
+    set_property(GLOBAL APPEND PROPERTY JOB_POOLS compile_job_pool=${WOWNERO_PARALLEL_COMPILE_JOBS})
+    set(CMAKE_JOB_POOL_COMPILE compile_job_pool)
+  endif ()
+
+  set(WOWNERO_PARALLEL_LINK_JOBS "" CACHE STRING "The maximum number of concurrent link jobs.")
+  if (WOWNERO_PARALLEL_LINK_JOBS)
+    set_property(GLOBAL APPEND PROPERTY JOB_POOLS link_job_pool=${WOWNERO_PARALLEL_LINK_JOBS})
+    set(CMAKE_JOB_POOL_LINK link_job_pool)
+  endif ()
+endif ()
+  
+option (USE_CLANG_TIDY_C   "Lint the code with clang-tidy - variant C"   OFF)
+option (USE_CLANG_TIDY_CXX "Lint the code with clang-tidy - variant C++" OFF)
+if (USE_CLANG_TIDY_C AND USE_CLANG_TIDY_CXX)
+	message(FATAL_ERROR "Enabling both USE_CLANG_TIDY_C and USE_CLANG_TIDY_CXX simultaneously crashes clang-tidy.")
+endif()
+if (USE_CLANG_TIDY_C OR USE_CLANG_TIDY_CXX)
+	include(SetClangTidy)
+endif()
+if (USE_CLANG_TIDY_C)
+	monero_clang_tidy("C")
+elseif (USE_CLANG_TIDY_CXX)
+	monero_clang_tidy("CXX")
+endif()
 
 enable_language(C ASM)
 
@@ -111,6 +147,24 @@ function (add_definition_if_library_exists library function header var)
   endif()
 endfunction()
 
+option(RELINK_TARGETS "Relink targets, when just a dependant .so changed, but not its header?" OFF)
+function (monero_set_target_no_relink target)
+  if (RELINK_TARGETS MATCHES OFF)
+    # Will not relink the target, when just its dependant .so has changed, but not it's interface
+    set_target_properties("${target}" PROPERTIES LINK_DEPENDS_NO_SHARED true)
+  endif()
+endfunction()
+
+function (monero_add_minimal_executable name)
+  source_group("${name}"
+    FILES
+      ${ARGN})
+
+  add_executable("${name}"
+    ${ARGN})
+    monero_set_target_no_relink( ${name} )
+endfunction()
+
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
   message(STATUS "Setting default build type: ${CMAKE_BUILD_TYPE}")
@@ -218,6 +272,7 @@ if(NOT MANUAL_SUBMODULES)
     check_submodule(external/unbound)
     check_submodule(external/rapidjson)
     check_submodule(external/RandomWOW)
+    check_submodule(external/supercop)
   endif()
 endif()
 
@@ -275,6 +330,12 @@ else()
 endif()
 option(BUILD_DEBUG_UTILITIES "Build debug utilities." DEFAULT_BUILD_DEBUG_UTILITIES)
 
+if(OSSFUZZ)
+  message(STATUS "Using OSS-Fuzz fuzzing system")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DOSSFUZZ")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DOSSFUZZ")
+endif()
+
 # Check whether we're on a 32-bit or 64-bit system
 if(CMAKE_SIZEOF_VOID_P EQUAL "8")
   set(DEFAULT_BUILD_64 ON)
@@ -313,7 +374,7 @@ endif()
 # elseif(CMAKE_SYSTEM_NAME MATCHES ".*BSDI.*")
 #   set(BSDI TRUE)
 
-include_directories(external/rapidjson/include external/easylogging++ src contrib/epee/include external)
+include_directories(external/rapidjson/include external/easylogging++ src contrib/epee/include external external/supercop/include)
 
 if(APPLE)
   include_directories(SYSTEM /usr/include/malloc)
@@ -396,7 +457,7 @@ elseif(CMAKE_C_COMPILER_ID STREQUAL "GNU" AND NOT MINGW)
   set(DEFAULT_STACK_TRACE ON)
   set(STACK_TRACE_LIB "easylogging++") # for diag output only
   set(LIBUNWIND_LIBRARIES "")
-elseif (ARM AND STATIC)
+elseif (ARM)
   set(DEFAULT_STACK_TRACE OFF)
   set(LIBUNWIND_LIBRARIES "")
 else()
@@ -433,7 +494,7 @@ endif ()
 
 if (APPLE AND NOT IOS)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=x86-64 -fvisibility=default -std=c++11")
-  if (NOT OpenSSL_DIR)
+  if (NOT OPENSSL_ROOT_DIR)
       EXECUTE_PROCESS(COMMAND brew --prefix openssl
         OUTPUT_VARIABLE OPENSSL_ROOT_DIR
         OUTPUT_STRIP_TRAILING_WHITESPACE)
@@ -458,6 +519,9 @@ add_definition_if_function_found(strptime HAVE_STRPTIME)
 
 add_definitions(-DAUTO_INITIALIZE_EASYLOGGINGPP)
 
+set(MONERO_GENERATED_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated_include")
+include_directories(${MONERO_GENERATED_HEADERS_DIR})
+
 # Generate header for embedded translations
 # Generate header for embedded translations, use target toolchain if depends, otherwise use the
 # lrelease and lupdate binaries from the host
@@ -471,14 +535,6 @@ ExternalProject_Add(generate_translations_header
 include_directories("${CMAKE_CURRENT_BINARY_DIR}/translations")
 add_subdirectory(external)
 
-# Final setup for miniupnpc
-if(UPNP_STATIC OR IOS)
-  add_definitions("-DUPNP_STATIC")
-else()
-  add_definitions("-DUPNP_DYNAMIC")
-  include_directories(${UPNP_INCLUDE})
-endif()
-
 # Final setup for libunbound
 include_directories(${UNBOUND_INCLUDE})
 link_directories(${UNBOUND_LIBRARY_DIRS})
@@ -667,7 +723,8 @@ else()
   endif()
 
   # linker
-  if (NOT (WIN32 AND (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS 9.1)))
+  if (NOT SANITIZE AND NOT OSSFUZZ AND NOT (WIN32 AND (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS 9.1)))
+    # PIE executables randomly crash at startup with ASAN
     # Windows binaries die on startup with PIE when compiled with GCC <9.x
     add_linker_flag_if_supported(-pie LD_SECURITY_FLAGS)
   endif()
@@ -789,7 +846,8 @@ else()
 
   endif(ARM)
 
-  if(ANDROID AND NOT BUILD_GUI_DEPS STREQUAL "ON" OR IOS)
+  # random crash on startup when asan is on if pie is enabled
+  if(NOT SANITIZE AND ANDROID AND NOT BUILD_GUI_DEPS STREQUAL "ON" OR IOS)
     #From Android 5: "only position independent executables (PIE) are supported" 
     message(STATUS "Enabling PIE executable")
     set(PIC_FLAG "")
@@ -924,10 +982,12 @@ list(APPEND EXTRA_LIBRARIES ${CMAKE_DL_LIBS})
 if (HIDAPI_FOUND OR LibUSB_COMPILE_TEST_PASSED)
   if (APPLE)
     if(DEPENDS)
-      list(APPEND EXTRA_LIBRARIES "-framework Foundation -framework IOKit")
+      list(APPEND EXTRA_LIBRARIES "-framework Foundation -framework AppKit -framework IOKit")
     else()
       find_library(COREFOUNDATION CoreFoundation)
+      find_library(APPKIT AppKit)
       find_library(IOKIT IOKit)
+      list(APPEND EXTRA_LIBRARIES ${APPKIT})
       list(APPEND EXTRA_LIBRARIES ${IOKIT})
       list(APPEND EXTRA_LIBRARIES ${COREFOUNDATION})
     endif()
@@ -973,12 +1033,10 @@ find_path(ZMQ_INCLUDE_PATH zmq.h)
 find_library(ZMQ_LIB zmq)
 find_library(PGM_LIBRARY pgm)
 find_library(NORM_LIBRARY norm)
+find_library(GSSAPI_LIBRARY gssapi_krb5)
 find_library(PROTOLIB_LIBRARY protolib)
 find_library(SODIUM_LIBRARY sodium)
 
-set(ZMQ_INCLUDE_PATH zmq_dummy_include_path)
-set(ZMQ_LIB zmq_dummy_lib_2)
-
 if(NOT ZMQ_INCLUDE_PATH)
   message(FATAL_ERROR "Could not find required header zmq.h")
 endif()
@@ -991,6 +1049,9 @@ endif()
 if(NORM_LIBRARY)
   set(ZMQ_LIB "${ZMQ_LIB};${NORM_LIBRARY}")
 endif()
+if(GSSAPI_LIBRARY)
+  set(ZMQ_LIB "${ZMQ_LIB};${GSSAPI_LIBRARY}")
+endif()
 if(PROTOLIB_LIBRARY)
   set(ZMQ_LIB "${ZMQ_LIB};${PROTOLIB_LIBRARY}")
 endif()
@@ -998,6 +1059,7 @@ if(SODIUM_LIBRARY)
   set(ZMQ_LIB "${ZMQ_LIB};${SODIUM_LIBRARY}")
 endif()
 
+include(external/supercop/functions.cmake) # place after setting flags and before src directory inclusion
 add_subdirectory(contrib)
 add_subdirectory(src)
 

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2014-2020, The Monero Project
+Copyright (c) 2014-2021, The Monero Project
 
 All rights reserved.
 

README.md

@@ -1,56 +1,34 @@
 [![Build Status](https://ci.wownero.com/api/badges/wownero/wownero/status.svg)](https://ci.wownero.com/wownero/wownero)
 # ~~Mo~~Wownero -  Such privacy! Many coins! Wow! 🐕
 
-Copyright (c) 2014-2020 The Monero Project.   
+Copyright (c) 2014-2021 The Monero Project.   
 Portions Copyright (c) 2012-2013 The Cryptonote developers.
 
-# CyberWOW
-An Android pruned full node for Wownero
-
-[<img src="https://git.wownero.com/wownero/meta/raw/branch/master/images/fdroid_badge.png"
-      alt="Get it on F-Droid"
-      height="80">](https://f-droid.org/en/packages/org.wownero.cyberwow/)
-<a href='https://play.google.com/store/apps/details?id=org.wownero.cyberwow'><img alt='Get it on Google Play' src='https://git.wownero.com/wownero/meta/raw/branch/master/images/google_badge.png' height='80'/></a>
-
-# Wownerujo
-An Android Wallet for Wownero
-
-[<img src="https://git.wownero.com/wownero/meta/raw/branch/master/images/fdroid_badge.png"
-      alt="Get it on F-Droid"
-      height="80">](https://f-droid.org/packages/com.wownero.wownerujo/)
-<a href='https://play.google.com/store/apps/details?id=com.wownero.wownerujo'><img alt='Get it on Google Play' src='https://git.wownero.com/wownero/meta/raw/branch/master/images/google_badge.png' height='80'/></a>
-
 ## Resources
 
-- Web: [wownero.org](http://wownero.org)
+- Web: [wownero.org](https://wownero.org)
 - Twitter: [@w0wn3r0](https://twitter.com/w0wn3r0)
 - Reddit: [/r/wownero](https://www.reddit.com/r/wownero)
 - Mail: [wownero@protonmail.com](mailto:wownero@protonmail.com)
-- GitHub: [git.wownero.com/wownero/wownero](https://git.wownero.com/wownero/wownero)
-- IRC: [#wownero on Freenode](https://kiwiirc.com/client/irc.freenode.net/?nick=suchchatter|?#wownero)
-- Bitmessage Chan: wownero (`BM-2cSzWtrj2pzLva9GF1Jp2TYsnLjrnJpvba`)
+- Git: [git.wownero.com/wownero/wownero](https://git.wownero.com/wownero/wownero)
 - Wownero Funding System: [funding.wownero.com](https://funding.wownero.com)
 - Wownero Forum: [forum.wownero.com](https://forum.wownero.com)
-- Discord: [discord.gg/abgaJbN](https://discord.gg/abgaJbN)
+- Discord: [discord.gg/ENbgme4bWq](https://discord.com/invite/ENbgme4bWq)
 - Telegram: [t.me/wownero](https://t.me/wownero)
+- Feather-WOW Desktop Wallet: [featherwallet.org/wownero](https://featherwallet.org/wownero)
+- WOW Stash Web Wallet: [wowstash.app](https://wowstash.app)
+- Public Node Status: [monero.fail](https://monero.fail/?crypto=wownero)
+- Map of Nodes: [wownero.fyi](https://wownero.fyi)
+- Wownero Memes: [suchwow.xyz]https://suchwow.xyz/posts/top)
+- XMR/WOW Swap: [nero Swap](https://neroswap.com)
+- Mining Pools: [miningpoolstats.stream](https://miningpoolstats.stream/wownero)
+- Market Info: [coinmarketcap.com](https://coinmarketcap.com/currencies/wownero), [coingecko.com](https://www.coingecko.com/en/coins/wownero/usd)
 
-Blockchain Explorers
+### Blockchain Explorers
 - https://explore.wownero.com
-- https://explorer.wowkira.com
 - http://wow5eqtzqvsg5jctqzg5g7uk3u62sfqiacj5x6lo4by7bvnj6jkvubyd.onion
-- https://wownero.xmrauctions.com
-
-Free Public Nodes
-- wow7dhbgiljnkspkzpjyy66auegbrye2ptfv4gucgbhireg5rrjza5ad.onion:34568
-- so.wow.candy.surf:34568 (US)
-- such.wow.candy.surf:34568 (CAN)
-- very.wow.candy.surf:34568 (IN)
-- much.wow.candy.surf:34568 (UK)
-- node.suchwow.xyz:34568 (US)
-
-Tor Peers
-- wowp2p5gelm6vhl2d5tvfqills63jilgy6hkvlrqljooov5ktaxgqdad.onion
-- f3moshycuklu3mb3wnlfjwn26nsgzreqtfzfuyjjk46u7jqxlhf7d5id.onion
+- https://wownero.club
+- https://explorer.wownero.fyi
 
 ## Introduction
 
@@ -63,10 +41,13 @@ Wownero is a 100% community-sponsored endeavor. Supporting services are also gra
 [<img src="https://git.wownero.com/wownero/meta/raw/branch/master/images/macstadium.png"
       alt="MacStadium"
       height="100">](https://www.macstadium.com)
+[<img src="https://git.wownero.com/wownero/meta/raw/branch/master/images/jetbrains.png"
+      alt="JetBrains"
+      height="100">](https://www.jetbrains.com)
 
 Developers are volunteers doing this mostly for shits and giggles. If you would like to support our shenanigans and stimulant addictions, please consider donating to [WFS proposals](https://funding.wownero.com/proposals) or the dev slush fund.
 
-Donation Addresses
+### Donation Addresses
 
 WOW: `Wo3MWeKwtA918DU4c69hVSNgejdWFCRCuWjShRY66mJkU2Hv58eygJWDJS1MNa2Ge5M1WjUkGHuLqHkweDxwZZU42d16v94mP`
 
@@ -103,7 +84,7 @@ Dates are provided in the format YYYY-MM-DD.
 | 114,969                          | 2019-06-14 | F For Fappening    | v0.6.1.0               | v0.6.1.2                  | RandomWOW, new block weight algorithm, slightly more efficient RingCT format
 | 160,777                          | 2019-11-20 | Gaping Goatse    | v0.7.0.0               | v0.7.1.0                  | Only allow >= 2 outputs, change to the block median used to calculate penalty, rct sigs in coinbase forbidden, 4 unlock time as protocol rule
 | -                         | 2020-06-28 | Hallucinogenic Hypnotoad  | v0.8.0.0               | v0.8.0.2                  | Dandelion++ support
-| XXXX                         | 2020-09-XX | XXXXX  | v0.9.0.0               | v0.9.0.0                  | RandomWOW v2 (WOWv2), increase coinbase unlock to 8,640 (1 mo.), CLSAG
+| 253,999                         | 2020-10-09 | Illiterate Illuminati  | v0.9.0.0               | v0.9.3.3                  | Dynamic coinbase unlock (up to 1 mo.), Deterministic unlock times, Enforce maximum coinbase amount, show_qr_code wallet command, CLSAG
 
 X's indicate that these details have not been determined as of commit date.
 
@@ -117,7 +98,7 @@ Packages are available for
 
         yay -S wownero-git
 
-* Gentoo
+* Gentoo - Russian hacking tool
 
         emerge --noreplace eselect-repository
         eselect repository enable monero
@@ -155,7 +136,6 @@ Packaging for your favorite distribution would be a welcome contribution!
         git clone https://git.wownero.com/wownero/wownero && cd wownero
         make -j2
 
-
 * Debian/Ubuntu
 
         sudo apt update && sudo apt install build-essential cmake pkg-config libboost-all-dev libssl-dev libzmq3-dev libunbound-dev libsodium-dev libunwind8-dev liblzma-dev libreadline6-dev libldns-dev libexpat1-dev libpgm-dev libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler libudev-dev git -y
@@ -166,7 +146,7 @@ Packaging for your favorite distribution would be a welcome contribution!
 ## Running Binaries
 
 The build places the binary in `bin/` sub-directory within the build directory
-from which cmake was invoked (repository root by default). To run in
+from which cmake was invoked (repository root by default). To run in the
 foreground:
 
     ./bin/wownerod
@@ -175,7 +155,7 @@ To list all available options, run `./bin/wownerod --help`.  Options can be
 specified either on the command line or in a configuration file passed by the
 `--config-file` argument.  To specify an option in the configuration file, add
 a line with the syntax `argumentname=value`, where `argumentname` is the name
-of the argument without the leading dashes, for example `log-level=1`.
+of the argument without the leading dashes, for example, `log-level=1`.
 
 To run in background:
 

ZMQ.md

@@ -0,0 +1,61 @@
+# The Current/Future Status of ZMQ in Monero
+
+## ZMQ Pub/Sub
+Client `ZMQ_SUB` sockets must "subscribe" to topics before it receives any data.
+This allows filtering on the server side, so network traffic is reduced. Monero
+allows for filtering on: (1) format, (2) context, and (3) event.
+
+ * **format** refers to the _wire_ format (i.e. JSON) used to send event
+   information.
+ * **context** allows for a reduction in fields for the event, so the
+   daemon doesn't waste cycles serializing fields that get ignored.
+ * **event** refers to status changes occurring within the daemon (i.e. new
+   block to main chain).
+
+ * Formats:
+   * `json`
+ * Contexts:
+   * `full` - the entire block or transaction is transmitted (the hash can be
+     computed remotely).
+   * `minimal` - the bare minimum for a remote client to react to an event is
+     sent.
+ * Events:
+   * `chain_main` - changes to the primary/main blockchain.
+   * `txpool_add` - new _publicly visible_ transactions in the mempool.
+     Includes previously unseen transactions in a block but _not_ the
+     `miner_tx`. Does not "re-publish" after a reorg. Includes `do_not_relay`
+     transactions.
+
+The subscription topics are formatted as `format-context-event`, with prefix
+matching supported by both Monero and ZMQ. The `format`, `context` and `event`
+will _never_ have hyphens or colons in their name. For example, subscribing to
+`json-minimal-chain_main` will send minimal information in JSON when changes
+to the main/primary blockchain occur. Whereas, subscribing to `json-minimal`
+will send minimal information in JSON on all available events supported by the
+daemon.
+
+The Monero daemon will ensure that events prefixed by `chain` will be sent in
+"chain-order" - the `prev_id` (hash) field will _always_ refer to a previous
+block. On rollbacks/reorgs, the event will reference an earlier block in the
+chain instead of the last block. The Monero daemon also ensures that
+`txpool_add` events are sent before `chain_*` events - the `chain_*` messages
+will only serialize miner transactions since the other transactions were
+previously published via `txpool_add`. This prevents transactions from being
+serialized twice, even when the transaction was first observed in a block.
+
+ZMQ Pub/Sub will drop messages if the network is congested, so the above rules
+for send order are used for detecting lost messages. A missing gap in `height`
+or `prev_id` for `chain_*` events indicates a lost pub message. Missing
+`txpool_add` messages can only be detected at the next `chain_` message.
+
+Since blockchain events can be dropped, clients will likely want to have a
+timeout against `chain_main` events. The `GetLastBlockHeader` RPC is useful
+for checking the current chain state. Dropped messages should be rare in most
+conditions.
+
+The Monero daemon will send a `txpool_add` pub exactly once for each
+transaction, even after a reorg or restarts. Clients should use the
+`GetTransactionPool` after a reorg to get all transactions that have been put
+back into the tx pool or been invalidated due to a double-spend.
+
+

cmake/GitVersion.cmake

@@ -48,29 +48,21 @@ function (get_version_tag_from_git GIT)
         message(STATUS "You are currently on commit ${COMMIT}")
 
         # Get all the tags
-        execute_process(COMMAND "${GIT}" rev-list --tags --max-count=1 --abbrev-commit 
+        execute_process(COMMAND "${GIT}" tag -l --points-at HEAD
                         WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
                         RESULT_VARIABLE RET
-                        OUTPUT_VARIABLE TAGGEDCOMMIT
+                        OUTPUT_VARIABLE TAG
                         OUTPUT_STRIP_TRAILING_WHITESPACE)
 
-        if(NOT TAGGEDCOMMIT)
-            message(WARNING "Cannot determine most recent tag. Make sure that you are building either from a Git working tree or from a source archive.")
+        # Check if we're building that tagged commit or a different one
+        if(TAG)
+            message(STATUS "You are building a tagged release")
+            set(VERSIONTAG "release")
+            set(VERSION_IS_RELEASE "true")
+        else()
+            message(STATUS "You are ahead of or behind a tagged release")
             set(VERSIONTAG "${COMMIT}")
             set(VERSION_IS_RELEASE "false")
-        else()
-            message(STATUS "The most recent tag was at ${TAGGEDCOMMIT}")
-
-            # Check if we're building that tagged commit or a different one
-            if(COMMIT STREQUAL TAGGEDCOMMIT)
-                message(STATUS "You are building a tagged release")
-                set(VERSIONTAG "release")
-                set(VERSION_IS_RELEASE "true")
-            else()
-                message(STATUS "You are ahead of or behind a tagged release")
-                set(VERSIONTAG "${COMMIT}")
-                set(VERSION_IS_RELEASE "false")
-            endif()
         endif()	    
     endif()
 

cmake/SetClangTidy.cmake

@@ -0,0 +1,72 @@
+# Copyright (c) 2014-2020, The Monero Project
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# https://cmake.org/cmake/help/latest/variable/CMAKE_LANG_CLANG_TIDY.html
+# This module sets the following variables:
+# CMAKE_C_CLANG_TIDY
+# CMAKE_CXX_CLANG_TIDY
+# when clang-tidy is found in PATH. Afterwards, the code is being linted by the tool.
+# The checks to be enabled can be manipulated with the variable MONERO_CLANG_TIDY_CHECKS
+
+macro (monero_clang_tidy LANGUAGE)
+	set(TOOL_NAME "clang-tidy")
+	set(MONERO_CLANG_TIDY_MIN_VERSION "3.6")
+	if(${CMAKE_VERSION} VERSION_LESS "${MONERO_CLANG_TIDY_MIN_VERSION}")
+		message(FATAL_ERROR "Sorry, ${TOOL_NAME} is available for CMake from version ${MONERO_CLANG_TIDY_MIN_VERSION}")
+	else()
+		message(STATUS "Trying to enable ${TOOL_NAME}")
+		find_program(MONERO_CLANG_BIN ${TOOL_NAME})
+		if(NOT MONERO_CLANG_BIN)
+			message(FATAL_ERROR "${TOOL_NAME} not found! Try running: sudo apt install ${TOOL_NAME}")
+		else()
+			message(STATUS "Found ${MONERO_CLANG_BIN}")
+			set(MONERO_CLANG_TIDY_CHECKS
+			    -header-filter=.; 	# By default the headers are excluded. This line enables them.
+			    -checks=*; 		# Currently enabling all checks
+			    # An example of selectively enabling checks:
+			    #-checks=bugprone-*,cppcoreguidelines-avoid-goto # Have to be in one line :(
+			)
+			# Current list of checks is avaibale under:
+			# https://clang.llvm.org/extra/clang-tidy/
+			if (${LANGUAGE} STREQUAL "C")
+				set(CMAKE_C_CLANG_TIDY
+					${MONERO_CLANG_BIN}; # Mind the semicolon
+					${MONERO_CLANG_TIDY_CHECKS}
+				)
+			elseif (${LANGUAGE} STREQUAL "CXX")
+				set(CMAKE_CXX_CLANG_TIDY
+					${MONERO_CLANG_BIN}; # Mind the semicolon
+					${MONERO_CLANG_TIDY_CHECKS}
+				)
+			else()
+				message(FATAL_ERROR "${TOOL_NAME}: Unsupported language: ${LANGUAGE}")
+			endif()
+		endif()
+	endif()
+endmacro()
+

cmake/Version.cmake

@@ -28,7 +28,7 @@
 
 function (write_version tag)
   set(VERSIONTAG "${tag}" CACHE STRING "The tag portion of the Monero software version" FORCE)
-  configure_file("${CMAKE_SOURCE_DIR}/src/version.cpp.in" "${CMAKE_BINARY_DIR}/version.cpp")
+  configure_file("${CMAKE_CURRENT_LIST_DIR}/../src/version.cpp.in" "${CMAKE_BINARY_DIR}/version.cpp")
 endfunction ()
 
 find_package(Git QUIET)

contrib/depends/packages/expat.mk

@@ -1,6 +1,6 @@
 package=expat
 $(package)_version=2.2.4
-$(package)_download_path=https://downloads.sourceforge.net/project/expat/expat/$($(package)_version)
+$(package)_download_path=https://github.com/libexpat/libexpat/releases/download/R_2_2_4
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=03ad85db965f8ab2d27328abcf0bc5571af6ec0a414874b2066ee3fdd372019e
 

contrib/depends/packages/native_ds_store.mk

@@ -1,9 +1,9 @@
 package=native_ds_store
 $(package)_version=1.1.0
-$(package)_download_path=https://bitbucket.org/al45tair/ds_store/get
-$(package)_download_file=v$($(package)_version).tar.bz2
-$(package)_file_name=$(package)-$($(package)_version).tar.bz2
-$(package)_sha256_hash=921596764d71d1bbd3297a90ef6d286f718794d667e4f81d91d14053525d64c1
+$(package)_download_path=https://github.com/al45tair/ds_store/archive/
+$(package)_download_file=v$($(package)_version).tar.gz
+$(package)_file_name=$(package)-$($(package)_version).tar.gz
+$(package)_sha256_hash=a9f4c0755c6be7224ff7029e188dd262e830bb81e801424841db9eb0780ec8ed
 $(package)_install_libdir=$(build_prefix)/lib/python/dist-packages
 $(package)_dependencies=native_biplist
 

contrib/depends/packages/native_mac_alias.mk

@@ -1,9 +1,9 @@
 package=native_mac_alias
 $(package)_version=1.1.0
-$(package)_download_path=https://bitbucket.org/al45tair/mac_alias/get
-$(package)_download_file=v$($(package)_version).tar.bz2
-$(package)_file_name=$(package)-$($(package)_version).tar.bz2
-$(package)_sha256_hash=87ad827e66790028361e43fc754f68ed041a9bdb214cca03c853f079b04fb120
+$(package)_download_path=https://github.com/al45tair/mac_alias/archive/
+$(package)_download_file=v$($(package)_version).tar.gz
+$(package)_file_name=$(package)-$($(package)_version).tar.gz
+$(package)_sha256_hash=b10cb44ecb64fc25283fae7a9cf365d2829377d84e37b9c21100aca8757509be
 $(package)_install_libdir=$(build_prefix)/lib/python/dist-packages
 $(package)_patches=python3.patch
 

contrib/depends/packages/openssl.mk

@@ -1,36 +1,31 @@
 package=openssl
-$(package)_version=1.0.2r
-$(package)_download_path=https://ftp.openssl.org/source/old/1.0.2
+$(package)_version=1.1.1i
+$(package)_download_path=https://www.openssl.org/source
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
-$(package)_sha256_hash=ae51d08bba8a83958e894946f15303ff894d75c2b8bbd44a852b64e3fe11d0d6
-$(package)_patches=fix_arflags.patch
+$(package)_sha256_hash=e8be6a35fe41d10603c3cc635e93289ed00bf34b79671a3a4de64fcee00d5242
 
 define $(package)_set_vars
 $(package)_config_env=AR="$($(package)_ar)" ARFLAGS=$($(package)_arflags) RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)"
+$(package)_config_env_arm_android=ANDROID_NDK_HOME="$(host_prefix)/native" PATH="$(host_prefix)/native/bin" CC=clang AR=ar RANLIB=ranlib
+$(package)_config_env_aarch64_android=ANDROID_NDK_HOME="$(host_prefix)/native" PATH="$(host_prefix)/native/bin" CC=clang AR=ar RANLIB=ranlib
+$(package)_build_env_arm_android=ANDROID_NDK_HOME="$(host_prefix)/native"
+$(package)_build_env_aarch64_android=ANDROID_NDK_HOME="$(host_prefix)/native"
 $(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/openssl
 $(package)_config_opts+=no-capieng
 $(package)_config_opts+=no-dso
 $(package)_config_opts+=no-dtls1
 $(package)_config_opts+=no-ec_nistp_64_gcc_128
 $(package)_config_opts+=no-gost
-$(package)_config_opts+=no-gmp
 $(package)_config_opts+=no-heartbeats
-$(package)_config_opts+=no-jpake
-$(package)_config_opts+=no-krb5
-$(package)_config_opts+=no-libunbound
 $(package)_config_opts+=no-md2
 $(package)_config_opts+=no-rc5
 $(package)_config_opts+=no-rdrand
 $(package)_config_opts+=no-rfc3779
-$(package)_config_opts+=no-rsax
 $(package)_config_opts+=no-sctp
-$(package)_config_opts+=no-sha0
 $(package)_config_opts+=no-shared
 $(package)_config_opts+=no-ssl-trace
 $(package)_config_opts+=no-ssl2
 $(package)_config_opts+=no-ssl3
-$(package)_config_opts+=no-static_engine
-$(package)_config_opts+=no-store
 $(package)_config_opts+=no-unit-test
 $(package)_config_opts+=no-weak-ssl-ciphers
 $(package)_config_opts+=no-zlib
@@ -42,8 +37,8 @@ $(package)_config_opts_x86_64_linux=linux-x86_64
 $(package)_config_opts_i686_linux=linux-generic32
 $(package)_config_opts_arm_linux=linux-generic32
 $(package)_config_opts_aarch64_linux=linux-generic64
-$(package)_config_opts_arm_android=--static android-armv7 no-asm
-$(package)_config_opts_aarch64_android=--static android no-asm
+$(package)_config_opts_arm_android=--static android-arm
+$(package)_config_opts_aarch64_android=--static android-arm64
 $(package)_config_opts_riscv64_linux=linux-generic64
 $(package)_config_opts_mipsel_linux=linux-generic32
 $(package)_config_opts_mips_linux=linux-generic32
@@ -55,10 +50,8 @@ $(package)_config_opts_x86_64_freebsd=BSD-x86_64
 endef
 
 define $(package)_preprocess_cmds
-  sed -i.old "/define DATE/d" util/mkbuildinf.pl && \
-  sed -i.old "s|engines apps test|engines|" Makefile.org && \
-  sed -i -e "s/-mandroid //" Configure && \
-  patch < $($(package)_patch_dir)/fix_arflags.patch
+  sed -i.old 's|"engines", "apps", "test", "util", "tools", "fuzz"|"engines", "tools"|' Configure && \
+  sed -i -e 's|cflags --sysroot.*",|cflags",|' Configurations/15-android.conf
 endef
 
 define $(package)_config_cmds
@@ -70,7 +63,7 @@ define $(package)_build_cmds
 endef
 
 define $(package)_stage_cmds
-  $(MAKE) INSTALL_PREFIX=$($(package)_staging_dir) -j1 install_sw
+  $(MAKE) DESTDIR=$($(package)_staging_dir) -j1 install_sw
 endef
 
 define $(package)_postprocess_cmds

contrib/depends/packages/qt.mk

@@ -1,18 +1,18 @@
 PACKAGE=qt
-$(package)_version=5.7.1
-$(package)_download_path=http://linorg.usp.br/Qt/archive/qt/5.7/5.7.1/submodules
-$(package)_suffix=opensource-src-$($(package)_version).tar.gz
+$(package)_version=5.15.1
+$(package)_download_path=https://download.qt.io/official_releases/qt/5.15/$($(package)_version)/submodules
+$(package)_suffix=everywhere-src-$($(package)_version).tar.xz
 $(package)_file_name=qtbase-$($(package)_suffix)
-$(package)_sha256_hash=95f83e532d23b3ddbde7973f380ecae1bac13230340557276f75f2e37984e410
+$(package)_sha256_hash=33960404d579675b7210de103ed06a72613bfc4305443e278e2d32a3eb1f3d8c
 $(package)_build_subdir=qtbase
 $(package)_qt_libs=corelib
-$(package)_patches=pidlist_absolute.patch fix_qt_pkgconfig.patch qfixed-coretext.patch
+$(package)_patches=fix_qt_pkgconfig.patch fix_no_printer.patch fix_rcc_determinism.patch no-xlib.patch
 
 $(package)_qttranslations_file_name=qttranslations-$($(package)_suffix)
-$(package)_qttranslations_sha256_hash=3a15aebd523c6d89fb97b2d3df866c94149653a26d27a00aac9b6d3020bc5a1d
+$(package)_qttranslations_sha256_hash=46e0c0e3a511fbcc803a4146204062e47f6ed43b34d98a3c27372a03b8746bd8
 
 $(package)_qttools_file_name=qttools-$($(package)_suffix)
-$(package)_qttools_sha256_hash=22d67de915cb8cd93e16fdd38fa006224ad9170bd217c2be1e53045a8dd02f0f
+$(package)_qttools_sha256_hash=c98ee5f0f980bf68cbf0c94d62434816a92441733de50bd9adbe9b9055f03498
 
 $(package)_extra_sources  = $($(package)_qttranslations_file_name)
 $(package)_extra_sources += $($(package)_qttools_file_name)
@@ -24,28 +24,26 @@ $(package)_config_opts += -bindir $(build_prefix)/bin
 $(package)_config_opts += -c++std c++11
 $(package)_config_opts += -confirm-license
 $(package)_config_opts += -dbus-runtime
-$(package)_config_opts += -no-alsa
-$(package)_config_opts += -no-audio-backend
+$(package)_config_opts += -hostprefix $(build_prefix)
+$(package)_config_opts += -no-compile-examples
 $(package)_config_opts += -no-cups
 $(package)_config_opts += -no-egl
 $(package)_config_opts += -no-eglfs
-$(package)_config_opts += -no-feature-style-windowsmobile
-$(package)_config_opts += -no-feature-style-windowsce
+$(package)_config_opts += -no-evdev
+$(package)_config_opts += -no-gui
 $(package)_config_opts += -no-freetype
 $(package)_config_opts += -no-gif
 $(package)_config_opts += -no-glib
-$(package)_config_opts += -no-gstreamer
 $(package)_config_opts += -no-icu
+$(package)_config_opts += -no-ico
 $(package)_config_opts += -no-iconv
 $(package)_config_opts += -no-kms
 $(package)_config_opts += -no-linuxfb
+$(package)_config_opts += -no-libjpeg
 $(package)_config_opts += -no-libudev
-$(package)_config_opts += -no-mitshm
 $(package)_config_opts += -no-mtdev
-$(package)_config_opts += -no-pulseaudio
 $(package)_config_opts += -no-openvg
 $(package)_config_opts += -no-reduce-relocations
-$(package)_config_opts += -no-qml-debug
 $(package)_config_opts += -no-sql-db2
 $(package)_config_opts += -no-sql-ibase
 $(package)_config_opts += -no-sql-oci
@@ -56,8 +54,6 @@ $(package)_config_opts += -no-sql-psql
 $(package)_config_opts += -no-sql-sqlite
 $(package)_config_opts += -no-sql-sqlite2
 $(package)_config_opts += -no-use-gold-linker
-$(package)_config_opts += -no-xinput2
-$(package)_config_opts += -no-xrender
 $(package)_config_opts += -nomake examples
 $(package)_config_opts += -nomake tests
 $(package)_config_opts += -opensource
@@ -65,25 +61,46 @@ $(package)_config_opts += -no-openssl
 $(package)_config_opts += -optimized-qmake
 $(package)_config_opts += -pch
 $(package)_config_opts += -pkg-config
+$(package)_config_opts += -prefix $(host_prefix)
 $(package)_config_opts += -no-libpng
-$(package)_config_opts += -no-libjpeg
 $(package)_config_opts += -qt-pcre
+$(package)_config_opts += -qt-harfbuzz
 $(package)_config_opts += -no-zlib
-$(package)_config_opts += -reduce-exports
 $(package)_config_opts += -static
 $(package)_config_opts += -silent
 $(package)_config_opts += -v
-$(package)_config_opts += -no-feature-printer
+$(package)_config_opts += -no-feature-bearermanagement
+$(package)_config_opts += -no-feature-colordialog
+$(package)_config_opts += -no-feature-dial
+$(package)_config_opts += -no-feature-filesystemwatcher
+$(package)_config_opts += -no-feature-fontcombobox
+$(package)_config_opts += -no-feature-ftp
+$(package)_config_opts += -no-feature-image_heuristic_mask
+$(package)_config_opts += -no-feature-keysequenceedit
+$(package)_config_opts += -no-feature-lcdnumber
+$(package)_config_opts += -no-feature-pdf
 $(package)_config_opts += -no-feature-printdialog
-$(package)_config_opts += -no-gui
-$(package)_config_opts += -no-freetype
-$(package)_config_opts += -no-sm
-$(package)_config_opts += -no-fontconfig
-$(package)_config_opts += -no-opengl
-$(package)_config_opts += -no-xkb
-$(package)_config_opts += -no-xcb
-$(package)_config_opts += -no-xshape
-$(package)_build_env  = QT_RCC_TEST=1
+$(package)_config_opts += -no-feature-printer
+$(package)_config_opts += -no-feature-printpreviewdialog
+$(package)_config_opts += -no-feature-printpreviewwidget
+$(package)_config_opts += -no-feature-sessionmanager
+$(package)_config_opts += -no-feature-sql
+$(package)_config_opts += -no-feature-statemachine
+$(package)_config_opts += -no-feature-syntaxhighlighter
+$(package)_config_opts += -no-feature-textbrowser
+$(package)_config_opts += -no-feature-textodfwriter
+$(package)_config_opts += -no-feature-topleveldomain
+$(package)_config_opts += -no-feature-udpsocket
+$(package)_config_opts += -no-feature-undocommand
+$(package)_config_opts += -no-feature-undogroup
+$(package)_config_opts += -no-feature-undostack
+$(package)_config_opts += -no-feature-undoview
+$(package)_config_opts += -no-feature-vnc
+$(package)_config_opts += -no-feature-wizard
+$(package)_config_opts_linux = -no-fontconfig
+$(package)_config_opts_linux += -no-opengl
+$(package)_config_opts_linux += -no-xcb
+$(package)_config_opts_linux += -no-feature-xlib
 endef
 
 define $(package)_fetch_cmds
@@ -108,14 +125,24 @@ endef
 
 
 define $(package)_preprocess_cmds
+  sed -i.old "s|FT_Get_Font_Format|FT_Get_X11_Font_Format|" qtbase/src/platformsupport/fontdatabases/freetype/qfontengine_ft.cpp && \
   sed -i.old "s|updateqm.commands = \$$$$\$$$$LRELEASE|updateqm.commands = $($(package)_extract_dir)/qttools/bin/lrelease|" qttranslations/translations/translations.pro && \
   sed -i.old "/updateqm.depends =/d" qttranslations/translations/translations.pro && \
-  patch -p1 < $($(package)_patch_dir)/pidlist_absolute.patch && \
-  patch -p1 < $($(package)_patch_dir)/fix_qt_pkgconfig.patch && \
-  patch -p1 < $($(package)_patch_dir)/qfixed-coretext.patch && \
+  sed -i.old "s/src_plugins.depends = src_sql src_network/src_plugins.depends = src_network/" qtbase/src/src.pro && \
+  cp -r qtbase/mkspecs/linux-arm-gnueabi-g++ qtbase/mkspecs/bitcoin-linux-g++ && \
+  sed -i.old "s/arm-linux-gnueabi-/$(host)-/g" qtbase/mkspecs/bitcoin-linux-g++/qmake.conf && \
+  patch -p1 -i $($(package)_patch_dir)/fix_qt_pkgconfig.patch && \
+  patch -p1 -i $($(package)_patch_dir)/fix_no_printer.patch && \
   echo "!host_build: QMAKE_CFLAGS     += $($(package)_cflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \
   echo "!host_build: QMAKE_CXXFLAGS   += $($(package)_cxxflags) $($(package)_cppflags)" >> qtbase/mkspecs/common/gcc-base.conf && \
-  echo "!host_build: QMAKE_LFLAGS     += $($(package)_ldflags)" >> qtbase/mkspecs/common/gcc-base.conf
+  echo "!host_build: QMAKE_LFLAGS     += $($(package)_ldflags)" >> qtbase/mkspecs/common/gcc-base.conf && \
+  patch -p1 -i $($(package)_patch_dir)/no-xlib.patch && \
+  echo "QMAKE_LINK_OBJECT_MAX = 10" >> qtbase/mkspecs/win32-g++/qmake.conf && \
+  echo "QMAKE_LINK_OBJECT_SCRIPT = object_script" >> qtbase/mkspecs/win32-g++/qmake.conf && \
+  sed -i.old "s|QMAKE_CFLAGS           += |!host_build: QMAKE_CFLAGS            = $($(package)_cflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \
+  sed -i.old "s|QMAKE_CXXFLAGS         += |!host_build: QMAKE_CXXFLAGS            = $($(package)_cxxflags) $($(package)_cppflags) |" qtbase/mkspecs/win32-g++/qmake.conf && \
+  sed -i.old "0,/^QMAKE_LFLAGS_/s|^QMAKE_LFLAGS_|!host_build: QMAKE_LFLAGS            = $($(package)_ldflags)\n&|" qtbase/mkspecs/win32-g++/qmake.conf && \
+  sed -i.old "s/LIBRARY_PATH/(CROSS_)?\0/g" qtbase/mkspecs/features/toolchain.prf
 endef
 
 define $(package)_config_cmds

contrib/depends/patches/openssl/fix_arflags.patch

@@ -1,24 +0,0 @@
---- Makefile.org.O	2019-02-26 14:20:20.000000000 +0000
-+++ Makefile.org	2019-11-15 13:05:54.370086856 +0000
-@@ -63,8 +63,8 @@
- PEX_LIBS= 
- EX_LIBS= 
- EXE_EXT= 
--ARFLAGS=
--AR=ar $(ARFLAGS) r
-+ARFLAGS= r
-+AR=ar $(ARFLAGS)
- RANLIB= ranlib
- RC= windres
- NM= nm
---- Configure.O	2019-02-26 14:20:20.000000000 +0000
-+++ Configure	2019-11-16 07:43:14.933990774 +0000
-@@ -1251,7 +1251,7 @@
- my $shared_extension = $fields[$idx_shared_extension];
- my $ranlib = $ENV{'RANLIB'} || $fields[$idx_ranlib];
- my $ar = $ENV{'AR'} || "ar";
--my $arflags = $fields[$idx_arflags];
-+my $arflags = $ENV{'ARFLAGS'} || $fields[$idx_arflags];
- my $windres = $ENV{'RC'} || $ENV{'WINDRES'} || "windres";
- my $multilib = $fields[$idx_multilib];
- 

contrib/depends/patches/qt/fix_no_printer.patch

@@ -0,0 +1,19 @@
+--- x/qtbase/src/plugins/platforms/cocoa/qprintengine_mac_p.h
++++ y/qtbase/src/plugins/platforms/cocoa/qprintengine_mac_p.h
+@@ -52,6 +52,7 @@
+ //
+
+ #include <QtCore/qglobal.h>
++#include <qpa/qplatformprintdevice.h>
+
+ #ifndef QT_NO_PRINTER
+
+--- x/qtbase/src/plugins/plugins.pro
++++ y/qtbase/src/plugins/plugins.pro
+@@ -9,6 +9,3 @@ qtHaveModule(gui) {
+     !android:qtConfig(library): SUBDIRS *= generic
+ }
+ qtHaveModule(widgets): SUBDIRS += styles
+-
+-!winrt:qtHaveModule(printsupport): \
+-    SUBDIRS += printsupport

contrib/depends/patches/qt/fix_qt_pkgconfig.patch

@@ -1,11 +1,11 @@
 --- old/qtbase/mkspecs/features/qt_module.prf
 +++ new/qtbase/mkspecs/features/qt_module.prf
-@@ -245,7 +245,7 @@
+@@ -269,7 +269,7 @@ load(qt_installs)
  load(qt_targets)
  
  # this builds on top of qt_common
--!internal_module:!lib_bundle:if(unix|mingw) {
-+unix|mingw {
+-!internal_module:if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) {
++if(unix|mingw):!if(darwin:debug_and_release:CONFIG(debug, debug|release)) {
      CONFIG += create_pc
      QMAKE_PKGCONFIG_DESTDIR = pkgconfig
      host_build: \

contrib/depends/patches/qt/fix_rcc_determinism.patch

@@ -0,0 +1,15 @@
+--- old/qtbase/src/tools/rcc/rcc.cpp
++++ new/qtbase/src/tools/rcc/rcc.cpp
+@@ -207,7 +207,11 @@ void RCCFileInfo::writeDataInfo(RCCResourceLibrary &lib)
+     if (lib.formatVersion() >= 2) {
+         // last modified time stamp
+         const QDateTime lastModified = m_fileInfo.lastModified();
+-        lib.writeNumber8(quint64(lastModified.isValid() ? lastModified.toMSecsSinceEpoch() : 0));
++        quint64 lastmod = quint64(lastModified.isValid() ? lastModified.toMSecsSinceEpoch() : 0);
++        static const quint64 sourceDate = 1000 * qgetenv("QT_RCC_SOURCE_DATE_OVERRIDE").toULongLong();
++        if (sourceDate != 0)
++            lastmod = sourceDate;
++        lib.writeNumber8(lastmod);
+         if (text || pass1)
+             lib.writeChar('\n');
+     }

contrib/depends/patches/qt/no-xlib.patch

@@ -0,0 +1,69 @@
+From 9563cef873ae82e06f60708d706d054717e801ce Mon Sep 17 00:00:00 2001
+From: Carl Dong <contact@carldong.me>
+Date: Thu, 18 Jul 2019 17:22:05 -0400
+Subject: [PATCH] Wrap xlib related code blocks in #if's
+
+They are not necessary to compile QT.
+---
+ qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp | 8 ++++++++
+ 1 file changed, 8 insertions(+)
+
+diff --git a/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp b/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp
+index 7c62c2e2b3..c05c6c0a07 100644
+--- a/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp
++++ b/qtbase/src/plugins/platforms/xcb/qxcbcursor.cpp
+@@ -49,7 +49,9 @@
+ #include <QtGui/QWindow>
+ #include <QtGui/QBitmap>
+ #include <QtGui/private/qguiapplication_p.h>
++#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library)
+ #include <X11/cursorfont.h>
++#endif
+ #include <xcb/xfixes.h>
+ #include <xcb/xcb_image.h>
+ 
+@@ -391,6 +393,7 @@ void QXcbCursor::changeCursor(QCursor *cursor, QWindow *window)
+     xcb_flush(xcb_connection());
+ }
+ 
++#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library)
+ static int cursorIdForShape(int cshape)
+ {
+     int cursorId = 0;
+@@ -444,6 +447,7 @@ static int cursorIdForShape(int cshape)
+     }
+     return cursorId;
+ }
++#endif
+ 
+ xcb_cursor_t QXcbCursor::createNonStandardCursor(int cshape)
+ {
+@@ -556,7 +560,9 @@ static xcb_cursor_t loadCursor(void *dpy, int cshape)
+ xcb_cursor_t QXcbCursor::createFontCursor(int cshape)
+ {
+     xcb_connection_t *conn = xcb_connection();
++#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library)
+     int cursorId = cursorIdForShape(cshape);
++#endif
+     xcb_cursor_t cursor = XCB_NONE;
+ 
+     // Try Xcursor first
+@@ -586,6 +592,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape)
+     // Non-standard X11 cursors are created from bitmaps
+     cursor = createNonStandardCursor(cshape);
+ 
++#if QT_CONFIG(xcb_xlib) && QT_CONFIG(library)
+     // Create a glpyh cursor if everything else failed
+     if (!cursor && cursorId) {
+         cursor = xcb_generate_id(conn);
+@@ -593,6 +600,7 @@ xcb_cursor_t QXcbCursor::createFontCursor(int cshape)
+                                 cursorId, cursorId + 1,
+                                 0xFFFF, 0xFFFF, 0xFFFF, 0, 0, 0);
+     }
++#endif
+ 
+     if (cursor && cshape >= 0 && cshape < Qt::LastCursor && connection()->hasXFixes()) {
+         const char *name = cursorNames[cshape].front();
+---
+2.22.0
+

contrib/depends/patches/qt/pidlist_absolute.patch

@@ -1,37 +0,0 @@
-diff -dur old/qtbase/src/plugins/platforms/windows/qwindowscontext.h new/qtbase/src/plugins/platforms/windows/qwindowscontext.h
---- old/qtbase/src/plugins/platforms/windows/qwindowscontext.h
-+++ new/qtbase/src/plugins/platforms/windows/qwindowscontext.h
-@@ -136,10 +136,18 @@
-     inline void init();
- 
-     typedef HRESULT (WINAPI *SHCreateItemFromParsingName)(PCWSTR, IBindCtx *, const GUID&, void **);
-+#if defined(Q_CC_MINGW) && (!defined(__MINGW64_VERSION_MAJOR) || __MINGW64_VERSION_MAJOR < 3)
-+    typedef HRESULT (WINAPI *SHGetKnownFolderIDList)(const GUID &, DWORD, HANDLE, ITEMIDLIST **);
-+#else
-     typedef HRESULT (WINAPI *SHGetKnownFolderIDList)(const GUID &, DWORD, HANDLE, PIDLIST_ABSOLUTE *);
-+#endif
-     typedef HRESULT (WINAPI *SHGetStockIconInfo)(int , int , _SHSTOCKICONINFO *);
-     typedef HRESULT (WINAPI *SHGetImageList)(int, REFIID , void **);
-+#if defined(Q_CC_MINGW) && (!defined(__MINGW64_VERSION_MAJOR) || __MINGW64_VERSION_MAJOR < 3)
-+    typedef HRESULT (WINAPI *SHCreateItemFromIDList)(const ITEMIDLIST *, REFIID, void **);
-+#else
-     typedef HRESULT (WINAPI *SHCreateItemFromIDList)(PCIDLIST_ABSOLUTE, REFIID, void **);
-+#endif
- 
-     SHCreateItemFromParsingName sHCreateItemFromParsingName;
-     SHGetKnownFolderIDList sHGetKnownFolderIDList;
-diff -dur old/qtbase/src/plugins/platforms/windows/qwindowsdialoghelpers.cpp new/qtbase/src/plugins/platforms/windows/qwindowsdialoghelpers.cpp
---- old/qtbase/src/plugins/platforms/windows/qwindowsdialoghelpers.cpp
-+++ new/qtbase/src/plugins/platforms/windows/qwindowsdialoghelpers.cpp
-@@ -1016,7 +1016,11 @@
-             qWarning() << __FUNCTION__ << ": Invalid CLSID: " << url.path();
-             return Q_NULLPTR;
-         }
-+#if defined(Q_CC_MINGW) && (!defined(__MINGW64_VERSION_MAJOR) || __MINGW64_VERSION_MAJOR < 3)
-+        ITEMIDLIST *idList;
-+#else
-         PIDLIST_ABSOLUTE idList;
-+#endif
-         HRESULT hr = QWindowsContext::shell32dll.sHGetKnownFolderIDList(uuid, 0, 0, &idList);
-         if (FAILED(hr)) {
-             qErrnoWarning("%s: SHGetKnownFolderIDList(%s)) failed", __FUNCTION__, qPrintable(url.toString()));

contrib/depends/patches/qt/qfixed-coretext.patch

@@ -1,34 +0,0 @@
-From dbdd5f0ffbce52c8b789ed09f1aa3f1da6c02e23 Mon Sep 17 00:00:00 2001
-From: Gabriel de Dietrich <gabriel.dedietrich@qt.io>
-Date: Fri, 30 Mar 2018 11:58:16 -0700
-Subject: [PATCH] QCoreTextFontEngine: Fix build with Xcode 9.3
-
-Apple LLVM version 9.1.0 (clang-902.0.39.1)
-
-Error message:
-
-.../qfontengine_coretext.mm:827:20: error: qualified reference to
-      'QFixed' is a constructor name rather than a type in this context
-    return QFixed::QFixed(int(CTFontGetUnitsPerEm(ctfont)));
-
-Change-Id: Iebe26b3b087a16b10664208fc8851cbddb47f043
-Reviewed-by: Konstantin Ritt <ritt.ks@gmail.com>
----
- src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git old/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm new/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm
-index 25ff69d877d..98b753eff96 100644
---- old/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm
-+++ new/qtbase/src/platformsupport/fontdatabases/mac/qfontengine_coretext.mm
-@@ -824,7 +824,7 @@ void QCoreTextFontEngine::getUnscaledGlyph(glyph_t glyph, QPainterPath *path, gl
- 
- QFixed QCoreTextFontEngine::emSquareSize() const
- {
--    return QFixed::QFixed(int(CTFontGetUnitsPerEm(ctfont)));
-+    return QFixed(int(CTFontGetUnitsPerEm(ctfont)));
- }
- 
- QFontEngine *QCoreTextFontEngine::cloneWithSize(qreal pixelSize) const
--- 
-2.16.3

contrib/depends/toolchain.cmake.in

@@ -1,5 +1,6 @@
 # Set the system name to one of Android, Darwin, FreeBSD, Linux, or Windows
 SET(CMAKE_SYSTEM_NAME @depends@)
+SET(CMAKE_SYSTEM_PROCESSOR @arch@)
 SET(CMAKE_BUILD_TYPE @release_type@)
 
 OPTION(STATIC "Link libraries statically" ON)
@@ -55,7 +56,7 @@ SET(Boost_NO_SYSTEM_PATHS ON)
 SET(Boost_USE_STATIC_LIBS ON)
 SET(Boost_USE_STATIC_RUNTIME ON)
 
-SET(OpenSSL_DIR @prefix@/lib)
+SET(OPENSSL_ROOT_DIR @prefix@)
 SET(ARCHITECTURE @arch@)
 
 # for libraries and headers in the target directories
@@ -63,14 +64,14 @@ set (CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) # Find programs on host
 set (CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) # Find libs in target
 set (CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) # Find includes in target
 
-set(CMAKE_SYSTEM_PROCESSOR ${CMAKE_HOST_SYSTEM_PROCESSOR} CACHE STRING "" FORCE)
-
 # specify the cross compiler to be used. Darwin uses clang provided by the SDK.
 if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
   SET(CMAKE_C_COMPILER @prefix@/native/bin/clang)
   SET(CMAKE_C_COMPILER_TARGET x86_64-apple-darwin11)
   SET(CMAKE_CXX_COMPILER @prefix@/native/bin/clang++ -stdlib=libc++)
   SET(CMAKE_CXX_COMPILER_TARGET x86_64-apple-darwin11)
+  SET(CMAKE_ASM_COMPILER_TARGET x86_64-apple-darwin11)
+  SET(CMAKE_ASM-ATT_COMPILER_TARGET x86_64-apple-darwin11)
   SET(_CMAKE_TOOLCHAIN_PREFIX x86_64-apple-darwin11-)
   SET(APPLE True)
   SET(BUILD_TAG "mac-x64")

contrib/epee/include/byte_stream.h

@@ -58,7 +58,6 @@ namespace epee
     byte_buffer buffer_;        //! Beginning of buffer
     std::uint8_t* next_write_;  //! Current write position
     const std::uint8_t* end_;   //! End of buffer
-    std::size_t increase_size_; //! Minimum buffer size increase
 
     //! \post `requested <= available()`
     void overflow(const std::size_t requested);
@@ -75,29 +74,17 @@ namespace epee
     using char_type = std::uint8_t;
     using Ch = char_type;
 
-    //! \return Default minimum size increase on buffer overflow
-    static constexpr std::size_t default_increase() noexcept { return 4096; }
-
     //! Increase internal buffer by at least `byte_stream_increase` bytes.
     byte_stream() noexcept
-      : byte_stream(default_increase())
-    {}
-
-    //! Increase internal buffer by at least `increase` bytes.
-    explicit byte_stream(const std::size_t increase) noexcept
       : buffer_(nullptr),
         next_write_(nullptr),
-        end_(nullptr),
-        increase_size_(increase)
+        end_(nullptr)
     {}
 
     byte_stream(byte_stream&& rhs) noexcept;
     ~byte_stream() noexcept = default;
     byte_stream& operator=(byte_stream&& rhs) noexcept;
 
-    //! \return The minimum increase size on buffer overflow
-    std::size_t increase_size() const noexcept { return increase_size_; }
-
     const std::uint8_t* data() const noexcept { return buffer_.get(); }
     std::uint8_t* tellp() const noexcept { return next_write_; }
     std::size_t available() const noexcept { return end_ - next_write_; }
@@ -117,6 +104,9 @@ namespace epee
       check(more);
     }
 
+    //! Reset write position, but do not release internal memory. \post `size() == 0`.
+    void clear() noexcept { next_write_ = buffer_.get(); }
+
     /*! Copy `length` bytes starting at `ptr` to end of stream.
         \throw std::range_error If exceeding max size_t value.
         \throw std::bad_alloc If allocation fails. */
@@ -185,7 +175,7 @@ namespace epee
     void put_n(const std::uint8_t ch, const std::size_t count)
     {
       check(count);
-      std::memset(tellp(), count, ch);
+      std::memset(tellp(), ch, count);
       next_write_ += count;
     }
 

contrib/epee/include/console_handler.h

@@ -543,6 +543,31 @@ eof:
       return it->second.second;
     }
 
+    std::vector<std::string> get_command_list(const std::vector<std::string>& keywords = std::vector<std::string>())
+    {
+      std::vector<std::string> list;
+      list.reserve(m_command_handlers.size());
+      for(auto const& x:m_command_handlers)
+      {
+        bool take = true;
+        for(auto const& y:keywords)
+        {
+          bool in_usage = x.second.second.first.find(y) != std::string::npos;
+          bool in_description = x.second.second.second.find(y) != std::string::npos;
+          if (!(in_usage || in_description))
+          {
+            take = false;
+            break;
+          }
+        }
+        if (take)
+        {
+          list.push_back(x.first);
+        }
+      }
+      return list;
+    }
+
     void set_handler(const std::string& cmd, const callback& hndlr, const std::string& usage = "", const std::string& description = "")
     {
       lookup::mapped_type & vt = m_command_handlers[cmd];

contrib/epee/include/misc_language.h

@@ -106,6 +106,14 @@ namespace misc_utils
 		return true;
 	}
 
+  template <typename T>
+  T get_mid(const T &a, const T &b)
+  {
+    //returns the average of two numbers; overflow safe and works with at least all integral and floating point types
+    //(a+b)/2 = (a/2) + (b/2) + ((a - 2*(a/2)) + (b - 2*(b/2)))/2
+    return (a/2) + (b/2) + ((a - 2*(a/2)) + (b - 2*(b/2)))/2;
+  }
+
   template<class type_vec_type>
   type_vec_type median(std::vector<type_vec_type> &v)
   {
@@ -122,7 +130,7 @@ namespace misc_utils
       return v[n];
     }else 
     {//2, 4, 6...
-      return (v[n-1] + v[n])/2;
+      return get_mid<type_vec_type>(v[n-1],v[n]);
     }
 
   }

contrib/epee/include/net/abstract_http_client.h

@@ -64,12 +64,13 @@ namespace http
     abstract_http_client() {}
     virtual ~abstract_http_client() {}
     bool set_server(const std::string& address, boost::optional<login> user, ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect);
+    virtual bool set_proxy(const std::string& address);
     virtual void set_server(std::string host, std::string port, boost::optional<login> user, ssl_options_t ssl_options = ssl_support_t::e_ssl_support_autodetect) = 0;
     virtual void set_auto_connect(bool auto_connect) = 0;
     virtual bool connect(std::chrono::milliseconds timeout) = 0;
     virtual bool disconnect() = 0;
     virtual bool is_connected(bool *ssl = NULL) = 0;
-    virtual bool invoke(const boost::string_ref uri, const boost::string_ref method, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0;
+    virtual bool invoke(const boost::string_ref uri, const boost::string_ref method, const boost::string_ref body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0;
     virtual bool invoke_get(const boost::string_ref uri, std::chrono::milliseconds timeout, const std::string& body = std::string(), const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0;
     virtual bool invoke_post(const boost::string_ref uri, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0;
     virtual uint64_t get_bytes_sent() const = 0;

contrib/epee/include/net/abstract_tcp_server2.inl

@@ -207,7 +207,6 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     buffer_ssl_init_fill = 0;
     if (is_income && m_ssl_support != epee::net_utils::ssl_support_t::e_ssl_support_disabled)
       socket().async_receive(boost::asio::buffer(buffer_),
-        boost::asio::socket_base::message_peek,
         strand_.wrap(
           std::bind(&connection<t_protocol_handler>::handle_receive, self,
             std::placeholders::_1,
@@ -334,6 +333,9 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     TRY_ENTRY();
     //_info("[sock " << socket().native_handle() << "] Async read calledback.");
     
+    if (m_was_shutdown)
+        return;
+
     if (!e)
     {
         double current_speed_down;
@@ -360,6 +362,9 @@ PRAGMA_WARNING_DISABLE_VS(4355)
 					CRITICAL_REGION_LOCAL(	epee::net_utils::network_throttle_manager::m_lock_get_global_throttle_in );
 					delay = epee::net_utils::network_throttle_manager::get_global_throttle_in().get_sleep_time_after_tick( bytes_transferred );
 				}
+
+				if (m_was_shutdown)
+					return;
 				
 				delay *= 0.5;
 				long int ms = (long int)(delay * 100);
@@ -431,6 +436,9 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     std::size_t bytes_transferred)
   {
     TRY_ENTRY();
+
+    if (m_was_shutdown) return;
+
     if (e)
     {
       // offload the error case
@@ -438,13 +446,11 @@ PRAGMA_WARNING_DISABLE_VS(4355)
       return;
     }
 
-    reset_timer(get_timeout_from_bytes_read(bytes_transferred), false);
-
     buffer_ssl_init_fill += bytes_transferred;
-    if (buffer_ssl_init_fill <= get_ssl_magic_size())
+    MTRACE("we now have " << buffer_ssl_init_fill << "/" << get_ssl_magic_size() << " bytes needed to detect SSL");
+    if (buffer_ssl_init_fill < get_ssl_magic_size())
     {
       socket().async_receive(boost::asio::buffer(buffer_.data() + buffer_ssl_init_fill, buffer_.size() - buffer_ssl_init_fill),
-        boost::asio::socket_base::message_peek,
         strand_.wrap(
           boost::bind(&connection<t_protocol_handler>::handle_receive, connection<t_protocol_handler>::shared_from_this(),
             boost::asio::placeholders::error,
@@ -470,7 +476,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     if (m_ssl_support == epee::net_utils::ssl_support_t::e_ssl_support_enabled)
     {
       // Handshake
-      if (!handshake(boost::asio::ssl::stream_base::server))
+      if (!handshake(boost::asio::ssl::stream_base::server, boost::asio::const_buffer(buffer_.data(), buffer_ssl_init_fill)))
       {
         MERROR("SSL handshake failed");
         boost::interprocess::ipcdetail::atomic_write32(&m_want_close_connection, 1);
@@ -485,6 +491,11 @@ PRAGMA_WARNING_DISABLE_VS(4355)
         return;
       }
     }
+    else
+    {
+      handle_read(e, buffer_ssl_init_fill);
+      return;
+    }
 
     async_read_some(boost::asio::buffer(buffer_),
       strand_.wrap(
@@ -651,6 +662,8 @@ PRAGMA_WARNING_DISABLE_VS(4355)
         boost::this_thread::sleep(boost::posix_time::milliseconds( ms ) );
         m_send_que_lock.lock();
         _dbg1("sleep for queue: " << ms);
+	if (m_was_shutdown)
+		return false;
 
         if (retry > retry_limit) {
             MWARNING("send que size is more than ABSTRACT_SERVER_SEND_QUE_MAX_COUNT(" << ABSTRACT_SERVER_SEND_QUE_MAX_COUNT << "), shutting down connection");
@@ -748,7 +761,8 @@ PRAGMA_WARNING_DISABLE_VS(4355)
   template<class t_protocol_handler>
   void connection<t_protocol_handler>::reset_timer(boost::posix_time::milliseconds ms, bool add)
   {
-    if (ms.total_milliseconds() < 0)
+    const auto tms = ms.total_milliseconds();
+    if (tms < 0 || (add && tms == 0))
     {
       MWARNING("Ignoring negative timeout " << ms);
       return;

contrib/epee/include/net/connection_basic.hpp

@@ -132,10 +132,10 @@ class connection_basic { // not-templated base class for rapid developmet of som
 		ssl_support_t get_ssl_support() const { return m_ssl_support; }
 		void disable_ssl() { m_ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_disabled; }
 
-		bool handshake(boost::asio::ssl::stream_base::handshake_type type)
+		bool handshake(boost::asio::ssl::stream_base::handshake_type type, boost::asio::const_buffer buffer = {})
 		{
 			//m_state != nullptr verified in constructor
-			return m_state->ssl_options().handshake(socket_, type);
+			return m_state->ssl_options().handshake(socket_, type, buffer);
 		}
 
 		template<typename MutableBufferSequence, typename ReadHandler>

contrib/epee/include/net/http_client.h

@@ -233,7 +233,7 @@ namespace net_utils
 			}
 
 			//---------------------------------------------------------------------------
-			inline bool invoke(const boost::string_ref uri, const boost::string_ref method, const std::string& body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) override
+			inline bool invoke(const boost::string_ref uri, const boost::string_ref method, const boost::string_ref body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) override
 			{
 				CRITICAL_REGION_LOCAL(m_lock);
 				if(!is_connected())
@@ -885,14 +885,6 @@ namespace net_utils
 			}
 		};
 		typedef http_simple_client_template<blocked_mode_client> http_simple_client;
-
-    class http_simple_client_factory : public http_client_factory
-    {
-    public:
-      std::unique_ptr<abstract_http_client> create() override {
-        return std::unique_ptr<epee::net_utils::http::abstract_http_client>(new epee::net_utils::http::http_simple_client());
-      }
-    };
 	}
 }
 }

contrib/epee/include/net/http_server_handlers_map2.h

@@ -42,8 +42,17 @@
   MINFO("HTTP [" << m_conn_context.m_remote_address.host_str() << "] " << query_info.m_http_method_str << " " << query_info.m_URI); \
   response.m_response_code = 200; \
   response.m_response_comment = "Ok"; \
-  if(!handle_http_request_map(query_info, response, m_conn_context)) \
-  {response.m_response_code = 404;response.m_response_comment = "Not found";} \
+  try \
+  { \
+    if(!handle_http_request_map(query_info, response, m_conn_context)) \
+    {response.m_response_code = 404;response.m_response_comment = "Not found";} \
+  } \
+  catch (const std::exception &e) \
+  { \
+    MERROR(m_conn_context << "Exception in handle_http_request_map: " << e.what()); \
+    response.m_response_code = 500; \
+    response.m_response_comment = "Internal Server Error"; \
+  } \
   return true; \
 }
 
@@ -69,9 +78,11 @@
       uint64_t ticks1 = epee::misc_utils::get_tick_count(); \
       boost::value_initialized<command_type::response> resp;\
       MINFO(m_conn_context << "calling " << s_pattern); \
-      if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context)) \
+      bool res = false; \
+      try { res = callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context); } \
+      catch (const std::exception &e) { MERROR(m_conn_context << "Failed to " << #callback_f << "(): " << e.what()); } \
+      if (!res) \
       { \
-        MERROR(m_conn_context << "Failed to " << #callback_f << "()"); \
         response_info.m_response_code = 500; \
         response_info.m_response_comment = "Internal Server Error"; \
         return true; \
@@ -97,16 +108,20 @@
       uint64_t ticks1 = misc_utils::get_tick_count(); \
       boost::value_initialized<command_type::response> resp;\
       MINFO(m_conn_context << "calling " << s_pattern); \
-      if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context)) \
+      bool res = false; \
+      try { res = callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context); } \
+      catch (const std::exception &e) { MERROR(m_conn_context << "Failed to " << #callback_f << "()"); } \
+      if (!res) \
       { \
-        MERROR(m_conn_context << "Failed to " << #callback_f << "()"); \
         response_info.m_response_code = 500; \
         response_info.m_response_comment = "Internal Server Error"; \
         return true; \
       } \
       uint64_t ticks2 = misc_utils::get_tick_count(); \
-      epee::serialization::store_t_to_binary(static_cast<command_type::response&>(resp), response_info.m_body); \
+      epee::byte_slice buffer; \
+      epee::serialization::store_t_to_binary(static_cast<command_type::response&>(resp), buffer, 64 * 1024); \
       uint64_t ticks3 = epee::misc_utils::get_tick_count(); \
+      response_info.m_body.assign(reinterpret_cast<const char*>(buffer.data()), buffer.size()); \
       response_info.m_mime_tipe = " application/octet-stream"; \
       response_info.m_header_info.m_content_type = " application/octet-stream"; \
       MDEBUG( s_pattern << "() processed with " << ticks1-ticks << "/"<< ticks2-ticks1 << "/" << ticks3-ticks2 << "ms"); \
@@ -184,7 +199,10 @@
   fail_resp.jsonrpc = "2.0"; \
   fail_resp.id = req.id; \
   MINFO(m_conn_context << "Calling RPC method " << method_name); \
-  if(!callback_f(req.params, resp.result, fail_resp.error, &m_conn_context)) \
+  bool res = false; \
+  try { res = callback_f(req.params, resp.result, fail_resp.error, &m_conn_context); } \
+  catch (const std::exception &e) { MERROR(m_conn_context << "Failed to " << #callback_f << "(): " << e.what()); } \
+  if (!res) \
   { \
     epee::serialization::store_t_to_json(static_cast<epee::json_rpc::error_response&>(fail_resp), response_info.m_body); \
     return true; \
@@ -203,7 +221,10 @@
   fail_resp.jsonrpc = "2.0"; \
   fail_resp.id = req.id; \
   MINFO(m_conn_context << "calling RPC method " << method_name); \
-  if(!callback_f(req.params, resp.result, fail_resp.error, response_info, &m_conn_context)) \
+  bool res = false; \
+  try { res = callback_f(req.params, resp.result, fail_resp.error, response_info, &m_conn_context); } \
+  catch (const std::exception &e) { MERROR(m_conn_context << "Failed to " << #callback_f << "(): " << e.what()); } \
+  if (!res) \
   { \
     epee::serialization::store_t_to_json(static_cast<epee::json_rpc::error_response&>(fail_resp), response_info.m_body); \
     return true; \
@@ -217,7 +238,10 @@
 { \
   PREPARE_OBJECTS_FROM_JSON(command_type) \
   MINFO(m_conn_context << "calling RPC method " << method_name); \
-  if(!callback_f(req.params, resp.result, &m_conn_context)) \
+  bool res = false; \
+  try { res = callback_f(req.params, resp.result, &m_conn_context); } \
+  catch (const std::exception &e) { MERROR(m_conn_context << "Failed to " << #callback_f << "(): " << e.what()); } \
+  if (!res) \
   { \
     epee::json_rpc::error_response fail_resp = AUTO_VAL_INIT(fail_resp); \
     fail_resp.jsonrpc = "2.0"; \

contrib/epee/include/net/levin_base.h

@@ -31,7 +31,6 @@
 
 #include <cstdint>
 
-#include "byte_slice.h"
 #include "net_utils_base.h"
 #include "span.h"
 
@@ -39,6 +38,7 @@
 
 namespace epee
 {
+class byte_slice;
 namespace levin
 {
 #pragma pack(push)
@@ -72,7 +72,8 @@ namespace levin
 
 
 #define LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED 0
-#define LEVIN_DEFAULT_MAX_PACKET_SIZE 100000000      //100MB by default
+#define LEVIN_INITIAL_MAX_PACKET_SIZE  256*1024      // 256 KiB before handshake
+#define LEVIN_DEFAULT_MAX_PACKET_SIZE 100000000      //100MB by default after handshake
 
 #define LEVIN_PACKET_REQUEST			0x00000001
 #define LEVIN_PACKET_RESPONSE		0x00000002
@@ -86,7 +87,7 @@ namespace levin
   template<class t_connection_context = net_utils::connection_context_base>
   struct levin_commands_handler
   {
-    virtual int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, t_connection_context& context)=0;
+    virtual int invoke(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, t_connection_context& context)=0;
     virtual int notify(int command, const epee::span<const uint8_t> in_buff, t_connection_context& context)=0;
     virtual void callback(t_connection_context& context){};
 

contrib/epee/include/net/levin_protocol_handler_async.h

@@ -84,7 +84,8 @@ class async_protocol_handler_config
 
 public:
   typedef t_connection_context connection_context;
-  uint64_t m_max_packet_size; 
+  uint64_t m_initial_max_packet_size;
+  uint64_t m_max_packet_size;
   uint64_t m_invoke_timeout;
 
   int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, boost::uuids::uuid connection_id);
@@ -105,7 +106,7 @@ public:
   size_t get_in_connections_count();
   void set_handler(levin_commands_handler<t_connection_context>* handler, void (*destroy)(levin_commands_handler<t_connection_context>*) = NULL);
 
-  async_protocol_handler_config():m_pcommands_handler(NULL), m_pcommands_handler_destroy(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE), m_invoke_timeout(LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED)
+  async_protocol_handler_config():m_pcommands_handler(NULL), m_pcommands_handler_destroy(NULL), m_initial_max_packet_size(LEVIN_INITIAL_MAX_PACKET_SIZE), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE), m_invoke_timeout(LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED)
   {}
   ~async_protocol_handler_config() { set_handler(NULL, NULL); }
   void del_out_connections(size_t count);
@@ -162,6 +163,7 @@ public:
   net_utils::i_service_endpoint* m_pservice_endpoint; 
   config_type& m_config;
   t_connection_context& m_connection_context;
+  std::atomic<uint64_t> m_max_packet_size;
 
   net_utils::buffer m_cache_in_buffer;
   stream_state m_state;
@@ -289,7 +291,8 @@ public:
             m_current_head(bucket_head2()),
             m_pservice_endpoint(psnd_hndlr), 
             m_config(config), 
-            m_connection_context(conn_context), 
+            m_connection_context(conn_context),
+            m_max_packet_size(config.m_initial_max_packet_size),
             m_cache_in_buffer(4 * 1024),
             m_state(stream_state_head)
   {
@@ -399,13 +402,14 @@ public:
     }
 
     // these should never fail, but do runtime check for safety
-    CHECK_AND_ASSERT_MES(m_config.m_max_packet_size >= m_cache_in_buffer.size(), false, "Bad m_cache_in_buffer.size()");
-    CHECK_AND_ASSERT_MES(m_config.m_max_packet_size - m_cache_in_buffer.size() >= m_fragment_buffer.size(), false, "Bad m_cache_in_buffer.size() + m_fragment_buffer.size()");
+    const uint64_t max_packet_size = m_max_packet_size;
+    CHECK_AND_ASSERT_MES(max_packet_size >= m_cache_in_buffer.size(), false, "Bad m_cache_in_buffer.size()");
+    CHECK_AND_ASSERT_MES(max_packet_size - m_cache_in_buffer.size() >= m_fragment_buffer.size(), false, "Bad m_cache_in_buffer.size() + m_fragment_buffer.size()");
 
     // flipped to subtraction; prevent overflow since m_max_packet_size is variable and public
-    if(cb > m_config.m_max_packet_size - m_cache_in_buffer.size() - m_fragment_buffer.size())
+    if(cb > max_packet_size - m_cache_in_buffer.size() - m_fragment_buffer.size())
     {
-      MWARNING(m_connection_context << "Maximum packet size exceed!, m_max_packet_size = " << m_config.m_max_packet_size
+      MWARNING(m_connection_context << "Maximum packet size exceed!, m_max_packet_size = " << max_packet_size
                           << ", packet received " << m_cache_in_buffer.size() +  cb 
                           << ", connection will be closed.");
       return false;
@@ -430,7 +434,7 @@ public:
               //async call scenario
               boost::shared_ptr<invoke_response_handler_base> response_handler = m_invoke_response_handlers.front();
               response_handler->reset_timer();
-              MDEBUG(m_connection_context << "LEVIN_PACKET partial msg received. len=" << cb);
+              MDEBUG(m_connection_context << "LEVIN_PACKET partial msg received. len=" << cb << ", current total " << m_cache_in_buffer.size() << "/" << m_current_head.m_cb << " (" << (100.0f * m_cache_in_buffer.size() / (m_current_head.m_cb ? m_current_head.m_cb : 1)) << "%)");
             }
           }
           break;
@@ -465,6 +469,14 @@ public:
             temp = std::move(m_fragment_buffer);
             m_fragment_buffer.clear();
             std::memcpy(std::addressof(m_current_head), std::addressof(temp[0]), sizeof(bucket_head2));
+            const size_t max_bytes = m_connection_context.get_max_bytes(m_current_head.m_command);
+            if(m_current_head.m_cb > std::min<size_t>(max_packet_size, max_bytes))
+            {
+              MERROR(m_connection_context << "Maximum packet size exceed!, m_max_packet_size = " << std::min<size_t>(max_packet_size, max_bytes)
+                << ", packet header received " << m_current_head.m_cb << ", command " << m_current_head.m_command
+                << ", connection will be closed.");
+              return false;
+            }
             buff_to_invoke = {reinterpret_cast<const uint8_t*>(temp.data()) + sizeof(bucket_head2), temp.size() - sizeof(bucket_head2)};
           }
 
@@ -514,16 +526,19 @@ public:
           {
             if(m_current_head.m_have_to_return_data)
             {
-              std::string return_buff;
+              byte_slice return_buff;
               const uint32_t return_code = m_config.m_pcommands_handler->invoke(
                 m_current_head.m_command, buff_to_invoke, return_buff, m_connection_context
               );
 
+              // peer_id remains unset if dropped
+              if (m_current_head.m_command == m_connection_context.handshake_command() && m_connection_context.handshake_complete())
+                m_max_packet_size = m_config.m_max_packet_size;
+
               bucket_head2 head = make_header(m_current_head.m_command, return_buff.size(), LEVIN_PACKET_RESPONSE, false);
               head.m_return_code = SWAP32LE(return_code);
-              return_buff.insert(0, reinterpret_cast<const char*>(&head), sizeof(head));
 
-              if(!m_pservice_endpoint->do_send(byte_slice{std::move(return_buff)}))
+              if(!m_pservice_endpoint->do_send(byte_slice{{epee::as_byte_span(head), epee::to_span(return_buff)}}))
                 return false;
 
               MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb
@@ -577,10 +592,11 @@ public:
           m_cache_in_buffer.erase(sizeof(bucket_head2));
           m_state = stream_state_body;
           m_oponent_protocol_ver = m_current_head.m_protocol_version;
-          if(m_current_head.m_cb > m_config.m_max_packet_size)
+          const size_t max_bytes = m_connection_context.get_max_bytes(m_current_head.m_command);
+          if(m_current_head.m_cb > std::min<size_t>(max_packet_size, max_bytes))
           {
-            LOG_ERROR_CC(m_connection_context, "Maximum packet size exceed!, m_max_packet_size = " << m_config.m_max_packet_size 
-              << ", packet header received " << m_current_head.m_cb 
+            LOG_ERROR_CC(m_connection_context, "Maximum packet size exceed!, m_max_packet_size = " << std::min<size_t>(max_packet_size, max_bytes)
+              << ", packet header received " << m_current_head.m_cb << ", command " << m_current_head.m_command
               << ", connection will be closed.");
             return false;
           }
@@ -634,6 +650,9 @@ public:
       boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0);
       CRITICAL_REGION_BEGIN(m_invoke_response_handlers_lock);
 
+      if (command == m_connection_context.handshake_command())
+        m_max_packet_size = m_config.m_max_packet_size;
+
       if(!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true))
       {
         LOG_ERROR_CC(m_connection_context, "Failed to do_send");
@@ -675,6 +694,9 @@ public:
 
     boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0);
 
+    if (command == m_connection_context.handshake_command())
+      m_max_packet_size = m_config.m_max_packet_size;
+
     if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true))
     {
       LOG_ERROR_CC(m_connection_context, "Failed to send request");

contrib/epee/include/net/local_ip.h

@@ -28,8 +28,6 @@
 #pragma once
 
 #include <string>
-#include <boost/algorithm/string/predicate.hpp>
-#include <boost/asio/ip/address_v6.hpp>
 #include "int-util.h"
 
 // IP addresses are kept in network byte order
@@ -40,30 +38,6 @@ namespace epee
 {
   namespace net_utils
   {
-
-    inline
-    bool is_ipv6_local(const std::string& ip)
-    {
-      auto addr = boost::asio::ip::address_v6::from_string(ip);
-
-      // ipv6 link-local unicast addresses are fe80::/10
-      bool is_link_local = addr.is_link_local();
-
-      auto addr_bytes = addr.to_bytes();
-
-      // ipv6 unique local unicast addresses start with fc00::/7 -- (fcXX or fdXX)
-      bool is_unique_local_unicast = (addr_bytes[0] == 0xfc || addr_bytes[0] == 0xfd);
-
-      return is_link_local || is_unique_local_unicast;
-    }
-
-    inline
-    bool is_ipv6_loopback(const std::string& ip)
-    {
-      // ipv6 loopback is ::1
-      return boost::asio::ip::address_v6::from_string(ip).is_loopback();
-    }
-
     inline
     bool is_ip_local(uint32_t ip)
     {

contrib/epee/include/net/net_helper.h

@@ -44,6 +44,7 @@
 #include <boost/lambda/lambda.hpp>
 #include <boost/interprocess/detail/atomic.hpp>
 #include <boost/system/error_code.hpp>
+#include <boost/utility/string_ref.hpp>
 #include <functional>
 #include "net/net_utils_base.h"
 #include "net/net_ssl.h"
@@ -178,7 +179,7 @@ namespace net_utils
 					// SSL Options
 					if (m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_enabled || m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect)
 					{
-						if (!m_ssl_options.handshake(*m_ssl_socket, boost::asio::ssl::stream_base::client, addr, timeout))
+						if (!m_ssl_options.handshake(*m_ssl_socket, boost::asio::ssl::stream_base::client, {}, addr, timeout))
 						{
 							if (m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect)
 							{
@@ -280,7 +281,7 @@ namespace net_utils
 
 
 		inline 
-		bool send(const std::string& buff, std::chrono::milliseconds timeout)
+		bool send(const boost::string_ref buff, std::chrono::milliseconds timeout)
 		{
 
 			try
@@ -298,7 +299,7 @@ namespace net_utils
 				// object is used as a callback and will update the ec variable when the
 				// operation completes. The blocking_udp_client.cpp example shows how you
 				// can use boost::bind rather than boost::lambda.
-				async_write(buff.c_str(), buff.size(), ec);
+				async_write(buff.data(), buff.size(), ec);
 
 				// Block until the asynchronous operation has completed.
 				while (ec == boost::asio::error::would_block)

contrib/epee/include/net/net_parse_helpers.h

@@ -131,8 +131,8 @@ namespace net_utils
   inline
     bool parse_url_ipv6(const std::string url_str, http::url_content& content)
   {
-    STATIC_REGEXP_EXPR_1(rexp_match_uri, "^((.*?)://)?(\\[(.*)\\](:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal);
-    //                                     12         3   4      5 6        7
+    STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(\\[(.*)\\](:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal);
+    //                                     12            3   4      5 6        7
 
     content.port = 0;
     boost::smatch result;
@@ -175,8 +175,8 @@ namespace net_utils
 
     ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash=
     //STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal);
-    STATIC_REGEXP_EXPR_1(rexp_match_uri, "^((.*?)://)?(([^/:]*)(:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal);
-    //                                     12         34      5 6        7
+    STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(([^/:]*)(:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal);
+    //                                     12            34       5 6        7
     content.port = 0;
     boost::smatch result;	
     if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched))

contrib/epee/include/net/net_ssl.h

@@ -132,6 +132,7 @@ namespace net_utils
     bool handshake(
       boost::asio::ssl::stream<boost::asio::ip::tcp::socket> &socket,
       boost::asio::ssl::stream_base::handshake_type type,
+      boost::asio::const_buffer buffer = {},
       const std::string& host = {},
       std::chrono::milliseconds timeout = std::chrono::seconds(15)) const;
   };

contrib/epee/include/rolling_median.h

@@ -34,6 +34,8 @@
 
 #pragma once
 
+#include "misc_language.h"
+
 #include <stdlib.h>
 #include <stdint.h>
 
@@ -139,7 +141,6 @@ public:
 
   rolling_median_t(rolling_median_t &&m)
   {
-    free(data);
     memcpy(this, &m, sizeof(rolling_median_t));
     m.data = NULL;
   }
@@ -226,7 +227,7 @@ public:
     Item v = data[heap[0]];
     if (minCt < maxCt)
     {
-      v = (v + data[heap[-1]]) / 2;
+      v = get_mid<Item>(v, data[heap[-1]]);
     }
     return v;
   }

contrib/epee/include/storages/http_abstract_invoke.h

@@ -29,6 +29,7 @@
 #include <boost/utility/string_ref.hpp>
 #include <chrono>
 #include <string>
+#include "byte_slice.h"
 #include "portable_storage_template_helper.h"
 #include "net/http_base.h"
 #include "net/http_server_handlers_map2.h"
@@ -74,12 +75,12 @@ namespace epee
     template<class t_request, class t_response, class t_transport>
     bool invoke_http_bin(const boost::string_ref uri, const t_request& out_struct, t_response& result_struct, t_transport& transport, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref method = "POST")
     {
-      std::string req_param;
-      if(!serialization::store_t_to_binary(out_struct, req_param))
+      byte_slice req_param;
+      if(!serialization::store_t_to_binary(out_struct, req_param, 16 * 1024))
         return false;
 
       const http::http_response_info* pri = NULL;
-      if(!transport.invoke(uri, method, req_param, timeout, std::addressof(pri)))
+      if(!transport.invoke(uri, method, boost::string_ref{reinterpret_cast<const char*>(req_param.data()), req_param.size()}, timeout, std::addressof(pri)))
       {
         LOG_PRINT_L1("Failed to invoke http request to  " << uri);
         return false;
@@ -97,7 +98,12 @@ namespace epee
         return false;
       }
 
-      return serialization::load_t_from_binary(result_struct, epee::strspan<uint8_t>(pri->m_body));
+      static const constexpr epee::serialization::portable_storage::limits_t default_http_bin_limits = {
+        65536 * 3, // objects
+        65536 * 3, // fields
+        65536 * 3, // strings
+      };
+      return serialization::load_t_from_binary(result_struct, epee::strspan<uint8_t>(pri->m_body), &default_http_bin_limits);
     }
 
     template<class t_request, class t_response, class t_transport>

contrib/epee/include/storages/levin_abstract_invoke2.h

@@ -27,8 +27,10 @@
 #pragma once
 
 #include "portable_storage_template_helper.h"
+#include <boost/utility/string_ref.hpp>
 #include <boost/utility/value_init.hpp>
 #include <functional>
+#include "byte_slice.h"
 #include "span.h"
 #include "net/levin_base.h"
 
@@ -50,6 +52,11 @@ namespace
     snprintf(buf, sizeof(buf),  "command-%u", command);
     return on_levin_traffic(context, initiator, sent, error, bytes, buf);
   }
+  static const constexpr epee::serialization::portable_storage::limits_t default_levin_limits = {
+    8192, // objects
+    16384, // fields
+    16384, // strings
+  };
 }
 
 namespace epee
@@ -75,7 +82,7 @@ namespace epee
         return false;
       }
       serialization::portable_storage stg_ret;
-      if(!stg_ret.load_from_binary(buff_to_recv))
+      if(!stg_ret.load_from_binary(buff_to_recv, &default_levin_limits))
       {
         LOG_ERROR("Failed to load_from_binary on command " << command);
         return false;
@@ -110,18 +117,19 @@ namespace epee
       const boost::uuids::uuid &conn_id = context.m_connection_id;
       typename serialization::portable_storage stg;
       out_struct.store(stg);
-      std::string buff_to_send, buff_to_recv;
-      stg.store_to_binary(buff_to_send);
+      byte_slice buff_to_send;
+      std::string buff_to_recv;
+      stg.store_to_binary(buff_to_send, 16 * 1024);
 
       on_levin_traffic(context, true, true, false, buff_to_send.size(), command);
-      int res = transport.invoke(command, buff_to_send, buff_to_recv, conn_id);
+      int res = transport.invoke(command, boost::string_ref{reinterpret_cast<const char*>(buff_to_send.data()), buff_to_send.size()}, buff_to_recv, conn_id);
       if( res <=0 )
       {
         LOG_PRINT_L1("Failed to invoke command " << command << " return code " << res);
         return false;
       }
       typename serialization::portable_storage stg_ret;
-      if(!stg_ret.load_from_binary(buff_to_recv))
+      if(!stg_ret.load_from_binary(buff_to_recv, &default_levin_limits))
       {
         on_levin_traffic(context, true, false, true, buff_to_recv.size(), command);
         LOG_ERROR("Failed to load_from_binary on command " << command);
@@ -137,10 +145,10 @@ namespace epee
       const boost::uuids::uuid &conn_id = context.m_connection_id;
       typename serialization::portable_storage stg;
       const_cast<t_arg&>(out_struct).store(stg);//TODO: add true const support to searilzation
-      std::string buff_to_send;
-      stg.store_to_binary(buff_to_send);
+      byte_slice buff_to_send;
+      stg.store_to_binary(buff_to_send, 16 * 1024);
       on_levin_traffic(context, true, true, false, buff_to_send.size(), command);
-      int res = transport.invoke_async(command, epee::strspan<uint8_t>(buff_to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool 
+      int res = transport.invoke_async(command, epee::to_span(buff_to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool
       {
         t_result result_struct = AUTO_VAL_INIT(result_struct);
         if( code <=0 )
@@ -152,7 +160,7 @@ namespace epee
           return false;
         }
         serialization::portable_storage stg_ret;
-        if(!stg_ret.load_from_binary(buff))
+        if(!stg_ret.load_from_binary(buff, &default_levin_limits))
         {
           on_levin_traffic(context, true, false, true, buff.size(), command);
           LOG_ERROR("Failed to load_from_binary on command " << command);
@@ -184,11 +192,11 @@ namespace epee
       const boost::uuids::uuid &conn_id = context.m_connection_id;
       serialization::portable_storage stg;
       out_struct.store(stg);
-      std::string buff_to_send;
+      byte_slice buff_to_send;
       stg.store_to_binary(buff_to_send);
 
       on_levin_traffic(context, true, true, false, buff_to_send.size(), command);
-      int res = transport.notify(command, epee::strspan<uint8_t>(buff_to_send), conn_id);
+      int res = transport.notify(command, epee::to_span(buff_to_send), conn_id);
       if(res <=0 )
       {
         MERROR("Failed to notify command " << command << " return code " << res);
@@ -199,10 +207,10 @@ namespace epee
     //----------------------------------------------------------------------------------------------------
     //----------------------------------------------------------------------------------------------------
     template<class t_owner, class t_in_type, class t_out_type, class t_context, class callback_t>
-    int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, callback_t cb, t_context& context )
+    int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, callback_t cb, t_context& context )
     {
       serialization::portable_storage strg;
-      if(!strg.load_from_binary(in_buff))
+      if(!strg.load_from_binary(in_buff, &default_levin_limits))
       {
         on_levin_traffic(context, false, false, true, in_buff.size(), command);
         LOG_ERROR("Failed to load_from_binary in command " << command);
@@ -222,7 +230,7 @@ namespace epee
       serialization::portable_storage strg_out;
       static_cast<t_out_type&>(out_struct).store(strg_out);
 
-      if(!strg_out.store_to_binary(buff_out))
+      if(!strg_out.store_to_binary(buff_out, 32 * 1024))
       {
         LOG_ERROR("Failed to store_to_binary in command" << command);
         return -1;
@@ -236,7 +244,7 @@ namespace epee
     int buff_to_t_adapter(t_owner* powner, int command, const epee::span<const uint8_t> in_buff, callback_t cb, t_context& context)
     {
       serialization::portable_storage strg;
-      if(!strg.load_from_binary(in_buff))
+      if(!strg.load_from_binary(in_buff, &default_levin_limits))
       {
         on_levin_traffic(context, false, false, true, in_buff.size(), command);
         LOG_ERROR("Failed to load_from_binary in notify " << command);
@@ -254,7 +262,7 @@ namespace epee
     }
 
 #define CHAIN_LEVIN_INVOKE_MAP2(context_type) \
-  int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, context_type& context) \
+  int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, context_type& context) \
   { \
   bool handled = false; \
   return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \
@@ -263,13 +271,13 @@ namespace epee
 #define CHAIN_LEVIN_NOTIFY_MAP2(context_type) \
   int notify(int command, const epee::span<const uint8_t> in_buff, context_type& context) \
   { \
-  bool handled = false; std::string fake_str;\
+  bool handled = false; epee::byte_slice fake_str; \
   return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \
   } 
 
 
 #define CHAIN_LEVIN_INVOKE_MAP() \
-  int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, epee::net_utils::connection_context_base& context) \
+  int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, epee::net_utils::connection_context_base& context) \
   { \
   bool handled = false; \
   return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \
@@ -289,8 +297,9 @@ namespace epee
   } 
 
 #define BEGIN_INVOKE_MAP2(owner_type) \
-  template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, t_context& context, bool& handled) \
+  template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, t_context& context, bool& handled) \
   { \
+  try { \
   typedef owner_type internal_owner_type_name;
 
 #define HANDLE_INVOKE2(command_id, func, type_name_in, typename_out) \
@@ -336,7 +345,13 @@ namespace epee
   LOG_ERROR("Unknown command:" << command); \
   on_levin_traffic(context, false, false, true, in_buff.size(), "invalid-command"); \
   return LEVIN_ERROR_CONNECTION_HANDLER_NOT_DEFINED; \
+  } \
+  catch (const std::exception &e) { \
+    MERROR("Error in handle_invoke_map: " << e.what()); \
+    return LEVIN_ERROR_CONNECTION_TIMEDOUT; /* seems kinda appropriate */ \
+  } \
   }
+
   }
 }
 

contrib/epee/include/storages/portable_storage.h

@@ -32,7 +32,6 @@
 
 #include "misc_language.h"
 #include "portable_storage_base.h"
-#include "portable_storage_to_bin.h"
 #include "portable_storage_from_bin.h"
 #include "portable_storage_to_json.h"
 #include "portable_storage_from_json.h"
@@ -42,6 +41,7 @@
 
 namespace epee
 {
+  class byte_slice;
   namespace serialization
   {
     /************************************************************************/
@@ -54,6 +54,13 @@ namespace epee
       typedef epee::serialization::harray  harray;
       typedef storage_entry meta_entry;
 
+      struct limits_t
+      {
+        size_t n_objects;
+        size_t n_fields;
+        size_t n_strings; // not counting field names
+      };
+
       portable_storage(){}
       virtual ~portable_storage(){}
       hsection   open_section(const std::string& section_name,  hsection hparent_section, bool create_if_notexist = false);
@@ -83,9 +90,9 @@ namespace epee
       bool        delete_entry(const std::string& pentry_name, hsection hparent_section = nullptr);
 
       //-------------------------------------------------------------------------------
-      bool		store_to_binary(binarybuffer& target);
-      bool		load_from_binary(const epee::span<const uint8_t> target);
-      bool		load_from_binary(const std::string& target) { return load_from_binary(epee::strspan<uint8_t>(target)); }
+      bool		store_to_binary(byte_slice& target, std::size_t initial_buffer_size = 8192);
+      bool		load_from_binary(const epee::span<const uint8_t> target, const limits_t *limits = NULL);
+      bool		load_from_binary(const std::string& target, const limits_t *limits = NULL) { return load_from_binary(epee::strspan<uint8_t>(target), limits); }
       template<class trace_policy>
       bool		  dump_as_xml(std::string& targetObj, const std::string& root_name = "");
       bool		  dump_as_json(std::string& targetObj, size_t indent = 0, bool insert_newlines = true);
@@ -133,24 +140,8 @@ namespace epee
     {
       return false;//TODO: don't think i ever again will use xml - ambiguous and "overtagged" format
     }
-
     inline
-    bool portable_storage::store_to_binary(binarybuffer& target)
-    {
-      TRY_ENTRY();
-      std::stringstream ss;
-      storage_block_header sbh = AUTO_VAL_INIT(sbh);
-      sbh.m_signature_a = SWAP32LE(PORTABLE_STORAGE_SIGNATUREA);
-      sbh.m_signature_b = SWAP32LE(PORTABLE_STORAGE_SIGNATUREB);
-      sbh.m_ver = PORTABLE_STORAGE_FORMAT_VER;
-      ss.write((const char*)&sbh, sizeof(storage_block_header));
-      pack_entry_to_buff(ss, m_root);
-      target = ss.str();
-      return true;
-      CATCH_ENTRY("portable_storage::store_to_binary", false)
-    }
-    inline
-    bool portable_storage::load_from_binary(const epee::span<const uint8_t> source)
+    bool portable_storage::load_from_binary(const epee::span<const uint8_t> source, const limits_t *limits)
     {
       m_root.m_entries.clear();
       if(source.size() < sizeof(storage_block_header))
@@ -173,6 +164,8 @@ namespace epee
       }
       TRY_ENTRY();
       throwable_buffer_reader buf_reader(source.data()+sizeof(storage_block_header), source.size()-sizeof(storage_block_header));
+      if (limits)
+        buf_reader.set_limits(limits->n_objects, limits->n_fields, limits->n_strings);
       buf_reader.read(m_root);
       return true;//TODO:
       CATCH_ENTRY("portable_storage::load_from_binary", false);
@@ -282,6 +275,7 @@ namespace epee
       static_assert(std::is_rvalue_reference<entry_type&&>(), "unexpected copy of value");
       TRY_ENTRY();
       CHECK_AND_ASSERT(psection, nullptr);
+      CHECK_AND_ASSERT(!pentry_name.empty(), nullptr);
       auto ins_res = psection->m_entries.emplace(pentry_name, std::forward<entry_type>(entry));
       return &ins_res.first->second;
       CATCH_ENTRY("portable_storage::insert_new_entry_get_storage_entry", nullptr);

contrib/epee/include/storages/portable_storage_from_bin.h

@@ -29,6 +29,7 @@
 #pragma once 
 
 #include "misc_language.h"
+#include "misc_log_ex.h"
 #include "portable_storage_base.h"
 #include "portable_storage_bin_utils.h"
 
@@ -42,6 +43,24 @@ namespace epee
 {
   namespace serialization
   {
+    template<typename T>
+    struct ps_min_bytes {
+      static constexpr const size_t strict = 4096; // actual low bound
+    };
+    template<> struct ps_min_bytes<uint64_t> { static constexpr const size_t strict = 8; };
+    template<> struct ps_min_bytes<int64_t> { static constexpr const size_t strict = 8; };
+    template<> struct ps_min_bytes<uint32_t> { static constexpr const size_t strict = 4; };
+    template<> struct ps_min_bytes<int32_t> { static constexpr const size_t strict = 4; };
+    template<> struct ps_min_bytes<uint16_t> { static constexpr const size_t strict = 2; };
+    template<> struct ps_min_bytes<int16_t> { static constexpr const size_t strict = 2; };
+    template<> struct ps_min_bytes<uint8_t> { static constexpr const size_t strict = 1; };
+    template<> struct ps_min_bytes<int8_t> { static constexpr const size_t strict = 1; };
+    template<> struct ps_min_bytes<double> { static constexpr const size_t strict = 8; };
+    template<> struct ps_min_bytes<bool> { static constexpr const size_t strict = 1; };
+    template<> struct ps_min_bytes<std::string> { static constexpr const size_t strict = 2; };
+    template<> struct ps_min_bytes<section> { static constexpr const size_t strict = 1; };
+    template<> struct ps_min_bytes<array_entry> { static constexpr const size_t strict = 1; };
+
     struct throwable_buffer_reader
     {
       throwable_buffer_reader(const void* ptr, size_t sz);
@@ -61,6 +80,9 @@ namespace epee
       void read(section& sec);
       void read(std::string& str);
       void read(array_entry &ae);
+      template<class t_type>
+      size_t min_bytes() const;
+      void set_limits(size_t objects, size_t fields, size_t strings);
     private:
       struct recursuion_limitation_guard
       {
@@ -81,6 +103,13 @@ namespace epee
       const uint8_t* m_ptr;
       size_t m_count;
       size_t m_recursion_count;
+      size_t m_objects;
+      size_t m_fields;
+      size_t m_strings;
+
+      size_t max_objects;
+      size_t max_fields;
+      size_t max_strings;
     };
 
     inline throwable_buffer_reader::throwable_buffer_reader(const void* ptr, size_t sz)
@@ -92,6 +121,12 @@ namespace epee
       m_ptr = (uint8_t*)ptr;
       m_count = sz;
       m_recursion_count = 0;
+      m_objects = 0;
+      m_fields = 0;
+      m_strings = 0;
+      max_objects = std::numeric_limits<size_t>::max();
+      max_fields = std::numeric_limits<size_t>::max();
+      max_strings = std::numeric_limits<size_t>::max();
     }
     inline 
     void throwable_buffer_reader::read(void* target, size_t count)
@@ -108,6 +143,7 @@ namespace epee
       RECURSION_LIMITATION();
       uint8_t name_len = 0;
       read(name_len);
+      CHECK_AND_ASSERT_THROW_MES(name_len > 0, "Section name is missing");
       sce_name.resize(name_len);
       read((void*)sce_name.data(), name_len);
     }
@@ -138,7 +174,18 @@ namespace epee
       //for pod types
       array_entry_t<type_name> sa;
       size_t size = read_varint();
-      CHECK_AND_ASSERT_THROW_MES(size <= m_count, "Size sanity check failed");
+      CHECK_AND_ASSERT_THROW_MES(size <= m_count / ps_min_bytes<type_name>::strict, "Size sanity check failed");
+      if (std::is_same<type_name, section>())
+      {
+        CHECK_AND_ASSERT_THROW_MES(size <= max_objects - m_objects, "Too many objects");
+        m_objects += size;
+      }
+      else if (std::is_same<type_name, std::string>())
+      {
+        CHECK_AND_ASSERT_THROW_MES(size <= max_strings - m_strings, "Too many strings");
+        m_strings += size;
+      }
+
       sa.reserve(size);
       //TODO: add some optimization here later
       while(size--)
@@ -204,6 +251,8 @@ namespace epee
     inline storage_entry throwable_buffer_reader::read_se<std::string>()
     {
       RECURSION_LIMITATION();
+      CHECK_AND_ASSERT_THROW_MES(m_strings + 1 <= max_strings, "Too many strings");
+      m_strings += 1;
       return storage_entry(read<std::string>());
     }
 
@@ -212,6 +261,8 @@ namespace epee
     inline storage_entry throwable_buffer_reader::read_se<section>()
     {
       RECURSION_LIMITATION();
+      CHECK_AND_ASSERT_THROW_MES(m_objects < max_objects, "Too many objects");
+      ++m_objects;
       section s;//use extra variable due to vs bug, line "storage_entry se(section()); " can't be compiled in visual studio
       storage_entry se(std::move(s));
       section& section_entry = boost::get<section>(se);
@@ -263,12 +314,16 @@ namespace epee
       RECURSION_LIMITATION();
       sec.m_entries.clear();
       size_t count = read_varint();
+      CHECK_AND_ASSERT_THROW_MES(count <= max_fields - m_fields, "Too many object fields");
+      m_fields += count;
       while(count--)
       {
         //read section name string
         std::string sec_name;
         read_sec_name(sec_name);
-        sec.m_entries.emplace(std::move(sec_name), load_storage_entry());
+        const auto insert_loc = sec.m_entries.lower_bound(sec_name);
+        CHECK_AND_ASSERT_THROW_MES(insert_loc == sec.m_entries.end() || insert_loc->first != sec_name, "duplicate key: " << sec_name);
+        sec.m_entries.emplace_hint(insert_loc, std::move(sec_name), load_storage_entry());
       }
     }
     inline 
@@ -289,5 +344,12 @@ namespace epee
       RECURSION_LIMITATION();
       CHECK_AND_ASSERT_THROW_MES(false, "Reading array entry is not supported");
     }
+    inline
+    void throwable_buffer_reader::set_limits(size_t objects, size_t fields, size_t strings)
+    {
+      max_objects = objects;
+      max_fields = fields;
+      max_strings = strings;
+    }
   }
 }

contrib/epee/include/storages/portable_storage_template_helper.h

@@ -28,6 +28,7 @@
 
 #include <string>
 
+#include "byte_slice.h"
 #include "parserse_base_utils.h"
 #include "portable_storage.h"
 #include "file_io_utils.h"
@@ -84,10 +85,10 @@ namespace epee
     }
     //-----------------------------------------------------------------------------------------------------------
     template<class t_struct>
-    bool load_t_from_binary(t_struct& out, const epee::span<const uint8_t> binary_buff)
+    bool load_t_from_binary(t_struct& out, const epee::span<const uint8_t> binary_buff, const epee::serialization::portable_storage::limits_t *limits = NULL)
     {
       portable_storage ps;
-      bool rs = ps.load_from_binary(binary_buff);
+      bool rs = ps.load_from_binary(binary_buff, limits);
       if(!rs)
         return false;
 
@@ -111,18 +112,18 @@ namespace epee
     }
     //-----------------------------------------------------------------------------------------------------------
     template<class t_struct>
-    bool store_t_to_binary(t_struct& str_in, std::string& binary_buff, size_t indent = 0)
+    bool store_t_to_binary(t_struct& str_in, byte_slice& binary_buff, size_t initial_buffer_size = 8192)
     {
       portable_storage ps;
       str_in.store(ps);
-      return ps.store_to_binary(binary_buff);
+      return ps.store_to_binary(binary_buff, initial_buffer_size);
     }
     //-----------------------------------------------------------------------------------------------------------
     template<class t_struct>
-    std::string store_t_to_binary(t_struct& str_in, size_t indent = 0)
+    byte_slice store_t_to_binary(t_struct& str_in, size_t initial_buffer_size = 8192)
     {
-      std::string binary_buff;
-      store_t_to_binary(str_in, binary_buff, indent);
+      byte_slice binary_buff;
+      store_t_to_binary(str_in, binary_buff, initial_buffer_size);
       return binary_buff;
     }
   }

contrib/epee/include/storages/portable_storage_to_bin.h

@@ -211,6 +211,7 @@ namespace epee
       for(const section_pair& se: sec.m_entries)
       {
         CHECK_AND_ASSERT_THROW_MES(se.first.size() < std::numeric_limits<uint8_t>::max(), "storage_entry_name is too long: " << se.first.size() << ", val: " << se.first);
+        CHECK_AND_ASSERT_THROW_MES(!se.first.empty(), "storage_entry_name is empty");
         uint8_t len = static_cast<uint8_t>(se.first.size());
         strm.write((const char*)&len, sizeof(len));
         strm.write(se.first.data(), size_t(len));

contrib/epee/src/CMakeLists.txt

@@ -29,7 +29,7 @@
 
 add_library(epee STATIC byte_slice.cpp byte_stream.cpp hex.cpp abstract_http_client.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp
     wipeable_string.cpp levin_base.cpp memwipe.c connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp
-    int-util.cpp)
+    int-util.cpp portable_storage.cpp)
 
 if (USE_READLINE AND (GNU_READLINE_FOUND OR (DEPENDS AND NOT MINGW)))
   add_library(epee_readline STATIC readline_buffer.cpp)

contrib/epee/src/abstract_http_client.cpp

@@ -137,6 +137,11 @@ namespace http
     set_server(std::move(parsed.host), std::to_string(parsed.port), std::move(user), std::move(ssl_options));
     return true;
   }
+
+  bool epee::net_utils::http::abstract_http_client::set_proxy(const std::string& address)
+  {
+    return false;
+  }
 }
 }
 }

contrib/epee/src/byte_stream.cpp

@@ -34,6 +34,11 @@
 
 #include <iostream>
 
+namespace
+{
+  constexpr const std::size_t minimum_increase = 4096;
+}
+
 namespace epee
 {
   void byte_stream::overflow(const std::size_t requested)
@@ -46,7 +51,7 @@ namespace epee
 
     const std::size_t len = size();
     const std::size_t cap = capacity();
-    const std::size_t increase = std::max(need, increase_size());
+    const std::size_t increase = std::max(std::max(need, cap), minimum_increase);
 
     next_write_ = nullptr;
     end_ = nullptr;
@@ -62,8 +67,7 @@ namespace epee
   byte_stream::byte_stream(byte_stream&& rhs) noexcept
     : buffer_(std::move(rhs.buffer_)),
       next_write_(rhs.next_write_),
-      end_(rhs.end_),
-      increase_size_(rhs.increase_size_)
+      end_(rhs.end_)
   {
     rhs.next_write_ = nullptr;
     rhs.end_ = nullptr;
@@ -76,7 +80,6 @@ namespace epee
       buffer_ = std::move(rhs.buffer_);
       next_write_ = rhs.next_write_;
       end_ = rhs.end_;
-      increase_size_ = rhs.increase_size_;
       rhs.next_write_ = nullptr;
       rhs.end_ = nullptr;
     }

contrib/epee/src/net_ssl.cpp

@@ -473,6 +473,7 @@ bool ssl_options_t::has_fingerprint(boost::asio::ssl::verify_context &ctx) const
 bool ssl_options_t::handshake(
   boost::asio::ssl::stream<boost::asio::ip::tcp::socket> &socket,
   boost::asio::ssl::stream_base::handshake_type type,
+  boost::asio::const_buffer buffer,
   const std::string& host,
   std::chrono::milliseconds timeout) const
 {
@@ -530,7 +531,7 @@ bool ssl_options_t::handshake(
   });
 
   boost::system::error_code ec = boost::asio::error::would_block;
-  socket.async_handshake(type, boost::lambda::var(ec) = boost::lambda::_1);
+  socket.async_handshake(type, boost::asio::buffer(buffer), boost::lambda::var(ec) = boost::lambda::_1);
   if (io_service.stopped())
   {
     io_service.reset();

contrib/epee/src/net_utils_base.cpp

@@ -6,6 +6,17 @@
 #include "string_tools.h"
 #include "net/local_ip.h"
 
+static inline uint32_t make_address_v4_from_v6(const boost::asio::ip::address_v6& a)
+{
+  const auto &bytes = a.to_bytes();
+  uint32_t v4 = 0;
+  v4 = (v4 << 8) | bytes[12];
+  v4 = (v4 << 8) | bytes[13];
+  v4 = (v4 << 8) | bytes[14];
+  v4 = (v4 << 8) | bytes[15];
+  return htonl(v4);
+}
+
 namespace epee { namespace net_utils
 {
 	bool ipv4_network_address::equal(const ipv4_network_address& other) const noexcept
@@ -83,8 +94,28 @@ namespace epee { namespace net_utils
 		network_address::interface const* const other_self = other.self.get();
 		if (self_ == other_self) return true;
 		if (!self_ || !other_self) return false;
-		if (typeid(*self_) != typeid(*other_self)) return false;
-		return self_->is_same_host(*other_self);
+		if (typeid(*self_) == typeid(*other_self))
+			return self_->is_same_host(*other_self);
+		const auto this_id = get_type_id();
+		if (this_id == ipv4_network_address::get_type_id() && other.get_type_id() == ipv6_network_address::get_type_id())
+		{
+			const boost::asio::ip::address_v6 &actual_ip = other.as<const epee::net_utils::ipv6_network_address>().ip();
+			if (actual_ip.is_v4_mapped())
+			{
+				const uint32_t v4ip = make_address_v4_from_v6(actual_ip);
+				return is_same_host(ipv4_network_address(v4ip, 0));
+			}
+		}
+		else if (this_id == ipv6_network_address::get_type_id() && other.get_type_id() == ipv4_network_address::get_type_id())
+		{
+			const boost::asio::ip::address_v6 &actual_ip = this->as<const epee::net_utils::ipv6_network_address>().ip();
+			if (actual_ip.is_v4_mapped())
+			{
+				const uint32_t v4ip = make_address_v4_from_v6(actual_ip);
+				return other.is_same_host(ipv4_network_address(v4ip, 0));
+			}
+		}
+		return false;
 	}
 
   std::string print_connection_context(const connection_context_base& ctx)

contrib/epee/src/portable_storage.cpp

@@ -0,0 +1,29 @@
+
+#include "byte_slice.h"
+#include "byte_stream.h"
+#include "misc_log_ex.h"
+#include "span.h"
+#include "storages/portable_storage.h"
+#include "storages/portable_storage_to_bin.h"
+
+namespace epee
+{
+namespace serialization
+{
+  bool portable_storage::store_to_binary(byte_slice& target, const std::size_t initial_buffer_size)
+  {
+    TRY_ENTRY();
+    byte_stream ss;
+    ss.reserve(initial_buffer_size);
+    storage_block_header sbh{};
+    sbh.m_signature_a = SWAP32LE(PORTABLE_STORAGE_SIGNATUREA);
+    sbh.m_signature_b = SWAP32LE(PORTABLE_STORAGE_SIGNATUREB);
+    sbh.m_ver = PORTABLE_STORAGE_FORMAT_VER;
+    ss.write(epee::as_byte_span(sbh));
+    pack_entry_to_buff(ss, m_root);
+    target = epee::byte_slice{std::move(ss)};
+    return true;
+    CATCH_ENTRY("portable_storage::store_to_binary", false)
+  }
+}
+}

contrib/epee/src/readline_buffer.cpp

@@ -6,6 +6,7 @@
 #include <boost/thread/lock_guard.hpp>
 #include <boost/algorithm/string.hpp>
 
+static bool same_as_last_line(const std::string&);
 static void install_line_handler();
 static void remove_line_handler();
 
@@ -51,6 +52,7 @@ rdln::readline_buffer::readline_buffer()
 
 void rdln::readline_buffer::start()
 {
+  boost::lock_guard<boost::mutex> lock(sync_mutex);
   if(m_cout_buf != NULL)
     return;
   m_cout_buf = std::cout.rdbuf();
@@ -60,6 +62,7 @@ void rdln::readline_buffer::start()
 
 void rdln::readline_buffer::stop()
 {
+  boost::lock_guard<boost::mutex> lock(sync_mutex);
   if(m_cout_buf == NULL)
     return;
   std::cout.rdbuf(m_cout_buf);
@@ -88,9 +91,9 @@ rdln::linestatus rdln::readline_buffer::get_line(std::string& line) const
 
 void rdln::readline_buffer::set_prompt(const std::string& prompt)
 {
+  boost::lock_guard<boost::mutex> lock(sync_mutex);
   if(m_cout_buf == NULL)
     return;
-  boost::lock_guard<boost::mutex> lock(sync_mutex);
   rl_set_prompt(std::string(m_prompt_length, ' ').c_str());
   rl_redisplay();
   rl_set_prompt(prompt.c_str());
@@ -113,6 +116,12 @@ const std::vector<std::string>& rdln::readline_buffer::get_completions()
 int rdln::readline_buffer::sync()
 {
   boost::lock_guard<boost::mutex> lock(sync_mutex);
+
+  if (m_cout_buf == nullptr)
+  {
+    return -1;
+  }
+
 #if RL_READLINE_VERSION < 0x0700
   char lbuf[2] = {0,0};
   char *line = NULL;
@@ -167,8 +176,11 @@ static void handle_line(char* line)
     boost::trim_right(test_line);
     if(!test_line.empty())
     {
-      add_history(test_line.c_str());
-      history_set_pos(history_length);
+      if (!same_as_last_line(test_line))
+      {
+        add_history(test_line.c_str());
+        history_set_pos(history_length);
+      }
       if (test_line == "exit" || test_line == "q")
         exit = true;
     }
@@ -184,6 +196,16 @@ static void handle_line(char* line)
   return;
 }
 
+// same_as_last_line returns true, if the last line in the history is
+// equal to test_line.
+static bool same_as_last_line(const std::string& test_line)
+{
+  // Note that state->offset == state->length, when a new line was entered.
+  HISTORY_STATE* state = history_get_history_state();
+  return state->length > 0
+    && test_line.compare(state->entries[state->length-1]->line) == 0;
+}
+
 static char* completion_matches(const char* text, int state)
 {
   static size_t list_index;

contrib/fuzz_testing/fuzz.sh

@@ -14,8 +14,8 @@ then
   exit 1
 fi
 case "$type" in
-  block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof) ;;
-  *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof"; exit 1 ;;
+  block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof|utf8) ;;
+  *) echo "usage: $0 block|transaction|signature|cold-outputs|cold-transaction|load-from-binary|load-from-json|base58|parse-url|http-client|levin|bulletproof|utf8"; exit 1 ;;
 esac
 
 if test -d "fuzz-out/$type"

contrib/gitian/README.md

@@ -126,7 +126,7 @@ Setup for LXC:
 
 ```bash
 GH_USER=fluffypony
-VERSION=v0.15.0.0
+VERSION=v0.17.0.0
 
 ./gitian-build.py --setup $GH_USER $VERSION
 ```
@@ -182,12 +182,13 @@ If you chose to do detached signing using `--detach-sign` above (recommended), y
 
 ```bash
 GH_USER=fluffypony
-VERSION=v0.15.0.0
+VERSION=v0.17.0.0
 
 gpg --detach-sign ${VERSION}-linux/${GH_USER}/monero-linux-*-build.assert
 gpg --detach-sign ${VERSION}-win/${GH_USER}/monero-win-*-build.assert
 gpg --detach-sign ${VERSION}-osx/${GH_USER}/monero-osx-*-build.assert
 gpg --detach-sign ${VERSION}-android/${GH_USER}/monero-android-*-build.assert
+gpg --detach-sign ${VERSION}-freebsd/${GH_USER}/monero-freebsd-*-build.assert
 ```
 
 This will create a `.sig` file for each `.assert` file above (2 files for each platform).

contrib/gitian/gitian-android.yml

@@ -1,5 +1,5 @@
 ---
-name: "wownero-android-0.8"
+name: "wownero-android-0.9"
 enable_cache: true
 suites:
 - "bionic"
@@ -32,7 +32,7 @@ packages:
 - "python3-zmq"
 - "unzip"
 remotes:
-- "url": "https://github.com/wownero/wownero.git"
+- "url": "https://git.wownero.com/wownero/wownero.git"
   "dir": "wownero"
 files: []
 script: |

contrib/gitian/gitian-build.py

@@ -5,8 +5,8 @@ import os
 import subprocess
 import sys
 
-gsigs = 'https://github.com/wownero/gitian.sigs.git'
-gbrepo = 'https://github.com/wownero/gitian-builder.git'
+gsigs = 'https://git.wownero.com/qvqc/gitian.sigs.git'
+gbrepo = 'https://git.wownero.com/qvqc/gitian-builder.git'
 
 platforms = {'l': ['Linux', 'linux', 'tar.bz2'],
         'a': ['Android', 'android', 'tar.bz2'],
@@ -118,7 +118,7 @@ def main():
     parser = argparse.ArgumentParser(description='Script for running full Gitian builds.', usage='%(prog)s [options] signer version')
     parser.add_argument('-c', '--commit', action='store_true', dest='commit', help='Indicate that the version argument is for a commit or branch')
     parser.add_argument('-p', '--pull', action='store_true', dest='pull', help='Indicate that the version argument is the number of a github repository pull request')
-    parser.add_argument('-u', '--url', dest='url', default='https://github.com/wownero/wownero', help='Specify the URL of the repository. Default is %(default)s')
+    parser.add_argument('-u', '--url', dest='url', default='https://git.wownero.com/wownero/wownero', help='Specify the URL of the repository. Default is %(default)s')
     parser.add_argument('-v', '--verify', action='store_true', dest='verify', help='Verify the Gitian build')
     parser.add_argument('-b', '--build', action='store_true', dest='build', help='Do a Gitian build')
     parser.add_argument('-B', '--buildsign', action='store_true', dest='buildsign', help='Build both signed and unsigned binaries')

contrib/gitian/gitian-freebsd.yml

@@ -1,5 +1,5 @@
 ---
-name: "wownero-freebsd-0.8"
+name: "wownero-freebsd-0.9"
 enable_cache: true
 suites:
 - "bionic"
@@ -32,7 +32,7 @@ packages:
 - "libprotobuf-dev"
 - "python3-zmq"
 remotes:
-- "url": "https://github.com/wownero/wownero.git"
+- "url": "https://git.wownero.com/wownero/wownero.git"
   "dir": "wownero"
 files: []
 script: |
@@ -119,7 +119,7 @@ script: |
   for i in ${HOSTS}; do
     export PATH=${WRAP_DIR}:${BASEPREFIX}/${i}/native/bin:${ORIGPATH}
     mkdir build && cd build
-    cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DCMAKE_BUILD_TYPE=Release
+    cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_SKIP_RPATH=ON
     make ${MAKEOPTS}
     chmod 755 bin/*
     cp ../LICENSE bin

contrib/gitian/gitian-linux.yml

@@ -1,5 +1,5 @@
 ---
-name: "wownero-linux-0.8"
+name: "wownero-linux-0.9"
 enable_cache: true
 suites:
 - "bionic"
@@ -43,13 +43,13 @@ packages:
 - "libprotobuf-dev"
 - "python3-zmq"
 remotes:
-- "url": "https://github.com/wownero/wownero.git"
+- "url": "https://git.wownero.com/wownero/wownero.git"
   "dir": "wownero"
 files: []
 script: |
 
   WRAP_DIR=$HOME/wrapped
-  HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu i686-linux-gnu"
+  HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu"
   FAKETIME_HOST_PROGS=""
   FAKETIME_PROGS="date"
   HOST_CFLAGS="-O2 -g"
@@ -111,6 +111,11 @@ script: |
   rm -f $WRAP_DIR/extra_includes/i686-linux-gnu/asm
   ln -s /usr/include/x86_64-linux-gnu/asm $EXTRA_INCLUDES_BASE/i686-linux-gnu/asm
 
+  # glibc 2.23 breaks compatibility with <=2.19 use of lgamma function.
+  # Hack the math header to restore the old behavior.
+  mkdir $EXTRA_INCLUDES_BASE/bits
+  sed -e '/__REDIRFROM .lgamma,/,+3s/_USE_/_DONTUSE_/g' /usr/include/x86_64-linux-gnu/bits/math-finite.h > $EXTRA_INCLUDES_BASE/bits/math-finite.h
+
   # gcc 7+ honors SOURCE_DATE_EPOCH, no faketime needed
   export SOURCE_DATE_EPOCH=`date -d 2000-01-01T12:00:00 +%s`
 
@@ -127,14 +132,14 @@ script: |
   # Build dependencies for each host
   export TAR_OPTIONS=--mtime=2000-01-01T12:00:00
   for i in $HOSTS; do
-    EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i"
-    if [ -d "$EXTRA_INCLUDES" ]; then
-      export C_INCLUDE_PATH="$EXTRA_INCLUDES"
-      export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES"
+    ARCH_INCLUDES="$EXTRA_INCLUDES_BASE/$i"
+    if [ -d "$ARCH_INCLUDES" ]; then
+      EXTRA_INCLUDES="${EXTRA_INCLUDES_BASE}:${ARCH_INCLUDES}"
     else
-      unset C_INCLUDE_PATH
-      unset CPLUS_INCLUDE_PATH
+      EXTRA_INCLUDES="${EXTRA_INCLUDES_BASE}"
     fi
+    export C_INCLUDE_PATH="$EXTRA_INCLUDES"
+    export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES"
     make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" V=1
   done
 
@@ -151,15 +156,15 @@ script: |
   for i in ${HOSTS}; do
     export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH}
     mkdir build && cd build
-    EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i"
-    if [ -d "$EXTRA_INCLUDES" ]; then
-      export C_INCLUDE_PATH="$EXTRA_INCLUDES"
-      export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES"
+    ARCH_INCLUDES="$EXTRA_INCLUDES_BASE/$i"
+    if [ -d "$ARCH_INCLUDES" ]; then
+      EXTRA_INCLUDES="${EXTRA_INCLUDES_BASE}:${ARCH_INCLUDES}"
     else
-      unset C_INCLUDE_PATH
-      unset CPLUS_INCLUDE_PATH
+      EXTRA_INCLUDES="${EXTRA_INCLUDES_BASE}"
     fi
-    cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=ON
+    export C_INCLUDE_PATH="$EXTRA_INCLUDES"
+    export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES"
+    cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=ON -DCMAKE_SKIP_RPATH=ON
     make ${MAKEOPTS}
     chmod 755 bin/*
     cp ../LICENSE bin

contrib/gitian/gitian-osx.yml

@@ -1,5 +1,5 @@
 ---
-name: "wownero-osx-0.8"
+name: "wownero-osx-0.9"
 enable_cache: true
 suites:
 - "bionic"
@@ -24,7 +24,7 @@ packages:
 - "python-dev"
 - "python-setuptools"
 remotes:
-- "url": "https://github.com/wownero/wownero.git"
+- "url": "https://git.wownero.com/wownero/wownero.git"
   "dir": "wownero"
 files:
 - "MacOSX10.11.sdk.tar.gz"

contrib/gitian/gitian-win.yml

@@ -1,5 +1,5 @@
 ---
-name: "wownero-win-0.8"
+name: "wownero-win-0.9"
 enable_cache: true
 suites:
 - "bionic"
@@ -36,12 +36,12 @@ alternatives:
   package: "x86_64-w64-mingw32-gcc"
   path: "/usr/bin/x86_64-w64-mingw32-gcc-posix"
 remotes:
-- "url": "https://github.com/wownero/wownero.git"
+- "url": "https://git.wownero.com/wownero/wownero.git"
   "dir": "wownero"
 files: []
 script: |
   WRAP_DIR=$HOME/wrapped
-  HOSTS="i686-w64-mingw32 x86_64-w64-mingw32"
+  HOSTS="x86_64-w64-mingw32"
   FAKETIME_HOST_PROGS="windres objcopy"
   FAKETIME_PROGS="date zip"
   HOST_CFLAGS="-O2 -g"

external/CMakeLists.txt

@@ -34,42 +34,22 @@
 # We always compile if we are building statically to reduce static dependency issues...
 # ...except for FreeBSD, because FreeBSD is a special case that doesn't play well with
 # others.
-if(NOT IOS)
-    find_package(Miniupnpc QUIET)
+
+find_package(Miniupnpc REQUIRED)
+
+message(STATUS "Using in-tree miniupnpc")
+add_subdirectory(miniupnp/miniupnpc)
+set_property(TARGET libminiupnpc-static PROPERTY FOLDER "external")
+if(MSVC)
+  set_property(TARGET libminiupnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " -wd4244 -wd4267")
+elseif(NOT MSVC)
+  set_property(TARGET libminiupnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-undef -Wno-unused-result -Wno-unused-value")
+endif()
+if(CMAKE_SYSTEM_NAME MATCHES "NetBSD")
+	set_property(TARGET libminiupnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " -D_NETBSD_SOURCE")
 endif()
 
-# If we have the correct shared version and we're not building static, use it
-if(STATIC OR IOS)
- set(USE_SHARED_MINIUPNPC false)
-elseif(MINIUPNP_FOUND AND MINIUPNPC_VERSION_1_7_OR_HIGHER)
- set(USE_SHARED_MINIUPNPC true)
-endif()
-
-if(USE_SHARED_MINIUPNPC)
-  message(STATUS "Using shared miniupnpc found at ${MINIUPNP_INCLUDE_DIR}")
-
-  set(UPNP_STATIC false PARENT_SCOPE)
-  set(UPNP_INCLUDE ${MINIUPNP_INCLUDE_DIR} PARENT_SCOPE)
-  set(UPNP_LIBRARIES ${MINIUPNP_LIBRARY} PARENT_SCOPE)
-else()
-  if(STATIC)
-    message(STATUS "Using miniupnpc from local source tree for static build")
-  else()
-    message(STATUS "Using miniupnpc from local source tree (/external/miniupnp/miniupnpc)")
-  endif()
-
-  add_subdirectory(miniupnp/miniupnpc)
-
-  set_property(TARGET libminiupnpc-static PROPERTY FOLDER "external")
-  if(MSVC)
-    set_property(TARGET libminiupnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " -wd4244 -wd4267")
-  elseif(NOT MSVC)
-    set_property(TARGET libminiupnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-undef -Wno-unused-result -Wno-unused-value")
-  endif()
-
-  set(UPNP_STATIC true PARENT_SCOPE)
-  set(UPNP_LIBRARIES "libminiupnpc-static" PARENT_SCOPE)
-endif()
+set(UPNP_LIBRARIES "libminiupnpc-static" PARENT_SCOPE)
 
 find_package(Unbound)
 

external/easylogging++/ea_config.h

@@ -11,7 +11,7 @@
 #define ELPP_UTC_DATETIME
 
 #ifdef EASYLOGGING_CC
-#if !(!defined __GLIBC__ || !defined __GNUC__ || defined __MINGW32__ || defined __MINGW64__ || defined __ANDROID__)
+#if !(!defined __GLIBC__ || !defined __GNUC__ || defined __MINGW32__ || defined __MINGW64__ || defined __ANDROID__ || defined __NetBSD__)
 #define ELPP_FEATURE_CRASH_LOG
 #endif
 #endif

external/easylogging++/easylogging++.cc

@@ -2968,6 +2968,16 @@ void Writer::initializeLogger(Logger *logger, bool needLock) {
 }
 
 void Writer::processDispatch() {
+  static std::atomic_flag in_dispatch;
+  if (in_dispatch.test_and_set())
+  {
+    if (m_proceed && m_logger != NULL)
+    {
+      m_logger->stream().str(ELPP_LITERAL(""));
+      m_logger->releaseLock();
+    }
+    return;
+  }
 #if ELPP_LOGGING_ENABLED
   if (ELPP->hasFlag(LoggingFlag::MultiLoggerSupport)) {
     bool firstDispatched = false;
@@ -3006,6 +3016,7 @@ void Writer::processDispatch() {
     m_logger->releaseLock();
   }
 #endif // ELPP_LOGGING_ENABLED
+  in_dispatch.clear();
 }
 
 void Writer::triggerDispatch(void) {

external/qrcodegen/CMakeLists.txt

@@ -1,7 +1,8 @@
 project(libqrcodegen)
 
 add_library(qrcodegen STATIC QrCode.cpp)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
+set_target_properties(qrcodegen PROPERTIES POSITION_INDEPENDENT_CODE ON)
+set_target_properties(qrcodegen PROPERTIES CXX_STANDARD 11)
 
 target_include_directories(qrcodegen PUBLIC 
     ${CMAKE_CURRENT_SOURCE_DIR})

src/CMakeLists.txt

@@ -75,6 +75,8 @@ function (monero_add_executable name)
     PROPERTY
       RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
   enable_stack_trace("${name}")
+  
+  monero_set_target_no_relink("${name}")
 endfunction ()
 
 function (monero_add_library name)
@@ -92,6 +94,7 @@ function (monero_add_library_with_deps)
   set(objlib obj_${MONERO_ADD_LIBRARY_NAME})
   add_library(${objlib} OBJECT ${MONERO_ADD_LIBRARY_SOURCES})
   add_library("${MONERO_ADD_LIBRARY_NAME}" $<TARGET_OBJECTS:${objlib}>)
+  monero_set_target_no_relink("${MONERO_ADD_LIBRARY_NAME}")
   if (MONERO_ADD_LIBRARY_DEPENDS)
     add_dependencies(${objlib} ${MONERO_ADD_LIBRARY_DEPENDS})
   endif()

src/blockchain_db/blockchain_db.cpp

@@ -63,6 +63,7 @@ bool matches_category(relay_method method, relay_category category) noexcept
   {
     default:
     case relay_method::local:
+    case relay_method::forward:
     case relay_method::stem:
       return false;
     case relay_method::block:
@@ -79,6 +80,7 @@ void txpool_tx_meta_t::set_relay_method(relay_method method) noexcept
   kept_by_block = 0;
   do_not_relay = 0;
   is_local = 0;
+  is_forwarding = 0;
   dandelionpp_stem = 0;
 
   switch (method)
@@ -89,8 +91,8 @@ void txpool_tx_meta_t::set_relay_method(relay_method method) noexcept
     case relay_method::local:
       is_local = 1;
       break;
-    default:
-    case relay_method::fluff:
+    case relay_method::forward:
+      is_forwarding = 1;
       break;
     case relay_method::stem:
       dandelionpp_stem = 1;
@@ -98,26 +100,45 @@ void txpool_tx_meta_t::set_relay_method(relay_method method) noexcept
     case relay_method::block:
       kept_by_block = 1;
       break;
+    default:
+    case relay_method::fluff:
+      break;
   }
 }
 
 relay_method txpool_tx_meta_t::get_relay_method() const noexcept
 {
-  if (kept_by_block)
-    return relay_method::block;
-  if (do_not_relay)
-    return relay_method::none;
-  if (is_local)
-    return relay_method::local;
-  if (dandelionpp_stem)
-    return relay_method::stem;
+  const uint8_t state =
+    uint8_t(kept_by_block) +
+    (uint8_t(do_not_relay) << 1) +
+    (uint8_t(is_local) << 2) +
+    (uint8_t(is_forwarding) << 3) +
+    (uint8_t(dandelionpp_stem) << 4);
+
+  switch (state)
+  {
+    default: // error case
+    case 0:
+      break;
+    case 1:
+      return relay_method::block;
+    case 2:
+      return relay_method::none;
+    case 4:
+      return relay_method::local;
+    case 8:
+      return relay_method::forward;
+    case 16:
+      return relay_method::stem;
+  };
   return relay_method::fluff;
 }
 
 bool txpool_tx_meta_t::upgrade_relay_method(relay_method method) noexcept
 {
   static_assert(relay_method::none < relay_method::local, "bad relay_method value");
-  static_assert(relay_method::local < relay_method::stem, "bad relay_method value");
+  static_assert(relay_method::local < relay_method::forward, "bad relay_method value");
+  static_assert(relay_method::forward < relay_method::stem, "bad relay_method value");
   static_assert(relay_method::stem < relay_method::fluff, "bad relay_method value");
   static_assert(relay_method::fluff < relay_method::block, "bad relay_method value");
 
@@ -158,7 +179,7 @@ void BlockchainDB::pop_block()
   pop_block(blk, txs);
 }
 
-void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& txp, const crypto::hash* tx_hash_ptr, const crypto::hash* tx_prunable_hash_ptr)
+void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& txp, const crypto::hash* tx_hash_ptr, const crypto::hash* tx_prunable_hash_ptr)
 {
   const transaction &tx = txp.first;
 
@@ -260,12 +281,13 @@ uint64_t BlockchainDB::add_block( const std::pair<block, blobdata>& blck
   time1 = epee::misc_utils::get_tick_count();
 
   uint64_t num_rct_outs = 0;
-  add_transaction(blk_hash, std::make_pair(blk.miner_tx, tx_to_blob(blk.miner_tx)));
+  blobdata miner_bd = tx_to_blob(blk.miner_tx);
+  add_transaction(blk_hash, std::make_pair(blk.miner_tx, blobdata_ref(miner_bd)));
   if (blk.miner_tx.version == 2)
     num_rct_outs += blk.miner_tx.vout.size();
   int tx_i = 0;
   crypto::hash tx_hash = crypto::null_hash;
-  for (const std::pair<transaction, blobdata>& tx : txs)
+  for (const std::pair<transaction, blobdata_ref>& tx : txs)
   {
     tx_hash = blk.tx_hashes[tx_i];
     add_transaction(blk_hash, tx, &tx_hash);

src/blockchain_db/blockchain_db.h

@@ -160,7 +160,7 @@ struct txpool_tx_meta_t
   uint64_t max_used_block_height;
   uint64_t last_failed_height;
   uint64_t receive_time;
-  uint64_t last_relayed_time; //!< If Dandelion++ stem, randomized embargo timestamp. Otherwise, last relayed timestmap.
+  uint64_t last_relayed_time; //!< If received over i2p/tor, randomized forward time. If Dandelion++stem, randomized embargo time. Otherwise, last relayed timestamp
   // 112 bytes
   uint8_t kept_by_block;
   uint8_t relayed;
@@ -169,7 +169,8 @@ struct txpool_tx_meta_t
   uint8_t pruned: 1;
   uint8_t is_local: 1;
   uint8_t dandelionpp_stem : 1;
-  uint8_t bf_padding: 4;
+  uint8_t is_forwarding: 1;
+  uint8_t bf_padding: 3;
 
   uint8_t padding[76]; // till 192 bytes
 
@@ -439,7 +440,7 @@ private:
    * @param tx_prunable_hash the hash of the prunable part of the transaction
    * @return the transaction ID
    */
-  virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) = 0;
+  virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) = 0;
 
   /**
    * @brief remove data about a transaction
@@ -567,7 +568,7 @@ protected:
    * @param tx_hash_ptr the hash of the transaction, if already calculated
    * @param tx_prunable_hash_ptr the hash of the prunable part of the transaction, if already calculated
    */
-  void add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& tx, const crypto::hash* tx_hash_ptr = NULL, const crypto::hash* tx_prunable_hash_ptr = NULL);
+  void add_transaction(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& tx, const crypto::hash* tx_hash_ptr = NULL, const crypto::hash* tx_prunable_hash_ptr = NULL);
 
   mutable uint64_t time_tx_exists = 0;  //!< a performance metric
   uint64_t time_commit1 = 0;  //!< a performance metric
@@ -1311,17 +1312,20 @@ public:
    * height. The number of blocks returned is variable, based on the max_size passed.
    *
    * @param start_height the height of the first block
-   * @param min_count the minimum number of blocks to return, if they exist
-   * @param max_count the maximum number of blocks to return
+   * @param min_block_count the minimum number of blocks to return, if they exist
+   * @param max_block_count the maximum number of blocks to return
+   * @param max_tx_count the maximum number of txes to return
    * @param max_size the maximum size of block/transaction data to return (will be exceeded by one blocks's worth at most, if min_count is met)
    * @param blocks the returned block/transaction data
    * @param pruned whether to return full or pruned tx data
    * @param skip_coinbase whether to return or skip coinbase transactions (they're in blocks regardless)
    * @param get_miner_tx_hash whether to calculate and return the miner (coinbase) tx hash
    *
+   * The call will return at least min_block_count if possible, even if this contravenes max_tx_count
+   *
    * @return true iff the blocks and transactions were found
    */
-  virtual bool get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const = 0;
+  virtual bool get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const = 0;
 
   /**
    * @brief fetches the prunable transaction blob with the given hash
@@ -1523,7 +1527,7 @@ public:
    *
    * @param details the details of the transaction to add
    */
-  virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t& details) = 0;
+  virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata_ref &blob, const txpool_tx_meta_t& details) = 0;
 
   /**
    * @brief update a txpool transaction's metadata
@@ -1643,7 +1647,7 @@ public:
    * @param: data: the metadata for the block
    * @param: blob: the block's blob
    */
-  virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata &blob) = 0;
+  virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata_ref &blob) = 0;
 
   /**
    * @brief get an alternative block by hash
@@ -1686,7 +1690,7 @@ public:
    *
    * @return false if the function returns false for any transaction, otherwise true
    */
-  virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false, relay_category category = relay_category::broadcasted) const = 0;
+  virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata_ref*)>, bool include_blob = false, relay_category category = relay_category::broadcasted) const = 0;
 
   /**
    * @brief runs a function over all key images stored
@@ -1778,7 +1782,7 @@ public:
    *
    * @return false if the function returns false for any output, otherwise true
    */
-  virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata *blob)> f, bool include_blob = false) const = 0;
+  virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata_ref *blob)> f, bool include_blob = false) const = 0;
 
 
   //

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -856,7 +856,7 @@ void BlockchainLMDB::remove_block()
       throw1(DB_ERROR(lmdb_error("Failed to add removal of block info to db transaction: ", result).c_str()));
 }
 
-uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& txp, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash)
+uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& txp, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -896,7 +896,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons
   if (result)
     throw0(DB_ERROR(lmdb_error("Failed to add tx data to db transaction: ", result).c_str()));
 
-  const cryptonote::blobdata &blob = txp.second;
+  const cryptonote::blobdata_ref &blob = txp.second;
   MDB_val_sized(blobval, blob);
 
   unsigned int unprunable_size = tx.unprunable_size;
@@ -1756,7 +1756,7 @@ void BlockchainLMDB::unlock()
       auto_txn.commit(); \
   } while(0)
 
-void BlockchainLMDB::add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t &meta)
+void BlockchainLMDB::add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata_ref &blob, const txpool_tx_meta_t &meta)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -2308,7 +2308,7 @@ bool BlockchainLMDB::check_pruning()
   return prune_worker(prune_mode_check, 0);
 }
 
-bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob, relay_category category) const
+bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata_ref*)> f, bool include_blob, relay_category category) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -2334,8 +2334,7 @@ bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&,
     const txpool_tx_meta_t &meta = *(const txpool_tx_meta_t*)v.mv_data;
     if (!meta.matches(category))
       continue;
-    const cryptonote::blobdata *passed_bd = NULL;
-    cryptonote::blobdata bd;
+    cryptonote::blobdata_ref bd;
     if (include_blob)
     {
       MDB_val b;
@@ -2344,11 +2343,10 @@ bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&,
         throw0(DB_ERROR("Failed to find txpool tx blob to match metadata"));
       if (result)
         throw0(DB_ERROR(lmdb_error("Failed to enumerate txpool tx blob: ", result).c_str()));
-      bd.assign(reinterpret_cast<const char*>(b.mv_data), b.mv_size);
-      passed_bd = &bd;
+      bd = {reinterpret_cast<const char*>(b.mv_data), b.mv_size};
     }
 
-    if (!f(txid, meta, passed_bd)) {
+    if (!f(txid, meta, &bd)) {
       ret = false;
       break;
     }
@@ -2359,7 +2357,7 @@ bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&,
   return ret;
 }
 
-bool BlockchainLMDB::for_all_alt_blocks(std::function<bool(const crypto::hash&, const alt_block_data_t&, const cryptonote::blobdata*)> f, bool include_blob) const
+bool BlockchainLMDB::for_all_alt_blocks(std::function<bool(const crypto::hash&, const alt_block_data_t&, const cryptonote::blobdata_ref*)> f, bool include_blob) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -2384,15 +2382,13 @@ bool BlockchainLMDB::for_all_alt_blocks(std::function<bool(const crypto::hash&,
     if (v.mv_size < sizeof(alt_block_data_t))
       throw0(DB_ERROR("alt_blocks record is too small"));
     const alt_block_data_t *data = (const alt_block_data_t*)v.mv_data;
-    const cryptonote::blobdata *passed_bd = NULL;
-    cryptonote::blobdata bd;
+    cryptonote::blobdata_ref bd;
     if (include_blob)
     {
-      bd.assign(reinterpret_cast<const char*>(v.mv_data) + sizeof(alt_block_data_t), v.mv_size - sizeof(alt_block_data_t));
-      passed_bd = &bd;
+      bd = {reinterpret_cast<const char*>(v.mv_data) + sizeof(alt_block_data_t), v.mv_size - sizeof(alt_block_data_t)};
     }
 
-    if (!f(blkid, *data, passed_bd)) {
+    if (!f(blkid, *data, &bd)) {
       ret = false;
       break;
     }
@@ -3005,7 +3001,7 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const
 
   if (! tx_found)
   {
-    LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
+    LOG_PRINT_L3("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
     return false;
   }
 
@@ -3036,7 +3032,7 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h, uint64_t& tx_id) const
   bool ret = false;
   if (get_result == MDB_NOTFOUND)
   {
-    LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
+    LOG_PRINT_L3("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
   }
   else if (get_result)
     throw0(DB_ERROR(lmdb_error("DB error attempting to fetch transaction from hash", get_result).c_str()));
@@ -3175,7 +3171,7 @@ bool BlockchainLMDB::get_pruned_tx_blobs_from(const crypto::hash& h, size_t coun
   return true;
 }
 
-bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const
+bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -3189,14 +3185,15 @@ bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_count, si
     RCURSOR(txs_prunable);
   }
 
-  blocks.reserve(std::min<size_t>(max_count, 10000)); // guard against very large max count if only checking bytes
+  blocks.reserve(std::min<size_t>(max_block_count, 10000)); // guard against very large max count if only checking bytes
   const uint64_t blockchain_height = height();
   uint64_t size = 0;
+  size_t num_txes = 0;
   MDB_val_copy<uint64_t> key(start_height);
   MDB_val k, v, val_tx_id;
   uint64_t tx_id = ~0;
   MDB_cursor_op op = MDB_SET;
-  for (uint64_t h = start_height; h < blockchain_height && blocks.size() < max_count && (size < max_size || blocks.size() < min_count); ++h)
+  for (uint64_t h = start_height; h < blockchain_height && blocks.size() < max_block_count && (size < max_size || blocks.size() < min_block_count); ++h)
   {
     MDB_cursor_op op = h == start_height ? MDB_SET : MDB_NEXT;
     int result = mdb_cursor_get(m_cur_blocks, &key, &v, op);
@@ -3247,6 +3244,7 @@ bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_count, si
     op = MDB_NEXT;
 
     current_block.second.reserve(b.tx_hashes.size());
+    num_txes += b.tx_hashes.size() + (skip_coinbase ? 0 : 1);
     for (const auto &tx_hash: b.tx_hashes)
     {
       // get pruned data
@@ -3266,6 +3264,9 @@ bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_count, si
       current_block.second.push_back(std::make_pair(tx_hash, std::move(tx_blob)));
       size += current_block.second.back().second.size();
     }
+
+    if (blocks.size() >= min_block_count && num_txes >= max_tx_count)
+      break;
   }
 
   TXN_POSTFIX_RDONLY();
@@ -3604,8 +3605,7 @@ bool BlockchainLMDB::for_blocks_range(const uint64_t& h1, const uint64_t& h2, st
     if (ret)
       throw0(DB_ERROR("Failed to enumerate blocks"));
     uint64_t height = *(const uint64_t*)k.mv_data;
-    blobdata bd;
-    bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
+    blobdata_ref bd{reinterpret_cast<char*>(v.mv_data), v.mv_size};
     block b;
     if (!parse_and_validate_block_from_blob(bd, b))
       throw0(DB_ERROR("Failed to parse block from blob retrieved from the db"));
@@ -3660,15 +3660,16 @@ bool BlockchainLMDB::for_all_transactions(std::function<bool(const crypto::hash&
     if (ret)
       throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str()));
     transaction tx;
-    blobdata bd;
-    bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
     if (pruned)
     {
+      blobdata_ref bd{reinterpret_cast<char*>(v.mv_data), v.mv_size};
       if (!parse_and_validate_tx_base_from_blob(bd, tx))
         throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db"));
     }
     else
     {
+      blobdata bd;
+      bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
       ret = mdb_cursor_get(m_cur_txs_prunable, &k, &v, MDB_SET);
       if (ret)
         throw0(DB_ERROR(lmdb_error("Failed to get prunable tx data the db: ", ret).c_str()));
@@ -4402,7 +4403,7 @@ uint8_t BlockchainLMDB::get_hard_fork_version(uint64_t height) const
   return ret;
 }
 
-void BlockchainLMDB::add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata &blob)
+void BlockchainLMDB::add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata_ref &blob)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -4969,7 +4970,7 @@ void BlockchainLMDB::migrate_0_1()
     }
 
     MDB_dbi o_txs;
-    blobdata bd;
+    blobdata_ref bd;
     block b;
     MDB_val hk;
 
@@ -5051,7 +5052,7 @@ void BlockchainLMDB::migrate_0_1()
       } else if (result)
         throw0(DB_ERROR(lmdb_error("Failed to get a record from blocks: ", result).c_str()));
 
-      bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
+      bd = {reinterpret_cast<char*>(v.mv_data), v.mv_size};
       if (!parse_and_validate_block_from_blob(bd, b))
         throw0(DB_ERROR("Failed to parse block from blob retrieved from the db"));
 
@@ -5062,7 +5063,7 @@ void BlockchainLMDB::migrate_0_1()
         result = mdb_cursor_get(c_txs, &hk, &v, MDB_SET);
         if (result)
           throw0(DB_ERROR(lmdb_error("Failed to get record from txs: ", result).c_str()));
-        bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
+        bd = {reinterpret_cast<char*>(v.mv_data), v.mv_size};
         if (!parse_and_validate_tx_from_blob(bd, tx))
           throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db"));
         add_transaction(null_hash, std::make_pair(std::move(tx), bd), &b.tx_hashes[j]);
@@ -5184,8 +5185,7 @@ void BlockchainLMDB::migrate_1_2()
       else if (result)
         throw0(DB_ERROR(lmdb_error("Failed to get a record from txs: ", result).c_str()));
 
-      cryptonote::blobdata bd;
-      bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size);
+      cryptonote::blobdata bd{reinterpret_cast<char*>(v.mv_data), v.mv_size};
       transaction tx;
       if (!parse_and_validate_tx_from_blob(bd, tx))
         throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db"));

src/blockchain_db/lmdb/db_lmdb.h

@@ -257,7 +257,7 @@ public:
   virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const;
   virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const;
   virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<cryptonote::blobdata> &bd) const;
-  virtual bool get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const;
+  virtual bool get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const;
   virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const;
   virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const;
 
@@ -283,7 +283,7 @@ public:
 
   virtual bool has_key_image(const crypto::key_image& img) const;
 
-  virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t& meta);
+  virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata_ref &blob, const txpool_tx_meta_t& meta);
   virtual void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t& meta);
   virtual uint64_t get_txpool_tx_count(relay_category category = relay_category::broadcasted) const;
   virtual bool txpool_has_tx(const crypto::hash &txid, relay_category tx_category) const;
@@ -296,20 +296,20 @@ public:
   virtual bool update_pruning();
   virtual bool check_pruning();
 
-  virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata &blob);
+  virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata_ref &blob);
   virtual bool get_alt_block(const crypto::hash &blkid, alt_block_data_t *data, cryptonote::blobdata *blob);
   virtual void remove_alt_block(const crypto::hash &blkid);
   virtual uint64_t get_alt_block_count();
   virtual void drop_alt_blocks();
 
-  virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob = false, relay_category category = relay_category::broadcasted) const;
+  virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata_ref*)> f, bool include_blob = false, relay_category category = relay_category::broadcasted) const;
 
   virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const;
   virtual bool for_blocks_range(const uint64_t& h1, const uint64_t& h2, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const;
   virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>, bool pruned) const;
   virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const;
   virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const;
-  virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata *blob)> f, bool include_blob = false) const;
+  virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata_ref *blob)> f, bool include_blob = false) const;
 
   virtual uint64_t add_block( const std::pair<block, blobdata>& blk
                             , size_t block_weight
@@ -376,7 +376,7 @@ private:
 
   virtual void remove_block();
 
-  virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash);
+  virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<transaction, blobdata_ref>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash);
 
   virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx);
 

src/blockchain_db/testdb.h

@@ -70,7 +70,7 @@ public:
   virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; }
   virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; }
   virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<cryptonote::blobdata> &bd) const override { return false; }
-  virtual bool get_blocks_from(uint64_t start_height, size_t min_count, size_t max_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const override { return false; }
+  virtual bool get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const override { return false; }
   virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; }
   virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const override { return false; }
   virtual uint64_t get_block_height(const crypto::hash& h) const override { return 0; }
@@ -111,7 +111,7 @@ public:
   virtual std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(const uint64_t tx_index, size_t n_txes) const override { return std::vector<std::vector<uint64_t>>(); }
   virtual bool has_key_image(const crypto::key_image& img) const override { return false; }
   virtual void remove_block() override { }
-  virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<cryptonote::transaction, cryptonote::blobdata>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) override {return 0;}
+  virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const std::pair<cryptonote::transaction, cryptonote::blobdata_ref>& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) override {return 0;}
   virtual void remove_transaction_data(const crypto::hash& tx_hash, const cryptonote::transaction& tx) override {}
   virtual uint64_t add_output(const crypto::hash& tx_hash, const cryptonote::tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) override {return 0;}
   virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector<uint64_t>& amount_output_indices) override {}
@@ -127,7 +127,7 @@ public:
   virtual std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> get_output_histogram(const std::vector<uint64_t> &amounts, bool unlocked, uint64_t recent_cutoff, uint64_t min_count) const override { return std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>>(); }
   virtual bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, std::vector<uint64_t> &distribution, uint64_t &base) const override { return false; }
 
-  virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const cryptonote::txpool_tx_meta_t& details) override {}
+  virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata_ref &blob, const cryptonote::txpool_tx_meta_t& details) override {}
   virtual void update_txpool_tx(const crypto::hash &txid, const cryptonote::txpool_tx_meta_t& details) override {}
   virtual uint64_t get_txpool_tx_count(relay_category tx_relay = relay_category::broadcasted) const override { return 0; }
   virtual bool txpool_has_tx(const crypto::hash &txid, relay_category tx_category) const override { return  false; }
@@ -136,7 +136,7 @@ public:
   virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd, relay_category tx_category) const override { return false; }
   virtual uint64_t get_database_size() const override { return 0; }
   virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid, relay_category tx_category) const override { return ""; }
-  virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const cryptonote::txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false, relay_category category = relay_category::broadcasted) const override { return false; }
+  virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const cryptonote::txpool_tx_meta_t&, const cryptonote::blobdata_ref*)>, bool include_blob = false, relay_category category = relay_category::broadcasted) const override { return false; }
 
   virtual void add_block( const cryptonote::block& blk
                         , size_t block_weight
@@ -160,12 +160,12 @@ public:
   virtual uint64_t get_max_block_size() override { return 100000000; }
   virtual void add_max_block_size(uint64_t sz) override { }
 
-  virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata &blob) override {}
+  virtual void add_alt_block(const crypto::hash &blkid, const cryptonote::alt_block_data_t &data, const cryptonote::blobdata_ref &blob) override {}
   virtual bool get_alt_block(const crypto::hash &blkid, alt_block_data_t *data, cryptonote::blobdata *blob) override { return false; }
   virtual void remove_alt_block(const crypto::hash &blkid) override {}
   virtual uint64_t get_alt_block_count() override { return 0; }
   virtual void drop_alt_blocks() override {}
-  virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata *blob)> f, bool include_blob = false) const override { return true; }
+  virtual bool for_all_alt_blocks(std::function<bool(const crypto::hash &blkid, const alt_block_data_t &data, const cryptonote::blobdata_ref *blob)> f, bool include_blob = false) const override { return true; }
 };
 
 }

src/blockchain_utilities/blockchain_blackball.cpp

@@ -28,8 +28,6 @@
 
 #include <boost/range/adaptor/transformed.hpp>
 #include <boost/algorithm/string.hpp>
-#include <boost/archive/portable_binary_iarchive.hpp>
-#include <boost/archive/portable_binary_oarchive.hpp>
 #include "common/unordered_containers_boost_serialization.h"
 #include "common/command_line.h"
 #include "common/varint.h"

src/blockchain_utilities/blockchain_export.cpp

@@ -47,6 +47,7 @@ int main(int argc, char* argv[])
   epee::string_tools::set_module_name_and_folder(argv[0]);
 
   uint32_t log_level = 0;
+  uint64_t block_start = 0;
   uint64_t block_stop = 0;
   bool blocks_dat = false;
 
@@ -58,6 +59,7 @@ int main(int argc, char* argv[])
   po::options_description desc_cmd_sett("Command line options and settings options");
   const command_line::arg_descriptor<std::string> arg_output_file = {"output-file", "Specify output file", "", true};
   const command_line::arg_descriptor<std::string> arg_log_level  = {"log-level",  "0-4 or categories", ""};
+  const command_line::arg_descriptor<uint64_t> arg_block_start = {"block-start", "Start at block number", block_start};
   const command_line::arg_descriptor<uint64_t> arg_block_stop = {"block-stop", "Stop at block number", block_stop};
   const command_line::arg_descriptor<bool> arg_blocks_dat = {"blocksdat", "Output in blocks.dat format", blocks_dat};
 
@@ -67,6 +69,7 @@ int main(int argc, char* argv[])
   command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on);
   command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on);
   command_line::add_arg(desc_cmd_sett, arg_log_level);
+  command_line::add_arg(desc_cmd_sett, arg_block_start);
   command_line::add_arg(desc_cmd_sett, arg_block_stop);
   command_line::add_arg(desc_cmd_sett, arg_blocks_dat);
 
@@ -97,6 +100,7 @@ int main(int argc, char* argv[])
     mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str());
   else
     mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
+  block_start = command_line::get_arg(vm, arg_block_start);
   block_stop = command_line::get_arg(vm, arg_block_stop);
 
   LOG_PRINT_L0("Starting...");
@@ -178,7 +182,7 @@ int main(int argc, char* argv[])
   else
   {
     BootstrapFile bootstrap;
-    r = bootstrap.store_blockchain_raw(core_storage, NULL, output_file_path, block_stop);
+    r = bootstrap.store_blockchain_raw(core_storage, NULL, output_file_path, block_start, block_stop);
   }
   CHECK_AND_ASSERT_MES(r, 1, "Failed to export blockchain raw data");
   LOG_PRINT_L0("Blockchain raw data exported OK");

src/blockchain_utilities/blockchain_import.cpp

@@ -227,6 +227,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
     return false;
   }
 
+  uint64_t block_first, block_last;
   uint64_t start_height = 1, seek_height;
   if (opt_resume)
     start_height = core.get_blockchain_storage().get_current_blockchain_height();
@@ -235,10 +236,10 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
   BootstrapFile bootstrap;
   std::streampos pos;
   // BootstrapFile bootstrap(import_file_path);
-  uint64_t total_source_blocks = bootstrap.count_blocks(import_file_path, pos, seek_height);
-  MINFO("bootstrap file last block number: " << total_source_blocks-1 << " (zero-based height)  total blocks: " << total_source_blocks);
+  uint64_t total_source_blocks = bootstrap.count_blocks(import_file_path, pos, seek_height, block_first);
+  MINFO("bootstrap file last block number: " << total_source_blocks+block_first-1 << " (zero-based height)  total blocks: " << total_source_blocks);
 
-  if (total_source_blocks-1 <= start_height)
+  if (total_source_blocks+block_first-1 <= start_height)
   {
     return false;
   }
@@ -260,7 +261,8 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
 
   // 4 byte magic + (currently) 1024 byte header structures
   uint8_t major_version, minor_version;
-  bootstrap.seek_to_first_chunk(import_file, major_version, minor_version);
+  uint64_t dummy;
+  bootstrap.seek_to_first_chunk(import_file, major_version, minor_version, dummy, dummy);
 
   std::string str1;
   char buffer1[1024];
@@ -275,7 +277,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
 
   if (! block_stop)
   {
-    block_stop = total_source_blocks - 1;
+    block_stop = total_source_blocks+block_first - 1;
   }
 
   // These are what we'll try to use, and they don't have to be a determination

src/blockchain_utilities/blockchain_stats.cpp

@@ -68,6 +68,9 @@ int main(int argc, char* argv[])
   const command_line::arg_descriptor<bool> arg_outputs  = {"with-outputs", "with output stats", false};
   const command_line::arg_descriptor<bool> arg_ringsize  = {"with-ringsize", "with ringsize stats", false};
   const command_line::arg_descriptor<bool> arg_hours  = {"with-hours", "with txns per hour", false};
+  const command_line::arg_descriptor<bool> arg_emission  = {"with-emission", "with coin emission", false};
+  const command_line::arg_descriptor<bool> arg_fees  = {"with-fees", "with txn fees", false};
+  const command_line::arg_descriptor<bool> arg_diff  = {"with-diff", "with difficulty", false};
 
   command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir);
   command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on);
@@ -79,6 +82,9 @@ int main(int argc, char* argv[])
   command_line::add_arg(desc_cmd_sett, arg_outputs);
   command_line::add_arg(desc_cmd_sett, arg_ringsize);
   command_line::add_arg(desc_cmd_sett, arg_hours);
+  command_line::add_arg(desc_cmd_sett, arg_emission);
+  command_line::add_arg(desc_cmd_sett, arg_fees);
+  command_line::add_arg(desc_cmd_sett, arg_diff);
   command_line::add_arg(desc_cmd_only, command_line::arg_help);
 
   po::options_description desc_options("Allowed options");
@@ -120,6 +126,9 @@ int main(int argc, char* argv[])
   bool do_outputs = command_line::get_arg(vm, arg_outputs);
   bool do_ringsize = command_line::get_arg(vm, arg_ringsize);
   bool do_hours = command_line::get_arg(vm, arg_hours);
+  bool do_emission = command_line::get_arg(vm, arg_emission);
+  bool do_fees = command_line::get_arg(vm, arg_fees);
+  bool do_diff = command_line::get_arg(vm, arg_diff);
 
   LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)");
   std::unique_ptr<Blockchain> core_storage;
@@ -177,12 +186,20 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
   // spit out a comment that GnuPlot can use as an index
   std::cout << ENDL << "# DATA" << ENDL;
   std::cout << "Date\tBlocks/day\tBlocks\tTxs/Day\tTxs\tBytes/Day\tBytes";
+  if (do_emission)
+    std::cout << "\tEmission/day\tEmission";
+  if (do_fees)
+    std::cout << "\tFees/day\tFees";
+  if (do_diff)
+    std::cout << "\tDiffMin\tDiffMax\tDiffAvg";
   if (do_inputs)
     std::cout << "\tInMin\tInMax\tInAvg";
   if (do_outputs)
     std::cout << "\tOutMin\tOutMax\tOutAvg";
   if (do_ringsize)
     std::cout << "\tRingMin\tRingMax\tRingAvg";
+  if (do_inputs || do_outputs || do_ringsize)
+    std::cout << std::setprecision(2) << std::fixed;
   if (do_hours) {
     char buf[8];
     unsigned int i;
@@ -193,14 +210,20 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
   }
   std::cout << ENDL;
 
+#define MAX_INOUT	0xffffffff
+#define MAX_RINGS	0xffffffff
+
   struct tm prevtm = {0}, currtm;
   uint64_t prevsz = 0, currsz = 0;
   uint64_t prevtxs = 0, currtxs = 0;
   uint64_t currblks = 0;
   uint64_t totins = 0, totouts = 0, totrings = 0;
-  uint32_t minins = 10, maxins = 0;
-  uint32_t minouts = 10, maxouts = 0;
-  uint32_t minrings = 50, maxrings = 0;
+  boost::multiprecision::uint128_t prevemission = 0, prevfees = 0;
+  boost::multiprecision::uint128_t emission = 0, fees = 0;
+  boost::multiprecision::uint128_t totdiff = 0, mindiff = 0, maxdiff = 0;
+  uint32_t minins = MAX_INOUT, maxins = 0;
+  uint32_t minouts = MAX_INOUT, maxouts = 0;
+  uint32_t minrings = MAX_RINGS, maxrings = 0;
   uint32_t io, tottxs = 0;
   uint32_t txhr[24] = {0};
   unsigned int i;
@@ -230,34 +253,50 @@ plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, ''
       std::cout << timebuf << "\t" << currblks << "\t" << h << "\t" << currtxs << "\t" << prevtxs + currtxs << "\t" << currsz << "\t" << prevsz + currsz;
       prevsz += currsz;
       currsz = 0;
-      currblks = 0;
       prevtxs += currtxs;
       currtxs = 0;
       if (!tottxs)
         tottxs = 1;
+      if (do_emission) {
+        std::cout << "\t" << print_money(emission) << "\t" << print_money(prevemission + emission);
+        prevemission += emission;
+        emission = 0;
+      }
+      if (do_fees) {
+        std::cout << "\t" << print_money(fees) << "\t" << print_money(prevfees + fees);
+        prevfees += fees;
+        fees = 0;
+      }
+      if (do_diff) {
+        std::cout << "\t" << (maxdiff ? mindiff : 0) << "\t" << maxdiff << "\t" << totdiff / currblks;
+        mindiff = 0; maxdiff = 0; totdiff = 0;
+      }
       if (do_inputs) {
-        std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins / tottxs;
-        minins = 10; maxins = 0; totins = 0;
+        std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins * 1.0 / tottxs;
+        minins = MAX_INOUT; maxins = 0; totins = 0;
       }
       if (do_outputs) {
-        std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts / tottxs;
-        minouts = 10; maxouts = 0; totouts = 0;
+        std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts * 1.0 / tottxs;
+        minouts = MAX_INOUT; maxouts = 0; totouts = 0;
       }
       if (do_ringsize) {
-        std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings / tottxs;
-        minrings = 50; maxrings = 0; totrings = 0;
+        std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings * 1.0 / tottxs;
+        minrings = MAX_RINGS; maxrings = 0; totrings = 0;
       }
-      tottxs = 0;
       if (do_hours) {
         for (i=0; i<24; i++) {
           std::cout << "\t" << txhr[i];
           txhr[i] = 0;
         }
       }
+      currblks = 0;
+      tottxs = 0;
       std::cout << ENDL;
     }
 skip:
     currsz += bd.size();
+    uint64_t coinbase_amount;
+    uint64_t tx_fee_amount = 0;
     for (const auto& tx_id : blk.tx_hashes)
     {
       if (tx_id == crypto::null_hash)
@@ -275,7 +314,12 @@ skip:
         return 1;
       }
       currsz += bd.size();
+      if (db->get_prunable_tx_blob(tx_id, bd))
+        currsz += bd.size();
       currtxs++;
+      if (do_fees || do_emission) {
+        tx_fee_amount += get_tx_fee(tx);
+      }
       if (do_hours)
         txhr[currtm.tm_hour]++;
       if (do_inputs) {
@@ -306,6 +350,21 @@ skip:
       }
       tottxs++;
     }
+    if (do_diff) {
+      difficulty_type diff = db->get_block_difficulty(h);
+      if (!mindiff || diff < mindiff)
+        mindiff = diff;
+      if (diff > maxdiff)
+        maxdiff = diff;
+      totdiff += diff;
+    }
+    if (do_emission) {
+      coinbase_amount = get_outs_money_amount(blk.miner_tx);
+      emission += coinbase_amount - tx_fee_amount;
+    }
+    if (do_fees) {
+      fees += tx_fee_amount;
+    }
     currblks++;
 
     if (stop_requested)

src/blockchain_utilities/bootstrap_file.cpp

@@ -52,7 +52,7 @@ namespace
 
 
 
-bool BootstrapFile::open_writer(const boost::filesystem::path& file_path)
+bool BootstrapFile::open_writer(const boost::filesystem::path& file_path, uint64_t start_block, uint64_t stop_block)
 {
   const boost::filesystem::path dir_path = file_path.parent_path();
   if (!dir_path.empty())
@@ -78,7 +78,7 @@ bool BootstrapFile::open_writer(const boost::filesystem::path& file_path)
   m_raw_data_file = new std::ofstream();
 
   bool do_initialize_file = false;
-  uint64_t num_blocks = 0;
+  uint64_t num_blocks = 0, block_first = 0;
 
   if (! boost::filesystem::exists(file_path))
   {
@@ -88,10 +88,12 @@ bool BootstrapFile::open_writer(const boost::filesystem::path& file_path)
   }
   else
   {
-    num_blocks = count_blocks(file_path.string());
-    MDEBUG("appending to existing file with height: " << num_blocks-1 << "  total blocks: " << num_blocks);
+    std::streampos dummy_pos;
+    uint64_t dummy_height = 0;
+    num_blocks = count_blocks(file_path.string(), dummy_pos, dummy_height, block_first);
+    MDEBUG("appending to existing file with height: " << num_blocks+block_first-1 << "  total blocks: " << num_blocks);
   }
-  m_height = num_blocks;
+  m_height = num_blocks+block_first;
 
   if (do_initialize_file)
     m_raw_data_file->open(file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::trunc);
@@ -106,13 +108,12 @@ bool BootstrapFile::open_writer(const boost::filesystem::path& file_path)
     return false;
 
   if (do_initialize_file)
-    initialize_file();
+    initialize_file(start_block, stop_block);
 
   return true;
 }
 
-
-bool BootstrapFile::initialize_file()
+bool BootstrapFile::initialize_file(uint64_t first_block, uint64_t last_block)
 {
   const uint32_t file_magic = blockchain_raw_magic;
 
@@ -129,8 +130,8 @@ bool BootstrapFile::initialize_file()
   bfi.header_size = header_size;
 
   bootstrap::blocks_info bbi;
-  bbi.block_first = 0;
-  bbi.block_last = 0;
+  bbi.block_first = first_block;
+  bbi.block_last = last_block;
   bbi.block_last_pos = 0;
 
   buffer_type buffer2;
@@ -261,7 +262,7 @@ bool BootstrapFile::close()
 }
 
 
-bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_file, uint64_t requested_block_stop)
+bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_file, uint64_t start_block, uint64_t requested_block_stop)
 {
   uint64_t num_blocks_written = 0;
   m_max_chunk = 0;
@@ -269,17 +270,11 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
   m_tx_pool = _tx_pool;
   uint64_t progress_interval = 100;
   MINFO("Storing blocks raw data...");
-  if (!BootstrapFile::open_writer(output_file))
-  {
-    MFATAL("failed to open raw file for write");
-    return false;
-  }
   block b;
 
   // block_start, block_stop use 0-based height. m_height uses 1-based height. So to resume export
   // from last exported block, block_start doesn't need to add 1 here, as it's already at the next
   // height.
-  uint64_t block_start = m_height;
   uint64_t block_stop = 0;
   MINFO("source blockchain height: " <<  m_blockchain_storage->get_current_blockchain_height()-1);
   if ((requested_block_stop > 0) && (requested_block_stop < m_blockchain_storage->get_current_blockchain_height()))
@@ -292,6 +287,13 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
     block_stop = m_blockchain_storage->get_current_blockchain_height() - 1;
     MINFO("Using block height of source blockchain: " << block_stop);
   }
+  if (!BootstrapFile::open_writer(output_file, start_block, block_stop))
+  {
+    MFATAL("failed to open raw file for write");
+    return false;
+  }
+  uint64_t block_start = m_height ? m_height : start_block;
+  MINFO("Starting block height: " << block_start);
   for (m_cur_height = block_start; m_cur_height <= block_stop; ++m_cur_height)
   {
     // this method's height refers to 0-based height (genesis block = height 0)
@@ -323,7 +325,8 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
   return BootstrapFile::close();
 }
 
-uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version)
+uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version,
+	uint64_t &block_first, uint64_t &block_last)
 {
   uint32_t file_magic;
 
@@ -368,11 +371,35 @@ uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file, uint8_t
   MINFO("bootstrap magic size: " << sizeof(file_magic));
   MINFO("bootstrap header size: " << bfi.header_size);
 
+  uint32_t buflen_blocks_info;
+
+  import_file.read(buf1, sizeof(buflen_blocks_info));
+  str1.assign(buf1, sizeof(buflen_blocks_info));
+  if (! import_file)
+    throw std::runtime_error("Error reading expected number of bytes");
+  if (! ::serialization::parse_binary(str1, buflen_blocks_info))
+    throw std::runtime_error("Error in deserialization of buflen_blocks_info");
+  MINFO("bootstrap::blocks_info size: " << buflen_blocks_info);
+
+  if (buflen_blocks_info > sizeof(buf1))
+    throw std::runtime_error("Error: bootstrap::blocks_info size exceeds buffer size");
+  import_file.read(buf1, buflen_blocks_info);
+  if (! import_file)
+    throw std::runtime_error("Error reading expected number of bytes");
+  str1.assign(buf1, buflen_blocks_info);
+  bootstrap::blocks_info bbi;
+  if (! ::serialization::parse_binary(str1, bbi))
+    throw std::runtime_error("Error in deserialization of bootstrap::blocks_info");
+  MINFO("bootstrap first block:" << bbi.block_first);
+  MINFO("bootstrap last block:" << bbi.block_last);
+
   uint64_t full_header_size = sizeof(file_magic) + bfi.header_size;
   import_file.seekg(full_header_size);
 
   major_version = bfi.major_version;
   minor_version = bfi.minor_version;
+  block_first = bbi.block_first;
+  block_last = bbi.block_last;
   return full_header_size;
 }
 
@@ -436,13 +463,14 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path)
 {
   std::streampos dummy_pos;
   uint64_t dummy_height = 0;
-  return count_blocks(import_file_path, dummy_pos, dummy_height);
+  return count_blocks(import_file_path, dummy_pos, dummy_height, dummy_height);
 }
 
 // If seek_height is non-zero on entry, return a stream position <= this height when finished.
 // And return the actual height corresponding to this position. Allows the caller to locate its
 // starting position without having to reread the entire file again.
-uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::streampos &start_pos, uint64_t& seek_height)
+uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::streampos &start_pos,
+	uint64_t& seek_height, uint64_t &block_first)
 {
   boost::filesystem::path raw_file_path(import_file_path);
   boost::system::error_code ec;
@@ -464,7 +492,8 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::s
 
   uint64_t full_header_size; // 4 byte magic + length of header structures
   uint8_t major_version, minor_version;
-  full_header_size = seek_to_first_chunk(import_file, major_version, minor_version);
+  uint64_t block_last;
+  full_header_size = seek_to_first_chunk(import_file, major_version, minor_version, block_first, block_last);
 
   MINFO("Scanning blockchain from bootstrap file...");
   bool quit = false;
@@ -473,11 +502,11 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::s
 
   while (! quit)
   {
-    if (start_height && h + progress_interval >= start_height - 1)
+    if (start_height && h + block_first + progress_interval >= start_height - 1)
     {
       start_height = 0;
       start_pos = import_file.tellg();
-      seek_height = h;
+      seek_height = h + block_first;
     }
     bytes_read += count_bytes(import_file, progress_interval, blocks, quit);
     h += blocks;

src/blockchain_utilities/bootstrap_file.h

@@ -57,12 +57,12 @@ class BootstrapFile
 public:
 
   uint64_t count_bytes(std::ifstream& import_file, uint64_t blocks, uint64_t& h, bool& quit);
-  uint64_t count_blocks(const std::string& dir_path, std::streampos& start_pos, uint64_t& seek_height);
+  uint64_t count_blocks(const std::string& dir_path, std::streampos& start_pos, uint64_t& seek_height, uint64_t& block_first);
   uint64_t count_blocks(const std::string& dir_path);
-  uint64_t seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version);
+  uint64_t seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version, uint64_t &block_first, uint64_t &block_last);
 
   bool store_blockchain_raw(cryptonote::Blockchain* cs, cryptonote::tx_memory_pool* txp,
-      boost::filesystem::path& output_file, uint64_t use_block_height=0);
+      boost::filesystem::path& output_file, uint64_t start_block=0, uint64_t stop_block=0);
 
 protected:
 
@@ -75,8 +75,8 @@ protected:
   boost::iostreams::stream<boost::iostreams::back_insert_device<buffer_type>>* m_output_stream;
 
   // open export file for write
-  bool open_writer(const boost::filesystem::path& file_path);
-  bool initialize_file();
+  bool open_writer(const boost::filesystem::path& file_path, uint64_t start_block, uint64_t stop_block);
+  bool initialize_file(uint64_t start_block, uint64_t stop_block);
   bool close();
   void write_block(block& block);
   void flush_chunk();

src/blocks/CMakeLists.txt

@@ -45,4 +45,4 @@ foreach(BLOB_NAME checkpoints)
     )
 endforeach()
 
-add_library(blocks STATIC blocks.cpp ${GENERATED_SOURCES})
+monero_add_library(blocks blocks.cpp ${GENERATED_SOURCES})

src/checkpoints/checkpoints.cpp

@@ -182,12 +182,26 @@ namespace cryptonote
   {
     if (nettype == TESTNET)
     {
+    ADD_CHECKPOINT2(1,   "c47b8effcc84e22ddd1b876d059c8fbd3e26e4bdffb6bed8b8f4fe283104a7af", "0x2");
+    ADD_CHECKPOINT2(5,   "292add330f6cf5f3845dc4cd2f28ce8d211dd487ddcfe1b5d1d545d7b90ff7d1", "0x5df");  //Hard fork to v8
+    ADD_CHECKPOINT2(10,  "fbdf7d812aa6dd4c5915aadcbf023e722eab1d8ad47c24a230634e688d314798", "0xfa3");  //Hard fork to v9
+    ADD_CHECKPOINT2(15,  "9b86e40b959d7cfe75366686dfd2657699e85b8c658aa8c20b23f7bc01b38af0", "0x1967"); //Hard fork to v10
+    ADD_CHECKPOINT2(20,  "8f65c309ba3ed2bf25df2b7d91e1db40c864230aa3caaf823db81bc19e0bd6a3", "0x232b"); //Hard fork to v11
+    ADD_CHECKPOINT2(25,  "35f2957fdba45d5421561ba2f126a61ec0101b5c2eb79b9e296310acbbdbbe58", "0x2cef"); //Hard fork to v12
+    ADD_CHECKPOINT2(30,  "32cc5aacce8bea10ae869313e194ba51a30720810f3665433ffeea2818938429", "0x36b3"); //Hard fork to v13
+    ADD_CHECKPOINT2(35,  "7f1cdd2c4a6002772343ad9de6c5dba743a0be1d326a6f63924511f08b15863a", "0x4077"); //Hard fork to v14
+    ADD_CHECKPOINT2(40,  "63cf4f703489a881baf63bacd6a3af2b5bde79b558ce123e97313890be6f747f", "0x4a3b"); //Hard fork to v15
+    ADD_CHECKPOINT2(45,  "65da4c966ae02983c6c8b7ab9b959df612863c379c5e48669061cbbb1e02a3ab", "0x53ff"); //Hard fork to v16
+    ADD_CHECKPOINT2(50,  "8c68a444b6743a14152f130357be9751e42e82a84f69f25320eda532a830c7b0", "0x5dc3"); //Hard fork to v17
       return true;
     }
     if (nettype == STAGENET)
     {
       return true;
     }
+    // make RPC call to daemon
+    // curl http://127.0.0.1:34568/json_rpc -d '{"jsonrpc":"2.0","id":"0","method":"get_block","params":{"height":307880}}' -H 'Content-Type: application/json'
+    // "wide_cumulative_difficulty": "0x14eb4d0131fe8",
     ADD_CHECKPOINT2(1,       "97f4ce4d7879b3bea54dcec738cd2ebb7952b4e9bb9743262310cd5fec749340", "0x2");
     ADD_CHECKPOINT2(6969,    "aa7b66e8c461065139b55c29538a39c33ceda93e587f84d490ed573d80511c87", "0x118eef693fd");  //Hard fork to v8
     ADD_CHECKPOINT2(53666,   "3f43f56f66ef0c43cf2fd14d0d28fa2aae0ef8f40716773511345750770f1255", "0xb677d6405ae");  //Hard fork to v9
@@ -197,7 +211,21 @@ namespace cryptonote
     ADD_CHECKPOINT2(114969,  "b48245956b87f243048fd61021f4b3e5443e57eee7ff8ba4762d18926e80b80c", "0x1ca552b3ec68"); //Hard fork to v13
     ADD_CHECKPOINT2(115257,  "338e056551087fe23d6c4b4280244bc5362b004716d85ec799a775f190f9fea9", "0x1cb25f5d4628"); //Hard fork to v14
     ADD_CHECKPOINT2(160777,  "9496690579af21f38f00e67e11c2e85a15912fe4f412aad33d1162be1579e755", "0x5376eaa196a8"); //Hard fork to v15
-    ADD_CHECKPOINT2(230200,  "0ebb018c452fa84997f573937db67ef3946a857ee57cd40fa51ffe368896c666", "0xbcc1409c7268");
+    ADD_CHECKPOINT2(253999,  "755a289fe8a68e96a0f69069ba4007b676ec87dce2e47dfb9647fe5691f49883", "0x172d026ef7fe8"); //Hard fork to v16
+    ADD_CHECKPOINT2(254287,  "b37cb55abe73965b424f8028bf71bef98d069645077ffa52f0c134907b7734e3", "0x1746622f56668"); //Hard fork to v17
+    ADD_CHECKPOINT2(256700,  "389a8ab95a80e84ec74639c1078bc67b33af208ef00f53bd9609cfc40efa7059", "0x185ace3c1bd68");
+    ADD_CHECKPOINT2(271600,  "9597cdbdc52ca57d7dbd8f9c0a23a73194ef2ebbcfdc75c21992672706108d43", "0x1e2d2d6a2a9e8");
+    ADD_CHECKPOINT2(278300,  "b10dcdf7a51651f60fbcc0447409773eef1458d2c706d9a61daf467571ac19c9", "0x20a83a16d3968");
+    ADD_CHECKPOINT2(282700,  "79c06cafd7cb5f76bcebbf8f1ae16203bb41fd75b284bcd0eb0b457991ab7d4a", "0x22e3baf142de8");
+    ADD_CHECKPOINT2(307686,  "dfd056b2739c132a07629409a59a028cb7414fac23e3419e79d2f49d66fc3af5", "0x305ba542e3ea8"); // "difficulty": 25800000
+    ADD_CHECKPOINT2(307692,  "d822cd72037f62824ec87c9dc11768b45dc2632f697fa372e1885789c90f37fc", "0x305e124633878"); // "difficulty": 1890000
+    ADD_CHECKPOINT2(307735,  "60970378aecdc0a78ccf5154edcc56f23aad8554b49e4716f820461a7588bfdc", "0x3070771b9ba58"); // "difficulty": 17900000
+    ADD_CHECKPOINT2(307742,  "0ed835bc9fcd949b5a184cf607dcc62ac4268c9e4cf220f8b09bcce58f10916b", "0x30732f1248978"); // "difficulty": 21300000
+    ADD_CHECKPOINT2(307750,  "7bcafbc757237125b70f569b181eb1b66c530b10d817d7b940f7a73dc827211c", "0x30766666b3d98"); // "difficulty": 10900000
+    ADD_CHECKPOINT2(307766,  "02fd6c7d6bae710cfa3efb08f50e4bc9a590f6ab61eabd87e5e951338c0c36f6", "0x307d2d47a7918"); // "difficulty": 2960000
+    ADD_CHECKPOINT2(307800,  "3594894b4231cfdfe911afed6552f9fb4cfe6048bacd0973a3a98623ec8548ce", "0x308b305ca7618");
+    ADD_CHECKPOINT2(307880,  "659274b698f680c6cae2716cbd4e15ad5def23b5de98e53734c4af2c2e74bb7a", "0x30af6e91e8018");
+    ADD_CHECKPOINT2(307883,  "9a8c35cd10963a14bba8a9628d1776df92fee5e3153b7249f5d15726efafaaea", "0x30b0965ba5a18");
     return true;
   }
 

src/common/dns_utils.cpp

@@ -519,14 +519,14 @@ bool load_txt_records_from_dns(std::vector<std::string> &good_records, const std
   // send all requests in parallel
   std::deque<bool> avail(dns_urls.size(), false), valid(dns_urls.size(), false);
   tools::threadpool& tpool = tools::threadpool::getInstance();
-  tools::threadpool::waiter waiter;
+  tools::threadpool::waiter waiter(tpool);
   for (size_t n = 0; n < dns_urls.size(); ++n)
   {
     tpool.submit(&waiter,[n, dns_urls, &records, &avail, &valid](){
       records[n] = tools::DNSResolver::instance().get_txt_record(dns_urls[n], avail[n], valid[n]); 
     });
   }
-  waiter.wait(&tpool);
+  waiter.wait();
 
   size_t cur_index = first_index;
   do

src/common/notify.cpp

@@ -62,7 +62,7 @@ static void replace(std::vector<std::string> &v, const char *tag, const char *s)
     boost::replace_all(str, tag, s);
 }
 
-int Notify::notify(const char *tag, const char *s, ...)
+int Notify::notify(const char *tag, const char *s, ...) const
 {
   std::vector<std::string> margs = args;
 

src/common/notify.h

@@ -38,8 +38,12 @@ class Notify
 {
 public:
   Notify(const char *spec);
+  Notify(const Notify&) = default;
+  Notify(Notify&&) = default;
+  Notify& operator=(const Notify&) = default;
+  Notify& operator=(Notify&&) = default;
 
-  int notify(const char *tag, const char *s, ...);
+  int notify(const char *tag, const char *s, ...) const;
 
 private:
   std::string filename;
@@ -47,3 +51,4 @@ private:
 };
 
 }
+

src/common/powerof.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include <stdint.h>
+
+namespace tools
+{
+  template<uint64_t a, uint64_t b>
+  struct PowerOf
+  {
+    enum Data : uint64_t
+    {
+      // a^b = a * a^(b-1)
+      Value = a * PowerOf<a, b - 1>::Value,
+    };
+  };
+
+  template<uint64_t a>
+  struct PowerOf<a, 0>
+  {
+    enum Data : uint64_t
+    {
+      // a^0 = 1
+      Value = 1,
+    };
+  };
+}

src/common/threadpool.cpp

@@ -120,7 +120,7 @@ threadpool::waiter::~waiter()
   catch (...) { /* ignore */ }
   try
   {
-    wait(NULL);
+    wait();
   }
   catch (const std::exception &e)
   {
@@ -128,12 +128,12 @@ threadpool::waiter::~waiter()
   }
 }
 
-void threadpool::waiter::wait(threadpool *tpool) {
-  if (tpool)
-    tpool->run(true);
+bool threadpool::waiter::wait() {
+  pool.run(true);
   boost::unique_lock<boost::mutex> lock(mt);
   while(num)
     cv.wait(lock);
+  return !error();
 }
 
 void threadpool::waiter::inc() {
@@ -166,7 +166,8 @@ void threadpool::run(bool flush) {
     lock.unlock();
     ++depth;
     is_leaf = e.leaf;
-    e.f();
+    try { e.f(); }
+    catch (const std::exception &ex) { e.wo->set_error(); try { MERROR("Exception in threadpool job: " << ex.what()); } catch (...) {} }
     --depth;
     is_leaf = false;
 

src/common/threadpool.h

@@ -55,12 +55,16 @@ public:
   class waiter {
     boost::mutex mt;
     boost::condition_variable cv;
+    threadpool &pool;
     int num;
+    bool error_flag;
     public:
     void inc();
     void dec();
-    void wait(threadpool *tpool);  //! Wait for a set of tasks to finish.
-    waiter() : num(0){}
+    bool wait();  //! Wait for a set of tasks to finish, returns false iff any error
+    void set_error() noexcept { error_flag = true; }
+    bool error() const noexcept { return error_flag; }
+    waiter(threadpool &pool) : pool(pool), num(0), error_flag(false) {}
     ~waiter();
   };
 

src/common/updates.cpp

@@ -98,6 +98,8 @@ namespace tools
     const char *base = user ? "" : "";
 #ifdef _WIN32
     static const char *extension = strncmp(buildtag.c_str(), "source", 6) ? (strncmp(buildtag.c_str(), "install-", 8) ? ".zip" : ".exe") : ".tar.bz2";
+#elif defined(__APPLE__)
+    static const char *extension = strncmp(software.c_str(), "monero-gui", 10) ? ".tar.bz2" : ".dmg";
 #else
     static const char extension[] = ".tar.bz2";
 #endif

src/common/util.cpp

@@ -871,10 +871,19 @@ std::string get_nix_version_display_string()
     return max_concurrency;
   }
 
+  bool is_privacy_preserving_network(const std::string &address)
+  {
+    if (boost::ends_with(address, ".onion"))
+      return true;
+    if (boost::ends_with(address, ".i2p"))
+      return true;
+    return false;
+  }
+
   bool is_local_address(const std::string &address)
   {
     // always assume Tor/I2P addresses to be untrusted by default
-    if (boost::ends_with(address, ".onion") || boost::ends_with(address, ".i2p"))
+    if (is_privacy_preserving_network(address))
     {
       MDEBUG("Address '" << address << "' is Tor/I2P, non local");
       return false;
@@ -1000,13 +1009,13 @@ std::string get_nix_version_display_string()
     for (char c: val)
     {
       if (c == '*')
-        newval += escape ? "*" : ".*";
+        newval += escape ? "*" : ".*", escape = false;
       else if (c == '?')
-        newval += escape ? "?" : ".";
+        newval += escape ? "?" : ".", escape = false;
       else if (c == '\\')
         newval += '\\', escape = !escape;
       else
-        newval += c;
+        newval += c, escape = false;
     }
     return newval;
   }
@@ -1116,7 +1125,7 @@ std::string get_nix_version_display_string()
     static constexpr const byte_map sizes[] =
     {
         {"%.0f B", 1024},
-        {"%.2f KB", 1024 * 1024},
+        {"%.2f kB", 1024 * 1024},
         {"%.2f MB", std::uint64_t(1024) * 1024 * 1024},
         {"%.2f GB", std::uint64_t(1024) * 1024 * 1024 * 1024},
         {"%.2f TB", std::uint64_t(1024) * 1024 * 1024 * 1024 * 1024}

src/common/util.h

@@ -228,6 +228,7 @@ namespace tools
   unsigned get_max_concurrency();
 
   bool is_local_address(const std::string &address);
+  bool is_privacy_preserving_network(const std::string &address);
   int vercmp(const char *v0, const char *v1); // returns < 0, 0, > 0, similar to strcmp, but more human friendly than lexical - does not attempt to validate
 
   bool sha256sum(const uint8_t *data, size_t len, crypto::hash &hash);

src/crypto/CMakeLists.txt

@@ -116,3 +116,6 @@ endif()
 
 # cheat because cmake and ccache hate each other
 set_property(SOURCE CryptonightR_template.S PROPERTY LANGUAGE C)
+
+# Must be done last, because it references libraries in this directory
+add_subdirectory(wallet)

src/crypto/crypto-ops.c

@@ -1234,6 +1234,56 @@ void ge_double_scalarmult_base_vartime(ge_p2 *r, const unsigned char *a, const g
   }
 }
 
+// Computes aG + bB + cC (G is the fixed basepoint)
+void ge_triple_scalarmult_base_vartime(ge_p2 *r, const unsigned char *a, const unsigned char *b, const ge_dsmp Bi, const unsigned char *c, const ge_dsmp Ci) {
+  signed char aslide[256];
+  signed char bslide[256];
+  signed char cslide[256];
+  ge_p1p1 t;
+  ge_p3 u;
+  int i;
+
+  slide(aslide, a);
+  slide(bslide, b);
+  slide(cslide, c);
+
+  ge_p2_0(r);
+
+  for (i = 255; i >= 0; --i) {
+    if (aslide[i] || bslide[i] || cslide[i]) break;
+  }
+
+  for (; i >= 0; --i) {
+    ge_p2_dbl(&t, r);
+
+    if (aslide[i] > 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_madd(&t, &u, &ge_Bi[aslide[i]/2]);
+    } else if (aslide[i] < 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_msub(&t, &u, &ge_Bi[(-aslide[i])/2]);
+    }
+
+    if (bslide[i] > 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_add(&t, &u, &Bi[bslide[i]/2]);
+    } else if (bslide[i] < 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_sub(&t, &u, &Bi[(-bslide[i])/2]);
+    }
+
+    if (cslide[i] > 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_add(&t, &u, &Ci[cslide[i]/2]);
+    } else if (cslide[i] < 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_sub(&t, &u, &Ci[(-cslide[i])/2]);
+    }
+
+    ge_p1p1_to_p2(r, &t);
+  }
+}
+
 void ge_double_scalarmult_base_vartime_p3(ge_p3 *r3, const unsigned char *a, const ge_p3 *A, const unsigned char *b) {
   signed char aslide[256];
   signed char bslide[256];
@@ -2148,6 +2198,56 @@ void ge_double_scalarmult_precomp_vartime2(ge_p2 *r, const unsigned char *a, con
   }
 }
 
+// Computes aA + bB + cC (all points require precomputation)
+void ge_triple_scalarmult_precomp_vartime(ge_p2 *r, const unsigned char *a, const ge_dsmp Ai, const unsigned char *b, const ge_dsmp Bi, const unsigned char *c, const ge_dsmp Ci) {
+  signed char aslide[256];
+  signed char bslide[256];
+  signed char cslide[256];
+  ge_p1p1 t;
+  ge_p3 u;
+  int i;
+
+  slide(aslide, a);
+  slide(bslide, b);
+  slide(cslide, c);
+
+  ge_p2_0(r);
+
+  for (i = 255; i >= 0; --i) {
+    if (aslide[i] || bslide[i] || cslide[i]) break;
+  }
+
+  for (; i >= 0; --i) {
+    ge_p2_dbl(&t, r);
+
+    if (aslide[i] > 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_add(&t, &u, &Ai[aslide[i]/2]);
+    } else if (aslide[i] < 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_sub(&t, &u, &Ai[(-aslide[i])/2]);
+    }
+
+    if (bslide[i] > 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_add(&t, &u, &Bi[bslide[i]/2]);
+    } else if (bslide[i] < 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_sub(&t, &u, &Bi[(-bslide[i])/2]);
+    }
+
+    if (cslide[i] > 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_add(&t, &u, &Ci[cslide[i]/2]);
+    } else if (cslide[i] < 0) {
+      ge_p1p1_to_p3(&u, &t);
+      ge_sub(&t, &u, &Ci[(-cslide[i])/2]);
+    }
+
+    ge_p1p1_to_p2(r, &t);
+  }
+}
+
 void ge_double_scalarmult_precomp_vartime2_p3(ge_p3 *r3, const unsigned char *a, const ge_dsmp Ai, const unsigned char *b, const ge_dsmp Bi) {
   signed char aslide[256];
   signed char bslide[256];

src/crypto/crypto-ops.h

@@ -79,6 +79,7 @@ typedef ge_cached ge_dsmp[8];
 extern const ge_precomp ge_Bi[8];
 void ge_dsm_precomp(ge_dsmp r, const ge_p3 *s);
 void ge_double_scalarmult_base_vartime(ge_p2 *, const unsigned char *, const ge_p3 *, const unsigned char *);
+void ge_triple_scalarmult_base_vartime(ge_p2 *, const unsigned char *, const unsigned char *, const ge_dsmp, const unsigned char *, const ge_dsmp);
 void ge_double_scalarmult_base_vartime_p3(ge_p3 *, const unsigned char *, const ge_p3 *, const unsigned char *);
 
 /* From ge_frombytes.c, modified */
@@ -130,6 +131,7 @@ void sc_reduce(unsigned char *);
 void ge_scalarmult(ge_p2 *, const unsigned char *, const ge_p3 *);
 void ge_scalarmult_p3(ge_p3 *, const unsigned char *, const ge_p3 *);
 void ge_double_scalarmult_precomp_vartime(ge_p2 *, const unsigned char *, const ge_p3 *, const unsigned char *, const ge_dsmp);
+void ge_triple_scalarmult_precomp_vartime(ge_p2 *, const unsigned char *, const ge_dsmp, const unsigned char *, const ge_dsmp, const unsigned char *, const ge_dsmp);
 void ge_double_scalarmult_precomp_vartime2(ge_p2 *, const unsigned char *, const ge_dsmp, const unsigned char *, const ge_dsmp);
 void ge_double_scalarmult_precomp_vartime2_p3(ge_p3 *, const unsigned char *, const ge_dsmp, const unsigned char *, const ge_dsmp);
 void ge_mul8(ge_p1p1 *, const ge_p2 *);

src/crypto/crypto.cpp

@@ -43,6 +43,8 @@
 #include "crypto.h"
 #include "hash.h"
 
+#include "cryptonote_config.h"
+
 namespace {
   static void local_abort(const char *msg)
   {
@@ -121,13 +123,17 @@ namespace crypto {
   void random32_unbiased(unsigned char *bytes)
   {
     // l = 2^252 + 27742317777372353535851937790883648493.
-    // it fits 15 in 32 bytes
+    // l fits 15 times in 32 bytes (iow, 15 l is the highest multiple of l that fits in 32 bytes)
     static const unsigned char limit[32] = { 0xe3, 0x6a, 0x67, 0x72, 0x8b, 0xce, 0x13, 0x29, 0x8f, 0x30, 0x82, 0x8c, 0x0b, 0xa4, 0x10, 0x39, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0 };
-    do
+    while(1)
     {
       generate_random_bytes_thread_safe(32, bytes);
-    } while (!sc_isnonzero(bytes) && !less32(bytes, limit)); // should be good about 15/16 of the time
-    sc_reduce32(bytes);
+      if (!less32(bytes, limit))
+        continue;
+      sc_reduce32(bytes);
+      if (sc_isnonzero(bytes))
+        break;
+    }
   }
   /* generate a random 32-byte (256-bit) integer and copy it to res */
   static inline void random_scalar(ec_scalar &res) {
@@ -261,11 +267,24 @@ namespace crypto {
     ec_point comm;
   };
 
+  // Used in v1 tx proofs
+  struct s_comm_2_v1 {
+    hash msg;
+    ec_point D;
+    ec_point X;
+    ec_point Y;
+  };
+
+  // Used in v1/v2 tx proofs
   struct s_comm_2 {
     hash msg;
     ec_point D;
     ec_point X;
     ec_point Y;
+    hash sep; // domain separation
+    ec_point R;
+    ec_point A;
+    ec_point B;
   };
 
   void crypto_ops::generate_signature(const hash &prefix_hash, const public_key &pub, const secret_key &sec, signature &sig) {
@@ -321,6 +340,86 @@ namespace crypto {
     return sc_isnonzero(&c) == 0;
   }
 
+  // Generate a proof of knowledge of `r` such that (`R = rG` and `D = rA`) or (`R = rB` and `D = rA`) via a Schnorr proof
+  // This handles use cases for both standard addresses and subaddresses
+  //
+  // NOTE: This generates old v1 proofs, and is for TESTING ONLY
+  void crypto_ops::generate_tx_proof_v1(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const secret_key &r, signature &sig) {
+    // sanity check
+    ge_p3 R_p3;
+    ge_p3 A_p3;
+    ge_p3 B_p3;
+    ge_p3 D_p3;
+    if (ge_frombytes_vartime(&R_p3, &R) != 0) throw std::runtime_error("tx pubkey is invalid");
+    if (ge_frombytes_vartime(&A_p3, &A) != 0) throw std::runtime_error("recipient view pubkey is invalid");
+    if (B && ge_frombytes_vartime(&B_p3, &*B) != 0) throw std::runtime_error("recipient spend pubkey is invalid");
+    if (ge_frombytes_vartime(&D_p3, &D) != 0) throw std::runtime_error("key derivation is invalid");
+#if !defined(NDEBUG)
+    {
+      assert(sc_check(&r) == 0);
+      // check R == r*G or R == r*B
+      public_key dbg_R;
+      if (B)
+      {
+        ge_p2 dbg_R_p2;
+        ge_scalarmult(&dbg_R_p2, &r, &B_p3);
+        ge_tobytes(&dbg_R, &dbg_R_p2);
+      }
+      else
+      {
+        ge_p3 dbg_R_p3;
+        ge_scalarmult_base(&dbg_R_p3, &r);
+        ge_p3_tobytes(&dbg_R, &dbg_R_p3);
+      }
+      assert(R == dbg_R);
+      // check D == r*A
+      ge_p2 dbg_D_p2;
+      ge_scalarmult(&dbg_D_p2, &r, &A_p3);
+      public_key dbg_D;
+      ge_tobytes(&dbg_D, &dbg_D_p2);
+      assert(D == dbg_D);
+    }
+#endif
+
+    // pick random k
+    ec_scalar k;
+    random_scalar(k);
+    
+    s_comm_2_v1 buf;
+    buf.msg = prefix_hash;
+    buf.D = D;
+    
+    if (B)
+    {
+      // compute X = k*B
+      ge_p2 X_p2;
+      ge_scalarmult(&X_p2, &k, &B_p3);
+      ge_tobytes(&buf.X, &X_p2);
+    }
+    else
+    {
+      // compute X = k*G
+      ge_p3 X_p3;
+      ge_scalarmult_base(&X_p3, &k);
+      ge_p3_tobytes(&buf.X, &X_p3);
+    }
+    
+    // compute Y = k*A
+    ge_p2 Y_p2;
+    ge_scalarmult(&Y_p2, &k, &A_p3);
+    ge_tobytes(&buf.Y, &Y_p2);
+
+    // sig.c = Hs(Msg || D || X || Y) 
+    hash_to_scalar(&buf, sizeof(buf), sig.c);
+
+    // sig.r = k - sig.c*r
+    sc_mulsub(&sig.r, &sig.c, &unwrap(r), &k);
+  }
+
+  // Generate a proof of knowledge of `r` such that (`R = rG` and `D = rA`) or (`R = rB` and `D = rA`) via a Schnorr proof
+  // This handles use cases for both standard addresses and subaddresses
+  //
+  // Generates only proofs for InProofV2 and OutProofV2
   void crypto_ops::generate_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const secret_key &r, signature &sig) {
     // sanity check
     ge_p3 R_p3;
@@ -362,10 +461,20 @@ namespace crypto {
     ec_scalar k;
     random_scalar(k);
     
+    // if B is not present
+    static const ec_point zero = {{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }};
+
     s_comm_2 buf;
     buf.msg = prefix_hash;
     buf.D = D;
-
+    buf.R = R;
+    buf.A = A;
+    if (B)
+        buf.B = *B;
+    else
+        buf.B = zero;
+    cn_fast_hash(config::HASH_KEY_TXPROOF_V2, sizeof(config::HASH_KEY_TXPROOF_V2)-1, buf.sep);
+    
     if (B)
     {
       // compute X = k*B
@@ -386,7 +495,7 @@ namespace crypto {
     ge_scalarmult(&Y_p2, &k, &A_p3);
     ge_tobytes(&buf.Y, &Y_p2);
 
-    // sig.c = Hs(Msg || D || X || Y)
+    // sig.c = Hs(Msg || D || X || Y || sep || R || A || B) 
     hash_to_scalar(&buf, sizeof(buf), sig.c);
 
     // sig.r = k - sig.c*r
@@ -395,7 +504,8 @@ namespace crypto {
     memwipe(&k, sizeof(k));
   }
 
-  bool crypto_ops::check_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const signature &sig) {
+  // Verify a proof: either v1 (version == 1) or v2 (version == 2)
+  bool crypto_ops::check_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const signature &sig, const int version) {
     // sanity check
     ge_p3 R_p3;
     ge_p3 A_p3;
@@ -467,14 +577,31 @@ namespace crypto {
     ge_p2 Y_p2;
     ge_p1p1_to_p2(&Y_p2, &Y_p1p1);
 
-    // compute c2 = Hs(Msg || D || X || Y)
+    // Compute hash challenge
+    // for v1, c2 = Hs(Msg || D || X || Y)
+    // for v2, c2 = Hs(Msg || D || X || Y || sep || R || A || B)
+
+    // if B is not present
+    static const ec_point zero = {{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }};
+
     s_comm_2 buf;
     buf.msg = prefix_hash;
     buf.D = D;
+    buf.R = R;
+    buf.A = A;
+    if (B)
+        buf.B = *B;
+    else
+        buf.B = zero;
+    cn_fast_hash(config::HASH_KEY_TXPROOF_V2, sizeof(config::HASH_KEY_TXPROOF_V2)-1, buf.sep);
     ge_tobytes(&buf.X, &X_p2);
     ge_tobytes(&buf.Y, &Y_p2);
     ec_scalar c2;
-    hash_to_scalar(&buf, sizeof(s_comm_2), c2);
+
+    // Hash depends on version
+    if (version == 1) hash_to_scalar(&buf, sizeof(s_comm_2) - 3*sizeof(ec_point) - sizeof(hash), c2);
+    else if (version == 2) hash_to_scalar(&buf, sizeof(s_comm_2), c2);
+    else return false;
 
     // test if c2 == sig.c
     sc_sub(&c2, &c2, &sig.c);

src/crypto/crypto.h

@@ -132,8 +132,10 @@ namespace crypto {
     friend bool check_signature(const hash &, const public_key &, const signature &);
     static void generate_tx_proof(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const secret_key &, signature &);
     friend void generate_tx_proof(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const secret_key &, signature &);
-    static bool check_tx_proof(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const signature &);
-    friend bool check_tx_proof(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const signature &);
+    static void generate_tx_proof_v1(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const secret_key &, signature &);
+    friend void generate_tx_proof_v1(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const secret_key &, signature &);
+    static bool check_tx_proof(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const signature &, const int);
+    friend bool check_tx_proof(const hash &, const public_key &, const public_key &, const boost::optional<public_key> &, const public_key &, const signature &, const int);
     static void generate_key_image(const public_key &, const secret_key &, key_image &);
     friend void generate_key_image(const public_key &, const secret_key &, key_image &);
     static void generate_ring_signature(const hash &, const key_image &,
@@ -248,8 +250,11 @@ namespace crypto {
   inline void generate_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const secret_key &r, signature &sig) {
     crypto_ops::generate_tx_proof(prefix_hash, R, A, B, D, r, sig);
   }
-  inline bool check_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const signature &sig) {
-    return crypto_ops::check_tx_proof(prefix_hash, R, A, B, D, sig);
+  inline void generate_tx_proof_v1(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const secret_key &r, signature &sig) {
+    crypto_ops::generate_tx_proof_v1(prefix_hash, R, A, B, D, r, sig);
+  }
+  inline bool check_tx_proof(const hash &prefix_hash, const public_key &R, const public_key &A, const boost::optional<public_key> &B, const public_key &D, const signature &sig, const int version) {
+    return crypto_ops::check_tx_proof(prefix_hash, R, A, B, D, sig, version);
   }
 
   /* To send money to a key:

src/crypto/hash.h

@@ -73,7 +73,7 @@ namespace crypto {
   inline void cn_slow_hash(const void *data, std::size_t length, hash &hash, int variant = 0, uint64_t height = 0) {
     cn_slow_hash(data, length, reinterpret_cast<char *>(&hash), variant, 0/*prehashed*/, height);
   }
-  
+
   inline void cn_slow_hash_prehashed(const void *data, std::size_t length, hash &hash, int variant = 0, uint64_t height = 0) {
     cn_slow_hash(data, length, reinterpret_cast<char *>(&hash), variant, 1/*prehashed*/, height);
   }

src/crypto/rx-slow-hash.c

@@ -116,6 +116,46 @@ static inline int enabled_flags(void) {
 #define SEEDHASH_EPOCH_BLOCKS	2048	/* Must be same as BLOCKS_SYNCHRONIZING_MAX_COUNT in cryptonote_config.h */
 #define SEEDHASH_EPOCH_LAG		64
 
+static inline int is_power_of_2(uint64_t n) { return n && (n & (n-1)) == 0; }
+
+static int get_seedhash_epoch_lag(void)
+{
+  static unsigned int lag = (unsigned int)-1;
+  if (lag != (unsigned int)-1)
+    return lag;
+  const char *e = getenv("SEEDHASH_EPOCH_LAG");
+  if (e)
+  {
+    lag = atoi(e);
+    if (lag > SEEDHASH_EPOCH_LAG || !is_power_of_2(lag))
+      lag = SEEDHASH_EPOCH_LAG;
+  }
+  else
+  {
+    lag = SEEDHASH_EPOCH_LAG;
+  }
+  return lag;
+}
+
+static unsigned int get_seedhash_epoch_blocks(void)
+{
+  static unsigned int blocks = (unsigned int)-1;
+  if (blocks != (unsigned int)-1)
+    return blocks;
+  const char *e = getenv("SEEDHASH_EPOCH_BLOCKS");
+  if (e)
+  {
+    blocks = atoi(e);
+    if (blocks < 2 || blocks > SEEDHASH_EPOCH_BLOCKS || !is_power_of_2(blocks))
+      blocks = SEEDHASH_EPOCH_BLOCKS;
+  }
+  else
+  {
+    blocks = SEEDHASH_EPOCH_BLOCKS;
+  }
+  return blocks;
+}
+
 void rx_reorg(const uint64_t split_height) {
   int i;
   CTHR_MUTEX_LOCK(rx_mutex);
@@ -130,14 +170,16 @@ void rx_reorg(const uint64_t split_height) {
 }
 
 uint64_t rx_seedheight(const uint64_t height) {
-  uint64_t s_height =  (height <= SEEDHASH_EPOCH_BLOCKS+SEEDHASH_EPOCH_LAG) ? 0 :
-                       (height - SEEDHASH_EPOCH_LAG - 1) & ~(SEEDHASH_EPOCH_BLOCKS-1);
+  const uint64_t seedhash_epoch_lag = get_seedhash_epoch_lag();
+  const uint64_t seedhash_epoch_blocks = get_seedhash_epoch_blocks();
+  uint64_t s_height =  (height <= seedhash_epoch_blocks+seedhash_epoch_lag) ? 0 :
+                       (height - seedhash_epoch_lag - 1) & ~(seedhash_epoch_blocks-1);
   return s_height;
 }
 
 void rx_seedheights(const uint64_t height, uint64_t *seedheight, uint64_t *nextheight) {
   *seedheight = rx_seedheight(height);
-  *nextheight = rx_seedheight(height + SEEDHASH_EPOCH_LAG);
+  *nextheight = rx_seedheight(height + get_seedhash_epoch_lag());
 }
 
 typedef struct seedinfo {
@@ -161,11 +203,11 @@ static void rx_initdata(randomx_cache *rs_cache, const int miners, const uint64_
     CTHR_THREAD_TYPE *st;
     si = malloc(miners * sizeof(seedinfo));
     if (si == NULL)
-      local_abort("Couldn't allocate RandomWOW mining threadinfo");
+      local_abort("Couldn't allocate RandomX mining threadinfo");
     st = malloc(miners * sizeof(CTHR_THREAD_TYPE));
     if (st == NULL) {
       free(si);
-      local_abort("Couldn't allocate RandomWOW mining threadlist");
+      local_abort("Couldn't allocate RandomX mining threadlist");
     }
     for (i=0; i<miners-1; i++) {
       si[i].si_cache = rs_cache;
@@ -194,7 +236,7 @@ static void rx_initdata(randomx_cache *rs_cache, const int miners, const uint64_
 void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const char *seedhash, const void *data, size_t length,
   char *hash, int miners, int is_alt) {
   uint64_t s_height = rx_seedheight(mainheight);
-  int toggle = (s_height & SEEDHASH_EPOCH_BLOCKS) != 0;
+  int toggle = (s_height & get_seedhash_epoch_blocks()) != 0;
   randomx_flags flags = enabled_flags() & ~disabled_flags();
   rx_state *rx_sp;
   randomx_cache *cache;
@@ -225,11 +267,11 @@ void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const ch
     if (cache == NULL) {
       cache = randomx_alloc_cache(flags | RANDOMX_FLAG_LARGE_PAGES);
       if (cache == NULL) {
-        mdebug(RX_LOGCAT, "Couldn't use largePages for RandomWOW cache");
+        mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX cache");
         cache = randomx_alloc_cache(flags);
       }
       if (cache == NULL)
-        local_abort("Couldn't allocate RandomWOW cache");
+        local_abort("Couldn't allocate RandomX cache");
     }
   }
   if (rx_sp->rs_height != seedheight || rx_sp->rs_cache == NULL || memcmp(seedhash, rx_sp->rs_hash, HASH_SIZE)) {
@@ -251,7 +293,7 @@ void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const ch
         if (rx_dataset == NULL) {
           rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_LARGE_PAGES);
           if (rx_dataset == NULL) {
-            mdebug(RX_LOGCAT, "Couldn't use largePages for RandomWOW dataset");
+            mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX dataset");
             rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_DEFAULT);
           }
           if (rx_dataset != NULL)
@@ -264,14 +306,14 @@ void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const ch
         miners = 0;
         if (!rx_dataset_nomem) {
           rx_dataset_nomem = 1;
-          mwarning(RX_LOGCAT, "Couldn't allocate RandomWOW dataset for miner");
+          mwarning(RX_LOGCAT, "Couldn't allocate RandomX dataset for miner");
         }
       }
       CTHR_MUTEX_UNLOCK(rx_dataset_mutex);
     }
     rx_vm = randomx_create_vm(flags | RANDOMX_FLAG_LARGE_PAGES, rx_sp->rs_cache, rx_dataset);
     if(rx_vm == NULL) { //large pages failed
-      mdebug(RX_LOGCAT, "Couldn't use largePages for RandomWOW VM");
+      mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX VM");
       rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset);
     }
     if(rx_vm == NULL) {//fallback if everything fails
@@ -279,7 +321,7 @@ void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const ch
       rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset);
     }
     if (rx_vm == NULL)
-      local_abort("Couldn't allocate RandomWOW VM");
+      local_abort("Couldn't allocate RandomX VM");
   } else if (miners) {
     CTHR_MUTEX_LOCK(rx_dataset_mutex);
     if (rx_dataset != NULL && rx_dataset_height != seedheight)

src/crypto/wallet/CMakeLists.txt

@@ -0,0 +1,62 @@
+# Copyright (c) 2020, The Monero Project
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#
+# Possibly user defined values.
+#
+set(MONERO_WALLET_CRYPTO_LIBRARY "auto" CACHE STRING "Select a wallet crypto library")
+
+#
+# If the user specified "auto", detect best library defaulting to internal.
+#
+if (${MONERO_WALLET_CRYPTO_LIBRARY} STREQUAL "auto")
+   monero_crypto_autodetect(AVAILABLE BEST)
+  if (DEFINED BEST)
+    message("Wallet crypto is using ${BEST} backend")
+    set(MONERO_WALLET_CRYPTO_LIBRARY ${BEST})
+  else ()
+    message("Defaulting to internal crypto library for wallet")
+    set(MONERO_WALLET_CRYPTO_LIBRARY "cn")
+  endif ()
+endif ()
+
+#
+# Configure library target "wallet-crypto" - clients will use this as a
+# library dependency which in turn will depend on the crypto library selected.
+#
+if (${MONERO_WALLET_CRYPTO_LIBRARY} STREQUAL "cn")
+  configure_file(${CMAKE_CURRENT_SOURCE_DIR}/empty.h.in ${MONERO_GENERATED_HEADERS_DIR}/crypto/wallet/ops.h)
+  add_library(wallet-crypto ALIAS cncrypto)
+else ()
+  monero_crypto_generate_header(${MONERO_WALLET_CRYPTO_LIBRARY} "${MONERO_GENERATED_HEADERS_DIR}/crypto/wallet/ops.h")
+  monero_crypto_get_target(${MONERO_WALLET_CRYPTO_LIBRARY} CRYPTO_TARGET)
+  add_library(wallet-crypto $<TARGET_OBJECTS:${CRYPTO_TARGET}>)
+  target_link_libraries(wallet-crypto cncrypto)
+endif ()
+
+

src/crypto/wallet/crypto.h

@@ -0,0 +1,56 @@
+// Copyright (c) 2020, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <cstddef>
+#include "crypto/wallet/ops.h"
+
+namespace crypto {
+  namespace wallet {
+// if C functions defined from external/supercop - cmake generates crypto/wallet/ops.h
+#if defined(monero_crypto_generate_key_derivation)
+      inline
+      bool generate_key_derivation(const public_key &tx_pub, const secret_key &view_sec, key_derivation &out)
+      {
+        return monero_crypto_generate_key_derivation(out.data, tx_pub.data, view_sec.data) == 0;
+      }
+
+      inline
+      bool derive_subaddress_public_key(const public_key &output_pub, const key_derivation &d, std::size_t index, public_key &out)
+      {
+        ec_scalar scalar;
+        derivation_to_scalar(d, index, scalar);
+        return monero_crypto_generate_subaddress_public_key(out.data, output_pub.data, scalar.data) == 0;
+      }
+#else
+    using ::crypto::generate_key_derivation;
+    using ::crypto::derive_subaddress_public_key;
+#endif
+  }
+}