Merge pull request #185 from wowario/fix

wallet2: fix mishandling rct outputs in coinbase tx

.gitmodules

@@ -9,3 +9,6 @@
 [submodule "external/rapidjson"]
 	path = external/rapidjson
 	url = https://github.com/Tencent/rapidjson
+[submodule "external/trezor-common"]
+	path = external/trezor-common
+	url = https://github.com/trezor/trezor-common.git

.travis.yml

@@ -23,9 +23,9 @@ env:
     - DOCKER_PACKAGES="build-essential libtool cmake autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache"
   matrix:
 # ARM v7
-    - HOST=arm-linux-gnueabihf PACKAGES="gperf g++-arm-linux-gnueabihf"
+    - HOST=arm-linux-gnueabihf PACKAGES="python3 gperf g++-arm-linux-gnueabihf"
 # ARM v8
-    - HOST=aarch64-linux-gnu PACKAGES="gperf g++-aarch64-linux-gnu"
+    - HOST=aarch64-linux-gnu PACKAGES="python3 gperf g++-aarch64-linux-gnu"
 # i686 Win
     - HOST=i686-w64-mingw32 PACKAGES="python3 nsis g++-mingw-w64-i686"
 # i686 Linux
@@ -33,7 +33,7 @@ env:
 # Win64
     - HOST=x86_64-w64-mingw32 PACKAGES="cmake python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64 bc" RUN_TESTS=true
 # x86_64 Linux
-    - HOST=x86_64-unknown-linux-gnu PACKAGES="gperf cmake python3-zmq protobuf-compiler libdbus-1-dev libharfbuzz-dev libprotobuf-dev" RUN_TESTS=true
+    - HOST=x86_64-unknown-linux-gnu PACKAGES="gperf cmake python3-zmq libdbus-1-dev libharfbuzz-dev" RUN_TESTS=true
 # Cross-Mac
     - HOST=x86_64-apple-darwin11 PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git" OSX_SDK=10.11
 
@@ -52,13 +52,14 @@ before_script:
     - if [ -n "$OSX_SDK" -a ! -f contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi
     - if [ -n "$OSX_SDK" -a -f contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then tar -C contrib/depends/SDKs -xf contrib/depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi
     - if [[ $HOST = *-mingw32 ]]; then $DOCKER_EXEC bash -c "update-alternatives --set $HOST-g++ \$(which $HOST-g++-posix)"; fi
+    - if [[ $HOST = *-mingw32 ]]; then $DOCKER_EXEC bash -c "update-alternatives --set $HOST-gcc \$(which $HOST-gcc-posix)"; fi
     - if [ -z "$NO_DEPENDS" ]; then $DOCKER_EXEC bash -c "CONFIG_SHELL= make $MAKEJOBS -C contrib/depends HOST=$HOST $DEP_OPTS"; fi
 script:
     - git submodule init && git submodule update
     - export TRAVIS_COMMIT_LOG=`git log --format=fuller -1`
     - OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST
     - if [ -z "$NO_DEPENDS" ]; then $DOCKER_EXEC ccache --max-size=$CCACHE_SIZE; fi
-    - $DOCKER_EXEC bash -c "mkdir build && cd build && cmake -DCMAKE_TOOLCHAIN_FILE=$TRAVIS_BUILD_DIR/contrib/depends/$HOST/share/toolchain.cmake -DTRAVIS=true .. && make $MAKEJOBS"
+    - $DOCKER_EXEC bash -c "mkdir build && cd build && cmake -DCMAKE_TOOLCHAIN_FILE=$TRAVIS_BUILD_DIR/contrib/depends/$HOST/share/toolchain.cmake .. && make $MAKEJOBS"
     - export LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/contrib/depends/$HOST/lib
 after_script:
     - echo $TRAVIS_COMMIT_RANGE

CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 #
 # All rights reserved.
 #
@@ -113,9 +113,6 @@ string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWER)
 # to identify the target architecture, to direct logic in this cmake script.
 # Since ARCH is a cached variable, it will not be set on first cmake invocation.
 if (NOT ARCH OR ARCH STREQUAL "" OR ARCH STREQUAL "native" OR ARCH STREQUAL "default")
-  if(CMAKE_SYSTEM_PROCESSOR STREQUAL "")
-    set(CMAKE_SYSTEM_PROCESSOR ${CMAKE_HOST_SYSTEM_PROCESSOR})
-  endif()
   set(ARCH_ID "${CMAKE_SYSTEM_PROCESSOR}")
 else()
   set(ARCH_ID "${ARCH}")
@@ -151,7 +148,7 @@ if(ARCH_ID STREQUAL "powerpc64" OR ARCH_ID STREQUAL "ppc64")
   set(PPC     0)
 endif()
 
-if(ARCH_ID STREQUAL "powerpc")
+if(ARCH_ID STREQUAL "powerpc" OR ARCH_ID STREQUAL "ppc")
   set(PPC64LE 0)
   set(PPC64   0)
   set(PPC     1)
@@ -187,19 +184,22 @@ endif()
 if(NOT MANUAL_SUBMODULES)
   find_package(Git)
   if(GIT_FOUND)
+    function (check_submodule relative_path)
+      execute_process(COMMAND git rev-parse "HEAD" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/${relative_path} OUTPUT_VARIABLE localHead)
+      execute_process(COMMAND git rev-parse "HEAD:${relative_path}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE checkedHead)
+      string(COMPARE EQUAL "${localHead}" "${checkedHead}" upToDate)
+      if (upToDate)
+        message(STATUS "Submodule '${relative_path}' is up-to-date")
+      else()
+        message(FATAL_ERROR "Submodule '${relative_path}' is not up-to-date. Please update with\ngit submodule update --init --force ${relative_path}\nor run cmake with -DMANUAL_SUBMODULES=1")
+      endif()
+    endfunction ()
+    
     message(STATUS "Checking submodules")
-    execute_process(COMMAND bash -c "cd ${CMAKE_CURRENT_SOURCE_DIR}/external/miniupnp && git rev-parse HEAD" OUTPUT_VARIABLE miniupnpLocalHead WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
-    execute_process(COMMAND bash -c "cd ${CMAKE_CURRENT_SOURCE_DIR}/external/unbound && git rev-parse HEAD" OUTPUT_VARIABLE unboundLocalHead WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
-    execute_process(COMMAND bash -c "cd ${CMAKE_CURRENT_SOURCE_DIR}/external/rapidjson && git rev-parse HEAD" OUTPUT_VARIABLE rapidjsonLocalHead WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
-    execute_process(COMMAND bash -c "git ls-tree HEAD ${CMAKE_CURRENT_SOURCE_DIR}/external/miniupnp | awk '{print $3}'" OUTPUT_VARIABLE miniupnpCheckedHead WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
-    execute_process(COMMAND bash -c "git ls-tree HEAD ${CMAKE_CURRENT_SOURCE_DIR}/external/unbound | awk '{print $3}'" OUTPUT_VARIABLE unboundCheckedHead WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
-    execute_process(COMMAND bash -c "git ls-tree HEAD ${CMAKE_CURRENT_SOURCE_DIR}/external/rapidjson | awk '{print $3}'" OUTPUT_VARIABLE rapidjsonCheckedHead WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
-    string(COMPARE EQUAL "${miniupnpLocalHead}" "${miniupnpCheckedHead}" miniupnpUpToDate)
-    string(COMPARE EQUAL "${unboundLocalHead}" "${unboundCheckedHead}" unboundUpToDate)
-    string(COMPARE EQUAL "${rapidjsonLocalHead}" "${rapidjsonCheckedHead}" rapidjsonUpToDate)
-    if (NOT miniupnpUpToDate OR NOT unboundUpToDate OR NOT rapidjsonUpToDate)
-      message(FATAL_ERROR "Submodules not up to date. Please update with git submodule init && git submodule update, or run cmake with -DMANUAL_SUBMODULES=1")
-    endif()
+    check_submodule(external/miniupnp)
+    check_submodule(external/unbound)
+    check_submodule(external/rapidjson)
+    check_submodule(external/trezor-common)
   endif()
 endif()
 
@@ -211,6 +211,9 @@ set(PER_BLOCK_CHECKPOINT 1)
 
 if(PER_BLOCK_CHECKPOINT)
   add_definitions("-DPER_BLOCK_CHECKPOINT")
+  set(Blocks "blocks")
+else()
+  set(Blocks "")
 endif()
 
 list(INSERT CMAKE_MODULE_PATH 0
@@ -527,6 +530,9 @@ else (HIDAPI_FOUND)
   message(STATUS "Could not find HIDAPI")
 endif()
 
+# Trezor support check
+include(CheckTrezor)
+
 if(MSVC)
   add_definitions("/bigobj /MP /W3 /GS- /D_CRT_SECURE_NO_WARNINGS /wd4996 /wd4345 /D_WIN32_WINNT=0x0600 /DWIN32_LEAN_AND_MEAN /DGTEST_HAS_TR1_TUPLE=0 /FIinline_c.h /D__SSE4_1__")
   # set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /Dinline=__inline")
@@ -603,9 +609,6 @@ else()
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARCH_FLAG}")
 
   set(WARNINGS "-Wall -Wextra -Wpointer-arith -Wundef -Wvla -Wwrite-strings -Wno-error=extra -Wno-error=deprecated-declarations -Wno-unused-parameter -Wno-unused-variable -Wno-error=unused-variable -Wno-error=undef -Wno-error=uninitialized")
-  if(NOT MINGW)
-    set(WARNINGS_AS_ERRORS_FLAG "-Werror")
-  endif()
   if(CMAKE_C_COMPILER_ID STREQUAL "Clang")
     if(ARM)
       set(WARNINGS "${WARNINGS} -Wno-error=inline-asm")
@@ -694,12 +697,17 @@ else()
   add_linker_flag_if_supported(-Wl,-z,noexecstack noexecstack_SUPPORTED)
   if (noexecstack_SUPPORTED)
     set(LD_SECURITY_FLAGS "${LD_SECURITY_FLAGS} -Wl,-z,noexecstack")
-    set(LD_RAW_FLAGS ${LD_RAW_FLAGS} -z noexecstack)
   endif()
   add_linker_flag_if_supported(-Wl,-z,noexecheap noexecheap_SUPPORTED)
   if (noexecheap_SUPPORTED)
     set(LD_SECURITY_FLAGS "${LD_SECURITY_FLAGS} -Wl,-z,noexecheap")
-    set(LD_RAW_FLAGS ${LD_RAW_FLAGS} -z noexecheap)
+  endif()
+
+  if(BACKCOMPAT)
+      add_definitions(-DFDELT_TYPE=long\ int)
+      add_linker_flag_if_supported(-Wl,--wrap=__divmoddi4 LD_BACKCOMPAT_FLAGS)
+      add_linker_flag_if_supported(-Wl,--wrap=glob LD_BACKCOMPAT_FLAGS)
+      message(STATUS "Using Lib C back compat flags: ${LD_BACKCOMPAT_FLAGS}")
   endif()
 
   # some windows linker bits
@@ -714,7 +722,7 @@ else()
 
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_FLAG} ${WARNINGS} ${C_WARNINGS} ${COVERAGE_FLAGS} ${PIC_FLAG} ${C_SECURITY_FLAGS}")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_CPP_FLAG} ${WARNINGS} ${CXX_WARNINGS} ${COVERAGE_FLAGS} ${PIC_FLAG} ${CXX_SECURITY_FLAGS}")
-  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${LD_SECURITY_FLAGS}")
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${LD_SECURITY_FLAGS} ${LD_BACKCOMPAT_FLAGS}")
 
   # With GCC 6.1.1 the compiled binary malfunctions due to aliasing. Until that
   # is fixed in the code (Issue #847), force compiler to be conservative.
@@ -822,6 +830,9 @@ else()
     set(DEBUG_FLAGS "${DEBUG_FLAGS} -O0 ")
   endif()
 
+  # At least some CLANGs default to not enough for monero
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ftemplate-depth=900")
+
   if(NOT DEFINED USE_LTO_DEFAULT)
     set(USE_LTO_DEFAULT false)
   endif()
@@ -902,11 +913,11 @@ endif()
 include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
 if(MINGW)
   set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wa,-mbig-obj")
-  set(EXTRA_LIBRARIES mswsock;ws2_32;iphlpapi;crypt32)
+  set(EXTRA_LIBRARIES mswsock;ws2_32;iphlpapi;crypt32;bcrypt)
   if(DEPENDS)
-    set(ICU_LIBRARIES ${Boost_LOCALE_LIBRARY} sicuio sicuin sicuuc sicudt sicutu iconv)
+    set(ICU_LIBRARIES sicuio sicuin sicuuc sicudt sicutu iconv)
   else()
-    set(ICU_LIBRARIES ${Boost_LOCALE_LIBRARY} icuio icuin icuuc icudt icutu iconv)
+    set(ICU_LIBRARIES icuio icuin icuuc icudt icutu iconv)
   endif()
 elseif(APPLE OR OPENBSD OR ANDROID)
   set(EXTRA_LIBRARIES "")
@@ -924,7 +935,7 @@ endif()
 
 list(APPEND EXTRA_LIBRARIES ${CMAKE_DL_LIBS})
 
-if (HIDAPI_FOUND) 
+if (HIDAPI_FOUND OR LibUSB_COMPILE_TEST_PASSED)
   if (APPLE)
     if(DEPENDS)
       list(APPEND EXTRA_LIBRARIES "-framework Foundation -framework IOKit")
@@ -959,7 +970,9 @@ if(ANDROID)
 endif()
 if(CMAKE_C_COMPILER_ID STREQUAL "Clang" AND ARCH_WIDTH EQUAL "32" AND NOT IOS AND NOT FREEBSD)
   find_library(ATOMIC atomic)
-  list(APPEND EXTRA_LIBRARIES ${ATOMIC})
+  if (ATOMIC_FOUND)
+    list(APPEND EXTRA_LIBRARIES ${ATOMIC})
+  endif()
 endif()
 
 find_path(ZMQ_INCLUDE_PATH zmq.hpp)

CONTRIBUTING.md

@@ -19,7 +19,8 @@ posted to #monero-dev on irc.freenode.net).
 
 Patches should be self contained. A good rule of thumb is to have
 one patch per separate issue, feature, or logical change. Also, no
-other changes, such as random whitespace changes or reindentation.
+other changes, such as random whitespace changes, reindentation,
+or fixing typoes, spelling, or wording, unless user visible.
 Following the code style of the particular chunk of code you're
 modifying is encouraged. Proper squashing should be done (eg, if
 you're making a buggy patch, then a later patch to fix the bug,

Dockerfile

@@ -18,14 +18,17 @@ RUN set -ex && \
         libtool-bin \
         autoconf \
         automake \
-        bzip2
+        bzip2 \
+        xsltproc \
+        gperf \
+        unzip
 
 WORKDIR /usr/local
 
 #Cmake
-ARG CMAKE_VERSION=3.12.1
-ARG CMAKE_VERSION_DOT=v3.12
-ARG CMAKE_HASH=c53d5c2ce81d7a957ee83e3e635c8cda5dfe20c9d501a4828ee28e1615e57ab2
+ARG CMAKE_VERSION=3.13.0
+ARG CMAKE_VERSION_DOT=v3.13
+ARG CMAKE_HASH=4058b2f1a53c026564e8936698d56c3b352d90df067b195cb749a97a3d273c90 
 RUN set -ex \
     && curl -s -O https://cmake.org/files/${CMAKE_VERSION_DOT}/cmake-${CMAKE_VERSION}.tar.gz \
     && echo "${CMAKE_HASH}  cmake-${CMAKE_VERSION}.tar.gz" | sha256sum -c \
@@ -49,8 +52,8 @@ RUN set -ex \
 ENV BOOST_ROOT /usr/local/boost_${BOOST_VERSION}
 
 # OpenSSL
-ARG OPENSSL_VERSION=1.1.0h
-ARG OPENSSL_HASH=5835626cde9e99656585fc7aaa2302a73a7e1340bf8c14fd635a62c66802a517
+ARG OPENSSL_VERSION=1.1.0j
+ARG OPENSSL_HASH=31bec6c203ce1a8e93d5994f4ed304c63ccf07676118b6634edded12ad1b3246
 RUN set -ex \
     && curl -s -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \
     && echo "${OPENSSL_HASH}  openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c \
@@ -76,8 +79,8 @@ RUN set -ex \
     && ldconfig
 
 # zmq.hpp
-ARG CPPZMQ_VERSION=v4.2.3
-ARG CPPZMQ_HASH=6aa3ab686e916cb0e62df7fa7d12e0b13ae9fae6
+ARG CPPZMQ_VERSION=v4.3.0
+ARG CPPZMQ_HASH=213da0b04ae3b4d846c9abc46bab87f86bfb9cf4
 RUN set -ex \
     && git clone https://github.com/zeromq/cppzmq.git -b ${CPPZMQ_VERSION} \
     && cd cppzmq \
@@ -109,6 +112,56 @@ RUN set -ex \
     && make check \
     && make install
 
+# Udev
+ARG UDEV_VERSION=v3.2.6
+ARG UDEV_HASH=0c35b136c08d64064efa55087c54364608e65ed6
+RUN set -ex \
+    && git clone https://github.com/gentoo/eudev -b ${UDEV_VERSION} \
+    && cd eudev \
+    && test `git rev-parse HEAD` = ${UDEV_HASH} || exit 1 \
+    && ./autogen.sh \
+    && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --disable-gudev --disable-introspection --disable-hwdb --disable-manpages --disable-shared \
+    && make \
+    && make install
+
+# Libusb
+ARG USB_VERSION=v1.0.22
+ARG USB_HASH=0034b2afdcdb1614e78edaa2a9e22d5936aeae5d
+RUN set -ex \
+    && git clone https://github.com/libusb/libusb.git -b ${USB_VERSION} \
+    && cd libusb \
+    && test `git rev-parse HEAD` = ${USB_HASH} || exit 1 \
+    && ./autogen.sh \
+    && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --disable-shared \
+    && make \
+    && make install
+
+# Hidapi
+ARG HIDAPI_VERSION=hidapi-0.8.0-rc1
+ARG HIDAPI_HASH=40cf516139b5b61e30d9403a48db23d8f915f52c
+RUN set -ex \
+    && git clone https://github.com/signal11/hidapi -b ${HIDAPI_VERSION} \
+    && cd hidapi \
+    && test `git rev-parse HEAD` = ${HIDAPI_HASH} || exit 1 \
+    && ./bootstrap \
+    && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --enable-static --disable-shared \
+    && make \
+    && make install
+
+# Protobuf
+ARG PROTOBUF_VERSION=v3.6.1
+ARG PROTOBUF_HASH=48cb18e5c419ddd23d9badcfe4e9df7bde1979b2
+RUN set -ex \
+    && git clone https://github.com/protocolbuffers/protobuf -b ${PROTOBUF_VERSION} \
+    && cd protobuf \
+    && test `git rev-parse HEAD` = ${PROTOBUF_HASH} || exit 1 \
+    && git submodule update --init --recursive \
+    && ./autogen.sh \
+    && CFLAGS="-fPIC" CXXFLAGS="-fPIC" ./configure --enable-static --disable-shared \
+    && make \
+    && make install \
+    && ldconfig
+
 WORKDIR /src
 COPY . .
 

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2014-2018, The Monero Project
+Copyright (c) 2014-2019, The Monero Project
 
 All rights reserved.
 

Makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 #
 # All rights reserved.
 #
@@ -46,6 +46,10 @@ endif
 
 all: release-all
 
+depends:
+	cd contrib/depends && $(MAKE) HOST=$(target) && cd ../.. && mkdir -p build/$(target)/release
+	cd build/$(target)/release && cmake -DCMAKE_TOOLCHAIN_FILE=$(CURDIR)/contrib/depends/$(target)/share/toolchain.cmake ../../.. && $(MAKE)
+
 cmake-debug:
 	mkdir -p $(builddir)/debug
 	cd $(builddir)/debug && cmake -D CMAKE_BUILD_TYPE=Debug $(topdir)

PKGBUILD

@@ -3,7 +3,8 @@
 
 pkgbase=('wownero-git')
 pkgname=('wownero-git')
-pkgver=0.5.0.2
+_gitbranch='dev-v0.5'
+pkgver=0.5.0.0
 pkgrel=1
 pkgdesc="a fairly launched privacy-centric meme coin with no premine and a finite supply"
 license=('custom:Cryptonote')
@@ -13,7 +14,7 @@ depends=('boost-libs' 'zeromq' 'unbound' 'libusb')
 makedepends=('git' 'cmake' 'boost')
 provides=('wownero-git')
 
-source=("${pkgname}"::"git+https://github.com/wownero/wownero")
+source=("${pkgname}"::"git+https://github.com/wownero/wownero#branch=${_gitbranch}")
 
 sha256sums=('SKIP')
 
@@ -25,7 +26,6 @@ pkgver() {
 
 build() {
   cd "${srcdir}/${pkgname}"
-  git fetch --tags && git checkout tags/v0.5.0.2 -b v0.5.0.2
   USE_SINGLE_BUILDDIR=1 make
 }
 

README.i18n.md

@@ -3,7 +3,7 @@ Monero daemon internationalization
 
 The Monero command line tools can be translated in various languages. If you wish to contribute and need help/support, contact the [Monero Localization Workgroup on Taiga](https://taiga.getmonero.org/project/erciccione-monero-localization/) or come chat on `#monero-translations` (Freenode/IRC, riot/matrix, MatterMost)
 
-In order to use the same translation workflow as the [Monero Core GUI](https://github.com/monero-project/monero-core), they use Qt Linguist translation files.  However, to avoid the dependencies on Qt this normally implies, they use a custom loader to read those files at runtime.
+In order to use the same translation workflow as the [Monero Core GUI](https://github.com/monero-project/monero-gui), they use Qt Linguist translation files.  However, to avoid the dependencies on Qt this normally implies, they use a custom loader to read those files at runtime.
 
 ### Tools for translators
 

README.md

@@ -41,7 +41,7 @@ Dates are provided in the format YYYY-MM-DD.
 | 6969                           | 2018-04-24 | Busty Brazzers                | v0.2.0.0               | v0.2.0.0                  | Bulletproofs, LWMA difficulty algorithm, ringsize >= 10, reduce unlock to 4
 | 53666                          | 2018-10-06 | Cool Cage                | v0.3.0.0               | v0.3.1.3                  | Cryptonight variant 2, LWMA v2, ringsize = 22, MMS
 | 63469                          | 2018-11-11 | Dank Doge               | v0.4.0.0               | v0.4.0.0                  | LWMA v4
-| 81769                          | 2019-02-19 | Erotic EggplantEmoji    | v0.5.0.0               | v0.5.1.0                  | Cryptonight/wow, LWMA v1 with N=144, Updated Bulletproofs, Fee Per Byte, Auto-churn
+| 81769                          | 2019-02-19 | Erotic EggplantEmoji    | v0.5.0.0               | v0.5.0.2                  | Cryptonight/wow, LWMA v1 with N=144, Updated Bulletproofs, Fee Per Byte, Auto-churn
 
 X's indicate that these details have not been determined as of commit date.
 

cmake/32-bit-toolchain.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 # 
 # All rights reserved.
 # 

cmake/64-bit-toolchain.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 # 
 # All rights reserved.
 # 

cmake/CheckTrezor.cmake

@@ -0,0 +1,166 @@
+OPTION(USE_DEVICE_TREZOR "Trezor support compilation" OFF)
+OPTION(USE_DEVICE_TREZOR_LIBUSB "Trezor LibUSB compilation" OFF)
+OPTION(USE_DEVICE_TREZOR_UDP_RELEASE "Trezor UdpTransport in release mode" OFF)
+
+# Helper function to fix cmake < 3.6.0 FindProtobuf variables
+function(_trezor_protobuf_fix_vars)
+    if(${CMAKE_VERSION} VERSION_LESS "3.6.0")
+        foreach(UPPER
+                PROTOBUF_SRC_ROOT_FOLDER
+                PROTOBUF_IMPORT_DIRS
+                PROTOBUF_DEBUG
+                PROTOBUF_LIBRARY
+                PROTOBUF_PROTOC_LIBRARY
+                PROTOBUF_INCLUDE_DIR
+                PROTOBUF_PROTOC_EXECUTABLE
+                PROTOBUF_LIBRARY_DEBUG
+                PROTOBUF_PROTOC_LIBRARY_DEBUG
+                PROTOBUF_LITE_LIBRARY
+                PROTOBUF_LITE_LIBRARY_DEBUG
+                )
+            if (DEFINED ${UPPER})
+                string(REPLACE "PROTOBUF_" "Protobuf_" Camel ${UPPER})
+                if (NOT DEFINED ${Camel})
+                    set(${Camel} ${${UPPER}} PARENT_SCOPE)
+                endif()
+            endif()
+        endforeach()
+    endif()
+endfunction()
+
+# Use Trezor master switch
+if (USE_DEVICE_TREZOR)
+    # Protobuf is required to build protobuf messages for Trezor
+    include(FindProtobuf OPTIONAL)
+    find_package(Protobuf)
+    _trezor_protobuf_fix_vars()
+
+    # Protobuf handling the cache variables set in docker.
+    if(NOT Protobuf_FOUND AND NOT Protobuf_LIBRARY AND NOT Protobuf_PROTOC_EXECUTABLE AND NOT Protobuf_INCLUDE_DIR)
+        message(STATUS "Could not find Protobuf")
+    elseif(NOT Protobuf_LIBRARY OR NOT EXISTS "${Protobuf_LIBRARY}")
+        message(STATUS "Protobuf library not found: ${Protobuf_LIBRARY}")
+        unset(Protobuf_FOUND)
+    elseif(NOT Protobuf_PROTOC_EXECUTABLE OR NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}")
+        message(STATUS "Protobuf executable not found: ${Protobuf_PROTOC_EXECUTABLE}")
+        unset(Protobuf_FOUND)
+    elseif(NOT Protobuf_INCLUDE_DIR OR NOT EXISTS "${Protobuf_INCLUDE_DIR}")
+        message(STATUS "Protobuf include dir not found: ${Protobuf_INCLUDE_DIR}")
+        unset(Protobuf_FOUND)
+    else()
+        message(STATUS "Protobuf lib: ${Protobuf_LIBRARY}, inc: ${Protobuf_INCLUDE_DIR}, protoc: ${Protobuf_PROTOC_EXECUTABLE}")
+        set(Protobuf_INCLUDE_DIRS ${Protobuf_INCLUDE_DIR})
+        set(Protobuf_FOUND 1)  # override found if all rquired info was provided by variables
+    endif()
+
+else()
+    message(STATUS "Trezor support disabled by USE_DEVICE_TREZOR")
+endif()
+
+if(Protobuf_FOUND AND USE_DEVICE_TREZOR)
+    if (NOT "$ENV{TREZOR_PYTHON}" STREQUAL "")
+        set(TREZOR_PYTHON "$ENV{TREZOR_PYTHON}" CACHE INTERNAL "Copied from environment variable TREZOR_PYTHON")
+    else()
+        find_package(Python QUIET COMPONENTS Interpreter)  # cmake 3.12+
+        if(Python_Interpreter_FOUND)
+            set(TREZOR_PYTHON "${Python_EXECUTABLE}")
+        endif()
+    endif()
+
+    if(NOT TREZOR_PYTHON)
+        find_package(PythonInterp)
+        if(PYTHONINTERP_FOUND AND PYTHON_EXECUTABLE)
+            set(TREZOR_PYTHON "${PYTHON_EXECUTABLE}")
+        endif()
+    endif()
+
+    if(NOT TREZOR_PYTHON)
+        message(STATUS "Trezor: Python not found")
+    endif()
+endif()
+
+# Protobuf compilation test
+if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON)
+    execute_process(COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I "${CMAKE_SOURCE_DIR}/cmake" -I "${Protobuf_INCLUDE_DIR}" "${CMAKE_SOURCE_DIR}/cmake/test-protobuf.proto" --cpp_out ${CMAKE_BINARY_DIR} RESULT_VARIABLE RET OUTPUT_VARIABLE OUT ERROR_VARIABLE ERR)
+    if(RET)
+        message(STATUS "Protobuf test generation failed: ${OUT} ${ERR}")
+    endif()
+
+    try_compile(Protobuf_COMPILE_TEST_PASSED
+        "${CMAKE_BINARY_DIR}"
+        SOURCES
+        "${CMAKE_BINARY_DIR}/test-protobuf.pb.cc"
+        "${CMAKE_SOURCE_DIR}/cmake/test-protobuf.cpp"
+        CMAKE_FLAGS
+        "-DINCLUDE_DIRECTORIES=${Protobuf_INCLUDE_DIR};${CMAKE_BINARY_DIR}"
+        "-DCMAKE_CXX_STANDARD=11"
+        LINK_LIBRARIES ${Protobuf_LIBRARY}
+        OUTPUT_VARIABLE OUTPUT
+    )
+    if(NOT Protobuf_COMPILE_TEST_PASSED)
+        message(STATUS "Protobuf Compilation test failed: ${OUTPUT}.")
+    endif()
+endif()
+
+# Try to build protobuf messages
+if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON AND Protobuf_COMPILE_TEST_PASSED)
+    set(ENV{PROTOBUF_INCLUDE_DIRS} "${Protobuf_INCLUDE_DIR}")
+    set(ENV{PROTOBUF_PROTOC_EXECUTABLE} "${Protobuf_PROTOC_EXECUTABLE}")
+    execute_process(COMMAND ${TREZOR_PYTHON} tools/build_protob.py WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/../src/device_trezor/trezor RESULT_VARIABLE RET OUTPUT_VARIABLE OUT ERROR_VARIABLE ERR)
+    if(RET)
+        message(WARNING "Trezor protobuf messages could not be regenerated (err=${RET}, python ${PYTHON})."
+                "OUT: ${OUT}, ERR: ${ERR}."
+                "Please read src/device_trezor/trezor/tools/README.md")
+    else()
+        message(STATUS "Trezor protobuf messages regenerated out: \"${OUT}.\"")
+        set(DEVICE_TREZOR_READY 1)
+        add_definitions(-DDEVICE_TREZOR_READY=1)
+        add_definitions(-DPROTOBUF_INLINE_NOT_IN_HEADERS=0)
+
+        if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+            add_definitions(-DTREZOR_DEBUG=1)
+        endif()
+
+        if(USE_DEVICE_TREZOR_UDP_RELEASE)
+            add_definitions(-DUSE_DEVICE_TREZOR_UDP_RELEASE=1)
+        endif()
+
+        if (Protobuf_INCLUDE_DIR)
+            include_directories(${Protobuf_INCLUDE_DIR})
+        endif()
+
+        # LibUSB support, check for particular version
+        # Include support only if compilation test passes
+        if (USE_DEVICE_TREZOR_LIBUSB)
+            find_package(LibUSB)
+        endif()
+
+        if (LibUSB_COMPILE_TEST_PASSED)
+            add_definitions(-DHAVE_TREZOR_LIBUSB=1)
+            if(LibUSB_INCLUDE_DIRS)
+                include_directories(${LibUSB_INCLUDE_DIRS})
+            endif()
+        endif()
+
+        set(TREZOR_LIBUSB_LIBRARIES "")
+        if(LibUSB_COMPILE_TEST_PASSED)
+            list(APPEND TREZOR_LIBUSB_LIBRARIES ${LibUSB_LIBRARIES})
+            message(STATUS "Trezor compatible LibUSB found at: ${LibUSB_INCLUDE_DIRS}")
+        endif()
+
+        if (BUILD_GUI_DEPS)
+            set(TREZOR_DEP_LIBS "")
+            set(TREZOR_DEP_LINKER "")
+
+            if (Protobuf_LIBRARY)
+                list(APPEND TREZOR_DEP_LIBS ${Protobuf_LIBRARY})
+                string(APPEND TREZOR_DEP_LINKER " -lprotobuf")
+            endif()
+
+            if (TREZOR_LIBUSB_LIBRARIES)
+                list(APPEND TREZOR_DEP_LIBS ${TREZOR_LIBUSB_LIBRARIES})
+                string(APPEND TREZOR_DEP_LINKER " -lusb-1.0")
+            endif()
+        endif()
+    endif()
+endif()

cmake/FindLibUSB.cmake

@@ -0,0 +1,140 @@
+# - Find libusb for portable USB support
+# This module will find libusb as published by
+#  http://libusb.sf.net and
+#  http://libusb-win32.sf.net
+#
+# It will use PkgConfig if present and supported, else search
+# it on its own. If the LibUSB_ROOT_DIR environment variable
+# is defined, it will be used as base path.
+# The following standard variables get defined:
+#  LibUSB_FOUND:        true if LibUSB was found
+#  LibUSB_HEADER_FILE:  the location of the C header file
+#  LibUSB_INCLUDE_DIRS: the directory that contains the include file
+#  LibUSB_LIBRARIES:    the library
+# source: https://github.com/IntelRealSense/librealsense
+
+include ( CheckLibraryExists )
+include ( CheckIncludeFile )
+
+find_package ( PkgConfig )
+if ( PKG_CONFIG_FOUND )
+    pkg_check_modules ( PKGCONFIG_LIBUSB libusb-1.0 )
+    if ( NOT PKGCONFIG_LIBUSB_FOUND )
+        pkg_check_modules ( PKGCONFIG_LIBUSB libusb )
+    endif ( NOT PKGCONFIG_LIBUSB_FOUND )
+endif ( PKG_CONFIG_FOUND )
+
+if ( PKGCONFIG_LIBUSB_FOUND )
+    set ( LibUSB_INCLUDE_DIRS ${PKGCONFIG_LIBUSB_INCLUDE_DIRS} )
+    foreach ( i ${PKGCONFIG_LIBUSB_LIBRARIES} )
+        string ( REGEX MATCH "[^-]*" ibase "${i}" )
+        find_library ( ${ibase}_LIBRARY
+                NAMES ${i}
+                PATHS ${PKGCONFIG_LIBUSB_LIBRARY_DIRS}
+                )
+        if ( ${ibase}_LIBRARY )
+            list ( APPEND LibUSB_LIBRARIES ${${ibase}_LIBRARY} )
+        endif ( ${ibase}_LIBRARY )
+        mark_as_advanced ( ${ibase}_LIBRARY )
+    endforeach ( i )
+
+else ( PKGCONFIG_LIBUSB_FOUND )
+    find_file ( LibUSB_HEADER_FILE
+            NAMES
+            libusb.h usb.h
+            PATHS
+            $ENV{ProgramFiles}/LibUSB-Win32
+            $ENV{LibUSB_ROOT_DIR}
+            PATH_SUFFIXES
+            include
+            libusb-1.0
+            include/libusb-1.0
+            )
+    mark_as_advanced ( LibUSB_HEADER_FILE )
+    get_filename_component ( LibUSB_INCLUDE_DIRS "${LibUSB_HEADER_FILE}" PATH )
+
+    if ( ${CMAKE_SYSTEM_NAME} STREQUAL "Windows" )
+        # LibUSB-Win32 binary distribution contains several libs.
+        # Use the lib that got compiled with the same compiler.
+        if ( MSVC )
+            if ( WIN32 )
+                set ( LibUSB_LIBRARY_PATH_SUFFIX lib/msvc )
+            else ( WIN32 )
+                set ( LibUSB_LIBRARY_PATH_SUFFIX lib/msvc_x64 )
+            endif ( WIN32 )
+        elseif ( BORLAND )
+            set ( LibUSB_LIBRARY_PATH_SUFFIX lib/bcc )
+        elseif ( CMAKE_COMPILER_IS_GNUCC )
+            set ( LibUSB_LIBRARY_PATH_SUFFIX lib/gcc )
+        endif ( MSVC )
+    endif ( ${CMAKE_SYSTEM_NAME} STREQUAL "Windows" )
+
+    find_library ( usb_LIBRARY
+            NAMES
+            usb-1.0 libusb usb
+            PATHS
+            $ENV{ProgramFiles}/LibUSB-Win32
+            $ENV{LibUSB_ROOT_DIR}
+            PATH_SUFFIXES
+            ${LibUSB_LIBRARY_PATH_SUFFIX}
+            )
+    mark_as_advanced ( usb_LIBRARY )
+    if ( usb_LIBRARY )
+        set ( LibUSB_LIBRARIES ${usb_LIBRARY} )
+    endif ( usb_LIBRARY )
+
+endif ( PKGCONFIG_LIBUSB_FOUND )
+
+if ( LibUSB_INCLUDE_DIRS AND LibUSB_LIBRARIES )
+    set ( LibUSB_FOUND true )
+endif ( LibUSB_INCLUDE_DIRS AND LibUSB_LIBRARIES )
+
+if ( LibUSB_FOUND )
+    set ( CMAKE_REQUIRED_INCLUDES "${LibUSB_INCLUDE_DIRS}" )
+    check_include_file ( "${LibUSB_HEADER_FILE}" LibUSB_FOUND )
+endif ( LibUSB_FOUND )
+
+if ( LibUSB_FOUND )
+    check_library_exists ( "${LibUSB_LIBRARIES}" usb_open "" LibUSB_FOUND )
+    check_library_exists ( "${LibUSB_LIBRARIES}" libusb_get_device_list "" LibUSB_VERSION_1.0 )
+    check_library_exists ( "${LibUSB_LIBRARIES}" libusb_get_port_numbers "" LibUSB_VERSION_1.0.16 )
+
+    # Library 1.0.16+ compilation test.
+    # The check_library_exists does not work well on Apple with shared libs.
+    if (APPLE OR LibUSB_VERSION_1.0.16)
+        if (APPLE)
+            if(DEPENDS)
+                list(APPEND TEST_COMPILE_EXTRA_LIBRARIES "-framework Foundation -framework IOKit")
+            else()
+                find_library(COREFOUNDATION CoreFoundation)
+                find_library(IOKIT IOKit)
+                list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${IOKIT})
+                list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${COREFOUNDATION})
+            endif()
+        endif()
+        if (WIN32)
+            list(APPEND TEST_COMPILE_EXTRA_LIBRARIES setupapi)
+        endif()
+        list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${LibUSB_LIBRARIES})
+
+        try_compile(LibUSB_COMPILE_TEST_PASSED
+                ${CMAKE_BINARY_DIR}
+                "${CMAKE_SOURCE_DIR}/cmake/test-libusb-version.c"
+                CMAKE_FLAGS
+                    "-DINCLUDE_DIRECTORIES=${LibUSB_INCLUDE_DIRS}"
+                    "-DLINK_DIRECTORIES=${LibUSB_LIBRARIES}"
+                LINK_LIBRARIES ${TEST_COMPILE_EXTRA_LIBRARIES}
+                OUTPUT_VARIABLE OUTPUT)
+        unset(TEST_COMPILE_EXTRA_LIBRARIES)
+        message(STATUS "LibUSB Compilation test: ${LibUSB_COMPILE_TEST_PASSED}")
+    endif()
+endif ( LibUSB_FOUND )
+
+if ( NOT LibUSB_FOUND )
+    if ( NOT LibUSB_FIND_QUIETLY )
+        message ( STATUS "LibUSB not found, try setting LibUSB_ROOT_DIR environment variable." )
+    endif ( NOT LibUSB_FIND_QUIETLY )
+    if ( LibUSB_FIND_REQUIRED )
+        message ( FATAL_ERROR "" )
+    endif ( LibUSB_FIND_REQUIRED )
+endif ( NOT LibUSB_FOUND )

cmake/FindReadline.cmake

@@ -66,7 +66,9 @@ check_function_exists(rl_copy_text HAVE_COPY_TEXT)
 check_function_exists(rl_filename_completion_function HAVE_COMPLETION_FUNCTION)
 
 if(NOT HAVE_COMPLETION_FUNCTION)
-  set(CMAKE_REQUIRED_LIBRARIES ${Readline_LIBRARY} ${Termcap_LIBRARY})
+  if (Readline_LIBRARY)
+    set(CMAKE_REQUIRED_LIBRARIES ${Readline_LIBRARY} ${Termcap_LIBRARY})
+  endif(Readline_LIBRARY)
   check_function_exists(rl_copy_text HAVE_COPY_TEXT_TC)
   check_function_exists(rl_filename_completion_function HAVE_COMPLETION_FUNCTION_TC)
   set(HAVE_COMPLETION_FUNCTION ${HAVE_COMPLETION_FUNCTION_TC})

cmake/FindUnbound.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 # All rights reserved.
 # 
 # Redistribution and use in source and binary forms, with or without modification, are

cmake/GenVersion.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 # 
 # All rights reserved.
 # 

cmake/Version.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 # 
 # All rights reserved.
 # 

cmake/test-libusb-version.c

@@ -0,0 +1,52 @@
+// Copyright (c) 2014-2019, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <libusb.h>
+
+#define UNUSED(expr) (void)(expr)
+
+int main(int argc, char *argv[]) {
+  libusb_device **devs;
+  libusb_context *ctx = NULL;
+
+  int r = libusb_init(&ctx); UNUSED(r);
+  ssize_t cnt = libusb_get_device_list(ctx, &devs); UNUSED(cnt);
+
+  struct libusb_device_descriptor desc;
+  r = libusb_get_device_descriptor(devs[0], &desc); UNUSED(r);
+  uint8_t bus_id = libusb_get_bus_number(devs[0]); UNUSED(bus_id);
+  uint8_t addr = libusb_get_device_address(devs[0]); UNUSED(addr);
+
+  uint8_t tmp_path[16];
+  r = libusb_get_port_numbers(devs[0], tmp_path, sizeof(tmp_path));
+  UNUSED(r);
+  UNUSED(tmp_path);
+
+  libusb_free_device_list(devs, 1);
+  libusb_exit(ctx);
+}

cmake/test-protobuf.cpp

@@ -0,0 +1,43 @@
+// Copyright (c) 2014-2019, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <string>
+#include <iostream>
+#include <google/protobuf/message.h>
+#include <google/protobuf/unknown_field_set.h>
+#include "test-protobuf.pb.h"
+
+int main(int argc, char *argv[]) {
+  google::protobuf::UnknownFieldSet ufs;
+  ufs.ClearAndFreeMemory();
+
+  Success sc;
+  sc.set_message("test");
+  sc.SerializeToOstream(&std::cerr);
+  return 0;
+}

cmake/test-protobuf.proto

@@ -0,0 +1,7 @@
+syntax = "proto2";
+
+import "google/protobuf/descriptor.proto";
+
+message Success {
+    optional string message = 1;
+}

cmake/test-static-assert.c

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

cmake/test-static-assert.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

contrib/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 # 
 # All rights reserved.
 # 
@@ -26,10 +26,5 @@
 # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
 # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-# warnings are cleared only for GCC on Linux
-if (NOT (MINGW OR APPLE OR FREEBSD OR OPENBSD OR DRAGONFLY))
-  add_compile_options("${WARNINGS_AS_ERRORS_FLAG}") # applies only to targets that follow
-endif()
-
 add_subdirectory(epee)
 

contrib/depends/Makefile

@@ -3,9 +3,6 @@
 SOURCES_PATH ?= $(BASEDIR)/sources
 BASE_CACHE ?= $(BASEDIR)/built
 SDK_PATH ?= $(BASEDIR)/SDKs
-NO_QT ?=
-NO_WALLET ?=
-NO_UPNP ?=
 FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources
 
 BUILD = $(shell ./config.guess)
@@ -97,17 +94,10 @@ $(host_arch)_$(host_os)_id_string+=$(shell $(host_CXX) --version 2>/dev/null)
 $(host_arch)_$(host_os)_id_string+=$(shell $(host_RANLIB) --version 2>/dev/null)
 $(host_arch)_$(host_os)_id_string+=$(shell $(host_STRIP) --version 2>/dev/null)
 
-qt_packages_$(NO_QT) = $(qt_packages) $(qt_$(host_os)_packages) $(qt_$(host_arch)_$(host_os)_packages)
-wallet_packages_$(NO_WALLET) = $(wallet_packages)
-upnp_packages_$(NO_UPNP) = $(upnp_packages)
 
-packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(qt_packages_) $(wallet_packages_) $(upnp_packages_)
+packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages)
 native_packages += $($(host_arch)_$(host_os)_native_packages) $($(host_os)_native_packages)
 
-ifneq ($(qt_packages_),)
-native_packages += $(qt_native_packages)
-endif
-
 all_packages = $(packages) $(native_packages)
 
 meta_depends = Makefile funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk
@@ -153,9 +143,6 @@ $(host_prefix)/share/config.site : config.site.in $(host_prefix)/.stamp_$(final_
             -e 's|@CPPFLAGS@|$(strip $(host_CPPFLAGS) $(host_$(release_type)_CPPFLAGS))|' \
             -e 's|@LDFLAGS@|$(strip $(host_LDFLAGS) $(host_$(release_type)_LDFLAGS))|' \
             -e 's|@allow_host_packages@|$(ALLOW_HOST_PACKAGES)|' \
-            -e 's|@no_qt@|$(NO_QT)|' \
-            -e 's|@no_wallet@|$(NO_WALLET)|' \
-            -e 's|@no_upnp@|$(NO_UPNP)|' \
             -e 's|@debug@|$(DEBUG)|' \
             $< > $@
 	$(AT)touch $@
@@ -176,9 +163,6 @@ $(host_prefix)/share/toolchain.cmake : toolchain.cmake.in $(host_prefix)/.stamp_
             -e 's|@CPPFLAGS@|$(strip $(host_CPPFLAGS) $(host_$(release_type)_CPPFLAGS))|' \
             -e 's|@LDFLAGS@|$(strip $(host_LDFLAGS) $(host_$(release_type)_LDFLAGS))|' \
             -e 's|@allow_host_packages@|$(ALLOW_HOST_PACKAGES)|' \
-            -e 's|@no_qt@|$(NO_QT)|' \
-            -e 's|@no_wallet@|$(NO_WALLET)|' \
-            -e 's|@no_upnp@|$(NO_UPNP)|' \
             -e 's|@debug@|$(DEBUG)|' \
             -e 's|@depends@|$(host_cmake)|' \
             -e 's|@prefix@|$($(host_arch)_$(host_os)_prefix)|'\

contrib/depends/README.md

@@ -12,11 +12,11 @@ For example:
 
     make HOST=x86_64-w64-mingw32 -j4
 
-A prefix will be generated that's suitable for plugging into Bitcoin's
-configure. In the above example, a dir named x86_64-w64-mingw32 will be
-created. To use it for Bitcoin:
+A toolchain will be generated that's suitable for plugging into Monero's
+cmake. In the above example, a dir named x86_64-w64-mingw32 will be
+created. To use it for Monero:
 
-    ./configure --prefix=`pwd`/depends/x86_64-w64-mingw32
+    cmake -DCMAKE_TOOLCHAIN=`pwd`/contrib/depends/x86_64-w64-mingw32
 
 Common `host-platform-triplets` for cross compilation are:
 
@@ -35,16 +35,10 @@ The following can be set when running make: make FOO=bar
     BASE_CACHE: built packages will be placed here
     SDK_PATH: Path where sdk's can be found (used by OSX)
     FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up
-    NO_QT: Don't download/build/cache qt and its dependencies
-    NO_WALLET: Don't download/build/cache libs needed to enable the wallet
-    NO_UPNP: Don't download/build/cache packages needed for enabling upnp
     DEBUG: disable some optimizations and enable more runtime checking
     HOST_ID_SALT: Optional salt to use when generating host package ids
     BUILD_ID_SALT: Optional salt to use when generating build package ids
 
-If some packages are not built, for example `make NO_WALLET=1`, the appropriate
-options will be passed to bitcoin's configure. In this case, `--disable-wallet`.
-
 Additional targets:
 
     download: run 'make download' to fetch all sources without building them
@@ -59,6 +53,16 @@ Download it from apple, or search for it on github. Create a new directoty calle
 directory and place the entire MacOSX10.11.sdk folder in it. The depends build will then pick it up automatically
 (without requiring SDK_PATH). 
 
+
+#Mingw builds
+
+Building for 32/64bit mingw requires switching alternatives to a posix mode
+
+```bash
+update-alternatives --set x86_64-w64-mingw32-g++ x86_64-w64-mingw32-g++-posix
+update-alternatives --set x86_64-w64-mingw32-gcc x86_64-w64-mingw32-gcc-posix
+```
+
 ### Other documentation
 
 - [description.md](description.md): General description of the depends system

contrib/depends/config.site.in

@@ -13,25 +13,6 @@ fi
 if test -z $with_qt_translationdir; then
   with_qt_translationdir=$depends_prefix/translations
 fi
-if test -z $with_qt_bindir && test -z "@no_qt@"; then
-  with_qt_bindir=$depends_prefix/native/bin
-fi
-if test -z $with_protoc_bindir && test -z "@no_qt@"; then
-  with_protoc_bindir=$depends_prefix/native/bin
-fi
-
-
-if test -z $enable_wallet && test -n "@no_wallet@"; then
-  enable_wallet=no
-fi
-
-if test -z $with_miniupnpc && test -n "@no_upnp@"; then
-  with_miniupnpc=no
-fi
-
-if test -z $with_gui && test -n "@no_qt@"; then
-  with_gui=no
-fi
 
 if test x@host_os@ = xdarwin; then
   BREW=no

contrib/depends/packages/bdb.mk

@@ -1,31 +0,0 @@
-package=bdb
-$(package)_version=4.8.30
-$(package)_download_path=http://download.oracle.com/berkeley-db
-$(package)_file_name=db-$($(package)_version).NC.tar.gz
-$(package)_sha256_hash=12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef
-$(package)_build_subdir=build_unix
-
-define $(package)_set_vars
-$(package)_config_opts=--disable-shared --enable-cxx --disable-replication
-$(package)_config_opts_mingw32=--enable-mingw
-$(package)_config_opts_linux=--with-pic
-$(package)_cxxflags=-std=c++11
-endef
-
-define $(package)_preprocess_cmds
-  sed -i.old 's/__atomic_compare_exchange/__atomic_compare_exchange_db/' dbinc/atomic.h && \
-  sed -i.old 's/atomic_init/atomic_init_db/' dbinc/atomic.h mp/mp_region.c mp/mp_mvcc.c mp/mp_fget.c mutex/mut_method.c mutex/mut_tas.c && \
-  cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub dist
-endef
-
-define $(package)_config_cmds
-  ../dist/$($(package)_autoconf)
-endef
-
-define $(package)_build_cmds
-  $(MAKE) libdb_cxx-4.8.a libdb-4.8.a
-endef
-
-define $(package)_stage_cmds
-  $(MAKE) DESTDIR=$($(package)_staging_dir) install_lib install_include
-endef

contrib/depends/packages/boost.mk

@@ -20,7 +20,7 @@ $(package)_archiver_$(host_os)=$($(package)_ar)
 $(package)_toolset_darwin=darwin
 $(package)_archiver_darwin=$($(package)_libtool)
 $(package)_config_libraries=chrono,filesystem,program_options,system,thread,test,date_time,regex,serialization,locale
-$(package)_cxxflags=-std=c++11 -fvisibility=hidden
+$(package)_cxxflags=-std=c++11
 $(package)_cxxflags_linux=-fPIC
 endef
 

contrib/depends/packages/icu4c.mk

@@ -6,7 +6,7 @@ $(package)_sha256_hash=1f912c54035533fb4268809701d65c7468d00e292efbc31e644490845
 $(package)_patches=icu-001-dont-build-static-dynamic-twice.patch
 
 define $(package)_set_vars
-  $(package)_build_opts=CFLAGS="$($(package)_cflags) $($(package)_cppflags) -DU_USING_ICU_NAMESPACE=0 --std=gnu++0x -DU_STATIC_IMPLEMENTATION -DU_COMBINED_IMPLEMENTATION -fPIC"
+  $(package)_build_opts=CFLAGS="$($(package)_cflags) $($(package)_cppflags) -DU_USING_ICU_NAMESPACE=0 --std=gnu++0x -DU_STATIC_IMPLEMENTATION -DU_COMBINED_IMPLEMENTATION -fPIC -DENABLE_STATIC=YES -DPGKDATA_MODE=static"
 endef
 
 define $(package)_config_cmds
@@ -17,7 +17,7 @@ define $(package)_config_cmds
   sh ../source/runConfigureICU Linux &&\
   make &&\
   cd ../buildb &&\
-  sh ../source/$($(package)_autoconf) --enable-static=yes --enable-shared=yes --disable-layoutex --prefix=$(host_prefix) --with-cross-build=`pwd`/../builda &&\
+  sh ../source/$($(package)_autoconf) --enable-static=yes --disable-shared --disable-layout --disable-layoutex --disable-tests --disable-samples --prefix=$(host_prefix) --with-cross-build=`pwd`/../builda &&\
   $(MAKE) $($(package)_build_opts)
 endef
 

contrib/depends/packages/ldns.mk

@@ -1,6 +1,6 @@
 package=ldns
 $(package)_version=1.6.17
-$(package)_download_path=http://www.nlnetlabs.nl/downloads/ldns/
+$(package)_download_path=https://www.nlnetlabs.nl/downloads/ldns/
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
 $(package)_sha256_hash=8b88e059452118e8949a2752a55ce59bc71fa5bc414103e17f5b6b06f9bcc8cd
 $(package)_dependencies=openssl

contrib/depends/packages/libICE.mk

@@ -1,6 +1,6 @@
 package=libICE
 $(package)_version=1.0.9
-$(package)_download_path=http://xorg.freedesktop.org/releases/individual/lib/
+$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=8f7032f2c1c64352b5423f6b48a8ebdc339cc63064af34d66a6c9aa79759e202
 $(package)_dependencies=xtrans xproto

contrib/depends/packages/libSM.mk

@@ -1,6 +1,6 @@
 package=libSM
 $(package)_version=1.2.2
-$(package)_download_path=http://xorg.freedesktop.org/releases/individual/lib/
+$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=0baca8c9f5d934450a70896c4ad38d06475521255ca63b717a6510fdb6e287bd
 $(package)_dependencies=xtrans xproto libICE

contrib/depends/packages/libevent.mk

@@ -1,30 +0,0 @@
-package=libevent
-$(package)_version=2.1.8-stable
-$(package)_download_path=https://github.com/libevent/libevent/archive/
-$(package)_file_name=release-$($(package)_version).tar.gz
-$(package)_sha256_hash=316ddb401745ac5d222d7c529ef1eada12f58f6376a66c1118eee803cb70f83d
-
-define $(package)_preprocess_cmds
-  ./autogen.sh
-endef
-
-define $(package)_set_vars
-  $(package)_config_opts=--disable-shared --disable-openssl --disable-libevent-regress --disable-samples
-  $(package)_config_opts_release=--disable-debug-mode
-  $(package)_config_opts_linux=--with-pic
-endef
-
-define $(package)_config_cmds
-  $($(package)_autoconf)
-endef
-
-define $(package)_build_cmds
-  $(MAKE)
-endef
-
-define $(package)_stage_cmds
-  $(MAKE) DESTDIR=$($(package)_staging_dir) install
-endef
-
-define $(package)_postprocess_cmds
-endef

contrib/depends/packages/libiconv.mk

@@ -3,9 +3,22 @@ $(package)_version=1.15
 $(package)_download_path=https://ftp.gnu.org/gnu/libiconv
 $(package)_file_name=libiconv-$($(package)_version).tar.gz
 $(package)_sha256_hash=ccf536620a45458d26ba83887a983b96827001e92a13847b45e4925cc8913178
+$(package)_patches=fix-whitespace.patch
+
+define $(package)_set_vars
+  $(package)_config_opts=--disable-nls
+  $(package)_config_opts=--enable-static
+  $(package)_config_opts=--disable-shared
+  $(package)_config_opts_linux=--with-pic
+endef
+
+define $(package)_preprocess_cmds
+  cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub build-aux/ &&\
+  patch -p1 < $($(package)_patch_dir)/fix-whitespace.patch
+endef
 
 define $(package)_config_cmds
-  $($(package)_autoconf) --disable-nls --enable-static --disable-shared
+  $($(package)_autoconf)
 endef
 
 define $(package)_build_cmds

contrib/depends/packages/libusb.mk

@@ -1,8 +1,8 @@
 package=libusb
-$(package)_version=1.0.9
-$(package)_download_path=http://sourceforge.net/projects/libusb/files/libusb-1.0/libusb-1.0.9/
+$(package)_version=1.0.22
+$(package)_download_path=https://sourceforge.net/projects/libusb/files/libusb-1.0/libusb-$($(package)_version)/
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
-$(package)_sha256_hash=e920eedc2d06b09606611c99ec7304413c6784cba6e33928e78243d323195f9b
+$(package)_sha256_hash=75aeb9d59a4fdb800d329a545c2e6799f732362193b465ea198f2aa275518157
 
 define $(package)_preprocess_cmds
   autoreconf -i
@@ -10,14 +10,22 @@ endef
 
 define $(package)_set_vars
   $(package)_config_opts=--disable-shared
-  $(package)_config_opts_linux=--with-pic
+  $(package)_config_opts_linux=--with-pic --disable-udev
+  $(package)_config_opts_mingw32=--disable-udev
+  $(package)_config_opts_darwin=--disable-udev
 endef
 
-define $(package)_config_cmds
-  cp -f $(BASEDIR)/config.guess config.guess &&\
-  cp -f $(BASEDIR)/config.sub config.sub &&\
-  $($(package)_autoconf)
-endef
+ifneq ($(host_os),darwin)
+  define $(package)_config_cmds
+    cp -f $(BASEDIR)/config.guess config.guess &&\
+    cp -f $(BASEDIR)/config.sub config.sub &&\
+    $($(package)_autoconf)
+  endef
+else
+  define $(package)_config_cmds
+    $($(package)_autoconf)
+  endef
+endif
 
 define $(package)_build_cmd
   $(MAKE)
@@ -27,5 +35,5 @@ define $(package)_stage_cmds
   $(MAKE) DESTDIR=$($(package)_staging_dir) install
 endef
 
-define $(package)_postprocess_cmds                                                                                                          cp -f lib/libusb-1.0.a lib/libusb.a
+define $(package)_postprocess_cmds  cp -f lib/libusb-1.0.a lib/libusb.a
 endef

contrib/depends/packages/miniupnpc.mk

@@ -1,28 +0,0 @@
-package=miniupnpc
-$(package)_version=2.0.20170509
-$(package)_download_path=http://miniupnp.free.fr/files
-$(package)_file_name=$(package)-$($(package)_version).tar.gz
-$(package)_sha256_hash=d3c368627f5cdfb66d3ebd64ca39ba54d6ff14a61966dbecb8dd296b7039f16a
-
-define $(package)_set_vars
-$(package)_build_opts=CC="$($(package)_cc)"
-$(package)_build_opts_darwin=OS=Darwin LIBTOOL="$($(package)_libtool)"
-$(package)_build_opts_mingw32=-f Makefile.mingw
-$(package)_build_env+=CFLAGS="$($(package)_cflags) $($(package)_cppflags)" AR="$($(package)_ar)"
-endef
-
-define $(package)_preprocess_cmds
-  mkdir dll && \
-  sed -e 's|MINIUPNPC_VERSION_STRING \"version\"|MINIUPNPC_VERSION_STRING \"$($(package)_version)\"|' -e 's|OS/version|$(host)|' miniupnpcstrings.h.in > miniupnpcstrings.h && \
-  sed -i.old "s|miniupnpcstrings.h: miniupnpcstrings.h.in wingenminiupnpcstrings|miniupnpcstrings.h: miniupnpcstrings.h.in|" Makefile.mingw
-endef
-
-define $(package)_build_cmds
-	$(MAKE) libminiupnpc.a $($(package)_build_opts)
-endef
-
-define $(package)_stage_cmds
-	mkdir -p $($(package)_staging_prefix_dir)/include/miniupnpc $($(package)_staging_prefix_dir)/lib &&\
-	install *.h $($(package)_staging_prefix_dir)/include/miniupnpc &&\
-	install libminiupnpc.a $($(package)_staging_prefix_dir)/lib
-endef

contrib/depends/packages/native_cdrkit.mk

@@ -1,6 +1,6 @@
 package=native_cdrkit
 $(package)_version=1.1.11
-$(package)_download_path=http://distro.ibiblio.org/fatdog/source/600/c
+$(package)_download_path=https://distro.ibiblio.org/fatdog/source/600/c
 $(package)_file_name=cdrkit-$($(package)_version).tar.bz2
 $(package)_sha256_hash=b50d64c214a65b1a79afe3a964c691931a4233e2ba605d793eb85d0ac3652564
 $(package)_patches=cdrkit-deterministic.patch

contrib/depends/packages/native_protobuf.mk

@@ -0,0 +1,28 @@
+package=protobuf3
+$(package)_version=3.6.1
+$(package)_download_path=https://github.com/protocolbuffers/protobuf/releases/download/v$($(package)_version)/
+$(package)_file_name=protobuf-cpp-$($(package)_version).tar.gz
+$(package)_sha256_hash=b3732e471a9bb7950f090fd0457ebd2536a9ba0891b7f3785919c654fe2a2529
+$(package)_cxxflags=-std=c++11
+
+define $(package)_set_vars
+  $(package)_config_opts=--disable-shared --prefix=$(build_prefix)
+  $(package)_config_opts_linux=--with-pic
+endef
+
+define $(package)_config_cmds
+  $($(package)_autoconf)
+endef
+
+define $(package)_build_cmds
+  $(MAKE) -C src libprotobuf.la all
+endef
+
+define $(package)_stage_cmds
+  $(MAKE) DESTDIR=$($(package)_staging_dir) -C src install install-libLTLIBRARIES install-nobase_includeHEADERS &&\
+  $(MAKE) DESTDIR=$($(package)_staging_dir) install-pkgconfigDATA
+endef
+
+define $(package)_postprocess_cmds
+  rm lib/libprotoc.a
+endef

contrib/depends/packages/packages.mk

@@ -1,12 +1,10 @@
-packages:=boost openssl libevent zeromq cppzmq zlib expat ldns cppzmq readline libiconv qt hidapi
-native_packages := native_ccache
-
-wallet_packages=bdb
+packages:=boost openssl zeromq cppzmq expat ldns cppzmq readline libiconv qt hidapi protobuf libusb
+native_packages := native_ccache native_protobuf
 
 darwin_native_packages = native_biplist native_ds_store native_mac_alias
 darwin_packages = sodium-darwin
 
-linux_packages = eudev libusb
+linux_packages = eudev
 
 ifeq ($(host_os),linux)
 packages += unwind
@@ -19,6 +17,5 @@ endif
 
 ifneq ($(build_os),darwin)
 darwin_native_packages += native_cctools native_cdrkit native_libdmg-hfsplus
-packages += readline
 endif
 

contrib/depends/packages/qt.mk

@@ -1,10 +1,9 @@
 PACKAGE=qt
 $(package)_version=5.7.1
-$(package)_download_path=http://download.qt.io/official_releases/qt/5.7/$($(package)_version)/submodules
+$(package)_download_path=https://download.qt.io/archive/qt/5.7/5.7.1/submodules
 $(package)_suffix=opensource-src-$($(package)_version).tar.gz
 $(package)_file_name=qtbase-$($(package)_suffix)
 $(package)_sha256_hash=95f83e532d23b3ddbde7973f380ecae1bac13230340557276f75f2e37984e410
-$(package)_dependencies=openssl zlib
 $(package)_build_subdir=qtbase
 $(package)_qt_libs=corelib
 $(package)_patches=pidlist_absolute.patch fix_qt_pkgconfig.patch qfixed-coretext.patch
@@ -62,14 +61,14 @@ $(package)_config_opts += -no-xrender
 $(package)_config_opts += -nomake examples
 $(package)_config_opts += -nomake tests
 $(package)_config_opts += -opensource
-$(package)_config_opts += -openssl-linked
+$(package)_config_opts += -no-openssl
 $(package)_config_opts += -optimized-qmake
 $(package)_config_opts += -pch
 $(package)_config_opts += -pkg-config
-$(package)_config_opts += -qt-libpng
-$(package)_config_opts += -qt-libjpeg
+$(package)_config_opts += -no-libpng
+$(package)_config_opts += -no-libjpeg
 $(package)_config_opts += -qt-pcre
-$(package)_config_opts += -system-zlib
+$(package)_config_opts += -no-zlib
 $(package)_config_opts += -reduce-exports
 $(package)_config_opts += -static
 $(package)_config_opts += -silent
@@ -124,7 +123,6 @@ define $(package)_config_cmds
   export PKG_CONFIG_LIBDIR=$(host_prefix)/lib/pkgconfig && \
   export PKG_CONFIG_PATH=$(host_prefix)/share/pkgconfig  && \
   ./configure $($(package)_config_opts) && \
-  echo "host_build: QT_CONFIG ~= s/system-zlib/zlib" >> mkspecs/qconfig.pri && \
   echo "CONFIG += force_bootstrap" >> mkspecs/qconfig.pri && \
   $(MAKE) sub-src-clean && \
   cd ../qttranslations && ../qtbase/bin/qmake qttranslations.pro -o Makefile && \

contrib/depends/packages/sodium-darwin.mk

@@ -1,8 +1,8 @@
 package=sodium-darwin
-$(package)_version=1.0.15
+$(package)_version=1.0.16
 $(package)_download_path=https://download.libsodium.org/libsodium/releases/
 $(package)_file_name=libsodium-$($(package)_version).tar.gz
-$(package)_sha256_hash=fb6a9e879a2f674592e4328c5d9f79f082405ee4bb05cb6e679b90afe9e178f4
+$(package)_sha256_hash=eeadc7e1e1bcef09680fb4837d448fbdf57224978f865ac1c16745868fbd0533
 
 define $(package)_set_vars
 $(package)_build_opts_darwin=OS=Darwin LIBTOOL="$($(package)_libtool)"

contrib/depends/packages/sodium.mk

@@ -1,8 +1,9 @@
 package=sodium
-$(package)_version=1.0.15
+$(package)_version=1.0.16
 $(package)_download_path=https://download.libsodium.org/libsodium/releases/
 $(package)_file_name=libsodium-$($(package)_version).tar.gz
-$(package)_sha256_hash=fb6a9e879a2f674592e4328c5d9f79f082405ee4bb05cb6e679b90afe9e178f4
+$(package)_sha256_hash=eeadc7e1e1bcef09680fb4837d448fbdf57224978f865ac1c16745868fbd0533
+$(package)_patches=fix-whitespace.patch
 
 define $(package)_set_vars
 $(package)_config_opts=--enable-static --disable-shared
@@ -11,6 +12,7 @@ endef
 
 define $(package)_config_cmds
   ./autogen.sh &&\
+  patch -p1 < $($(package)_patch_dir)/fix-whitespace.patch &&\
   $($(package)_autoconf) $($(package)_config_opts)
 endef
 

contrib/depends/packages/unbound.mk

@@ -1,6 +1,6 @@
 package=unbound
 $(package)_version=1.6.8
-$(package)_download_path=http://www.unbound.net/downloads/
+$(package)_download_path=https://www.unbound.net/downloads/
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
 $(package)_sha256_hash=e3b428e33f56a45417107448418865fe08d58e0e7fea199b855515f60884dd49
 $(package)_dependencies=openssl expat ldns

contrib/depends/packages/unwind.mk

@@ -1,6 +1,6 @@
 package=unwind
 $(package)_version=1.2
-$(package)_download_path=http://download.savannah.nongnu.org/releases/libunwind
+$(package)_download_path=https://download.savannah.nongnu.org/releases/libunwind
 $(package)_file_name=lib$(package)-$($(package)_version).tar.gz
 $(package)_sha256_hash=1de38ffbdc88bd694d10081865871cd2bfbb02ad8ef9e1606aee18d65532b992
 

contrib/depends/packages/xproto.mk

@@ -1,6 +1,6 @@
 package=xproto
 $(package)_version=7.0.26
-$(package)_download_path=http://xorg.freedesktop.org/releases/individual/proto
+$(package)_download_path=https://xorg.freedesktop.org/releases/individual/proto
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=636162c1759805a5a0114a369dffdeccb8af8c859ef6e1445f26a4e6e046514f
 

contrib/depends/packages/zlib.mk

@@ -1,27 +0,0 @@
-package=zlib
-$(package)_version=1.2.11
-$(package)_download_path=http://www.zlib.net
-$(package)_file_name=$(package)-$($(package)_version).tar.gz
-$(package)_sha256_hash=c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
-
-define $(package)_set_vars
-$(package)_build_opts= CC="$($(package)_cc)"
-$(package)_build_opts+=CFLAGS="$($(package)_cflags) $($(package)_cppflags) -fPIC"
-$(package)_build_opts+=RANLIB="$($(package)_ranlib)"
-$(package)_build_opts+=AR="$($(package)_ar)"
-$(package)_build_opts_darwin+=AR="$($(package)_libtool)"
-$(package)_build_opts_darwin+=ARFLAGS="-o"
-endef
-
-define $(package)_config_cmds
-  ./configure --static --prefix=$(host_prefix)
-endef
-
-define $(package)_build_cmds
-  $(MAKE) $($(package)_build_opts) libz.a
-endef
-
-define $(package)_stage_cmds
-  $(MAKE) DESTDIR=$($(package)_staging_dir) install $($(package)_build_opts)
-endef
-

contrib/depends/patches/libiconv/fix-whitespace.patch

@@ -0,0 +1,13 @@
+diff --git a/preload/configure b/preload/configure
+index aab5c77..e20b8f0 100755
+--- a/preload/configure
++++ b/preload/configure
+@@ -588,7 +588,7 @@ MAKEFLAGS=
+ PACKAGE_NAME='libiconv'
+ PACKAGE_TARNAME='libiconv'
+ PACKAGE_VERSION='0'
+-PACKAGE_STRING='libiconv 0'
++PACKAGE_STRING='libiconv0'
+ PACKAGE_BUGREPORT=''
+ PACKAGE_URL=''
+

contrib/depends/patches/sodium/fix-whitespace.patch

@@ -0,0 +1,13 @@
+diff --git a/configure b/configure
+index b29f769..ca008ae 100755
+--- a/configure
++++ b/configure
+@@ -591,7 +591,7 @@ MAKEFLAGS=
+ PACKAGE_NAME='libsodium'
+ PACKAGE_TARNAME='libsodium'
+ PACKAGE_VERSION='1.0.16'
+-PACKAGE_STRING='libsodium 1.0.16'
++PACKAGE_STRING='libsodium'
+ PACKAGE_BUGREPORT='https://github.com/jedisct1/libsodium/issues'
+ PACKAGE_URL='https://github.com/jedisct1/libsodium'
+ 

contrib/depends/toolchain.cmake.in

@@ -21,6 +21,12 @@ SET(LIBUNWIND_LIBRARY_DIRS @prefix@/lib)
 SET(LIBUSB-1.0_LIBRARY @prefix@/lib/libusb-1.0.a)
 SET(LIBUDEV_LIBRARY @prefix@/lib/libudev.a)
 
+SET(Protobuf_FOUND 1)
+SET(Protobuf_PROTOC_EXECUTABLE @prefix@/native/bin/protoc CACHE FILEPATH "Path to the native protoc")
+SET(Protobuf_INCLUDE_DIR @prefix@/include CACHE PATH "Protobuf include dir")
+SET(Protobuf_INCLUDE_DIRS @prefix@/include CACHE PATH "Protobuf include dir")
+SET(Protobuf_LIBRARY @prefix@/lib/libprotobuf.a CACHE FILEPATH "Protobuf library")
+
 SET(ZMQ_INCLUDE_PATH @prefix@/include)
 SET(ZMQ_LIB @prefix@/lib/libzmq.a)
 
@@ -41,8 +47,6 @@ set (CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) # Find programs on host
 set (CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) # Find libs in target
 set (CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) # Find includes in target
 
-set(CMAKE_SYSTEM_PROCESSOR ${CMAKE_HOST_SYSTEM_PROCESSOR} CACHE STRING "" FORCE)
-
 # specify the cross compiler to be used. Darwin uses clang provided by the SDK.
 if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
   SET(CMAKE_C_COMPILER @prefix@/native/bin/clang)
@@ -53,9 +57,6 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
   SET(APPLE True)
   SET(BUILD_TAG "mac-x64")
   SET(BUILD_64 ON)
-  if(NOT TRAVIS)
-  SET(ARCH "x86_64")
-  endif()
   SET(BREW OFF)
   SET(PORT OFF)
   SET(CMAKE_OSX_SYSROOT "@sdk@/MacOSX10.11.sdk/")
@@ -87,11 +88,6 @@ endif()
 
 if(ARCHITECTURE STREQUAL "i686" AND CMAKE_SYSTEM_NAME STREQUAL "Linux")
   SET(LINUX_32 ON)
-  SET(ARCH_ID "i386")
-endif()
-
-if(ARCHITECTURE STREQUAL "x86_64" AND CMAKE_SYSTEM_NAME STREQUAL "Linux")
-  SET(ARCH_ID "x86_64")
 endif()
 
 #Create a new global cmake flag that indicates building with depends

contrib/epee/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 # 
 # All rights reserved.
 # 

contrib/epee/include/int-util.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

contrib/epee/include/math_helper.h

@@ -230,35 +230,56 @@ namespace math_helper
 		}
 
 	}
-	template<int default_interval, bool start_immediate = true>
-	class once_a_time_seconds
+	template<uint64_t scale, int default_interval, bool start_immediate = true>
+	class once_a_time
 	{
+    uint64_t get_time() const
+    {
+#ifdef _WIN32
+      FILETIME fileTime;
+      GetSystemTimeAsFileTime(&fileTime);
+      unsigned __int64 present = 0;
+      present |= fileTime.dwHighDateTime;
+      present = present << 32;
+      present |= fileTime.dwLowDateTime;
+      present /= 10;  // mic-sec
+#else
+      struct timeval tv;
+      gettimeofday(&tv, NULL);
+      return tv.tv_sec * 1000000 + tv.tv_usec;
+#endif
+    }
+
 	public:
-		once_a_time_seconds():m_interval(default_interval)
+		once_a_time():m_interval(default_interval * scale)
 		{
 			m_last_worked_time = 0;
       if(!start_immediate)
-        time(&m_last_worked_time);
+        m_last_worked_time = get_time();
 		}
 
 		template<class functor_t>
 		bool do_call(functor_t functr)
 		{
-			time_t current_time = 0;
-			time(&current_time);
+			uint64_t current_time = get_time();
 
       if(current_time - m_last_worked_time > m_interval)
 			{
 				bool res = functr();
-				time(&m_last_worked_time);
+				m_last_worked_time = get_time();
 				return res;
 			}
 			return true;
 		}
 
 	private:
-		time_t m_last_worked_time;
-		time_t m_interval;
+		uint64_t m_last_worked_time;
+		uint64_t m_interval;
 	};
+
+  template<int default_interval, bool start_immediate = true>
+  class once_a_time_seconds: public once_a_time<1000000, default_interval, start_immediate> {};
+  template<int default_interval, bool start_immediate = true>
+  class once_a_time_milliseconds: public once_a_time<1000, default_interval, start_immediate> {};
 }
 }

contrib/epee/include/misc_language.h

@@ -147,7 +147,8 @@ namespace misc_utils
     {}
     ~call_befor_die()
     {
-      m_func();
+      try { m_func(); }
+      catch (...) { /* ignore */ }
     }
   };
 

contrib/epee/include/misc_log_ex.h

@@ -32,18 +32,27 @@
 
 #include "easylogging++.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
 #define MONERO_DEFAULT_LOG_CATEGORY "default"
+
 #define MAX_LOG_FILE_SIZE 104850000 // 100 MB - 7600 bytes
 #define MAX_LOG_FILES 50
 
-#define MCFATAL(cat,x) CLOG(FATAL,cat) << x
-#define MCERROR(cat,x) CLOG(ERROR,cat) << x
-#define MCWARNING(cat,x) CLOG(WARNING,cat) << x
-#define MCINFO(cat,x) CLOG(INFO,cat) << x
-#define MCDEBUG(cat,x) CLOG(DEBUG,cat) << x
-#define MCTRACE(cat,x) CLOG(TRACE,cat) << x
-#define MCLOG(level,cat,x) ELPP_WRITE_LOG(el::base::Writer, level, el::base::DispatchAction::NormalLog, cat) << x
-#define MCLOG_FILE(level,cat,x) ELPP_WRITE_LOG(el::base::Writer, level, el::base::DispatchAction::FileOnlyLog, cat) << x
+#define MCLOG_TYPE(level, cat, type, x) do { \
+    if (ELPP->vRegistry()->allowed(level, cat)) { \
+      el::base::Writer(level, __FILE__, __LINE__, ELPP_FUNC, type).construct(cat) << x; \
+    } \
+  } while (0)
+
+#define MCLOG(level, cat, x) MCLOG_TYPE(level, cat, el::base::DispatchAction::NormalLog, x)
+#define MCLOG_FILE(level, cat, x) MCLOG_TYPE(level, cat, el::base::DispatchAction::FileOnlyLog, x)
+
+#define MCFATAL(cat,x) MCLOG(el::Level::Fatal,cat, x)
+#define MCERROR(cat,x) MCLOG(el::Level::Error,cat, x)
+#define MCWARNING(cat,x) MCLOG(el::Level::Warning,cat, x)
+#define MCINFO(cat,x) MCLOG(el::Level::Info,cat, x)
+#define MCDEBUG(cat,x) MCLOG(el::Level::Debug,cat, x)
+#define MCTRACE(cat,x) MCLOG(el::Level::Trace,cat, x)
 
 #define MCLOG_COLOR(level,cat,color,x) MCLOG(level,cat,"\033[1;" color "m" << x << "\033[0m")
 #define MCLOG_RED(level,cat,x) MCLOG_COLOR(level,cat,"31",x)

contrib/epee/include/misc_os_dependent.h

@@ -122,6 +122,15 @@ namespace misc_utils
 		return boost::lexical_cast<std::string>(GetCurrentThreadId());
 #elif defined(__GNUC__)  
 		return boost::lexical_cast<std::string>(pthread_self());
+#endif
+	}
+
+	inline bool get_gmt_time(time_t t, struct tm &tm)
+	{
+#ifdef _WIN32
+		return gmtime_s(&tm, &t);
+#else
+		return gmtime_r(&t, &tm);
 #endif
 	}
 }

contrib/epee/include/mlocker.h

@@ -73,7 +73,7 @@ namespace epee
     mlocked(const T &&t): T(t) { mlocker::lock(this, sizeof(T)); }
     mlocked(const mlocked<T> &&mt): T(mt) { mlocker::lock(this, sizeof(T)); }
     mlocked<T> &operator=(const mlocked<T> &mt) { T::operator=(mt); return *this; }
-    ~mlocked() { mlocker::unlock(this, sizeof(T)); }
+    ~mlocked() { try { mlocker::unlock(this, sizeof(T)); } catch (...) { /* do not propagate */ } }
   };
 
   template<typename T>

contrib/epee/include/net/abstract_tcp_server2.h

@@ -36,7 +36,6 @@
 #define _ABSTRACT_TCP_SERVER2_H_ 
 
 
-#include <boost/asio.hpp>
 #include <string>
 #include <vector>
 #include <boost/noncopyable.hpp>
@@ -155,7 +154,8 @@ namespace net_utils
     //this should be the last one, because it could be wait on destructor, while other activities possible on other threads
     t_protocol_handler m_protocol_handler;
     //typename t_protocol_handler::config_type m_dummy_config;
-    std::list<boost::shared_ptr<connection<t_protocol_handler> > > m_self_refs; // add_ref/release support
+    size_t m_reference_count = 0; // reference count managed through add_ref/release support
+    boost::shared_ptr<connection<t_protocol_handler> > m_self_ref; // the reference to hold
     critical_section m_self_refs_lock;
     critical_section m_chunking_lock; // held while we add small chunks of the big do_send() to small do_send_chunk()
     critical_section m_shutdown_lock; // held while shutting down
@@ -246,7 +246,6 @@ namespace net_utils
                                                           m_timer(io_serice)
       {}
       boost::asio::deadline_timer m_timer;
-      uint64_t m_period;
     };
 
     template <class t_handler>
@@ -262,25 +261,27 @@ namespace net_utils
       {
         return m_handler();
       }
+      uint64_t m_period;
     };
 
     template<class t_handler>
     bool add_idle_handler(t_handler t_callback, uint64_t timeout_ms)
       {
-        boost::shared_ptr<idle_callback_conext_base> ptr(new idle_callback_conext<t_handler>(io_service_, t_callback, timeout_ms));
+        boost::shared_ptr<idle_callback_conext<t_handler>> ptr(new idle_callback_conext<t_handler>(io_service_, t_callback, timeout_ms));
         //needed call handler here ?...
         ptr->m_timer.expires_from_now(boost::posix_time::milliseconds(ptr->m_period));
-        ptr->m_timer.async_wait(boost::bind(&boosted_tcp_server<t_protocol_handler>::global_timer_handler, this, ptr));
+        ptr->m_timer.async_wait(boost::bind(&boosted_tcp_server<t_protocol_handler>::global_timer_handler<t_handler>, this, ptr));
         return true;
       }
 
-    bool global_timer_handler(/*const boost::system::error_code& err, */boost::shared_ptr<idle_callback_conext_base> ptr)
+    template<class t_handler>
+    bool global_timer_handler(/*const boost::system::error_code& err, */boost::shared_ptr<idle_callback_conext<t_handler>> ptr)
     {
       //if handler return false - he don't want to be called anymore
       if(!ptr->call_handler())
         return true;
       ptr->m_timer.expires_from_now(boost::posix_time::milliseconds(ptr->m_period));
-      ptr->m_timer.async_wait(boost::bind(&boosted_tcp_server<t_protocol_handler>::global_timer_handler, this, ptr));
+      ptr->m_timer.async_wait(boost::bind(&boosted_tcp_server<t_protocol_handler>::global_timer_handler<t_handler>, this, ptr));
       return true;
     }
 

contrib/epee/include/net/abstract_tcp_server2.inl

@@ -32,17 +32,16 @@
 
 
 
-//#include "net_utils_base.h"
-#include <boost/lambda/bind.hpp>
+#include <boost/bind.hpp>
 #include <boost/foreach.hpp>
-#include <boost/lambda/lambda.hpp>
 #include <boost/uuid/random_generator.hpp>
 #include <boost/chrono.hpp>
 #include <boost/utility/value_init.hpp>
 #include <boost/asio/deadline_timer.hpp>
 #include <boost/date_time/posix_time/posix_time.hpp> // TODO
-#include <boost/thread/thread.hpp> // TODO
 #include <boost/thread/condition_variable.hpp> // TODO
+#include "warnings.h"
+#include "string_tools.h"
 #include "misc_language.h"
 #include "net/local_ip.h"
 #include "pragma_comp_defs.h"
@@ -51,8 +50,6 @@
 #include <iomanip>
 #include <algorithm>
 
-#include "../../../../src/cryptonote_core/cryptonote_core.h" // e.g. for the send_stop_signal()
-
 #undef MONERO_DEFAULT_LOG_CATEGORY
 #define MONERO_DEFAULT_LOG_CATEGORY "net"
 
@@ -149,10 +146,8 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     const unsigned long ip_{boost::asio::detail::socket_ops::host_to_network_long(remote_ep.address().to_v4().to_ulong())};
     m_local = epee::net_utils::is_ip_loopback(ip_) || epee::net_utils::is_ip_local(ip_);
 
-    // create a random uuid
-    boost::uuids::uuid random_uuid;
-    // that stuff turns out to be included, even though it's from src... Taking advantage
-    random_uuid = crypto::rand<boost::uuids::uuid>();
+    // create a random uuid, we don't need crypto strength here
+    const boost::uuids::uuid random_uuid = boost::uuids::random_generator()();
 
     context.set_details(random_uuid, epee::net_utils::ipv4_network_address(ip_, remote_ep.port()), is_income);
     _dbg3("[sock " << socket_.native_handle() << "] new connection from " << print_connection_context_short(context) <<
@@ -232,7 +227,8 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     //_dbg3("[sock " << socket_.native_handle() << "] add_ref 2, m_peer_number=" << mI->m_peer_number);
     if(m_was_shutdown)
       return false;
-    m_self_refs.push_back(self);
+    ++m_reference_count;
+    m_self_ref = std::move(self);
     return true;
     CATCH_ENTRY_L0("connection<t_protocol_handler>::add_ref()", false);
   }
@@ -244,10 +240,12 @@ PRAGMA_WARNING_DISABLE_VS(4355)
     boost::shared_ptr<connection<t_protocol_handler> >  back_connection_copy;
     LOG_TRACE_CC(context, "[sock " << socket_.native_handle() << "] release");
     CRITICAL_REGION_BEGIN(m_self_refs_lock);
-    CHECK_AND_ASSERT_MES(m_self_refs.size(), false, "[sock " << socket_.native_handle() << "] m_self_refs empty at connection<t_protocol_handler>::release() call");
-    //erasing from container without additional copy can cause start deleting object, including m_self_refs
-    back_connection_copy = m_self_refs.back();
-    m_self_refs.pop_back();
+    CHECK_AND_ASSERT_MES(m_reference_count, false, "[sock " << socket_.native_handle() << "] m_reference_count already at 0 at connection<t_protocol_handler>::release() call");
+    // is this the last reference?
+    if (--m_reference_count == 0) {
+        // move the held reference to a local variable, keeping the object alive until the function terminates
+        std::swap(back_connection_copy, m_self_ref);
+    }
     CRITICAL_REGION_END();
     return true;
     CATCH_ENTRY_L0("connection<t_protocol_handler>::release()", false);
@@ -395,7 +393,7 @@ PRAGMA_WARNING_DISABLE_VS(4355)
       //ask it inside(!) critical region if we still able to go in event wait...
       size_t cnt = socket_.get_io_service().poll_one();     
       if(!cnt)
-        misc_utils::sleep_no_w(0);
+        misc_utils::sleep_no_w(1);
     }
     
     return true;
@@ -891,7 +889,9 @@ POP_WARNINGS
     {
       try
       {
-        io_service_.run();
+        size_t cnt = io_service_.run();
+        if (cnt == 0)
+          misc_utils::sleep_no_w(1);
       }
       catch(const std::exception& ex)
       {

contrib/epee/include/net/buffer.h

@@ -0,0 +1,62 @@
+// Copyright (c) 2018, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <vector>
+#include "misc_log_ex.h"
+#include "span.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net.buffer"
+
+//#define NET_BUFFER_LOG(x) MDEBUG(x)
+#define NET_BUFFER_LOG(x) ((void)0)
+
+namespace epee
+{
+namespace net_utils
+{
+class buffer
+{
+public:
+  buffer(size_t reserve = 0): offset(0) { storage.reserve(reserve); }
+
+  void append(const void *data, size_t sz);
+  void erase(size_t sz) { NET_BUFFER_LOG("erasing " << sz << "/" << size()); CHECK_AND_ASSERT_THROW_MES(offset + sz <= storage.size(), "erase: sz too large"); offset += sz; if (offset == storage.size()) { storage.resize(0); offset = 0; } }
+  epee::span<const uint8_t> span(size_t sz) const { CHECK_AND_ASSERT_THROW_MES(sz <= size(), "span is too large"); return epee::span<const uint8_t>(storage.data() + offset, sz); }
+  // carve must keep the data in scope till next call, other API calls (such as append, erase) can invalidate the carved buffer
+  epee::span<const uint8_t> carve(size_t sz) { CHECK_AND_ASSERT_THROW_MES(sz <= size(), "span is too large"); offset += sz; return epee::span<const uint8_t>(storage.data() + offset - sz, sz); }
+  size_t size() const { return storage.size() - offset; }
+
+private:
+  std::vector<uint8_t> storage;
+  size_t offset;
+};
+}
+}

contrib/epee/include/net/connection_basic.hpp

@@ -8,7 +8,7 @@
 // ! (how ever if in some wonderful juristdictions that is not the case, then why not make another sub-class withat that members and licence it as epee part)
 // ! Working on above premise, IF this is valid in your juristdictions, then consider this code as released as:
 
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 
@@ -42,22 +42,11 @@
 #define INCLUDED_p2p_connection_basic_hpp
 
 
-#include <boost/asio.hpp>
 #include <string>
-#include <vector>
-#include <boost/noncopyable.hpp>
-#include <boost/shared_ptr.hpp>
 #include <atomic>
+#include <memory>
 
 #include <boost/asio.hpp>
-#include <boost/array.hpp>
-#include <boost/noncopyable.hpp>
-#include <boost/shared_ptr.hpp>
-#include <boost/enable_shared_from_this.hpp>
-#include <boost/interprocess/detail/atomic.hpp>
-#include <boost/thread/thread.hpp>
-
-#include <memory>
 
 #include "net/net_utils_base.h"
 #include "syncobj.h"
@@ -92,7 +81,6 @@ class connection_basic { // not-templated base class for rapid developmet of som
     critical_section m_send_que_lock;
     std::list<std::string> m_send_que;
     volatile bool m_is_multithreaded;
-    double m_start_time;
     /// Strand to ensure the connection's handlers are not called concurrently.
     boost::asio::io_service::strand strand_;
     /// Socket for the connection.
@@ -112,8 +100,6 @@ class connection_basic { // not-templated base class for rapid developmet of som
 		void logger_handle_net_write(size_t size); // network data written
 		void logger_handle_net_read(size_t size); // network data read
 
-		void set_start_time();
-
 		// config for rate limit
 		
 		static void set_rate_up_limit(uint64_t limit);

contrib/epee/include/net/http_auth.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

contrib/epee/include/net/http_protocol_handler.inl

@@ -32,6 +32,7 @@
 #include "string_tools.h"
 #include "file_io_utils.h"
 #include "net_parse_helpers.h"
+#include "time_helper.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
 #define MONERO_DEFAULT_LOG_CATEGORY "net.http"

contrib/epee/include/net/http_server_handlers_map2.h

@@ -39,7 +39,7 @@
               epee::net_utils::http::http_response_info& response, \
               context_type& m_conn_context) \
 {\
-  LOG_PRINT_L2("HTTP [" << m_conn_context.m_remote_address.host_str() << "] " << query_info.m_http_method_str << " " << query_info.m_URI); \
+  MINFO("HTTP [" << m_conn_context.m_remote_address.host_str() << "] " << query_info.m_http_method_str << " " << query_info.m_URI); \
   response.m_response_code = 200; \
   response.m_response_comment = "Ok"; \
   if(!handle_http_request_map(query_info, response, m_conn_context)) \
@@ -54,7 +54,7 @@
   bool handled = false; \
   if(false) return true; //just a stub to have "else if"
 
-#define MAP_URI2(pattern, callback)  else if(std::string::npos != query_info.m_URI.find(pattern)) return callback(query_info, response_info, m_conn_context);
+#define MAP_URI2(pattern, callback)  else if(std::string::npos != query_info.m_URI.find(pattern)) return callback(query_info, response_info, &m_conn_context);
 
 #define MAP_URI_AUTO_XML2(s_pattern, callback_f, command_type) //TODO: don't think i ever again will use xml - ambiguous and "overtagged" format
 
@@ -68,7 +68,8 @@
       CHECK_AND_ASSERT_MES(parse_res, false, "Failed to parse json: \r\n" << query_info.m_body); \
       uint64_t ticks1 = epee::misc_utils::get_tick_count(); \
       boost::value_initialized<command_type::response> resp;\
-      if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp))) \
+      MINFO(m_conn_context << "calling " << s_pattern); \
+      if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context)) \
       { \
         LOG_ERROR("Failed to " << #callback_f << "()"); \
         response_info.m_response_code = 500; \
@@ -91,11 +92,12 @@
       handled = true; \
       uint64_t ticks = misc_utils::get_tick_count(); \
       boost::value_initialized<command_type::request> req; \
-      bool parse_res = epee::serialization::load_t_from_binary(static_cast<command_type::request&>(req), query_info.m_body); \
+      bool parse_res = epee::serialization::load_t_from_binary(static_cast<command_type::request&>(req), epee::strspan<uint8_t>(query_info.m_body)); \
       CHECK_AND_ASSERT_MES(parse_res, false, "Failed to parse bin body data, body size=" << query_info.m_body.size()); \
       uint64_t ticks1 = misc_utils::get_tick_count(); \
       boost::value_initialized<command_type::response> resp;\
-      if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp))) \
+      MINFO(m_conn_context << "calling " << s_pattern); \
+      if(!callback_f(static_cast<command_type::request&>(req), static_cast<command_type::response&>(resp), &m_conn_context)) \
       { \
         LOG_ERROR("Failed to " << #callback_f << "()"); \
         response_info.m_response_code = 500; \
@@ -179,7 +181,8 @@
   epee::json_rpc::error_response fail_resp = AUTO_VAL_INIT(fail_resp); \
   fail_resp.jsonrpc = "2.0"; \
   fail_resp.id = req.id; \
-  if(!callback_f(req.params, resp.result, fail_resp.error)) \
+  MINFO(m_conn_context << "Calling RPC method " << method_name); \
+  if(!callback_f(req.params, resp.result, fail_resp.error, &m_conn_context)) \
   { \
     epee::serialization::store_t_to_json(static_cast<epee::json_rpc::error_response&>(fail_resp), response_info.m_body); \
     return true; \
@@ -197,7 +200,8 @@
   epee::json_rpc::error_response fail_resp = AUTO_VAL_INIT(fail_resp); \
   fail_resp.jsonrpc = "2.0"; \
   fail_resp.id = req.id; \
-  if(!callback_f(req.params, resp.result, fail_resp.error, m_conn_context, response_info)) \
+  MINFO(m_conn_context << "calling RPC method " << method_name); \
+  if(!callback_f(req.params, resp.result, fail_resp.error, response_info, &m_conn_context)) \
   { \
     epee::serialization::store_t_to_json(static_cast<epee::json_rpc::error_response&>(fail_resp), response_info.m_body); \
     return true; \
@@ -210,7 +214,8 @@
     else if(callback_name == method_name) \
 { \
   PREPARE_OBJECTS_FROM_JSON(command_type) \
-  if(!callback_f(req.params, resp.result)) \
+  MINFO(m_conn_context << "calling RPC method " << method_name); \
+  if(!callback_f(req.params, resp.result, &m_conn_context)) \
   { \
     epee::json_rpc::error_response fail_resp = AUTO_VAL_INIT(fail_resp); \
     fail_resp.jsonrpc = "2.0"; \

contrib/epee/include/net/http_server_impl_base.h

@@ -33,7 +33,8 @@
 #include <boost/thread.hpp>
 #include <boost/bind.hpp> 
 
-#include "net/http_server_cp2.h"
+#include "net/abstract_tcp_server2.h"
+#include "http_protocol_handler.h"
 #include "net/http_server_handlers_map2.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY

contrib/epee/include/net/levin_base.h

@@ -80,8 +80,8 @@ namespace levin
   template<class t_connection_context = net_utils::connection_context_base>
   struct levin_commands_handler
   {
-    virtual int invoke(int command, const std::string& in_buff, std::string& buff_out, t_connection_context& context)=0;
-    virtual int notify(int command, const std::string& in_buff, t_connection_context& context)=0;
+    virtual int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, t_connection_context& context)=0;
+    virtual int notify(int command, const epee::span<const uint8_t> in_buff, t_connection_context& context)=0;
     virtual void callback(t_connection_context& context){};
 
     virtual void on_connection_new(t_connection_context& context){};

contrib/epee/include/net/levin_client.h

@@ -57,7 +57,7 @@ namespace levin
 		bool is_connected();
 		bool disconnect();
 
-		virtual int invoke(int command, const std::string& in_buff, std::string& buff_out);
+		virtual int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out);
 		virtual int notify(int command, const std::string& in_buff);
 
 	protected: 
@@ -72,7 +72,7 @@ namespace levin
   {
   public:
 
-    int invoke(int command, const std::string& in_buff, std::string& buff_out);
+    int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out);
     int notify(int command, const std::string& in_buff);
   };
 

contrib/epee/include/net/levin_client.inl

@@ -74,16 +74,16 @@ levin_client_impl::~levin_client_impl()
 }
 //------------------------------------------------------------------------------
 inline
-int levin_client_impl::invoke(int command, const std::string& in_buff, std::string& buff_out)
+int levin_client_impl::invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out)
 {
 	if(!is_connected())
 		return -1;
 
 	bucket_head head = {0};
-	head.m_signature = LEVIN_SIGNATURE;
-	head.m_cb = in_buff.size();
+	head.m_signature = SWAP64LE(LEVIN_SIGNATURE);
+	head.m_cb = SWAP64LE(in_buff.size());
 	head.m_have_to_return_data = true;
-	head.m_command = command;
+	head.m_command = SWAP32LE(command);
 	if(!m_transport.send(&head, sizeof(head)))
 		return -1;
 	
@@ -97,7 +97,7 @@ int levin_client_impl::invoke(int command, const std::string& in_buff, std::stri
 	head = *(bucket_head*)local_buff.data();
 
 
-	if(head.m_signature!=LEVIN_SIGNATURE) 
+	if(head.m_signature!=SWAP64LE(LEVIN_SIGNATURE))
 	{
 		LOG_PRINT_L1("Signature mismatch in response");
 		return -1;
@@ -116,10 +116,10 @@ int levin_client_impl::notify(int command, const std::string& in_buff)
 		return -1;
 
 	bucket_head head = {0};
-	head.m_signature = LEVIN_SIGNATURE;
-	head.m_cb = in_buff.size();
+	head.m_signature = SWAP64LE(LEVIN_SIGNATURE);
+	head.m_cb = SWAP64LE(in_buff.size());
 	head.m_have_to_return_data = false;
-	head.m_command = command;
+	head.m_command = SWAP32LE(command);
 	
 	if(!m_transport.send((const char*)&head, sizeof(head)))
 		return -1;
@@ -133,18 +133,19 @@ int levin_client_impl::notify(int command, const std::string& in_buff)
 //------------------------------------------------------------------------------
 //------------------------------------------------------------------------------
 inline
-  int levin_client_impl2::invoke(int command, const std::string& in_buff, std::string& buff_out)
+  int levin_client_impl2::invoke(int command, epee::span<const uint8_t>string& in_buff, std::string& buff_out)
 {
   if(!is_connected())
     return -1;
 
   bucket_head2 head = {0};
-  head.m_signature = LEVIN_SIGNATURE;
-  head.m_cb = in_buff.size();
+  head.m_signature = SWAP64LE(LEVIN_SIGNATURE);
+  head.m_cb = SWAP64LE(in_buff.size());
   head.m_have_to_return_data = true;
-  head.m_command = command;
-  head.m_protocol_version = LEVIN_PROTOCOL_VER_1;
-  head.m_flags = LEVIN_PACKET_REQUEST;
+  head.m_command = SWAP32LE(command);
+  head.m_return_code = SWAP32LE(0);
+  head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST);
+  head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1);
   if(!m_transport.send(&head, sizeof(head)))
     return -1;
 
@@ -157,14 +158,13 @@ inline
 
   head = *(bucket_head2*)local_buff.data();
 
-
-  if(head.m_signature!=LEVIN_SIGNATURE) 
+  if(head.m_signature != SWAP64LE(LEVIN_SIGNATURE))
   {
     LOG_PRINT_L1("Signature mismatch in response");
     return -1;
   }
 
-  if(!m_transport.recv_n(buff_out, head.m_cb))
+  if(!m_transport.recv_n(buff_out, SWAP64LE(head.m_cb)))
     return -1;
 
   return head.m_return_code;
@@ -177,12 +177,13 @@ inline
     return -1;
 
   bucket_head2 head = {0};
-  head.m_signature = LEVIN_SIGNATURE;
-  head.m_cb = in_buff.size();
+  head.m_signature = SWAP64LE(LEVIN_SIGNATURE);
+  head.m_cb = SWAP64LE(in_buff.size());
   head.m_have_to_return_data = false;
-  head.m_command = command;
-  head.m_protocol_version = LEVIN_PROTOCOL_VER_1;
-  head.m_flags = LEVIN_PACKET_REQUEST;
+  head.m_command = SWAP32LE(command);
+  head.m_return_code = SWAP32LE(0);
+  head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST);
+  head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1);
 
   if(!m_transport.send((const char*)&head, sizeof(head)))
     return -1;

contrib/epee/include/net/levin_client_async.h

@@ -431,7 +431,7 @@ namespace levin
 			}
 			CRITICAL_REGION_END();
 
-			LOG_PRINT_L4("LEVIN_PACKET_RECIEVED. [len=" << head.m_cb << ", flags=" << head.m_flags << ", is_cmd=" << head.m_have_to_return_data <<", cmd_id = " << head.m_command << ", pr_v=" << head.m_protocol_version << ", uid=" << string_tools::get_str_from_guid_a(head.m_id) << "]");
+			LOG_PRINT_L4("LEVIN_PACKET_RECEIVED. [len=" << head.m_cb << ", flags=" << head.m_flags << ", is_cmd=" << head.m_have_to_return_data <<", cmd_id = " << head.m_command << ", pr_v=" << head.m_protocol_version << ", uid=" << string_tools::get_str_from_guid_a(head.m_id) << "]");
 
 			if(is_request)
 			{

contrib/epee/include/net/levin_helper.h

@@ -30,6 +30,7 @@
 
 #include "levin_base.h"
 #include "serializeble_struct_helper.h"
+#include "int-util.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
 #define MONERO_DEFAULT_LOG_CATEGORY "net"
@@ -43,11 +44,11 @@ namespace levin
 	{
 		buff.resize(sizeof(levin::bucket_head));
 		levin::bucket_head& head = *(levin::bucket_head*)(&buff[0]);
-		head.m_signature = LEVIN_SIGNATURE;
+		head.m_signature = SWAP64LE(LEVIN_SIGNATURE);
 		head.m_cb = 0;
 		head.m_have_to_return_data = true;
-		head.m_command = command_id;
-		head.m_return_code = 1;
+		head.m_command = SWAP32LE(command_id);
+		head.m_return_code = SWAP32LE(1);
 		head.m_reservedA = rand(); //probably some flags in future
 		head.m_reservedB = rand(); //probably some check summ in future
 
@@ -55,7 +56,7 @@ namespace levin
 		if(!StorageNamed::save_struct_as_storage_to_buff_t<t_struct, StorageNamed::DefaultStorageType>(t, buff_strg))
 			return false;
 		
-		head.m_cb = buff_strg.size();
+		head.m_cb = SWAP64LE(buff_strg.size());
 		buff.append(buff_strg);
 		return true;
 	}
@@ -65,15 +66,15 @@ namespace levin
 	{
 		buff.resize(sizeof(levin::bucket_head));
 		levin::bucket_head& head = *(levin::bucket_head*)(&buff[0]);
-		head.m_signature = LEVIN_SIGNATURE;
+		head.m_signature = SWAP64LE(LEVIN_SIGNATURE);
 		head.m_cb = 0;
 		head.m_have_to_return_data = true;
-		head.m_command = command_id;
-		head.m_return_code = 1;
+		head.m_command = SWAP32LE(command_id);
+		head.m_return_code = SWAP32LE(1);
 		head.m_reservedA = rand(); //probably some flags in future
 		head.m_reservedB = rand(); //probably some check summ in future
 
-		head.m_cb = data.size();
+		head.m_cb = SWAP64LE(data.size());
 		buff.append(data);
 		return true;
 	}
@@ -86,7 +87,17 @@ namespace levin
 			return false;
 		}
 
-		levin::bucket_head& head = *(levin::bucket_head*)(&buff[0]);
+#if BYTE_ORDER == LITTLE_ENDIAN
+		levin::bucket_head &head = *(levin::bucket_head*)(&buff[0]);
+#else
+		levin::bucket_head head = *(levin::bucket_head*)(&buff[0]);
+		head.m_signature = SWAP64LE(head.m_signature);
+		head.m_cb = SWAP64LE(head.m_cb);
+		head.m_command = SWAP32LE(head.m_command);
+		head.m_return_code = SWAP32LE(head.m_return_code);
+		head.m_reservedA = SWAP32LE(head.m_reservedA);
+		head.m_reservedB = SWAP32LE(head.m_reservedB);
+#endif
 		if(head.m_signature != LEVIN_SIGNATURE)
 		{
 			LOG_PRINT_L3("Failed to read signature in levin message, at load_struct_from_levin_message");
@@ -113,7 +124,17 @@ namespace levin
 			return false;
 		}
 		
-		levin::bucket_head& head = *(levin::bucket_head*)(&buff[0]);
+#if BYTE_ORDER == LITTLE_ENDIAN
+		levin::bucket_head &head = *(levin::bucket_head*)(&buff[0]);
+#else
+		levin::bucket_head head = *(levin::bucket_head*)(&buff[0]);
+		head.m_signature = SWAP64LE(head.m_signature);
+		head.m_cb = SWAP64LE(head.m_cb);
+		head.m_command = SWAP32LE(head.m_command);
+		head.m_return_code = SWAP32LE(head.m_return_code);
+		head.m_reservedA = SWAP32LE(head.m_reservedA);
+		head.m_reservedB = SWAP32LE(head.m_reservedB);
+#endif
 		if(head.m_signature != LEVIN_SIGNATURE)
 		{
 			LOG_ERROR("Failed to read signature in levin message, at load_struct_from_levin_message");

contrib/epee/include/net/levin_protocol_handler.h

@@ -31,6 +31,7 @@
 
 #include <boost/uuid/uuid_generators.hpp>
 #include "levin_base.h"
+#include "int-util.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
 #define MONERO_DEFAULT_LOG_CATEGORY "net"
@@ -103,7 +104,7 @@ namespace levin
 			case conn_state_reading_head:
 				if(m_cach_in_buffer.size() < sizeof(bucket_head))
 				{
-					if(m_cach_in_buffer.size() >= sizeof(uint64_t) && *((uint64_t*)m_cach_in_buffer.data()) != LEVIN_SIGNATURE)
+					if(m_cach_in_buffer.size() >= sizeof(uint64_t) && *((uint64_t*)m_cach_in_buffer.data()) != SWAP64LE(LEVIN_SIGNATURE))
 					{
 						LOG_ERROR_CC(m_conn_context, "Signature mismatch on accepted connection");
 						return false;
@@ -112,13 +113,23 @@ namespace levin
 					break;
 				}
 				{
-					bucket_head* phead = (bucket_head*)m_cach_in_buffer.data();
-					if(LEVIN_SIGNATURE != phead->m_signature)
+#if BYTE_ORDER == LITTLE_ENDIAN
+					bucket_head &phead = *(bucket_head*)m_cach_in_buffer.data();
+#else
+					bucket_head phead = *(bucket_head*)m_cach_in_buffer.data();
+					phead.m_signature = SWAP64LE(phead.m_signature);
+					phead.m_cb = SWAP64LE(phead.m_cb);
+					phead.m_command = SWAP32LE(phead.m_command);
+					phead.m_return_code = SWAP32LE(phead.m_return_code);
+					phead.m_reservedA = SWAP32LE(phead.m_reservedA);
+					phead.m_reservedB = SWAP32LE(phead.m_reservedB);
+#endif
+					if(LEVIN_SIGNATURE != phead.m_signature)
 					{
 						LOG_ERROR_CC(m_conn_context, "Signature mismatch on accepted connection");
 						return false;
 					}
-					m_current_head = *phead;
+					m_current_head = phead;
 				}
 				m_cach_in_buffer.erase(0, sizeof(bucket_head));
 				m_state = conn_state_reading_body;

contrib/epee/include/net/levin_protocol_handler_async.h

@@ -34,9 +34,11 @@
 #include <atomic>
 
 #include "levin_base.h"
+#include "buffer.h"
 #include "misc_language.h"
 #include "syncobj.h"
 #include "misc_os_dependent.h"
+#include "int-util.h"
 
 #include <random>
 #include <chrono>
@@ -84,11 +86,11 @@ public:
   uint64_t m_max_packet_size; 
   uint64_t m_invoke_timeout;
 
-  int invoke(int command, const std::string& in_buff, std::string& buff_out, boost::uuids::uuid connection_id);
+  int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, boost::uuids::uuid connection_id);
   template<class callback_t>
-  int invoke_async(int command, const std::string& in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED);
+  int invoke_async(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED);
 
-  int notify(int command, const std::string& in_buff, boost::uuids::uuid connection_id);
+  int notify(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id);
   bool close(boost::uuids::uuid connection_id);
   bool update_connection_context(const t_connection_context& contxt);
   bool request_callback(boost::uuids::uuid connection_id);
@@ -99,7 +101,7 @@ public:
   size_t get_connections_count();
   void set_handler(levin_commands_handler<t_connection_context>* handler, void (*destroy)(levin_commands_handler<t_connection_context>*) = NULL);
 
-  async_protocol_handler_config():m_pcommands_handler(NULL), m_pcommands_handler_destroy(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE)
+  async_protocol_handler_config():m_pcommands_handler(NULL), m_pcommands_handler_destroy(NULL), m_max_packet_size(LEVIN_DEFAULT_MAX_PACKET_SIZE), m_invoke_timeout(LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED)
   {}
   ~async_protocol_handler_config() { set_handler(NULL, NULL); }
   void del_out_connections(size_t count);
@@ -142,7 +144,7 @@ public:
   config_type& m_config;
   t_connection_context& m_connection_context;
 
-  std::string m_cache_in_buffer;
+  net_utils::buffer m_cache_in_buffer;
   stream_state m_state;
 
   int32_t m_oponent_protocol_ver;
@@ -150,7 +152,7 @@ public:
 
   struct invoke_response_handler_base
   {
-    virtual bool handle(int res, const std::string& buff, connection_context& context)=0;
+    virtual bool handle(int res, const epee::span<const uint8_t> buff, connection_context& context)=0;
     virtual bool is_timer_started() const=0;
     virtual void cancel()=0;
     virtual bool cancel_timer()=0;
@@ -172,7 +174,7 @@ public:
           if(ec == boost::asio::error::operation_aborted)
             return;
           MINFO(con.get_context_ref() << "Timeout on invoke operation happened, command: " << command << " timeout: " << timeout);
-          std::string fake;
+          epee::span<const uint8_t> fake;
           cb(LEVIN_ERROR_CONNECTION_TIMEDOUT, fake, con.get_context_ref());
           con.close();
           con.finish_outer_call();
@@ -190,7 +192,7 @@ public:
     bool m_timer_cancelled;
     uint64_t m_timeout;
     int m_command;
-    virtual bool handle(int res, const std::string& buff, typename async_protocol_handler::connection_context& context)
+    virtual bool handle(int res, const epee::span<const uint8_t> buff, typename async_protocol_handler::connection_context& context)
     {
       if(!cancel_timer())
         return false;
@@ -206,7 +208,7 @@ public:
     {
       if(cancel_timer())
       {
-        std::string fake;
+        epee::span<const uint8_t> fake;
         m_cb(LEVIN_ERROR_CONNECTION_DESTROYED, fake, m_con.get_context_ref());
         m_con.finish_outer_call();
       }
@@ -236,7 +238,7 @@ public:
           if(ec == boost::asio::error::operation_aborted)
             return;
           MINFO(con.get_context_ref() << "Timeout on invoke operation happened, command: " << command << " timeout: " << timeout);
-          std::string fake;
+          epee::span<const uint8_t> fake;
           cb(LEVIN_ERROR_CONNECTION_TIMEDOUT, fake, con.get_context_ref());
           con.close();
           con.finish_outer_call();
@@ -264,6 +266,7 @@ public:
             m_pservice_endpoint(psnd_hndlr), 
             m_config(config), 
             m_connection_context(conn_context), 
+            m_cache_in_buffer(256 * 1024),
             m_state(stream_state_head)
   {
     m_close_called = 0;
@@ -272,6 +275,8 @@ public:
     m_wait_count = 0;
     m_oponent_protocol_ver = 0;
     m_connection_initialized = false;
+    m_invoke_buf_ready = 0;
+    m_invoke_result_code = LEVIN_ERROR_CONNECTION;
   }
   virtual ~async_protocol_handler()
   {
@@ -402,18 +407,11 @@ public:
           break;
         }
         {
-          std::string buff_to_invoke;
-          if(m_cache_in_buffer.size()  == m_current_head.m_cb)
-            buff_to_invoke.swap(m_cache_in_buffer);
-          else
-          {
-            buff_to_invoke.assign(m_cache_in_buffer, 0, (std::string::size_type)m_current_head.m_cb);
-            m_cache_in_buffer.erase(0, (std::string::size_type)m_current_head.m_cb);
-          }
+          epee::span<const uint8_t> buff_to_invoke = m_cache_in_buffer.carve((std::string::size_type)m_current_head.m_cb);
 
           bool is_response = (m_oponent_protocol_ver == LEVIN_PROTOCOL_VER_1 && m_current_head.m_flags&LEVIN_PACKET_RESPONSE);
 
-          MDEBUG(m_connection_context << "LEVIN_PACKET_RECIEVED. [len=" << m_current_head.m_cb
+          MDEBUG(m_connection_context << "LEVIN_PACKET_RECEIVED. [len=" << m_current_head.m_cb
             << ", flags" << m_current_head.m_flags 
             << ", r?=" << m_current_head.m_have_to_return_data 
             <<", cmd = " << m_current_head.m_command 
@@ -446,8 +444,8 @@ public:
               }else
               {
                 CRITICAL_REGION_BEGIN(m_local_inv_buff_lock);
-                buff_to_invoke.swap(m_local_inv_buff);
-                buff_to_invoke.clear();
+                m_local_inv_buff = std::string((const char*)buff_to_invoke.data(), buff_to_invoke.size());
+                buff_to_invoke = epee::span<const uint8_t>((const uint8_t*)NULL, 0);
                 m_invoke_result_code = m_current_head.m_return_code;
                 CRITICAL_REGION_END();
                 boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 1);
@@ -467,7 +465,18 @@ public:
               m_current_head.m_have_to_return_data = false;
               m_current_head.m_protocol_version = LEVIN_PROTOCOL_VER_1;
               m_current_head.m_flags = LEVIN_PACKET_RESPONSE;
+#if BYTE_ORDER == LITTLE_ENDIAN
               std::string send_buff((const char*)&m_current_head, sizeof(m_current_head));
+#else
+              bucket_head2 head = m_current_head;
+              head.m_signature = SWAP64LE(head.m_signature);
+              head.m_cb = SWAP64LE(head.m_cb);
+              head.m_command = SWAP32LE(head.m_command);
+              head.m_return_code = SWAP32LE(head.m_return_code);
+              head.m_flags = SWAP32LE(head.m_flags);
+              head.m_protocol_version = SWAP32LE(head.m_protocol_version);
+              std::string send_buff((const char*)&head, sizeof(head));
+#endif
               send_buff += return_buff;
               CRITICAL_REGION_BEGIN(m_send_lock);
               if(!m_pservice_endpoint->do_send(send_buff.data(), send_buff.size()))
@@ -489,7 +498,7 @@ public:
         {
           if(m_cache_in_buffer.size() < sizeof(bucket_head2))
           {
-            if(m_cache_in_buffer.size() >= sizeof(uint64_t) && *((uint64_t*)m_cache_in_buffer.data()) != LEVIN_SIGNATURE)
+            if(m_cache_in_buffer.size() >= sizeof(uint64_t) && *((uint64_t*)m_cache_in_buffer.span(8).data()) != SWAP64LE(LEVIN_SIGNATURE))
             {
               MWARNING(m_connection_context << "Signature mismatch, connection will be closed");
               return false;
@@ -498,15 +507,25 @@ public:
             break;
           }
 
-          bucket_head2* phead = (bucket_head2*)m_cache_in_buffer.data();
-          if(LEVIN_SIGNATURE != phead->m_signature)
+#if BYTE_ORDER == LITTLE_ENDIAN
+          bucket_head2& phead = *(bucket_head2*)m_cache_in_buffer.span(sizeof(bucket_head2)).data();
+#else
+          bucket_head2 phead = *(bucket_head2*)m_cache_in_buffer.span(sizeof(bucket_head2)).data();
+          phead.m_signature = SWAP64LE(phead.m_signature);
+          phead.m_cb = SWAP64LE(phead.m_cb);
+          phead.m_command = SWAP32LE(phead.m_command);
+          phead.m_return_code = SWAP32LE(phead.m_return_code);
+          phead.m_flags = SWAP32LE(phead.m_flags);
+          phead.m_protocol_version = SWAP32LE(phead.m_protocol_version);
+#endif
+          if(LEVIN_SIGNATURE != phead.m_signature)
           {
             LOG_ERROR_CC(m_connection_context, "Signature mismatch, connection will be closed");
             return false;
           }
-          m_current_head = *phead;
+          m_current_head = phead;
 
-          m_cache_in_buffer.erase(0, sizeof(bucket_head2));
+          m_cache_in_buffer.erase(sizeof(bucket_head2));
           m_state = stream_state_body;
           m_oponent_protocol_ver = m_current_head.m_protocol_version;
           if(m_current_head.m_cb > m_config.m_max_packet_size)
@@ -538,7 +557,7 @@ public:
   }
 
   template<class callback_t>
-  bool async_invoke(int command, const std::string& in_buff, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED)
+  bool async_invoke(int command, const epee::span<const uint8_t> in_buff, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED)
   {
     misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler(
       boost::bind(&async_protocol_handler::finish_outer_call, this));
@@ -564,13 +583,13 @@ public:
       }
 
       bucket_head2 head = {0};
-      head.m_signature = LEVIN_SIGNATURE;
-      head.m_cb = in_buff.size();
+      head.m_signature = SWAP64LE(LEVIN_SIGNATURE);
+      head.m_cb = SWAP64LE(in_buff.size());
       head.m_have_to_return_data = true;
 
-      head.m_flags = LEVIN_PACKET_REQUEST;
-      head.m_command = command;
-      head.m_protocol_version = LEVIN_PROTOCOL_VER_1;
+      head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST);
+      head.m_command = SWAP32LE(command);
+      head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1);
 
       boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0);
       CRITICAL_REGION_BEGIN(m_send_lock);
@@ -582,7 +601,7 @@ public:
         break;
       }
 
-      if(!m_pservice_endpoint->do_send(in_buff.data(), (int)in_buff.size()))
+      if(!m_pservice_endpoint->do_send(in_buff.data(), in_buff.size()))
       {
         LOG_ERROR_CC(m_connection_context, "Failed to do_send");
         err_code = LEVIN_ERROR_CONNECTION;
@@ -599,7 +618,7 @@ public:
 
     if (LEVIN_OK != err_code)
     {
-      std::string stub_buff;
+      epee::span<const uint8_t> stub_buff{(const uint8_t*)"", 0};
       // Never call callback inside critical section, that can cause deadlock
       cb(err_code, stub_buff, m_connection_context);
       return false;
@@ -608,7 +627,7 @@ public:
     return true;
   }
 
-  int invoke(int command, const std::string& in_buff, std::string& buff_out)
+  int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out)
   {
     misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler(
                                       boost::bind(&async_protocol_handler::finish_outer_call, this));
@@ -622,13 +641,13 @@ public:
       return LEVIN_ERROR_CONNECTION_DESTROYED;
 
     bucket_head2 head = {0};
-    head.m_signature = LEVIN_SIGNATURE;
-    head.m_cb = in_buff.size();
+    head.m_signature = SWAP64LE(LEVIN_SIGNATURE);
+    head.m_cb = SWAP64LE(in_buff.size());
     head.m_have_to_return_data = true;
 
-    head.m_flags = LEVIN_PACKET_REQUEST;
-    head.m_command = command;
-    head.m_protocol_version = LEVIN_PROTOCOL_VER_1;
+    head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST);
+    head.m_command = SWAP32LE(command);
+    head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1);
 
     boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0);
     CRITICAL_REGION_BEGIN(m_send_lock);
@@ -638,7 +657,7 @@ public:
       return LEVIN_ERROR_CONNECTION;
     }
 
-    if(!m_pservice_endpoint->do_send(in_buff.data(), (int)in_buff.size()))
+    if(!m_pservice_endpoint->do_send(in_buff.data(), in_buff.size()))
     {
       LOG_ERROR_CC(m_connection_context, "Failed to do_send");
       return LEVIN_ERROR_CONNECTION;
@@ -682,7 +701,7 @@ public:
     return m_invoke_result_code;
   }
 
-  int notify(int command, const std::string& in_buff)
+  int notify(int command, const epee::span<const uint8_t> in_buff)
   {
     misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler(
                           boost::bind(&async_protocol_handler::finish_outer_call, this));
@@ -696,13 +715,13 @@ public:
       return LEVIN_ERROR_CONNECTION_DESTROYED;
 
     bucket_head2 head = {0};
-    head.m_signature = LEVIN_SIGNATURE;
+    head.m_signature = SWAP64LE(LEVIN_SIGNATURE);
     head.m_have_to_return_data = false;
-    head.m_cb = in_buff.size();
+    head.m_cb = SWAP64LE(in_buff.size());
 
-    head.m_command = command;
-    head.m_protocol_version = LEVIN_PROTOCOL_VER_1;
-    head.m_flags = LEVIN_PACKET_REQUEST;
+    head.m_command = SWAP32LE(command);
+    head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1);
+    head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST);
     CRITICAL_REGION_BEGIN(m_send_lock);
     if(!m_pservice_endpoint->do_send(&head, sizeof(head)))
     {
@@ -710,7 +729,7 @@ public:
       return -1;
     }
 
-    if(!m_pservice_endpoint->do_send(in_buff.data(), (int)in_buff.size()))
+    if(!m_pservice_endpoint->do_send(in_buff.data(), in_buff.size()))
     {
       LOG_ERROR_CC(m_connection_context, "Failed to do_send()");
       return -1;
@@ -815,7 +834,7 @@ int async_protocol_handler_config<t_connection_context>::find_and_lock_connectio
 }
 //------------------------------------------------------------------------------------------
 template<class t_connection_context>
-int async_protocol_handler_config<t_connection_context>::invoke(int command, const std::string& in_buff, std::string& buff_out, boost::uuids::uuid connection_id)
+int async_protocol_handler_config<t_connection_context>::invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, boost::uuids::uuid connection_id)
 {
   async_protocol_handler<t_connection_context>* aph;
   int r = find_and_lock_connection(connection_id, aph);
@@ -823,7 +842,7 @@ int async_protocol_handler_config<t_connection_context>::invoke(int command, con
 }
 //------------------------------------------------------------------------------------------
 template<class t_connection_context> template<class callback_t>
-int async_protocol_handler_config<t_connection_context>::invoke_async(int command, const std::string& in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout)
+int async_protocol_handler_config<t_connection_context>::invoke_async(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout)
 {
   async_protocol_handler<t_connection_context>* aph;
   int r = find_and_lock_connection(connection_id, aph);
@@ -872,7 +891,7 @@ void async_protocol_handler_config<t_connection_context>::set_handler(levin_comm
 }
 //------------------------------------------------------------------------------------------
 template<class t_connection_context>
-int async_protocol_handler_config<t_connection_context>::notify(int command, const std::string& in_buff, boost::uuids::uuid connection_id)
+int async_protocol_handler_config<t_connection_context>::notify(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id)
 {
   async_protocol_handler<t_connection_context>* aph;
   int r = find_and_lock_connection(connection_id, aph);

contrib/epee/include/net/network_throttle-detail.hpp

@@ -2,7 +2,7 @@
 /// @author rfree (current maintainer in monero.cc project)
 /// @brief implementaion for throttling of connection (count and rate-limit speed etc)
 
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

contrib/epee/include/net/network_throttle.hpp

@@ -2,7 +2,7 @@
 /// @author rfree (current maintainer in monero.cc project)
 /// @brief interface for throttling of connection (count and rate-limit speed etc)
 
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

contrib/epee/include/readline_buffer.h

@@ -27,6 +27,7 @@ namespace rdln
 
   private:
     std::streambuf* m_cout_buf;
+    size_t m_prompt_length;
     static std::vector<std::string>& completion_commands();
   };
   

contrib/epee/include/span.h

@@ -109,6 +109,8 @@ namespace epee
     constexpr std::size_t size() const noexcept { return len; }
     constexpr std::size_t size_bytes() const noexcept { return size() * sizeof(value_type); }
 
+    const T &operator[](size_t idx) const { return ptr[idx]; }
+
   private:
     T* ptr;
     std::size_t len;
@@ -161,4 +163,12 @@ namespace epee
     static_assert(!has_padding<T>(), "source type may have padding");
     return {reinterpret_cast<std::uint8_t*>(std::addressof(src)), sizeof(T)};
   }
+
+  //! make a span from a std::string
+  template<typename T>
+  span<const T> strspan(const std::string &s) noexcept
+  {
+    static_assert(std::is_same<T, char>() || std::is_same<T, unsigned char>() || std::is_same<T, int8_t>() || std::is_same<T, uint8_t>(), "Unexpected type");
+    return {reinterpret_cast<const T*>(s.data()), s.size()};
+  }
 }

contrib/epee/include/storages/http_abstract_invoke.h

@@ -97,7 +97,7 @@ namespace epee
         return false;
       }
 
-      return serialization::load_t_from_binary(result_struct, pri->m_body);
+      return serialization::load_t_from_binary(result_struct, epee::strspan<uint8_t>(pri->m_body));
     }
 
     template<class t_request, class t_response, class t_transport>

contrib/epee/include/storages/levin_abstract_invoke2.h

@@ -28,6 +28,7 @@
 
 #include "portable_storage_template_helper.h"
 #include <boost/utility/value_init.hpp>
+#include "span.h"
 #include "net/levin_base.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
@@ -114,7 +115,7 @@ namespace epee
       const_cast<t_arg&>(out_struct).store(stg);//TODO: add true const support to searilzation
       std::string buff_to_send;
       stg.store_to_binary(buff_to_send);
-      int res = transport.invoke_async(command, buff_to_send, conn_id, [cb, command](int code, const std::string& buff, typename t_transport::connection_context& context)->bool 
+      int res = transport.invoke_async(command, epee::strspan<uint8_t>(buff_to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool 
       {
         t_result result_struct = AUTO_VAL_INIT(result_struct);
         if( code <=0 )
@@ -156,7 +157,7 @@ namespace epee
       std::string buff_to_send;
       stg.store_to_binary(buff_to_send);
 
-      int res = transport.notify(command, buff_to_send, conn_id);
+      int res = transport.notify(command, epee::strspan<uint8_t>(buff_to_send), conn_id);
       if(res <=0 )
       {
         MERROR("Failed to notify command " << command << " return code " << res);
@@ -167,7 +168,7 @@ namespace epee
     //----------------------------------------------------------------------------------------------------
     //----------------------------------------------------------------------------------------------------
     template<class t_owner, class t_in_type, class t_out_type, class t_context, class callback_t>
-    int buff_to_t_adapter(int command, const std::string& in_buff, std::string& buff_out, callback_t cb, t_context& context )
+    int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, callback_t cb, t_context& context )
     {
       serialization::portable_storage strg;
       if(!strg.load_from_binary(in_buff))
@@ -197,7 +198,7 @@ namespace epee
     }
 
     template<class t_owner, class t_in_type, class t_context, class callback_t>
-    int buff_to_t_adapter(t_owner* powner, int command, const std::string& in_buff, callback_t cb, t_context& context)
+    int buff_to_t_adapter(t_owner* powner, int command, const epee::span<const uint8_t> in_buff, callback_t cb, t_context& context)
     {
       serialization::portable_storage strg;
       if(!strg.load_from_binary(in_buff))
@@ -215,14 +216,14 @@ namespace epee
     }
 
 #define CHAIN_LEVIN_INVOKE_MAP2(context_type) \
-  int invoke(int command, const std::string& in_buff, std::string& buff_out, context_type& context) \
+  int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, context_type& context) \
   { \
   bool handled = false; \
   return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \
   } 
 
 #define CHAIN_LEVIN_NOTIFY_MAP2(context_type) \
-  int notify(int command, const std::string& in_buff, context_type& context) \
+  int notify(int command, const epee::span<const uint8_t> in_buff, context_type& context) \
   { \
   bool handled = false; std::string fake_str;\
   return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \
@@ -230,27 +231,27 @@ namespace epee
 
 
 #define CHAIN_LEVIN_INVOKE_MAP() \
-  int invoke(int command, const std::string& in_buff, std::string& buff_out, epee::net_utils::connection_context_base& context) \
+  int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, epee::net_utils::connection_context_base& context) \
   { \
   bool handled = false; \
   return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \
   } 
 
 #define CHAIN_LEVIN_NOTIFY_MAP() \
-  int notify(int command, const std::string& in_buff, epee::net_utils::connection_context_base& context) \
+  int notify(int command, const epee::span<const uint8_t> in_buff, epee::net_utils::connection_context_base& context) \
   { \
   bool handled = false; std::string fake_str;\
   return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \
   } 
 
 #define CHAIN_LEVIN_NOTIFY_STUB() \
-  int notify(int command, const std::string& in_buff, epee::net_utils::connection_context_base& context) \
+  int notify(int command, const epee::span<const uint8_t> in_buff, epee::net_utils::connection_context_base& context) \
   { \
   return -1; \
   } 
 
 #define BEGIN_INVOKE_MAP2(owner_type) \
-  template <class t_context> int handle_invoke_map(bool is_notify, int command, const std::string& in_buff, std::string& buff_out, t_context& context, bool& handled) \
+  template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, t_context& context, bool& handled) \
   { \
   typedef owner_type internal_owner_type_name;
 

contrib/epee/include/storages/parserse_base_utils.h

@@ -29,6 +29,7 @@
 #pragma once 
 
 #include <algorithm>
+#include <boost/utility/string_ref.hpp>
 
 namespace epee 
 {
@@ -36,15 +37,51 @@ namespace misc_utils
 {
   namespace parse
   {
+    // 1: digit
+    // 2: .eE (floating point)
+    // 4: alpha
+    // 8: whitespace
+    // 16: allowed in float but doesn't necessarily mean it's a float
+    static const constexpr uint8_t lut[256]={
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8, 0, 0, // 16
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 32
+      8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 16, 18, 0, // 48
+      17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 0, 0, 0, 0, 0, 0, // 64
+      0, 4, 4, 4, 4, 22, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, // 80
+      4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, // 96
+      0, 4, 4, 4, 4, 22, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, // 112
+      4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, // 128
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    };
+
+    inline bool isspace(char c)
+    {
+      return lut[(uint8_t)c] & 8;
+    }
+
+    inline bool isdigit(char c)
+    {
+      return lut[(uint8_t)c] & 1;
+    }
+
     inline std::string transform_to_escape_sequence(const std::string& src)
     {
       static const char escaped[] = "\b\f\n\r\t\v\"\\/";
-      if (std::find_first_of(src.begin(), src.end(), escaped, escaped + sizeof(escaped)) == src.end())
+      std::string::const_iterator it = std::find_first_of(src.begin(), src.end(), escaped, escaped + sizeof(escaped));
+      if (it == src.end())
         return src;
 
       std::string res;
       res.reserve(2 * src.size());
-      for(std::string::const_iterator it = src.begin(); it!=src.end(); ++it)
+      res.assign(src.begin(), it);
+      for(; it!=src.end(); ++it)
       {
         switch(*it)
         {
@@ -89,11 +126,15 @@ namespace misc_utils
       */
       inline void match_string2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val)
       {
-        val.clear();
-        val.reserve(std::distance(star_end_string, buf_end));
         bool escape_mode = false;
         std::string::const_iterator it = star_end_string;
         ++it;
+        std::string::const_iterator fi = it;
+        while (fi != buf_end && *fi != '\\' && *fi != '\"')
+          ++fi;
+        val.assign(it, fi);
+        val.reserve(std::distance(star_end_string, buf_end));
+        it = fi;
         for(;it != buf_end;it++)
         {
           if(escape_mode/*prev_ch == '\\'*/)
@@ -153,25 +194,34 @@ namespace misc_utils
           return false;
         }
       }
-      inline void match_number2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val, bool& is_float_val, bool& is_signed_val)
+      inline void match_number2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val, bool& is_float_val, bool& is_signed_val)
       {
         val.clear();
-        is_float_val = false;
-        for(std::string::const_iterator it = star_end_string;it != buf_end;it++)
+        uint8_t float_flag = 0;
+        is_signed_val = false;
+        size_t chars = 0;
+        std::string::const_iterator it = star_end_string;
+        if (it != buf_end && *it == '-')
         {
-          if(isdigit(*it) || (it == star_end_string && *it == '-') || (val.size() && *it == '.' ) || (is_float_val && (*it == 'e' || *it == 'E' || *it == '-' || *it == '+' )) )
+          is_signed_val = true;
+          ++chars;
+          ++it;
+        }
+        for(;it != buf_end;it++)
+        {
+          const uint8_t flags = lut[(uint8_t)*it];
+          if (flags & 16)
           {
-            if(!val.size() &&  *it == '-')
-              is_signed_val = true;
-            if(*it == '.' ) 
-              is_float_val = true;
-            val.push_back(*it);
+            float_flag |= flags;
+            ++chars;
           }
           else
           {
+            val = boost::string_ref(&*star_end_string, chars);
             if(val.size())
             {
               star_end_string = --it;
+              is_float_val = !!(float_flag & 2);
               return;
             }
             else 
@@ -180,7 +230,7 @@ namespace misc_utils
         }
         ASSERT_MES_AND_THROW("wrong number in json entry: " << std::string(star_end_string, buf_end));
       }
-      inline bool match_number(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val)
+      inline bool match_number(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val)
       {
         try
         {
@@ -193,15 +243,15 @@ namespace misc_utils
           return false;
         }
       }
-      inline void match_word2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val)
+      inline void match_word2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val)
       {
         val.clear();
 
         for(std::string::const_iterator it = star_end_string;it != buf_end;it++)
         {
-          if(!isalpha(*it))
+          if (!(lut[(uint8_t)*it] & 4))
           {
-            val.assign(star_end_string, it);
+            val = boost::string_ref(&*star_end_string, std::distance(star_end_string, it));
             if(val.size())
             {
               star_end_string = --it;
@@ -212,7 +262,7 @@ namespace misc_utils
         }
         ASSERT_MES_AND_THROW("failed to match word number in json entry: " << std::string(star_end_string, buf_end));
       }
-      inline bool match_word(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val)
+      inline bool match_word(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val)
       {
         try
         {

contrib/epee/include/storages/portable_storage.h

@@ -35,6 +35,8 @@
 #include "portable_storage_to_json.h"
 #include "portable_storage_from_json.h"
 #include "portable_storage_val_converters.h"
+#include "span.h"
+#include "int-util.h"
 
 namespace epee
 {
@@ -80,7 +82,8 @@ namespace epee
 
       //-------------------------------------------------------------------------------
       bool		store_to_binary(binarybuffer& target);
-      bool		load_from_binary(const binarybuffer& target);
+      bool		load_from_binary(const epee::span<const uint8_t> target);
+      bool		load_from_binary(const std::string& target) { return load_from_binary(epee::strspan<uint8_t>(target)); }
       template<class trace_policy>
       bool		  dump_as_xml(std::string& targetObj, const std::string& root_name = "");
       bool		  dump_as_json(std::string& targetObj, size_t indent = 0, bool insert_newlines = true);
@@ -135,8 +138,8 @@ namespace epee
       TRY_ENTRY();
       std::stringstream ss;
       storage_block_header sbh = AUTO_VAL_INIT(sbh);
-      sbh.m_signature_a = PORTABLE_STORAGE_SIGNATUREA;
-      sbh.m_signature_b = PORTABLE_STORAGE_SIGNATUREB;
+      sbh.m_signature_a = SWAP32LE(PORTABLE_STORAGE_SIGNATUREA);
+      sbh.m_signature_b = SWAP32LE(PORTABLE_STORAGE_SIGNATUREB);
       sbh.m_ver = PORTABLE_STORAGE_FORMAT_VER;
       ss.write((const char*)&sbh, sizeof(storage_block_header));
       pack_entry_to_buff(ss, m_root);
@@ -145,7 +148,7 @@ namespace epee
       CATCH_ENTRY("portable_storage::store_to_binary", false)
     }
     inline
-    bool portable_storage::load_from_binary(const binarybuffer& source)
+    bool portable_storage::load_from_binary(const epee::span<const uint8_t> source)
     {
       m_root.m_entries.clear();
       if(source.size() < sizeof(storage_block_header))
@@ -154,8 +157,8 @@ namespace epee
         return false;
       }
       storage_block_header* pbuff = (storage_block_header*)source.data();
-      if(pbuff->m_signature_a != PORTABLE_STORAGE_SIGNATUREA || 
-        pbuff->m_signature_b != PORTABLE_STORAGE_SIGNATUREB 
+      if(pbuff->m_signature_a != SWAP32LE(PORTABLE_STORAGE_SIGNATUREA) ||
+        pbuff->m_signature_b != SWAP32LE(PORTABLE_STORAGE_SIGNATUREB)
         )
       {
         LOG_ERROR("portable_storage: wrong binary format - signature mismatch");

contrib/epee/include/storages/portable_storage_base.h

@@ -31,7 +31,8 @@
 #include <boost/variant.hpp>
 #include <boost/any.hpp>
 #include <string>
-#include <list>
+#include <vector>
+#include <deque>
 
 #define PORTABLE_STORAGE_SIGNATUREA 0x01011101
 #define PORTABLE_STORAGE_SIGNATUREB 0x01020101 // bender's nightmare 
@@ -71,6 +72,9 @@ namespace epee
   {
     struct section;
 
+    template<typename T> struct entry_container { typedef std::vector<T> type; static void reserve(type &t, size_t n) { t.reserve(n); } };
+    template<> struct entry_container<bool> { typedef std::deque<bool> type; static void reserve(type &t, size_t n) {} };
+
     /************************************************************************/
     /*                                                                      */
     /************************************************************************/
@@ -119,8 +123,13 @@ namespace epee
         return m_array.back();
       }
 
-      std::list<t_entry_type> m_array;
-      mutable typename std::list<t_entry_type>::const_iterator m_it;
+      void reserve(size_t n)
+      {
+        entry_container<t_entry_type>::reserve(m_array, n);
+      }
+
+      typename entry_container<t_entry_type>::type m_array;
+      mutable typename entry_container<t_entry_type>::type::const_iterator m_it;
     };
 
 

contrib/epee/include/storages/portable_storage_from_bin.h

@@ -136,6 +136,7 @@ namespace epee
       //for pod types
       array_entry_t<type_name> sa;
       size_t size = read_varint();
+      sa.reserve(size);
       //TODO: add some optimization here later
       while(size--)
         sa.m_array.push_back(read<type_name>());        

contrib/epee/include/storages/portable_storage_from_json.h

@@ -39,7 +39,7 @@ namespace epee
   {
     namespace json
     {
-#define CHECK_ISSPACE()  if(!isspace(*it)){ ASSERT_MES_AND_THROW("Wrong JSON character at: " << std::string(it, buf_end));}
+#define CHECK_ISSPACE()  if(!epee::misc_utils::parse::isspace(*it)){ ASSERT_MES_AND_THROW("Wrong JSON character at: " << std::string(it, buf_end));}
 
       /*inline void parse_error()
       {
@@ -114,33 +114,39 @@ namespace epee
               std::string val;
               match_string2(it, buf_end, val);
               //insert text value 
-              stg.set_value(name, val, current_section);              
+              stg.set_value(name, std::move(val), current_section);
               state = match_state_wonder_after_value;
-            }else if (isdigit(*it) || *it == '-')
+            }else if (epee::misc_utils::parse::isdigit(*it) || *it == '-')
             {//just a named number value started
-              std::string val;
+              boost::string_ref val;
               bool is_v_float = false;bool is_signed = false;
               match_number2(it, buf_end, val, is_v_float, is_signed);
               if(!is_v_float)
               {
                 if(is_signed)
                 {
-                  int64_t nval = boost::lexical_cast<int64_t>(val);
+                  errno = 0;
+                  int64_t nval = strtoll(val.data(), NULL, 10);
+                  if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
                   stg.set_value(name, nval, current_section);              
                 }else
                 {
-                  uint64_t nval = boost::lexical_cast<uint64_t >(val);
+                  errno = 0;
+                  uint64_t nval = strtoull(val.data(), NULL, 10);
+                  if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
                   stg.set_value(name, nval, current_section);              
                 }
               }else
               {
-                double nval = boost::lexical_cast<double>(val);
+                errno = 0;
+                double nval = strtod(val.data(), NULL);
+                if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
                 stg.set_value(name, nval, current_section);              
               }
               state = match_state_wonder_after_value;
             }else if(isalpha(*it) )
             {// could be null, true or false
-              std::string word;
+              boost::string_ref word;
               match_word2(it, buf_end, word);
               if(boost::iequals(word, "null"))
               {
@@ -197,23 +203,36 @@ namespace epee
               //mean array of strings
               std::string val;
               match_string2(it, buf_end, val);
-              h_array = stg.insert_first_value(name, val, current_section);
+              h_array = stg.insert_first_value(name, std::move(val), current_section);
               CHECK_AND_ASSERT_THROW_MES(h_array, " failed to insert values entry");
               state = match_state_array_after_value;
               array_md = array_mode_string;
-            }else if (isdigit(*it) || *it == '-')
+            }else if (epee::misc_utils::parse::isdigit(*it) || *it == '-')
             {//array of numbers value started
-              std::string val;
+              boost::string_ref val;
               bool is_v_float = false;bool is_signed_val = false;
               match_number2(it, buf_end, val, is_v_float, is_signed_val);
               if(!is_v_float)
               {
-                int64_t nval = boost::lexical_cast<int64_t>(val);//bool res = string_tools::string_to_num_fast(val, nval);
-                h_array = stg.insert_first_value(name, nval, current_section);
+                if (is_signed_val)
+                {
+                  errno = 0;
+                  int64_t nval = strtoll(val.data(), NULL, 10);
+                  if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
+                  h_array = stg.insert_first_value(name, nval, current_section);
+                }else
+                {
+                  errno = 0;
+                  uint64_t nval = strtoull(val.data(), NULL, 10);
+                  if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
+                  h_array = stg.insert_first_value(name, nval, current_section);
+                }
                 CHECK_AND_ASSERT_THROW_MES(h_array, " failed to insert values section entry");
               }else
               {
-                double nval = boost::lexical_cast<double>(val);//bool res = string_tools::string_to_num_fast(val, nval);
+                errno = 0;
+                double nval = strtod(val.data(), NULL);
+                if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
                 h_array = stg.insert_first_value(name, nval, current_section);
                 CHECK_AND_ASSERT_THROW_MES(h_array, " failed to insert values section entry");
               }
@@ -226,7 +245,7 @@ namespace epee
               state = match_state_wonder_after_value;
             }else if(isalpha(*it) )
             {// array of booleans
-              std::string word;
+              boost::string_ref word;
               match_word2(it, buf_end, word);
               if(boost::iequals(word, "true"))
               {
@@ -272,27 +291,38 @@ namespace epee
               {
                 std::string val;
                 match_string2(it, buf_end, val);
-                bool res = stg.insert_next_value(h_array, val);
+                bool res = stg.insert_next_value(h_array, std::move(val));
                 CHECK_AND_ASSERT_THROW_MES(res, "failed to insert values");
                 state = match_state_array_after_value;
               }else CHECK_ISSPACE();
               break;
             case array_mode_numbers:
-              if (isdigit(*it) || *it == '-')
+              if (epee::misc_utils::parse::isdigit(*it) || *it == '-')
               {//array of numbers value started
-                std::string val;
+                boost::string_ref val;
                 bool is_v_float = false;bool is_signed_val = false;
                 match_number2(it, buf_end, val, is_v_float, is_signed_val);
                 bool insert_res = false;
                 if(!is_v_float)
                 {
-                  int64_t nval = boost::lexical_cast<int64_t>(val);  //bool res = string_tools::string_to_num_fast(val, nval);
-                  insert_res = stg.insert_next_value(h_array, nval);
-                  
+                  if (is_signed_val)
+                  {
+                    errno = 0;
+                    int64_t nval = strtoll(val.data(), NULL, 10);
+                    if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
+                    insert_res = stg.insert_next_value(h_array, nval);
+                  }else
+                  {
+                    errno = 0;
+                    uint64_t nval = strtoull(val.data(), NULL, 10);
+                    if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
+                    insert_res = stg.insert_next_value(h_array, nval);
+                  }
                 }else
                 {
-                  //TODO: optimize here if need
-                  double nval = boost::lexical_cast<double>(val); //string_tools::string_to_num_fast(val, nval);
+                  errno = 0;
+                  double nval = strtod(val.data(), NULL);
+                  if (errno) throw std::runtime_error("Invalid number: " + std::string(val));
                   insert_res = stg.insert_next_value(h_array, nval);              
                 }
                 CHECK_AND_ASSERT_THROW_MES(insert_res, "Failed to insert next value");
@@ -303,7 +333,7 @@ namespace epee
             case array_mode_booleans:
               if(isalpha(*it) )
               {// array of booleans
-                std::string word;
+                boost::string_ref word;
                 match_word2(it, buf_end, word);
                 if(boost::iequals(word, "true"))
                 {

contrib/epee/include/storages/portable_storage_template_helper.h

@@ -84,7 +84,7 @@ namespace epee
     }
     //-----------------------------------------------------------------------------------------------------------
     template<class t_struct>
-    bool load_t_from_binary(t_struct& out, const std::string& binary_buff)
+    bool load_t_from_binary(t_struct& out, const epee::span<const uint8_t> binary_buff)
     {
       portable_storage ps;
       bool rs = ps.load_from_binary(binary_buff);
@@ -95,6 +95,12 @@ namespace epee
     }
     //-----------------------------------------------------------------------------------------------------------
     template<class t_struct>
+    bool load_t_from_binary(t_struct& out, const std::string& binary_buff)
+    {
+      return load_t_from_binary(out, epee::strspan<uint8_t>(binary_buff));
+    }
+    //-----------------------------------------------------------------------------------------------------------
+    template<class t_struct>
     bool load_t_from_binary_file(t_struct& out, const std::string& binary_file)
     {
       std::string f_buff;

contrib/epee/include/string_tools.h

@@ -40,8 +40,6 @@
 #include <cstdlib>
 #include <string>
 #include <type_traits>
-#include <boost/uuid/uuid.hpp>
-#include <boost/uuid/uuid_io.hpp>
 #include <boost/lexical_cast.hpp>
 #include <boost/algorithm/string/predicate.hpp>
 #include "hex.h"
@@ -59,89 +57,64 @@
 #pragma comment (lib, "Rpcrt4.lib")
 #endif
 
+static const constexpr unsigned char isx[256] =
+{
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+    0,    1,    2,    3,    4,    5,    6,    7,    8,    9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff,   10,   11,   12,   13,   14,   15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff,   10,   11,   12,   13,   14,   15, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+};
+
 namespace epee
 {
 namespace string_tools
 {
-	//----------------------------------------------------------------------------
-	inline std::string get_str_from_guid_a(const boost::uuids::uuid& rid)
-	{
-		return boost::lexical_cast<std::string>(rid);
-	}
-	//----------------------------------------------------------------------------
-	inline bool get_guid_from_string(OUT boost::uuids::uuid& inetifer, const std::string& str_id)
-	{
-		std::string local_str_id = str_id;
-		if(local_str_id.size() < 36)
-			return false;
-
-		if('{' == *local_str_id.begin())
-			local_str_id.erase(0, 1);
-
-		if('}' == *(--local_str_id.end()))
-			local_str_id.erase(--local_str_id.end());
-
-		try
-		{
-			inetifer = boost::lexical_cast<boost::uuids::uuid>(local_str_id);
-			return true;
-		}
-		catch(...)
-		{
-			return false;
-		}
-	}
   //----------------------------------------------------------------------------
   inline std::string buff_to_hex_nodelimer(const std::string& src)
   {
     return to_hex::string(to_byte_span(to_span(src)));
   }
   //----------------------------------------------------------------------------
-  template<class CharT>
-  bool parse_hexstr_to_binbuff(const std::basic_string<CharT>& s, std::basic_string<CharT>& res, bool allow_partial_byte = false)
+  inline bool parse_hexstr_to_binbuff(const epee::span<const char> s, epee::span<char>& res)
   {
-    res.clear();
-    if (!allow_partial_byte && (s.size() & 1))
-      return false;
-    try
-    {
-      long v = 0;
-      for(size_t i = 0; i < (s.size() + 1) / 2; i++)
+      if (s.size() != res.size() * 2)
+        return false;
+
+      unsigned char *dst = (unsigned char *)&res[0];
+      const unsigned char *src = (const unsigned char *)s.data();
+      for(size_t i = 0; i < s.size(); i += 2)
       {
-        CharT byte_str[3];
-        size_t copied = s.copy(byte_str, 2, 2 * i);
-        byte_str[copied] = CharT(0);
-        CharT* endptr;
-        v = strtoul(byte_str, &endptr, 16);
-        if (v < 0 || 0xFF < v || endptr != byte_str + copied)
-        {
-          return false;
-        }
-        res.push_back(static_cast<unsigned char>(v));
+        int tmp = *src++;
+        tmp = isx[tmp];
+        if (tmp == 0xff) return false;
+        int t2 = *src++;
+        t2 = isx[t2];
+        if (t2 == 0xff) return false;
+        *dst++ = (tmp << 4) | t2;
       }
 
       return true;
-    }catch(...)
-    {
-      return false;
-    }
   }
   //----------------------------------------------------------------------------
-  template<class t_pod_type>
-  bool parse_tpod_from_hex_string(const std::string& str_hash, t_pod_type& t_pod)
+  inline bool parse_hexstr_to_binbuff(const std::string& s, std::string& res)
   {
-    static_assert(std::is_pod<t_pod_type>::value, "expected pod type");
-    std::string buf;
-    bool res = epee::string_tools::parse_hexstr_to_binbuff(str_hash, buf);
-    if (!res || buf.size() != sizeof(t_pod_type))
-    {
+    if (s.size() & 1)
       return false;
-    }
-    else
-    {
-      buf.copy(reinterpret_cast<char *>(&t_pod), sizeof(t_pod_type));
-      return true;
-    }
+    res.resize(s.size() / 2);
+    epee::span<char> rspan((char*)&res[0], res.size());
+    return parse_hexstr_to_binbuff(epee::to_span(s), rspan);
   }
   //----------------------------------------------------------------------------
 PUSH_WARNINGS
@@ -340,17 +313,10 @@ POP_WARNINGS
   bool hex_to_pod(const std::string& hex_str, t_pod_type& s)
   {
     static_assert(std::is_pod<t_pod_type>::value, "expected pod type");
-    std::string hex_str_tr = trim(hex_str);
     if(sizeof(s)*2 != hex_str.size())
       return false;
-    std::string bin_buff;
-    if(!parse_hexstr_to_binbuff(hex_str_tr, bin_buff))
-      return false;
-    if(bin_buff.size()!=sizeof(s))
-      return false;
-
-    s = *(t_pod_type*)bin_buff.data();
-    return true;
+    epee::span<char> rspan((char*)&s, sizeof(s));
+    return parse_hexstr_to_binbuff(epee::to_span(hex_str), rspan);
   }
   //----------------------------------------------------------------------------
   template<class t_pod_type>

contrib/epee/src/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 # 
 # All rights reserved.
 # 
@@ -27,7 +27,7 @@
 # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 add_library(epee STATIC hex.cpp http_auth.cpp mlog.cpp net_utils_base.cpp string_tools.cpp wipeable_string.cpp memwipe.c
-    connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp)
+    connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp)
 if (USE_READLINE AND GNU_READLINE_FOUND)
   add_library(epee_readline STATIC readline_buffer.cpp)
 endif()

contrib/epee/src/buffer.cpp

@@ -0,0 +1,97 @@
+// Copyright (c) 2018, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <string.h>
+#include "net/buffer.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net.buffer"
+
+namespace epee
+{
+namespace net_utils
+{
+
+void buffer::append(const void *data, size_t sz)
+{
+  const size_t capacity = storage.capacity();
+  const size_t avail = capacity - storage.size();
+
+  CHECK_AND_ASSERT_THROW_MES(storage.size() < std::numeric_limits<size_t>::max() - sz, "Too much data to append");
+
+  // decide when to move
+  if (sz > avail)
+  {
+    // we have to reallocate or move
+    const bool move = size() + sz <= capacity;
+    if (move)
+    {
+      const size_t bytes = storage.size() - offset;
+      NET_BUFFER_LOG("appending " << sz << " from " << size() << " by moving " << bytes << " from offset " << offset << " first (forced)");
+      memmove(storage.data(), storage.data() + offset, bytes);
+      storage.resize(bytes);
+      offset = 0;
+    }
+    else
+    {
+      NET_BUFFER_LOG("appending " << sz << " from " << size() << " by reallocating");
+      std::vector<uint8_t> new_storage;
+      size_t reserve = (((size() + sz) * 3 / 2) + 4095) & ~4095;
+      new_storage.reserve(reserve);
+      new_storage.resize(size());
+      memcpy(new_storage.data(), storage.data() + offset, storage.size() - offset);
+      offset = 0;
+      std::swap(storage, new_storage);
+    }
+  }
+  else
+  {
+    // we have space already
+    if (size() <= 4096 && offset > 4096 * 16 && offset >= capacity / 2)
+    {
+      // we have little to move, and we're far enough into the buffer that it's probably a win to move anyway
+      const size_t pos = storage.size() - offset;
+      NET_BUFFER_LOG("appending " << sz << " from " << size() << " by moving " << pos << " from offset " << offset << " first (unforced)");
+      memmove(storage.data(), storage.data() + offset, storage.size() - offset);
+      storage.resize(pos);
+      offset = 0;
+    }
+    else
+    {
+      NET_BUFFER_LOG("appending " << sz << " from " << size() << " by writing to existing capacity");
+    }
+  }
+
+  // add the new data
+  storage.insert(storage.end(), (const uint8_t*)data, (const uint8_t*)data + sz);
+
+  NET_BUFFER_LOG("storage now " << offset << "/" << storage.size() << "/" << storage.capacity());
+}
+
+}
+}

contrib/epee/src/connection_basic.cpp

@@ -2,7 +2,7 @@
 /// @author rfree (current maintainer in monero.cc project)
 /// @brief base for connection, contains e.g. the ratelimit hooks
 
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 
@@ -34,47 +34,15 @@
 
 #include "net/connection_basic.hpp"
 
-#include <boost/asio.hpp>
-#include <string>
-#include <vector>
-#include <boost/noncopyable.hpp>
-#include <boost/shared_ptr.hpp>
-#include <atomic>
-
-#include <boost/asio.hpp>
-#include <boost/array.hpp>
-#include <boost/noncopyable.hpp>
-#include <boost/shared_ptr.hpp>
-#include <boost/enable_shared_from_this.hpp>
-#include <boost/interprocess/detail/atomic.hpp>
-#include <boost/thread/thread.hpp>
-
-#include <memory>
-
-#include "syncobj.h"
-
 #include "net/net_utils_base.h" 
 #include "misc_log_ex.h" 
-#include <boost/lambda/bind.hpp>
-#include <boost/lambda/lambda.hpp>
-#include <boost/uuid/random_generator.hpp>
-#include <boost/chrono.hpp>
-#include <boost/utility/value_init.hpp>
-#include <boost/asio/deadline_timer.hpp>
 #include <boost/date_time/posix_time/posix_time.hpp>
 #include <boost/thread/thread.hpp>
-#include <boost/filesystem.hpp>
 #include "misc_language.h"
 #include "pragma_comp_defs.h"
-#include <fstream>
-#include <sstream>
 #include <iomanip>
-#include <algorithm>
-#include <mutex>
 
 #include <boost/asio/basic_socket.hpp>
-#include <boost/asio/ip/unicast.hpp>
-#include "net/abstract_tcp_server2.h"
 
 // TODO:
 #include "net/network_throttle-detail.hpp"
@@ -161,7 +129,6 @@ connection_basic::connection_basic(boost::asio::io_service& io_service, std::ato
 	try { boost::system::error_code e; remote_addr_str = socket_.remote_endpoint(e).address().to_string(); } catch(...){} ;
 
 	_note("Spawned connection p2p#"<<mI->m_peer_number<<" to " << remote_addr_str << " currently we have sockets count:" << m_ref_sock_count);
-	//boost::filesystem::create_directories("log/dr-monero/net/");
 }
 
 connection_basic::~connection_basic() noexcept(false) {
@@ -250,22 +217,15 @@ void connection_basic::sleep_before_packet(size_t packet_size, int phase,  int q
 	}
 
 }
-void connection_basic::set_start_time() {
-	CRITICAL_REGION_LOCAL(	network_throttle_manager::m_lock_get_global_throttle_out );
-	m_start_time = network_throttle_manager::get_global_throttle_out().get_time_seconds();
-}
 
 void connection_basic::do_send_handler_write(const void* ptr , size_t cb ) {
         // No sleeping here; sleeping is done once and for all in connection<t_protocol_handler>::handle_write
 	MTRACE("handler_write (direct) - before ASIO write, for packet="<<cb<<" B (after sleep)");
-	set_start_time();
 }
 
 void connection_basic::do_send_handler_write_from_queue( const boost::system::error_code& e, size_t cb, int q_len ) {
         // No sleeping here; sleeping is done once and for all in connection<t_protocol_handler>::handle_write
 	MTRACE("handler_write (after write, from queue="<<q_len<<") - before ASIO write, for packet="<<cb<<" B (after sleep)");
-
-	set_start_time();
 }
 
 void connection_basic::logger_handle_net_read(size_t size) { // network data read

contrib/epee/src/http_auth.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

contrib/epee/src/mlocker.cpp

@@ -38,6 +38,15 @@
 #include "syncobj.h"
 #include "mlocker.h"
 
+#include <atomic>
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "mlocker"
+
+// did an mlock operation previously fail? we only
+// want to log an error once and be done with it
+static std::atomic<bool> previously_failed{ false };
+
 static size_t query_page_size()
 {
 #if defined HAVE_MLOCK
@@ -58,8 +67,8 @@ static void do_lock(void *ptr, size_t len)
 {
 #if defined HAVE_MLOCK
   int ret = mlock(ptr, len);
-  if (ret < 0)
-    MERROR("Error locking page at " << ptr << ": " << strerror(errno));
+  if (ret < 0 && !previously_failed.exchange(true))
+    MERROR("Error locking page at " << ptr << ": " << strerror(errno) << ", subsequent mlock errors will be silenced");
 #else
 #warning Missing do_lock implementation
 #endif
@@ -69,7 +78,10 @@ static void do_unlock(void *ptr, size_t len)
 {
 #if defined HAVE_MLOCK
   int ret = munlock(ptr, len);
-  if (ret < 0)
+  // check whether we previously failed, but don't set it, this is just
+  // to pacify the errors of mlock()ing failed, in which case unlocking
+  // is also not going to work of course
+  if (ret < 0 && !previously_failed.load())
     MERROR("Error unlocking page at " << ptr << ": " << strerror(errno));
 #else
 #warning Missing implementation of page size detection
@@ -107,11 +119,14 @@ namespace epee
 
   mlocker::~mlocker()
   {
-    unlock(ptr, len);
+    try { unlock(ptr, len); }
+    catch (...) { /* ignore and do not propagate through the dtor */ }
   }
 
   void mlocker::lock(void *ptr, size_t len)
   {
+    TRY_ENTRY();
+
     size_t page_size = get_page_size();
     if (page_size == 0)
       return;
@@ -122,10 +137,14 @@ namespace epee
     for (size_t page = first; page <= last; ++page)
       lock_page(page);
     ++num_locked_objects;
+
+    CATCH_ENTRY_L1("mlocker::lock", void());
   }
 
   void mlocker::unlock(void *ptr, size_t len)
   {
+    TRY_ENTRY();
+
     size_t page_size = get_page_size();
     if (page_size == 0)
       return;
@@ -135,6 +154,8 @@ namespace epee
     for (size_t page = first; page <= last; ++page)
       unlock_page(page);
     --num_locked_objects;
+
+    CATCH_ENTRY_L1("mlocker::lock", void());
   }
 
   size_t mlocker::get_num_locked_pages()

contrib/epee/src/mlog.cpp

@@ -40,6 +40,7 @@
 #include <boost/filesystem.hpp>
 #include <boost/algorithm/string.hpp>
 #include "string_tools.h"
+#include "misc_os_dependent.h"
 #include "misc_log_ex.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
@@ -58,12 +59,7 @@ static std::string generate_log_filename(const char *base)
   char tmp[200];
   struct tm tm;
   time_t now = time(NULL);
-  if
-#ifdef WIN32
-  (!gmtime_s(&tm, &now))
-#else
-  (!gmtime_r(&now, &tm))
-#endif
+  if (!epee::misc_utils::get_gmt_time(now, tm))
     snprintf(tmp, sizeof(tmp), "part-%u", ++fallback_counter);
   else
     strftime(tmp, sizeof(tmp), "%Y-%m-%d-%H-%M-%S", &tm);
@@ -107,7 +103,7 @@ static const char *get_default_categories(int level)
       categories = "*:WARNING,net:FATAL,net.http:FATAL,net.p2p:FATAL,net.cn:FATAL,global:INFO,verify:FATAL,stacktrace:INFO,logging:INFO,msgwriter:INFO";
       break;
     case 1:
-      categories = "*:INFO,global:INFO,stacktrace:INFO,logging:INFO,msgwriter:INFO";
+      categories = "*:INFO,global:INFO,stacktrace:INFO,logging:INFO,msgwriter:INFO,perf.*:DEBUG";
       break;
     case 2:
       categories = "*:DEBUG";

contrib/epee/src/net_utils_base.cpp

@@ -1,9 +1,9 @@
 
 #include "net/net_utils_base.h"
-#include "string_tools.h"
 
-#include <cstring>
-#include <typeindex>
+#include <boost/uuid/uuid_io.hpp>
+
+#include "string_tools.h"
 #include "net/local_ip.h"
 
 namespace epee { namespace net_utils
@@ -75,7 +75,7 @@ namespace epee { namespace net_utils
   std::string print_connection_context(const connection_context_base& ctx)
   {
     std::stringstream ss;
-    ss << ctx.m_remote_address.str() << " " << epee::string_tools::get_str_from_guid_a(ctx.m_connection_id) << (ctx.m_is_income ? " INC":" OUT");
+    ss << ctx.m_remote_address.str() << " " << ctx.m_connection_id << (ctx.m_is_income ? " INC":" OUT");
     return ss.str();
   }
 

contrib/epee/src/network_throttle-detail.cpp

@@ -2,7 +2,7 @@
 /// @author rfree (current maintainer in monero.cc project)
 /// @brief implementaion for throttling of connection (count and rate-limit speed etc)
 
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 
@@ -32,20 +32,11 @@
 
 /* rfree: implementation for throttle details */
 
-#include <boost/asio.hpp>
 #include <string>
 #include <vector>
-#include <boost/noncopyable.hpp>
-#include <boost/shared_ptr.hpp>
 #include <atomic>
 
 #include <boost/asio.hpp>
-#include <boost/array.hpp>
-#include <boost/noncopyable.hpp>
-#include <boost/shared_ptr.hpp>
-#include <boost/enable_shared_from_this.hpp>
-#include <boost/interprocess/detail/atomic.hpp>
-#include <boost/thread/thread.hpp>
 
 #include <memory>
 
@@ -53,14 +44,7 @@
 
 #include "net/net_utils_base.h" 
 #include "misc_log_ex.h" 
-#include <boost/lambda/bind.hpp>
-#include <boost/lambda/lambda.hpp>
-#include <boost/uuid/random_generator.hpp>
 #include <boost/chrono.hpp>
-#include <boost/utility/value_init.hpp>
-#include <boost/asio/deadline_timer.hpp>
-#include <boost/date_time/posix_time/posix_time.hpp>
-#include <boost/thread/thread.hpp> 
 #include "misc_language.h"
 #include "pragma_comp_defs.h"
 #include <sstream>
@@ -146,9 +130,11 @@ network_throttle::network_throttle(const std::string &nameshort, const std::stri
 	m_network_add_cost = 128;
 	m_network_minimal_segment = 256;
 	m_network_max_segment = 1024*1024;
+	m_start_time = 0;
 	m_any_packet_yet = false;
 	m_slot_size = 1.0; // hard coded in few places
 	m_target_speed = 16 * 1024; // other defaults are probably defined in the command-line parsing code when this class is used e.g. as main global throttle
+	m_last_sample_time = 0;
 }
 
 void network_throttle::set_name(const std::string &name) 

contrib/epee/src/network_throttle.cpp

@@ -26,7 +26,7 @@ Throttling work by:
 
 */
 
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

contrib/epee/src/readline_buffer.cpp

@@ -1,9 +1,9 @@
 #include "readline_buffer.h"
 #include <readline/readline.h>
 #include <readline/history.h>
-#include <unistd.h>
 #include <iostream>
-#include <boost/thread.hpp>
+#include <boost/thread/mutex.hpp>
+#include <boost/thread/lock_guard.hpp>
 #include <boost/algorithm/string.hpp>
 
 static void install_line_handler();
@@ -44,7 +44,7 @@ std::vector<std::string>& rdln::readline_buffer::completion_commands()
 }
 
 rdln::readline_buffer::readline_buffer()
-: std::stringbuf(), m_cout_buf(NULL)
+: std::stringbuf(), m_cout_buf(NULL), m_prompt_length(0)
 {
   current = this;
 }
@@ -86,8 +86,11 @@ void rdln::readline_buffer::set_prompt(const std::string& prompt)
   if(m_cout_buf == NULL)
     return;
   boost::lock_guard<boost::mutex> lock(sync_mutex);
+  rl_set_prompt(std::string(m_prompt_length, ' ').c_str());
+  rl_redisplay();
   rl_set_prompt(prompt.c_str());
   rl_redisplay();
+  m_prompt_length = prompt.size();
 }
 
 void rdln::readline_buffer::add_completion(const std::string& command)
@@ -111,7 +114,7 @@ int rdln::readline_buffer::sync()
   int end = 0, point = 0;
 #endif
 
-  if (rl_end || *rl_prompt)
+  if (rl_end || (rl_prompt && *rl_prompt))
   {
 #if RL_READLINE_VERSION >= 0x0700
     rl_clear_visible_line();
@@ -134,7 +137,7 @@ int rdln::readline_buffer::sync()
   while ( this->snextc() != EOF );
 
 #if RL_READLINE_VERSION < 0x0700
-  if (end || *rl_prompt)
+  if (end || (rl_prompt && *rl_prompt))
   {
     rl_restore_prompt();
     rl_line_buffer = line;

contrib/epee/tests/src/CMakeLists.txt

@@ -14,8 +14,8 @@ IF (MSVC)
 	include_directories(SYSTEM platform/msvc)
 ELSE()
 	# set stuff for other systems
-	SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -Wall -Werror")
-	SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Werror -Wno-reorder")
+	SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -Wall")
+	SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -Wall -Wno-reorder")
 ENDIF()
 
 

contrib/epee/tests/src/net/test_net.h

@@ -29,7 +29,9 @@
 #include <boost/thread.hpp>
 #include <boost/bind.hpp> 
 
-#include "net/levin_server_cp2.h"
+#include "net/abstract_tcp_server2.h"
+#include "net/levin_protocol_handler.h"
+#include "net/levin_protocol_handler_async.h"
 #include "storages/abstract_invoke.h"
 
 namespace epee

contrib/gitian/README.md

@@ -0,0 +1,133 @@
+Gitian building
+================
+
+*Setup instructions for a Gitian build of Monero using a VM or physical system.*
+
+Gitian is the deterministic build process that is used to build the Monero CLI
+executables. It provides a way to be reasonably sure that the
+executables are really built from the git source. It also makes sure that
+the same, tested dependencies are used and statically built into the executable.
+
+Multiple developers build the source code by following a specific descriptor
+("recipe"), cryptographically sign the result, and upload the resulting signature.
+These results are compared and only if they match, the build is accepted and provided
+for download.
+
+More independent Gitian builders are needed, which is why this guide exists.
+It is preferred you follow these steps yourself instead of using someone else's
+VM image to avoid 'contaminating' the build.
+
+Preparing the Gitian builder host
+---------------------------------
+
+The first step is to prepare the host environment that will be used to perform the Gitian builds.
+This guide explains how to set up the environment, and how to start the builds.
+
+Gitian builds are for now executed on Ubuntu 18.04 "Bionic Beaver". A solution is being worked on to run 
+it in docker in the future. Please run Ubuntu in either a VM, or on your physical machine.
+You need to be logged in as the `gitianuser` in order to build gitian builds. If this user does not exist yet on your system, 
+create it. 
+
+Note that a version of `lxc-execute` higher or equal to 2.1.1 is required.
+You can check the version with `lxc-execute --version`.
+
+First we need to set up dependencies. Type/paste the following in the terminal:
+
+```bash
+sudo apt-get install git ruby apt-cacher-ng qemu-utils debootstrap lxc python-cheetah parted kpartx bridge-utils make ubuntu-archive-keyring curl firewalld
+```
+
+Then set up LXC and the rest with the following, which is a complex jumble of settings and workarounds:
+
+```bash
+sudo -s
+# the version of lxc-start in Debian needs to run as root, so make sure
+# that the build script can execute it without providing a password
+echo "%sudo ALL=NOPASSWD: /usr/bin/lxc-start" > /etc/sudoers.d/gitian-lxc
+echo "%sudo ALL=NOPASSWD: /usr/bin/lxc-execute" >> /etc/sudoers.d/gitian-lxc
+# make /etc/rc.local script that sets up bridge between guest and host
+echo '#!/bin/sh -e' > /etc/rc.local
+echo 'brctl addbr br0' >> /etc/rc.local
+echo 'ip addr add 10.0.3.1/24 broadcast 10.0.3.255 dev br0' >> /etc/rc.local
+echo 'ip link set br0 up' >> /etc/rc.local
+echo 'firewall-cmd --zone=trusted --add-interface=br0' >> /etc/rc.local
+echo 'exit 0' >> /etc/rc.local
+chmod +x /etc/rc.local
+# make sure that USE_LXC is always set when logging in as gitianuser,
+# and configure LXC IP addresses
+echo 'export USE_LXC=1' >> /home/gitianuser/.profile
+echo 'export GITIAN_HOST_IP=10.0.3.1' >> /home/gitianuser/.profile
+echo 'export LXC_GUEST_IP=10.0.3.5' >> /home/gitianuser/.profile
+reboot
+```
+
+This setup is required to enable networking in the container.
+
+
+Manual and Building
+-------------------
+The instructions below use the automated script [gitian-build.py](https://github.com/betcoin/bitcoin/blob/master/contrib/gitian-build.py) which only works in Ubuntu. 
+It calls all available descriptors. Help for the build steps taken can be accessed with `./gitian-build.py --help`.
+
+Initial Gitian Setup
+--------------------
+The `gitian-build.py` script will checkout different release tags, so it's best to copy it:
+
+```bash
+cp monero/contrib/gitian/gitian-build.py .
+```
+
+Setup the required environment, you only need to do this once:
+
+```
+./gitian-build.py --setup fluffypony v0.14.0
+```
+
+Where `fluffypony` is your Github name and `v0.14.0` is the version tag you want to build.
+
+While gitian and this build script does provide a way for you to sign the build directly, it is recommended to sign in a seperate step. 
+This script is only there for convenience. Seperate steps for building can still be taken.
+In order to sign gitian builds on your host machine, which has your PGP key, 
+fork the gitian.sigs repository and clone it on your host machine, 
+or pass the signed assert file back to your build machine.
+
+```
+git clone git@github.com:monero-project/gitian.sigs.git
+git remote add fluffypony git@github.com:fluffypony/gitian.sigs.git
+```
+
+Build Binaries
+-----------------------------
+To build the most recent tag:
+
+ `./gitian-build.py --detach-sign --no-commit -b fluffypony v0.14.0`
+
+To speed up the build, use `-j 5 -m 5000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 5000 is a little bit less than then the MB's of RAM you allocated. If there is memory corruption on your machine, try to tweak these values.
+
+If all went well, this produces a number of (uncommited) `.assert` files in the gitian.sigs repository.
+
+If you do detached, offline signing, you need to copy these uncommited changes to your host machine, where you can sign them. For example:
+
+```
+export NAME=fluffypony
+export VERSION=v0.14.0
+gpg --output $VERSION-linux/$NAME/monero-linux-$VERSION-build.assert.sig --detach-sign $VERSION-linux/$NAME/monero-linux-$VERSION-build.assert
+gpg --output $VERSION-osx-unsigned/$NAME/monero-osx-$VERSION-build.assert.sig --detach-sign $VERSION-osx-unsigned/$NAME/monero-osx-$VERSION-build.assert
+gpg --output $VERSION-win-unsigned/$NAME/monero-win-$VERSION-build.assert.sig --detach-sign $VERSION-win-unsigned/$NAME/monero-win-$VERSION-build.assert
+```
+
+Make a pull request (both the `.assert` and `.assert.sig` files) to the
+[monero-project/gitian.sigs](https://github.com/monero-project/gitian.sigs/) repository:
+
+```
+git checkout -b v0.14.0
+git commit -S -a -m "Add $NAME v0.14.0"
+git push --set-upstream $NAME v0.14.0
+```
+
+```bash
+    gpg --detach-sign ${VERSION}-linux/${SIGNER}/monero-linux-*-build.assert
+    gpg --detach-sign ${VERSION}-win-unsigned/${SIGNER}/monero-win-*-build.assert
+    gpg --detach-sign ${VERSION}-osx-unsigned/${SIGNER}/monero-osx-*-build.assert
+```
+

contrib/gitian/gitian-build.py

@@ -0,0 +1,192 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import subprocess
+import sys
+
+def setup():
+    global args, workdir
+    programs = ['ruby', 'git', 'apt-cacher-ng', 'make', 'wget']
+    if args.kvm:
+        programs += ['python-vm-builder', 'qemu-kvm', 'qemu-utils']
+    elif args.docker:
+        dockers = ['docker.io', 'docker-ce']
+        for i in dockers:
+            return_code = subprocess.call(['sudo', 'apt-get', 'install', '-qq', i])
+            if return_code == 0:
+                break
+        if return_code != 0:
+            print('Cannot find any way to install docker', file=sys.stderr)
+            exit(1)
+    else:
+        programs += ['lxc', 'debootstrap']
+    subprocess.check_call(['sudo', 'apt-get', 'install', '-qq'] + programs)
+    if not os.path.isdir('gitian.sigs'):
+        subprocess.check_call(['git', 'clone', 'https://github.com/monero-project/gitian.sigs.git'])
+    if not os.path.isdir('gitian-builder'):
+        subprocess.check_call(['git', 'clone', 'https://github.com/devrandom/gitian-builder.git'])
+    if not os.path.isdir('monero'):
+        subprocess.check_call(['git', 'clone', 'https://github.com/monero-project/monero.git'])
+    os.chdir('gitian-builder')
+    subprocess.check_call(['git', 'checkout', '963322de8420c50502c4cc33d4d7c0d84437b576'])
+    make_image_prog = ['bin/make-base-vm', '--suite', 'bionic', '--arch', 'amd64']
+    if args.docker:
+        make_image_prog += ['--docker']
+    elif not args.kvm:
+        make_image_prog += ['--lxc']
+    subprocess.check_call(make_image_prog)
+    os.chdir(workdir)
+    if args.is_bionic and not args.kvm and not args.docker:
+        subprocess.check_call(['sudo', 'sed', '-i', 's/lxcbr0/br0/', '/etc/default/lxc-net'])
+        print('Reboot is required')
+        exit(0)
+
+def build():
+    global args, workdir
+
+    os.makedirs('monero-binaries/' + args.version, exist_ok=True)
+    print('\nBuilding Dependencies\n')
+    os.chdir('gitian-builder')
+    os.makedirs('inputs', exist_ok=True)
+
+    subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz'])
+    subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch'])
+    subprocess.check_output(["echo 'a8c4e9cafba922f89de0df1f2152e7be286aba73f78505169bc351a7938dd911 inputs/osslsigncode-Backports-to-1.7.1.patch' | sha256sum -c"], shell=True)
+    subprocess.check_output(["echo 'f9a8cdb38b9c309326764ebc937cba1523a3a751a7ab05df3ecc99d18ae466c9 inputs/osslsigncode-1.7.1.tar.gz' | sha256sum -c"], shell=True)
+    subprocess.check_call(['make', '-C', '../monero/contrib/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common'])
+
+    if args.linux:
+        print('\nCompiling ' + args.version + ' Linux')
+        subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, '../monero/contrib/gitian/gitian-linux.yml'])
+        subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-linux', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-linux.yml'])
+        subprocess.check_call('mv build/out/monero-*.tar.gz ../monero-binaries/'+args.version, shell=True)
+
+    if args.windows:
+        print('\nCompiling ' + args.version + ' Windows')
+        subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero='+args.url, '../monero/contrib/gitian/gitian-win.yml'])
+        subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-win', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-win.yml'])
+        subprocess.check_call('mv build/out/monero*.zip ../monero-binaries/'+args.version, shell=True)
+
+    if args.macos:
+        print('\nCompiling ' + args.version + ' MacOS')
+        subprocess.check_call(['bin/gbuild', '-j', args.jobs, '-m', args.memory, '--commit', 'monero='+args.commit, '--url', 'monero'+args.url, '../monero/contrib/gitian/gitian-osx.yml'])
+        subprocess.check_call(['bin/gsign', '-p', args.sign_prog, '--signer', args.signer, '--release', args.version+'-osx', '--destination', '../gitian.sigs/', '../monero/contrib/gitian/gitian-osx.yml'])
+        subprocess.check_call('mv build/out/monero*.tar.gz ../monero-binaries/'+args.version, shell=True)
+
+    os.chdir(workdir)
+
+    if args.commit_files:
+        print('\nCommitting '+args.version+' Unsigned Sigs\n')
+        os.chdir('gitian.sigs')
+        subprocess.check_call(['git', 'add', args.version+'-linux/'+args.signer])
+        subprocess.check_call(['git', 'add', args.version+'-win/'+args.signer])
+        subprocess.check_call(['git', 'add', args.version+'-osx/'+args.signer])
+        subprocess.check_call(['git', 'commit', '-m', 'Add '+args.version+' unsigned sigs for '+args.signer])
+        os.chdir(workdir)
+
+def verify():
+    global args, workdir
+    os.chdir('gitian-builder')
+
+    print('\nVerifying v'+args.version+' Linux\n')
+    subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-linux', '../monero/contrib/gitian/gitian-linux.yml'])
+    print('\nVerifying v'+args.version+' Windows\n')
+    subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-win', '../monero/contrib/gitian/gitian-win.yml'])
+    print('\nVerifying v'+args.version+' MacOS\n')
+    subprocess.check_call(['bin/gverify', '-v', '-d', '../gitian.sigs/', '-r', args.version+'-osx', '../monero/contrib/gitian/gitian-osx.yml'])
+    os.chdir(workdir)
+
+def main():
+    global args, workdir
+
+    parser = argparse.ArgumentParser(usage='%(prog)s [options] signer version')
+    parser.add_argument('-c', '--commit', action='store_true', dest='commit', help='Indicate that the version argument is for a commit or branch')
+    parser.add_argument('-p', '--pull', action='store_true', dest='pull', help='Indicate that the version argument is the number of a github repository pull request')
+    parser.add_argument('-u', '--url', dest='url', default='https://github.com/monero-project/monero', help='Specify the URL of the repository. Default is %(default)s')
+    parser.add_argument('-v', '--verify', action='store_true', dest='verify', help='Verify the Gitian build')
+    parser.add_argument('-b', '--build', action='store_true', dest='build', help='Do a Gitian build')
+    parser.add_argument('-B', '--buildsign', action='store_true', dest='buildsign', help='Build both signed and unsigned binaries')
+    parser.add_argument('-o', '--os', dest='os', default='lwm', help='Specify which Operating Systems the build is for. Default is %(default)s. l for Linux, w for Windows, m for MacOS')
+    parser.add_argument('-j', '--jobs', dest='jobs', default='2', help='Number of processes to use. Default %(default)s')
+    parser.add_argument('-m', '--memory', dest='memory', default='2000', help='Memory to allocate in MiB. Default %(default)s')
+    parser.add_argument('-k', '--kvm', action='store_true', dest='kvm', help='Use KVM instead of LXC')
+    parser.add_argument('-d', '--docker', action='store_true', dest='docker', help='Use Docker instead of LXC')
+    parser.add_argument('-S', '--setup', action='store_true', dest='setup', help='Set up the Gitian building environment. Uses LXC. If you want to use KVM, use the --kvm option. Only works on Debian-based systems (Ubuntu, Debian)')
+    parser.add_argument('-D', '--detach-sign', action='store_true', dest='detach_sign', help='Create the assert file for detached signing. Will not commit anything.')
+    parser.add_argument('-n', '--no-commit', action='store_false', dest='commit_files', help='Do not commit anything to git')
+    parser.add_argument('signer', help='GPG signer to sign each build assert file')
+    parser.add_argument('version', help='Version number, commit, or branch to build. If building a commit or branch, the -c option must be specified')
+
+    args = parser.parse_args()
+    workdir = os.getcwd()
+
+    args.linux = 'l' in args.os
+    args.windows = 'w' in args.os
+    args.macos = 'm' in args.os
+
+    args.is_bionic = b'bionic' in subprocess.check_output(['lsb_release', '-cs'])
+
+    if args.buildsign:
+        args.build=True
+        args.sign=True
+
+    if args.kvm and args.docker:
+        raise Exception('Error: cannot have both kvm and docker')
+
+    args.sign_prog = 'true' if args.detach_sign else 'gpg --detach-sign'
+
+    # Set enviroment variable USE_LXC or USE_DOCKER, let gitian-builder know that we use lxc or docker
+    if args.docker:
+        os.environ['USE_DOCKER'] = '1'
+    elif not args.kvm:
+        os.environ['USE_LXC'] = '1'
+        if not 'GITIAN_HOST_IP' in os.environ.keys():
+            os.environ['GITIAN_HOST_IP'] = '10.0.3.1'
+        if not 'LXC_GUEST_IP' in os.environ.keys():
+            os.environ['LXC_GUEST_IP'] = '10.0.3.5'
+
+    # Disable for MacOS if no SDK found
+    if args.macos and not os.path.isfile('gitian-builder/inputs/MacOSX10.11.sdk.tar.gz'):
+        print('Cannot build for MacOS, SDK does not exist. Will build for other OSes')
+        args.macos = False
+
+    script_name = os.path.basename(sys.argv[0])
+    # Signer and version shouldn't be empty
+    if args.signer == '':
+        print(script_name+': Missing signer.')
+        print('Try '+script_name+' --help for more information')
+        exit(1)
+    if args.version == '':
+        print(script_name+': Missing version.')
+        print('Try '+script_name+' --help for more information')
+        exit(1)
+
+    # Add leading 'v' for tags
+    if args.commit and args.pull:
+        raise Exception('Cannot have both commit and pull')
+    args.commit = args.commit if args.commit else args.version
+
+    if args.setup:
+        setup()
+
+    os.chdir('monero')
+    if args.pull:
+        subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge'])
+        os.chdir('../gitian-builder/inputs/monero')
+        subprocess.check_call(['git', 'fetch', args.url, 'refs/pull/'+args.version+'/merge'])
+        args.commit = subprocess.check_output(['git', 'show', '-s', '--format=%H', 'FETCH_HEAD'], universal_newlines=True).strip()
+        args.version = 'pull-' + args.version
+    print(args.commit)
+    subprocess.check_call(['git', 'fetch'])
+    subprocess.check_call(['git', 'checkout', args.commit])
+    os.chdir(workdir)
+
+    if args.build:
+        build()
+
+    if args.verify:
+        verify()
+
+if __name__ == '__main__':
+    main()

contrib/gitian/gitian-linux.yml

@@ -0,0 +1,162 @@
+---
+name: "monero-linux-0.14"
+enable_cache: true
+suites:
+- "bionic"
+architectures:
+- "amd64"
+packages:
+- "curl"
+- "gperf"
+- "gcc-7"
+- "g++-7"
+- "gcc"
+- "g++"
+- "gcc-7-aarch64-linux-gnu"
+- "g++-7-aarch64-linux-gnu"
+- "gcc-aarch64-linux-gnu"
+- "g++-aarch64-linux-gnu"
+- "binutils-aarch64-linux-gnu"
+- "gcc-7-arm-linux-gnueabihf"
+- "g++-7-arm-linux-gnueabihf"
+- "gcc-arm-linux-gnueabihf"
+- "g++-arm-linux-gnueabihf"
+- "g++-7-multilib"
+- "gcc-7-multilib"
+- "binutils-arm-linux-gnueabihf"
+- "binutils-gold"
+- "git"
+- "pkg-config"
+- "build-essential"
+- "autoconf"
+- "libtool"
+- "automake"
+- "faketime"
+- "bsdmainutils"
+- "ca-certificates"
+- "python"
+- "cmake"
+- "ccache"
+- "protobuf-compiler"
+- "libdbus-1-dev"
+- "libharfbuzz-dev"
+- "libprotobuf-dev"
+- "python3-zmq"
+remotes:
+- "url": "https://github.com/monero-project/monero.git"
+  "dir": "monero"
+files: []
+script: |
+
+  WRAP_DIR=$HOME/wrapped
+  HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu i686-linux-gnu"
+  FAKETIME_HOST_PROGS="gcc g++"
+  FAKETIME_PROGS="date ar ranlib nm"
+  HOST_CFLAGS="-O2 -g"
+  HOST_CXXFLAGS="-O2 -g"
+  HOST_LDFLAGS=-static-libstdc++
+
+  export GZIP="-9n"
+  export TAR_OPTIONS="--mtime="$REFERENCE_DATE\\\ $REFERENCE_TIME""
+  export TZ="UTC"
+  export BUILD_DIR=`pwd`
+  mkdir -p ${WRAP_DIR}
+  if test -n "$GBUILD_CACHE_ENABLED"; then
+    export SOURCES_PATH=${GBUILD_COMMON_CACHE}
+    export BASE_CACHE=${GBUILD_PACKAGE_CACHE}
+    mkdir -p ${BASE_CACHE} ${SOURCES_PATH}
+  fi
+
+  function create_global_faketime_wrappers {
+  for prog in ${FAKETIME_PROGS}; do
+    echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog}
+    echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog}
+    echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog}
+    echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog}
+    echo "\$REAL \$@" >> $WRAP_DIR/${prog}
+    chmod +x ${WRAP_DIR}/${prog}
+  done
+  }
+
+  function create_per-host_faketime_wrappers {
+  for i in $HOSTS; do
+    for prog in ${FAKETIME_HOST_PROGS}; do
+        if which ${i}-${prog}-7
+        then
+            echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog}
+            echo "REAL=\`which -a ${i}-${prog}-7 | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
+            echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog}
+            echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog}
+            echo "\$REAL \$@" >> $WRAP_DIR/${i}-${prog}
+            chmod +x ${WRAP_DIR}/${i}-${prog}
+        fi
+    done
+  done
+  }
+
+  # Faketime for depends so intermediate results are comparable
+  export PATH_orig=${PATH}
+  create_global_faketime_wrappers "2000-01-01 12:00:00"
+  create_per-host_faketime_wrappers "2000-01-01 12:00:00"
+  export PATH=${WRAP_DIR}:${PATH}
+
+  EXTRA_INCLUDES_BASE=$WRAP_DIR/extra_includes
+  mkdir -p $EXTRA_INCLUDES_BASE
+
+  # x86 needs /usr/include/i386-linux-gnu/asm pointed to /usr/include/x86_64-linux-gnu/asm,
+  # but we can't write there. Instead, create a link here and force it to be included in the
+  # search paths by wrapping gcc/g++.
+
+  mkdir -p $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu
+  rm -f $WRAP_DIR/extra_includes/i686-pc-linux-gnu/asm
+  ln -s /usr/include/x86_64-linux-gnu/asm $EXTRA_INCLUDES_BASE/i686-pc-linux-gnu/asm
+
+  for prog in gcc g++; do
+  rm -f ${WRAP_DIR}/${prog}
+  cat << EOF > ${WRAP_DIR}/${prog}
+  #!/usr/bin/env bash
+  REAL="`which -a ${prog}-7 | grep -v ${WRAP_DIR}/${prog} | head -1`"
+  for var in "\$@"
+  do
+    if [ "\$var" = "-m32" ]; then
+      export C_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu"
+      export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES_BASE/i686-pc-linux-gnu"
+      break
+    fi
+  done
+  \$REAL \$@
+  EOF
+  chmod +x ${WRAP_DIR}/${prog}
+  done
+
+  cd monero
+  BASEPREFIX=`pwd`/contrib/depends
+  # Build dependencies for each host
+  for i in $HOSTS; do
+    EXTRA_INCLUDES="$EXTRA_INCLUDES_BASE/$i"
+    if [ -d "$EXTRA_INCLUDES" ]; then
+      export HOST_ID_SALT="$EXTRA_INCLUDES"
+    fi
+    make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" -j 4 V=1
+    unset HOST_ID_SALT
+  done
+
+  # Faketime for binaries
+  export PATH=${PATH_orig}
+  create_global_faketime_wrappers "${REFERENCE_DATETIME}"
+  create_per-host_faketime_wrappers "${REFERENCE_DATETIME}"
+  export PATH=${WRAP_DIR}:${PATH}
+
+  ORIGPATH="$PATH"
+  # Build in a new dir for each host
+  for i in ${HOSTS}; do
+    export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH}
+    mkdir build && cd build
+    cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=ON
+    make
+    DISTNAME=monero-${i}
+    mv bin ${DISTNAME}
+    find ${DISTNAME}/ | sort | tar --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}.tar.gz
+    cd ..
+    rm -rf build
+  done