enable refresh thread

Ensures both transfers and sweeps use a fee that's calculated
from the tx's weight. Using different logic could theoretically
enable distinguishability between the two types of txs. We don't
want that.

.github/workflows/build.yml

@@ -151,7 +151,7 @@ jobs:
     - name: install monero dependencies
       run: ${{env.APT_INSTALL_LINUX}}
     - name: install Python dependencies
-      run: pip install requests psutil monotonic zmq
+      run: pip install requests psutil monotonic zmq deepdiff
     - name: tests
       env:
         CTEST_OUTPUT_ON_FAILURE: ON

.gitmodules

@@ -15,5 +15,5 @@
 [submodule "external/randomwow"]
 	path = external/randomwow
 	url = https://git.wownero.com/wownero/RandomWOW
-        branch = 1.1.10-wow
+        branch = 1.2.1-wow
 

CMakeLists.txt

@@ -778,7 +778,7 @@ else()
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${ARCH_FLAG}")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARCH_FLAG}")
 
-  set(WARNINGS "-Wpointer-arith -Wundef -Wvla -Wwrite-strings -Wno-deprecated-declarations -Wno-unused-parameter -Wno-error=unused-variable -Wno-error=undef -Wno-error=uninitialized")
+  set(WARNINGS "-Wall -Wextra -Wpointer-arith -Wundef -Wvla -Wwrite-strings -Wno-error=extra -Wno-error=deprecated-declarations -Wno-unused-parameter -Wno-error=unused-variable -Wno-error=undef -Wno-error=uninitialized")
   if(CMAKE_C_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
     if(ARM)
       set(WARNINGS "${WARNINGS} -Wno-error=inline-asm")
@@ -1140,9 +1140,7 @@ if (HIDAPI_FOUND OR LibUSB_COMPILE_TEST_PASSED)
   endif()
 endif()
 
-if(NOT OPENBSD)
-  option(USE_READLINE "Build with GNU readline support." ON)
-endif()
+option(USE_READLINE "Build with GNU readline support." ON)
 if(USE_READLINE AND NOT DEPENDS)
   find_package(Readline)
   if(READLINE_FOUND AND GNU_READLINE_FOUND)

README.md

@@ -96,7 +96,7 @@ Dates are provided in the format YYYY-MM-DD.
 | -                             | 2020-06-28 | Hallucinogenic Hypnotoad  | v0.8.0.0                | v0.8.0.2                    | Dandelion++ support
 | 253,999                       | 2020-10-09 | Illiterate Illuminati  | v0.9.0.0                | v0.9.3.3                    | Dynamic coinbase unlock (up to 1 mo.), Deterministic unlock times, Enforce maximum coinbase amount, show_qr_code wallet command, CLSAG
 | 331,170                       | 2021-07-04 | Junkie Jeff  | v0.10.0.0               | v0.10.2.0                   | Bulletproofs+, Miner Block Header Signing, Vote by Block, Change coinbase unlock time to 1 day, Reset difficulty and switch back to Monero's difficulty algorithm
-| 514,000                       | 2023-04-01 | Kunty Karen  | v0.11.0.0               | v0.11.0.1                   | View tags, fee changes, adjusted dynamic block weight algorithm, multisig security fixes, RPC broadcast node donation sub-address, Limit tx_extra max size to ~1kb, 12-hour difficulty adjustment window
+| 514,000                       | 2023-04-01 | Kunty Karen  | v0.11.0.0               | v0.11.1.0                   | View tags, fee changes, adjusted dynamic block weight algorithm, multisig security fixes, RPC broadcast node donation sub-address, Limit tx_extra max size to ~1kb, 12-hour difficulty adjustment window
 
 X's indicate that these details have not been determined as of commit date.
 

contrib/depends/packages/native_biplist.mk

@@ -0,0 +1,20 @@
+package=native_biplist
+$(package)_version=0.9
+$(package)_download_path=https://pypi.python.org/packages/source/b/biplist
+$(package)_file_name=biplist-$($(package)_version).tar.gz
+$(package)_sha256_hash=b57cadfd26e4754efdf89e9e37de87885f9b5c847b2615688ca04adfaf6ca604
+$(package)_install_libdir=$(build_prefix)/lib/python/dist-packages
+$(package)_patches=sorted_list.patch
+
+define $(package)_preprocess_cmds
+  patch -p1 < $($(package)_patch_dir)/sorted_list.patch
+endef
+
+define $(package)_build_cmds
+    python setup.py build
+endef
+
+define $(package)_stage_cmds
+    mkdir -p $($(package)_install_libdir) && \
+    python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir)
+endef

contrib/depends/packages/native_cdrkit.mk

@@ -0,0 +1,26 @@
+package=native_cdrkit
+$(package)_version=1.1.11
+$(package)_download_path=https://distro.ibiblio.org/fatdog/source/600/c
+$(package)_file_name=cdrkit-$($(package)_version).tar.bz2
+$(package)_sha256_hash=b50d64c214a65b1a79afe3a964c691931a4233e2ba605d793eb85d0ac3652564
+$(package)_patches=cdrkit-deterministic.patch
+
+define $(package)_preprocess_cmds
+  patch -p1 < $($(package)_patch_dir)/cdrkit-deterministic.patch
+endef
+
+define $(package)_config_cmds
+  cmake -DCMAKE_INSTALL_PREFIX=$(build_prefix)
+endef
+
+define $(package)_build_cmds
+  $(MAKE) genisoimage
+endef
+
+define $(package)_stage_cmds
+  $(MAKE) DESTDIR=$($(package)_staging_dir) -C genisoimage install
+endef
+
+define $(package)_postprocess_cmds
+  rm bin/isovfy bin/isoinfo bin/isodump bin/isodebug bin/devdump
+endef

contrib/depends/packages/native_cmake-unused.mk

@@ -0,0 +1,23 @@
+package=native_cmake
+$(package)_version=3.14.0
+$(package)_version_dot=v3.14
+$(package)_download_path=https://cmake.org/files/$($(package)_version_dot)/
+$(package)_file_name=cmake-$($(package)_version).tar.gz
+$(package)_sha256_hash=aa76ba67b3c2af1946701f847073f4652af5cbd9f141f221c97af99127e75502
+
+define $(package)_set_vars
+$(package)_config_opts=
+endef
+
+define $(package)_config_cmds
+  ./bootstrap &&\
+  ./configure $($(package)_config_opts)
+endef
+
+define $(package)_build_cmd
+  $(MAKE)
+endef
+
+define $(package)_stage_cmds
+  $(MAKE) DESTDIR=$($(package)_staging_dir) install
+endef

contrib/depends/packages/native_ds_store.mk

@@ -0,0 +1,17 @@
+package=native_ds_store
+$(package)_version=1.1.0
+$(package)_download_path=https://github.com/al45tair/ds_store/archive/
+$(package)_download_file=v$($(package)_version).tar.gz
+$(package)_file_name=$(package)-$($(package)_version).tar.gz
+$(package)_sha256_hash=a9f4c0755c6be7224ff7029e188dd262e830bb81e801424841db9eb0780ec8ed
+$(package)_install_libdir=$(build_prefix)/lib/python/dist-packages
+$(package)_dependencies=native_biplist
+
+define $(package)_build_cmds
+    python setup.py build
+endef
+
+define $(package)_stage_cmds
+    mkdir -p $($(package)_install_libdir) && \
+    python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir)
+endef

contrib/depends/packages/native_libdmg-hfsplus.mk

@@ -0,0 +1,22 @@
+package=native_libdmg-hfsplus
+$(package)_version=0.1
+$(package)_download_path=https://github.com/theuni/libdmg-hfsplus/archive
+$(package)_file_name=libdmg-hfsplus-v$($(package)_version).tar.gz
+$(package)_sha256_hash=6569a02eb31c2827080d7d59001869ea14484c281efab0ae7f2b86af5c3120b3
+$(package)_build_subdir=build
+
+define $(package)_preprocess_cmds
+  mkdir build
+endef
+
+define $(package)_config_cmds
+  cmake -DCMAKE_INSTALL_PREFIX:PATH=$(build_prefix)/bin ..
+endef
+
+define $(package)_build_cmds
+  $(MAKE) -C dmg
+endef
+
+define $(package)_stage_cmds
+  $(MAKE) DESTDIR=$($(package)_staging_dir) -C dmg install
+endef

contrib/depends/packages/native_mac_alias.mk

@@ -0,0 +1,21 @@
+package=native_mac_alias
+$(package)_version=1.1.0
+$(package)_download_path=https://github.com/al45tair/mac_alias/archive/
+$(package)_download_file=v$($(package)_version).tar.gz
+$(package)_file_name=$(package)-$($(package)_version).tar.gz
+$(package)_sha256_hash=b10cb44ecb64fc25283fae7a9cf365d2829377d84e37b9c21100aca8757509be
+$(package)_install_libdir=$(build_prefix)/lib/python/dist-packages
+$(package)_patches=python3.patch
+
+define $(package)_preprocess_cmds
+  patch -p1 < $($(package)_patch_dir)/python3.patch
+endef
+
+define $(package)_build_cmds
+    python setup.py build
+endef
+
+define $(package)_stage_cmds
+    mkdir -p $($(package)_install_libdir) && \
+    python setup.py install --root=$($(package)_staging_dir) --prefix=$(build_prefix) --install-lib=$($(package)_install_libdir)
+endef

contrib/depends/packages/openssl.mk

@@ -1,20 +1,19 @@
 package=openssl
-$(package)_version=1.1.1t
+$(package)_version=3.0.11
 $(package)_download_path=https://www.openssl.org/source
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
-$(package)_sha256_hash=8dee9b24bdb1dcbf0c3d1e9b02fb8f6bf22165e807f45adeb7c9677536859d3b
+$(package)_sha256_hash=b3425d3bb4a2218d0697eb41f7fc0cdede016ed19ca49d168b78e8d947887f55
 
 define $(package)_set_vars
 $(package)_config_env=AR="$($(package)_ar)" ARFLAGS=$($(package)_arflags) RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)"
-$(package)_config_env_android=ANDROID_NDK_HOME="$(host_prefix)/native" PATH="$(host_prefix)/native/bin" CC=clang AR=ar RANLIB=ranlib
-$(package)_build_env_android=ANDROID_NDK_HOME="$(host_prefix)/native"
-$(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/openssl
+$(package)_config_env_android=ANDROID_NDK_ROOT="$(host_prefix)/native" PATH="$(host_prefix)/native/bin" CC=clang AR=ar RANLIB=ranlib
+$(package)_build_env_android=ANDROID_NDK_ROOT="$(host_prefix)/native"
+$(package)_config_opts=--prefix=$(host_prefix) --openssldir=$(host_prefix)/etc/openssl --libdir=$(host_prefix)/lib
 $(package)_config_opts+=no-capieng
 $(package)_config_opts+=no-dso
 $(package)_config_opts+=no-dtls1
 $(package)_config_opts+=no-ec_nistp_64_gcc_128
 $(package)_config_opts+=no-gost
-$(package)_config_opts+=no-heartbeats
 $(package)_config_opts+=no-md2
 $(package)_config_opts+=no-rc5
 $(package)_config_opts+=no-rdrand
@@ -22,8 +21,8 @@ $(package)_config_opts+=no-rfc3779
 $(package)_config_opts+=no-sctp
 $(package)_config_opts+=no-shared
 $(package)_config_opts+=no-ssl-trace
-$(package)_config_opts+=no-ssl2
 $(package)_config_opts+=no-ssl3
+$(package)_config_opts+=no-tests
 $(package)_config_opts+=no-unit-test
 $(package)_config_opts+=no-weak-ssl-ciphers
 $(package)_config_opts+=no-zlib
@@ -49,7 +48,7 @@ $(package)_config_opts_x86_64_freebsd=BSD-x86_64
 endef
 
 define $(package)_preprocess_cmds
-  sed -i.old 's|"engines", "apps", "test", "util", "tools", "fuzz"|"engines", "tools"|' Configure
+  sed -i.old 's|crypto ssl apps util tools fuzz providers doc|crypto ssl util tools providers|' build.info
 endef
 
 define $(package)_config_cmds

contrib/depends/packages/unbound.mk

@@ -11,6 +11,7 @@ define $(package)_set_vars
   $(package)_config_opts=--disable-shared --enable-static --without-pyunbound --prefix=$(host_prefix) --with-libexpat=$(host_prefix) --with-ssl=$(host_prefix) --with-libevent=no --without-pythonmodule --disable-flto --with-pthreads --with-libunbound-only
   $(package)_config_opts_linux=--with-pic
   $(package)_config_opts_w64=--enable-static-exe --sysconfdir=/etc --prefix=$(host_prefix) --target=$(host_prefix)
+  $(package)_config_opts_x86_64_darwin=ac_cv_func_SHA384_Init=yes
   $(package)_build_opts_mingw32=LDFLAGS="$($(package)_ldflags) -lpthread"
 endef
 

contrib/depends/patches/cmake/cmake-1-fixes.patch

@@ -0,0 +1,67 @@
+This file is part of MXE. See LICENSE.md for licensing information.
+
+Contains ad hoc patches for cross building.
+
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Tony Theodore <tonyt@logyst.com>
+Date: Fri, 12 Aug 2016 02:01:20 +1000
+Subject: [PATCH 1/3] fix windres invocation options
+
+windres doesn't recognise various gcc flags like -mms-bitfields,
+-fopenmp, -mthreads etc. (basically not `-D` or `-I`)
+
+diff --git a/Modules/Platform/Windows-windres.cmake b/Modules/Platform/Windows-windres.cmake
+index 1111111..2222222 100644
+--- a/Modules/Platform/Windows-windres.cmake
++++ b/Modules/Platform/Windows-windres.cmake
+@@ -1 +1 @@
+-set(CMAKE_RC_COMPILE_OBJECT "<CMAKE_RC_COMPILER> -O coff <DEFINES> <INCLUDES> <FLAGS> <SOURCE> <OBJECT>")
++set(CMAKE_RC_COMPILE_OBJECT "<CMAKE_RC_COMPILER> -O coff <DEFINES> <INCLUDES> <SOURCE> <OBJECT>")
+
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Tony Theodore <tonyt@logyst.com>
+Date: Tue, 25 Jul 2017 20:34:56 +1000
+Subject: [PATCH 2/3] add option to disable -isystem
+
+taken from (not accepted):
+https://gitlab.kitware.com/cmake/cmake/merge_requests/895
+
+see also:
+https://gitlab.kitware.com/cmake/cmake/issues/16291
+https://gitlab.kitware.com/cmake/cmake/issues/16919
+
+diff --git a/Modules/Compiler/GNU.cmake b/Modules/Compiler/GNU.cmake
+index 1111111..2222222 100644
+--- a/Modules/Compiler/GNU.cmake
++++ b/Modules/Compiler/GNU.cmake
+@@ -42,7 +42,7 @@ macro(__compiler_gnu lang)
+   string(APPEND CMAKE_${lang}_FLAGS_RELWITHDEBINFO_INIT " -O2 -g -DNDEBUG")
+   set(CMAKE_${lang}_CREATE_PREPROCESSED_SOURCE "<CMAKE_${lang}_COMPILER> <DEFINES> <INCLUDES> <FLAGS> -E <SOURCE> > <PREPROCESSED_SOURCE>")
+   set(CMAKE_${lang}_CREATE_ASSEMBLY_SOURCE "<CMAKE_${lang}_COMPILER> <DEFINES> <INCLUDES> <FLAGS> -S <SOURCE> -o <ASSEMBLY_SOURCE>")
+-  if(NOT APPLE OR NOT CMAKE_${lang}_COMPILER_VERSION VERSION_LESS 4) # work around #4462
++  if(NOT APPLE OR NOT CMAKE_${lang}_COMPILER_VERSION VERSION_LESS 4 AND (NOT MXE_DISABLE_INCLUDE_SYSTEM_FLAG)) # work around #4462
+     set(CMAKE_INCLUDE_SYSTEM_FLAG_${lang} "-isystem ")
+   endif()
+ endmacro()
+
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Tony Theodore <tonyt@logyst.com>
+Date: Tue, 15 Aug 2017 15:25:06 +1000
+Subject: [PATCH 3/3] add CPACK_NSIS_EXECUTABLE variable
+
+
+diff --git a/Source/CPack/cmCPackNSISGenerator.cxx b/Source/CPack/cmCPackNSISGenerator.cxx
+index 1111111..2222222 100644
+--- a/Source/CPack/cmCPackNSISGenerator.cxx
++++ b/Source/CPack/cmCPackNSISGenerator.cxx
+@@ -384,7 +384,9 @@ int cmCPackNSISGenerator::InitializeInternal()
+   }
+ #endif
+ 
+-  nsisPath = cmSystemTools::FindProgram("makensis", path, false);
++  this->SetOptionIfNotSet("CPACK_NSIS_EXECUTABLE", "makensis");
++  nsisPath = cmSystemTools::FindProgram(
++    this->GetOption("CPACK_NSIS_EXECUTABLE"), path, false);
+ 
+   if (nsisPath.empty()) {
+     cmCPackLogger(

contrib/depends/patches/native_biplist/sorted_list.patch

@@ -0,0 +1,29 @@
+--- a/biplist/__init__.py	2014-10-26 19:03:11.000000000 +0000
++++ b/biplist/__init__.py	2016-07-19 19:30:17.663521999 +0000
+@@ -541,7 +541,7 @@
+             return HashableWrapper(n)
+         elif isinstance(root, dict):
+             n = {}
+-            for key, value in iteritems(root):
++            for key, value in sorted(iteritems(root)):
+                 n[self.wrapRoot(key)] = self.wrapRoot(value)
+             return HashableWrapper(n)
+         elif isinstance(root, list):
+@@ -616,7 +616,7 @@
+             elif isinstance(obj, dict):
+                 size = proc_size(len(obj))
+                 self.incrementByteCount('dictBytes', incr=1+size)
+-                for key, value in iteritems(obj):
++                for key, value in sorted(iteritems(obj)):
+                     check_key(key)
+                     self.computeOffsets(key, asReference=True)
+                     self.computeOffsets(value, asReference=True)
+@@ -714,7 +714,7 @@
+                 keys = []
+                 values = []
+                 objectsToWrite = []
+-                for key, value in iteritems(obj):
++                for key, value in sorted(iteritems(obj)):
+                     keys.append(key)
+                     values.append(value)
+                 for key in keys:

contrib/depends/patches/native_cdrkit/cdrkit-deterministic.patch

@@ -0,0 +1,86 @@
+--- cdrkit-1.1.11.old/genisoimage/tree.c	2008-10-21 19:57:47.000000000 -0400
++++ cdrkit-1.1.11/genisoimage/tree.c	2013-12-06 00:23:18.489622668 -0500
+@@ -1139,8 +1139,9 @@
+ scan_directory_tree(struct directory *this_dir, char *path, 
+ 						  struct directory_entry *de)
+ {
+-	DIR		*current_dir;
++        int             current_file;
+ 	char		whole_path[PATH_MAX];
++        struct dirent  **d_list;
+ 	struct dirent	*d_entry;
+ 	struct directory *parent;
+ 	int		dflag;
+@@ -1164,7 +1165,8 @@
+ 	this_dir->dir_flags |= DIR_WAS_SCANNED;
+ 
+ 	errno = 0;	/* Paranoia */
+-	current_dir = opendir(path);
++	//current_dir = opendir(path);
++        current_file = scandir(path, &d_list, NULL, alphasort);
+ 	d_entry = NULL;
+ 
+ 	/*
+@@ -1173,12 +1175,12 @@
+ 	 */
+ 	old_path = path;
+ 
+-	if (current_dir) {
++	if (current_file >= 0) {
+ 		errno = 0;
+-		d_entry = readdir(current_dir);
++		d_entry = d_list[0];
+ 	}
+ 
+-	if (!current_dir || !d_entry) {
++	if (current_file < 0 || !d_entry) {
+ 		int	ret = 1;
+ 
+ #ifdef	USE_LIBSCHILY
+@@ -1191,8 +1193,8 @@
+ 			de->isorec.flags[0] &= ~ISO_DIRECTORY;
+ 			ret = 0;
+ 		}
+-		if (current_dir)
+-			closedir(current_dir);
++		if(d_list)
++			free(d_list);
+ 		return (ret);
+ 	}
+ #ifdef	ABORT_DEEP_ISO_ONLY
+@@ -1208,7 +1210,7 @@
+ 			errmsgno(EX_BAD, "use Rock Ridge extensions via -R or -r,\n");
+ 			errmsgno(EX_BAD, "or allow deep ISO9660 directory nesting via -D.\n");
+ 		}
+-		closedir(current_dir);
++		free(d_list);
+ 		return (1);
+ 	}
+ #endif
+@@ -1250,13 +1252,13 @@
+ 		 * The first time through, skip this, since we already asked
+ 		 * for the first entry when we opened the directory.
+ 		 */
+-		if (dflag)
+-			d_entry = readdir(current_dir);
++		if (dflag && current_file >= 0)
++			d_entry = d_list[current_file];
+ 		dflag++;
+ 
+-		if (!d_entry)
++		if (current_file < 0)
+ 			break;
+-
++                current_file--;
+ 		/* OK, got a valid entry */
+ 
+ 		/* If we do not want all files, then pitch the backups. */
+@@ -1348,7 +1350,7 @@
+ 		insert_file_entry(this_dir, whole_path, d_entry->d_name);
+ #endif	/* APPLE_HYB */
+ 	}
+-	closedir(current_dir);
++	free(d_list);
+ 
+ #ifdef APPLE_HYB
+ 	/*

contrib/depends/patches/native_mac_alias/python3.patch

@@ -0,0 +1,72 @@
+diff -dur a/mac_alias/alias.py b/mac_alias/alias.py
+--- a/mac_alias/alias.py	2015-10-19 12:12:48.000000000 +0200
++++ b/mac_alias/alias.py	2016-04-03 12:13:12.037159417 +0200
+@@ -243,10 +243,10 @@
+         alias = Alias()
+         alias.appinfo = appinfo
+             
+-        alias.volume = VolumeInfo (volname.replace('/',':'),
++        alias.volume = VolumeInfo (volname.decode().replace('/',':'),
+                                    voldate, fstype, disktype,
+                                    volattrs, volfsid)
+-        alias.target = TargetInfo (kind, filename.replace('/',':'),
++        alias.target = TargetInfo (kind, filename.decode().replace('/',':'),
+                                    folder_cnid, cnid,
+                                    crdate, creator_code, type_code)
+         alias.target.levels_from = levels_from
+@@ -261,9 +261,9 @@
+                 b.read(1)
+ 
+             if tag == TAG_CARBON_FOLDER_NAME:
+-                alias.target.folder_name = value.replace('/',':')
++                alias.target.folder_name = value.decode().replace('/',':')
+             elif tag == TAG_CNID_PATH:
+-                alias.target.cnid_path = struct.unpack(b'>%uI' % (length // 4),
++                alias.target.cnid_path = struct.unpack('>%uI' % (length // 4),
+                                                            value)
+             elif tag == TAG_CARBON_PATH:
+                 alias.target.carbon_path = value
+@@ -298,9 +298,9 @@
+                 alias.target.creation_date \
+                     = mac_epoch + datetime.timedelta(seconds=seconds)
+             elif tag == TAG_POSIX_PATH:
+-                alias.target.posix_path = value
++                alias.target.posix_path = value.decode()
+             elif tag == TAG_POSIX_PATH_TO_MOUNTPOINT:
+-                alias.volume.posix_path = value
++                alias.volume.posix_path = value.decode()
+             elif tag == TAG_RECURSIVE_ALIAS_OF_DISK_IMAGE:
+                 alias.volume.disk_image_alias = Alias.from_bytes(value)
+             elif tag == TAG_USER_HOME_LENGTH_PREFIX:
+@@ -422,13 +422,13 @@
+         #       (so doing so is ridiculous, and nothing could rely on it).
+         b.write(struct.pack(b'>h28pI2shI64pII4s4shhI2s10s',
+                             self.target.kind,
+-                            carbon_volname, voldate,
++                            carbon_volname, int(voldate),
+                             self.volume.fs_type,
+                             self.volume.disk_type,
+                             self.target.folder_cnid,
+                             carbon_filename,
+                             self.target.cnid,
+-                            crdate,
++                            int(crdate),
+                             self.target.creator_code,
+                             self.target.type_code,
+                             self.target.levels_from,
+@@ -449,12 +449,12 @@
+ 
+         b.write(struct.pack(b'>hhQhhQ',
+                 TAG_HIGH_RES_VOLUME_CREATION_DATE,
+-                8, long(voldate * 65536),
++                8, int(voldate * 65536),
+                 TAG_HIGH_RES_CREATION_DATE,
+-                8, long(crdate * 65536)))
++                8, int(crdate * 65536)))
+ 
+         if self.target.cnid_path:
+-            cnid_path = struct.pack(b'>%uI' % len(self.target.cnid_path),
++            cnid_path = struct.pack('>%uI' % len(self.target.cnid_path),
+                                     *self.target.cnid_path)
+             b.write(struct.pack(b'>hh', TAG_CNID_PATH,
+                                  len(cnid_path)))

contrib/depends/toolchain.cmake.in

@@ -144,8 +144,8 @@ elseif(ARCHITECTURE STREQUAL "aarch64")
 endif()
 
 if(ARCHITECTURE STREQUAL "riscv64")
-    set(NO_AES ON)
-    set(ARCH "rv64imafdc")
+    set(ARCH_ID "riscv64")
+    set(ARCH "rv64gc")
 endif()
 
 if(ARCHITECTURE STREQUAL "i686")

contrib/epee/include/file_io_utils.h

@@ -29,6 +29,7 @@
 
 #include <string>
 #include <ctime>
+#include <cstdint>
 
 namespace epee
 {

contrib/epee/include/net/abstract_tcp_server2.inl

@@ -583,11 +583,8 @@ namespace net_utils
             break;
         }
       }
-      else if (ec.value())
-        terminate();
       else {
-        cancel_timer();
-        on_interrupted();
+        terminate();
       }
     };
     m_strand.post(

contrib/epee/include/net/http_client.h

@@ -245,8 +245,18 @@ namespace net_utils
 					}
 				}
 
+				// This magic var determines the maximum length for when copying the body message in
+				// memory is faster/more preferable than the round-trip time for one packet
+				constexpr size_t BODY_NO_COPY_CUTOFF = 128 * 1024; // ~262 KB or ~175 packets
+
+				// Maximum expected total headers bytes
+				constexpr size_t HEADER_RESERVE_SIZE = 2048;
+
+				const bool do_copy_body = body.size() <= BODY_NO_COPY_CUTOFF;
+				const size_t req_buff_cap = HEADER_RESERVE_SIZE + (do_copy_body ? body.size() : 0);
+
 				std::string req_buff{};
-				req_buff.reserve(2048);
+				req_buff.reserve(req_buff_cap);
 				req_buff.append(method.data(), method.size()).append(" ").append(uri.data(), uri.size()).append(" HTTP/1.1\r\n");
 				add_field(req_buff, "Host", m_host_buff);
 				add_field(req_buff, "Content-Length", std::to_string(body.size()));
@@ -255,9 +265,7 @@ namespace net_utils
 				for(const auto& field : additional_params)
 					add_field(req_buff, field);
 
-				for (unsigned sends = 0; sends < 2; ++sends)
 				{
-					const std::size_t initial_size = req_buff.size();
 					const auto auth = m_auth.get_auth_field(method, uri);
 					if (auth)
 						add_field(req_buff, *auth);
@@ -265,11 +273,21 @@ namespace net_utils
 					req_buff += "\r\n";
 					//--
 
-					bool res = m_net_client.send(req_buff, timeout);
-					CHECK_AND_ASSERT_MES(res, false, "HTTP_CLIENT: Failed to SEND");
-					if(body.size())
+					if (do_copy_body) // small body
+					{
+						// Copy headers + body together and potentially send one fewer packet
+						req_buff.append(body.data(), body.size());
+						const bool res = m_net_client.send(req_buff, timeout);
+						CHECK_AND_ASSERT_MES(res, false, "HTTP_CLIENT: Failed to SEND");
+					}
+					else // large body
+					{
+						// Send headers and body seperately to avoid copying heavy body message
+						bool res = m_net_client.send(req_buff, timeout);
+						CHECK_AND_ASSERT_MES(res, false, "HTTP_CLIENT: Failed to SEND");
 						res = m_net_client.send(body, timeout);
-					CHECK_AND_ASSERT_MES(res, false, "HTTP_CLIENT: Failed to SEND");
+						CHECK_AND_ASSERT_MES(res, false, "HTTP_CLIENT: Failed to SEND");
+					}
 
 					m_response_info.clear();
 					m_state = reciev_machine_state_header;
@@ -282,19 +300,11 @@ namespace net_utils
 						return true;
 					}
 
-					switch (m_auth.handle_401(m_response_info))
+					if (m_auth.handle_401(m_response_info) == http_client_auth::kParseFailure)
 					{
-					case http_client_auth::kSuccess:
-						break;
-					case http_client_auth::kBadPassword:
-                                                sends = 2;
-						break;
-					default:
-					case http_client_auth::kParseFailure:
 						LOG_ERROR("Bad server response for authentication");
 						return false;
 					}
-					req_buff.resize(initial_size); // rollback for new auth generation
 				}
 				LOG_ERROR("Client has incorrect username/password for server requiring authentication");
 				return false;

contrib/epee/include/span.h

@@ -147,6 +147,16 @@ namespace epee
     return {reinterpret_cast<const std::uint8_t*>(src.data()), src.size_bytes()}; 
   }
 
+  //! \return `span<std::uint8_t>` from a STL compatible `src`.
+  template<typename T>
+  constexpr span<std::uint8_t> to_mut_byte_span(T& src)
+  {
+    using value_type = typename T::value_type;
+    static_assert(!std::is_empty<value_type>(), "empty value types will not work -> sizeof == 1");
+    static_assert(!has_padding<value_type>(), "source value type may have padding");
+    return {reinterpret_cast<std::uint8_t*>(src.data()), src.size() * sizeof(value_type)};
+  }
+
   //! \return `span<const std::uint8_t>` which represents the bytes at `&src`.
   template<typename T>
   span<const std::uint8_t> as_byte_span(const T& src) noexcept

contrib/epee/include/storages/parserse_base_utils.h

@@ -30,6 +30,7 @@
 
 #include <boost/utility/string_ref_fwd.hpp>
 #include <string>
+#include <cstdint>
 
 namespace epee 
 {

contrib/epee/include/storages/portable_storage_from_bin.h

@@ -33,6 +33,9 @@
 #include "portable_storage_base.h"
 #include "portable_storage_bin_utils.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "serialization"
+
 #ifdef EPEE_PORTABLE_STORAGE_RECURSION_LIMIT
 #define EPEE_PORTABLE_STORAGE_RECURSION_LIMIT_INTERNAL EPEE_PORTABLE_STORAGE_RECURSION_LIMIT
 #else 

contrib/epee/include/storages/portable_storage_from_json.h

@@ -31,6 +31,9 @@
 #include "parserse_base_utils.h"
 #include "file_io_utils.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "serialization"
+
 #define EPEE_JSON_RECURSION_LIMIT_INTERNAL 100
 
 namespace epee

contrib/epee/src/mlog.cpp

@@ -338,21 +338,11 @@ bool is_stdout_a_tty()
   return is_a_tty.load(std::memory_order_relaxed);
 }
 
-static bool is_nocolor()
-{
-  static const char *no_color_var = getenv("NO_COLOR");
-  static const bool no_color = no_color_var && *no_color_var; // apparently, NO_COLOR=0 means no color too (as per no-color.org)
-  return no_color;
-}
-
 void set_console_color(int color, bool bright)
 {
   if (!is_stdout_a_tty())
     return;
 
-  if (is_nocolor())
-    return;
-
   switch(color)
   {
   case console_color_default:
@@ -471,9 +461,6 @@ void reset_console_color() {
   if (!is_stdout_a_tty())
     return;
 
-  if (is_nocolor())
-    return;
-
 #ifdef WIN32
   HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
   SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE);

contrib/epee/src/net_ssl.cpp

@@ -496,6 +496,13 @@ void ssl_options_t::configure(
   const std::string& host) const
 {
   socket.next_layer().set_option(boost::asio::ip::tcp::no_delay(true));
+  {
+    // in case server is doing "virtual" domains, set hostname
+    SSL* const ssl_ctx = socket.native_handle();
+    if (type == boost::asio::ssl::stream_base::client && !host.empty() && ssl_ctx)
+      SSL_set_tlsext_host_name(ssl_ctx, host.c_str());
+  }
+
 
   /* Using system-wide CA store for client verification is funky - there is
      no expected hostname for server to verify against. If server doesn't have
@@ -513,11 +520,7 @@ void ssl_options_t::configure(
   {
     socket.set_verify_mode(boost::asio::ssl::verify_peer | boost::asio::ssl::verify_fail_if_no_peer_cert);
 
-    // in case server is doing "virtual" domains, set hostname
-    SSL* const ssl_ctx = socket.native_handle();
-    if (type == boost::asio::ssl::stream_base::client && !host.empty() && ssl_ctx)
-      SSL_set_tlsext_host_name(ssl_ctx, host.c_str());
-
+    
     socket.set_verify_callback([&](const bool preverified, boost::asio::ssl::verify_context &ctx)
     {
       // preverified means it passed system or user CA check. System CA is never loaded

contrib/epee/src/readline_buffer.cpp

@@ -238,10 +238,6 @@ static char** attempted_completion(const char* text, int start, int end)
 
 static void install_line_handler()
 {
-#if RL_READLINE_VERSION >= 0x0801
-  rl_variable_bind("enable-bracketed-paste", "off");
-#endif
-
   rl_attempted_completion_function = attempted_completion;
   rl_callback_handler_install("", handle_line);
   stifle_history(500);

contrib/gitian/DOCKRUN.md

@@ -57,7 +57,7 @@ The dockrun.sh script will do everything to build the binaries. Just specify the
 version to build as its only argument, e.g.
 
 ```bash
-VERSION=v0.18.2.0
+VERSION=v0.18.3.0
 ./dockrun.sh $VERSION
 ```
 

contrib/gitian/README.md

@@ -133,7 +133,7 @@ Common setup part:
 su - gitianuser
 
 GH_USER=YOUR_GITHUB_USER_NAME
-VERSION=v0.18.2.0
+VERSION=v0.18.3.0
 ```
 
 Where `GH_USER` is your GitHub user name and `VERSION` is the version tag you want to build. 

external/easylogging++/easylogging++.cc

@@ -149,11 +149,6 @@ static el::Color colorFromLevel(el::Level level)
 
 static void setConsoleColor(el::Color color, bool bright)
 {
-  static const char *no_color_var = getenv("NO_COLOR");
-  static const bool no_color = no_color_var && *no_color_var; // apparently, NO_COLOR=0 means no color too (as per no-color.org)
-  if (no_color)
-    return;
-
 #if ELPP_OS_WINDOWS
   HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE);
   switch (color)

src/checkpoints/checkpoints.cpp

@@ -236,6 +236,10 @@ namespace cryptonote
     ADD_CHECKPOINT2(500000,  "f4f771261b8c13cd83a9d8fa22e3cfe988564ad4b57dd90e79d5c0e77d61cf6a", "0x1185e7f2357a03");
     ADD_CHECKPOINT2(503500,  "776f36a17056c3e22bbfb51d5aeabb58000731e9ad549f0f2f8ad1e1bcedf312", "0x11a4884467f53d");
     ADD_CHECKPOINT2(509900,  "59520faa272fc68e0426c827a38b21cdc1df8f5a37c5fdcbbe00350fece5f3ae", "0x11dc780bd1b580");
+    ADD_CHECKPOINT2(514000,  "1569e712750f19e57aee3f73cc3091a0eea392740bd396ad3570bc96e0e6ffb5", "0x11fff58abe0a82"); //Hard fork to v20
+    ADD_CHECKPOINT2(516700,  "0443c47c5a4e344c3f68a491a3b2f6b78a769cdf9076249c93f187ececf9cc7c", "0x12128a532a59a6");
+    ADD_CHECKPOINT2(522000,  "8c15ae514063bf05e7662ab33b86a4131aa89df0784ce3da7876c5339bc1de3d", "0x123d9c604a7be6");
+    ADD_CHECKPOINT2(566000,  "136e37f5f130dd2dc2d8d30e46e16d74377cb834b7c5be32cd09ee6ad402556c", "0x1318d703b1bff7");
     return true;
   }
 

src/common/combinator.h

@@ -34,6 +34,7 @@
 #include <iostream>
 #include <vector>
 #include <stdexcept>
+#include <cstdint>
 
 namespace tools {
 

src/common/perf_timer.cpp

@@ -62,7 +62,7 @@ namespace tools
     while (1)
     {
       t1 = epee::misc_utils::get_ns_count();
-      if (t1 - t0 > 1*1000000000) break; // work one second
+      if (t1 - t0 > 1*100000000) break; // work 0.1 seconds
     }
 
     uint64_t r1 = get_tick_count();

src/common/util.cpp

@@ -882,13 +882,6 @@ std::string get_nix_version_display_string()
 
   bool is_local_address(const std::string &address)
   {
-    // always assume Tor/I2P addresses to be untrusted by default
-    if (is_privacy_preserving_network(address))
-    {
-      MDEBUG("Address '" << address << "' is Tor/I2P, non local");
-      return false;
-    }
-
     // extract host
     epee::net_utils::http::url_content u_c;
     if (!epee::net_utils::parse_url(address, u_c))
@@ -902,20 +895,22 @@ std::string get_nix_version_display_string()
       return false;
     }
 
-    // resolve to IP
-    boost::asio::io_service io_service;
-    boost::asio::ip::tcp::resolver resolver(io_service);
-    boost::asio::ip::tcp::resolver::query query(u_c.host, "");
-    boost::asio::ip::tcp::resolver::iterator i = resolver.resolve(query);
-    while (i != boost::asio::ip::tcp::resolver::iterator())
+    if (u_c.host == "localhost" || boost::ends_with(u_c.host, ".localhost")) { // RFC 6761 (6.3)
+      MDEBUG("Address '" << address << "' is local");
+      return true;
+    }
+
+    boost::system::error_code ec;
+    const auto parsed_ip = boost::asio::ip::address::from_string(u_c.host, ec);
+    if (ec) {
+      MDEBUG("Failed to parse '" << address << "' as IP address: " << ec.message() << ". Considering it not local");
+      return false;
+    }
+
+    if (parsed_ip.is_loopback())
     {
-      const boost::asio::ip::tcp::endpoint &ep = *i;
-      if (ep.address().is_loopback())
-      {
-        MDEBUG("Address '" << address << "' is local");
-        return true;
-      }
-      ++i;
+      MDEBUG("Address '" << address << "' is local");
+      return true;
     }
 
     MDEBUG("Address '" << address << "' is not local");
@@ -1067,7 +1062,7 @@ std::string get_nix_version_display_string()
     time_t tt = ts;
     struct tm tm;
     misc_utils::get_gmt_time(tt, tm);
-    strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%SZ", &tm);
+    strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", &tm);
     return std::string(buffer);
   }
 

src/crypto/c_threads.h

@@ -42,10 +42,11 @@
 #define CTHR_RWLOCK_TRYLOCK_READ(x)	TryAcquireSRWLockShared(&x)
 
 #define CTHR_THREAD_TYPE	HANDLE
-#define CTHR_THREAD_RTYPE	void
-#define CTHR_THREAD_RETURN	return
-#define CTHR_THREAD_CREATE(thr, func, arg)	((thr = (HANDLE)_beginthread(func, 0, arg)) != -1L)
-#define CTHR_THREAD_JOIN(thr)			WaitForSingleObject((HANDLE)thr, INFINITE)
+#define CTHR_THREAD_RTYPE	unsigned __stdcall
+#define CTHR_THREAD_RETURN	_endthreadex(0); return 0;
+#define CTHR_THREAD_CREATE(thr, func, arg)	((thr = (HANDLE)_beginthreadex(0, 0, func, arg, 0, 0)) != 0L)
+#define CTHR_THREAD_JOIN(thr)			do { WaitForSingleObject(thr, INFINITE); CloseHandle(thr); } while(0)
+#define CTHR_THREAD_CLOSE(thr)			CloseHandle((HANDLE)thr);
 
 #else
 
@@ -64,5 +65,6 @@
 #define CTHR_THREAD_RETURN	return NULL
 #define CTHR_THREAD_CREATE(thr, func, arg)	(pthread_create(&thr, NULL, func, arg) == 0)
 #define CTHR_THREAD_JOIN(thr)			pthread_join(thr, NULL)
+#define CTHR_THREAD_CLOSE(thr)
 
 #endif

src/crypto/rx-slow-hash.c

@@ -332,7 +332,7 @@ static void rx_init_dataset(size_t max_threads) {
       local_abort("Couldn't start RandomX seed thread");
     }
   }
-  rx_seedthread(&si[n1]);
+  randomx_init_dataset(main_dataset, si[n1].si_cache, si[n1].si_start, si[n1].si_count);
   for (size_t i = 0; i < n1; ++i) CTHR_THREAD_JOIN(st[i]);
   CTHR_RWLOCK_UNLOCK_READ(main_cache_lock);
 
@@ -402,6 +402,7 @@ void rx_set_main_seedhash(const char *seedhash, size_t max_dataset_init_threads)
   if (!CTHR_THREAD_CREATE(t, rx_set_main_seedhash_thread, info)) {
     local_abort("Couldn't start RandomX seed thread");
   }
+  CTHR_THREAD_CLOSE(t);
 }
 
 void rx_slow_hash(const char *seedhash, const void *data, size_t length, char *result_hash) {

src/cryptonote_basic/cryptonote_basic_impl.h

@@ -39,6 +39,15 @@ namespace cryptonote {
   /************************************************************************/
   /*                                                                      */
   /************************************************************************/
+  template<class t_array>
+  struct array_hasher: std::unary_function<t_array&, std::size_t>
+  {
+    std::size_t operator()(const t_array& val) const
+    {
+      return boost::hash_range(&val.data[0], &val.data[sizeof(val.data)]);
+    }
+  };
+
 
 #pragma pack(push, 1)
   struct public_address_outer_blob

src/cryptonote_basic/cryptonote_format_utils.cpp

@@ -1254,7 +1254,7 @@ namespace cryptonote
     char *end = NULL;
     errno = 0;
     const unsigned long long ull = strtoull(buf, &end, 10);
-    CHECK_AND_ASSERT_THROW_MES(ull != ULONG_MAX || errno == 0, "Failed to parse rounded amount: " << buf);
+    CHECK_AND_ASSERT_THROW_MES(ull != ULLONG_MAX || errno == 0, "Failed to parse rounded amount: " << buf);
     CHECK_AND_ASSERT_THROW_MES(ull != 0 || amount == 0, "Overflow in rounding");
     return ull;
   }

src/cryptonote_basic/verification_context.h

@@ -42,7 +42,12 @@ namespace cryptonote
     static_assert(unsigned(relay_method::none) == 0, "default m_relay initialization is not to relay_method::none");
 
     relay_method m_relay; // gives indication on how tx should be relayed (if at all)
-    bool m_verifivation_failed; //bad tx, should drop connection
+    bool m_verifivation_failed; //bad tx, tx should not enter mempool and connection should be dropped unless m_no_drop_offense
+    // Do not add to mempool, do not relay, but also do not punish the peer for sending or drop
+    // connections to them. Used for low fees, tx_extra too big, "relay-only rules". Not to be
+    // confused with breaking soft fork rules, because tx could be later added to the chain if mined
+    // because it does not violate consensus rules.
+    bool m_no_drop_offense;
     bool m_verifivation_impossible; //the transaction is related with an alternative blockchain
     bool m_added_to_pool; 
     bool m_low_mixin;

src/cryptonote_config.h

@@ -104,12 +104,12 @@
 
 #define BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT          10000  //by default, blocks ids count in synchronizing
 #define BLOCKS_IDS_SYNCHRONIZING_MAX_COUNT              25000  //max blocks ids count in synchronizing
-#define BLOCKS_SYNCHRONIZING_DEFAULT_COUNT_PRE_V4       50    //by default, blocks count in blocks downloading
+#define BLOCKS_SYNCHRONIZING_DEFAULT_COUNT_PRE_V4       100    //by default, blocks count in blocks downloading
 #define BLOCKS_SYNCHRONIZING_DEFAULT_COUNT              20     //by default, blocks count in blocks downloading
 #define BLOCKS_SYNCHRONIZING_MAX_COUNT                  2048   //must be a power of 2, greater than 128, equal to SEEDHASH_EPOCH_BLOCKS
 
-#define CRYPTONOTE_MEMPOOL_TX_LIVETIME                    86400 //seconds, one day
-#define CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME     86400 //seconds, one day
+#define CRYPTONOTE_MEMPOOL_TX_LIVETIME                    (86400*3) //seconds, three days
+#define CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME     604800 //seconds, one week
 
 
 #define CRYPTONOTE_DANDELIONPP_STEMS              2 // number of outgoing stem connections per epoch
@@ -141,7 +141,7 @@
 #define P2P_LOCAL_WHITE_PEERLIST_LIMIT                  1000
 #define P2P_LOCAL_GRAY_PEERLIST_LIMIT                   5000
 
-#define P2P_DEFAULT_CONNECTIONS_COUNT                   64
+#define P2P_DEFAULT_CONNECTIONS_COUNT                   12
 #define P2P_DEFAULT_HANDSHAKE_INTERVAL                  60           //secondes
 #define P2P_DEFAULT_PACKET_MAX_SIZE                     50000000     //50000000 bytes maximum packet size
 #define P2P_DEFAULT_PEERS_IN_HANDSHAKE                  250
@@ -154,8 +154,8 @@
 #define P2P_DEFAULT_WHITELIST_CONNECTIONS_PERCENT       70
 #define P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT            2
 #define P2P_DEFAULT_SYNC_SEARCH_CONNECTIONS_COUNT       2
-#define P2P_DEFAULT_LIMIT_RATE_UP                       1048576    // 1GB/s
-#define P2P_DEFAULT_LIMIT_RATE_DOWN                     1048576   // 1GB/s
+#define P2P_DEFAULT_LIMIT_RATE_UP                       2048       // kB/s
+#define P2P_DEFAULT_LIMIT_RATE_DOWN                     8192       // kB/s
 
 #define P2P_FAILED_ADDR_FORGET_SECONDS                  (60*60)     //1 hour
 #define P2P_IP_BLOCKTIME                                (60*60*24)  //24 hour

src/cryptonote_core/blockchain.cpp

@@ -2167,7 +2167,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id
       cryptonote::blobdata blob;
       if (m_tx_pool.have_tx(txid, relay_category::legacy))
       {
-        if (m_tx_pool.get_transaction_info(txid, td))
+        if (m_tx_pool.get_transaction_info(txid, td, true/*include_sensitive_data*/))
         {
           bei.block_cumulative_weight += td.weight;
         }
@@ -3823,7 +3823,7 @@ uint64_t Blockchain::get_dynamic_base_fee(uint64_t block_reward, size_t median_b
       div128_64(hi, lo, median_block_weight, &hi, &lo, NULL, NULL);
       assert(hi == 0);
       lo -= lo / 20;
-      return lo;
+      return lo == 0 ? 1 : lo;
     }
     else
     {
@@ -4736,40 +4736,9 @@ bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effecti
   }
   else
   {
-    const uint64_t block_weight = m_db->get_block_weight(db_height - 1);
+    const uint64_t nblocks = std::min<uint64_t>(m_long_term_block_weights_window, db_height);
+    const uint64_t long_term_median = get_long_term_block_weight_median(db_height - nblocks, nblocks);
 
-    uint64_t long_term_median;
-    if (db_height == 1)
-    {
-      long_term_median = CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5;
-    }
-    else
-    {
-      uint64_t nblocks = std::min<uint64_t>(m_long_term_block_weights_window, db_height);
-      if (nblocks == db_height)
-        --nblocks;
-      long_term_median = get_long_term_block_weight_median(db_height - nblocks - 1, nblocks);
-    }
-
-    m_long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median);
-
-    uint64_t short_term_constraint = m_long_term_effective_median_block_weight;
-    if (hf_version >= HF_VERSION_2021_SCALING)
-      short_term_constraint += m_long_term_effective_median_block_weight * 7 / 10;
-    else
-      short_term_constraint += m_long_term_effective_median_block_weight * 2 / 5;
-    uint64_t long_term_block_weight = std::min<uint64_t>(block_weight, short_term_constraint);
-
-    if (db_height == 1)
-    {
-      long_term_median = long_term_block_weight;
-    }
-    else
-    {
-      m_long_term_block_weights_cache_tip_hash = m_db->get_block_hash_from_height(db_height - 1);
-      m_long_term_block_weights_cache_rolling_median.insert(long_term_block_weight);
-      long_term_median = m_long_term_block_weights_cache_rolling_median.median();
-    }
     m_long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median);
 
     std::vector<uint64_t> weights;
@@ -5702,7 +5671,7 @@ void Blockchain::cancel()
 }
 
 #if defined(PER_BLOCK_CHECKPOINT)
-static const char expected_block_hashes_hash[] = "82164f48e44d04e7dc2fe7a42a08e64c40a68cb30c7cd6dcc68c88daa0406d5f";
+static const char expected_block_hashes_hash[] = "9ce2e9f40ff58954d6176e300af1d8da1804b829e1be8565805fea497df40f4b";
 void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints)
 {
   if (get_checkpoints == nullptr || !m_fast_sync)

src/cryptonote_core/cryptonote_core.cpp

@@ -1103,7 +1103,7 @@ namespace cryptonote
       else if(tvc[i].m_verifivation_impossible)
       {MERROR_VER("Transaction verification impossible: " << results[i].hash);}
 
-      if(tvc[i].m_added_to_pool)
+      if(tvc[i].m_added_to_pool && results[i].tx.extra.size() <= MAX_TX_EXTRA_SIZE)
       {
         MDEBUG("tx added: " << results[i].hash);
         valid_events = true;
@@ -1731,6 +1731,11 @@ namespace cryptonote
     return true;
   }
   //-----------------------------------------------------------------------------------------------
+  bool core::get_pool_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& txs, bool include_sensitive_txes) const
+  {
+    return m_mempool.get_transactions_info(txids, txs, include_sensitive_txes);
+  }
+  //-----------------------------------------------------------------------------------------------
   bool core::get_pool_transactions(std::vector<transaction>& txs, bool include_sensitive_data) const
   {
     m_mempool.get_transactions(txs, include_sensitive_data);
@@ -1743,6 +1748,11 @@ namespace cryptonote
     return true;
   }
   //-----------------------------------------------------------------------------------------------
+  bool core::get_pool_info(time_t start_time, bool include_sensitive_txes, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const
+  {
+    return m_mempool.get_pool_info(start_time, include_sensitive_txes, max_tx_count, added_txs, remaining_added_txids, removed_txs, incremental);
+  }
+  //-----------------------------------------------------------------------------------------------
   bool core::get_pool_transaction_stats(struct txpool_stats& stats, bool include_sensitive_data) const
   {
     m_mempool.get_transaction_stats(stats, include_sensitive_data);
@@ -1830,56 +1840,6 @@ namespace cryptonote
         main_message = "The daemon is running offline and will not attempt to sync to the Wownero network.";
       else
         main_message = "The daemon will start synchronizing with the network. This may take a long time to complete.";
-    MGINFO_MAGENTA(ENDL <<
-    "\n\n"
-    " ██   ██ ██    ██ ███    ██ ████████ ██    ██   \n"
-    " ██  ██  ██    ██ ████   ██    ██     ██  ██    \n"
-    " █████   ██    ██ ██ ██  ██    ██      ████     \n"
-    " ██  ██  ██    ██ ██  ██ ██    ██       ██      \n"
-    " ██   ██  ██████  ██   ████    ██       ██      \n"
-    << ENDL);
-    MGINFO_GREEN(ENDL <<
-    "                m                                   !5!                               \n"
-    "               !Y?~:            .:^~~~~^^::::...   ~7J5G:                             \n"
-    "              ~?JY5P5~     .:!JYJ?!~^::::^^^^^^^^^^::^^~^.                            \n"
-    "             :77?Y555P5~^7Y5P5J7!~~~!7?77!~^^^:::::...:^^^^^:.                        \n"
-    "            .77?JJJYYY5PPP5YJYJJYY55YJ?7!!~~~^^^:::.....^~!!!7!^.                     \n"
-    "            !?J?77??JYY?J55Y5Y55PPP5J??7!~!??7!!~^:.... .:~7???p?!.                   \n"
-    "            !?J?!!!!77JJ77?YY5PGP55YYYJ??Y5PP5J7!~~~~!~..:~~7J55YJJ7:                 \n"
-    "            ~?JJ7^::~!?J?!?5555555PGGP55PGB#BPYJJJY55P5Y?7?J?Y5PP555Y?^               \n"
-    "            .JJY?~:::~!7?YYYJY5PPGGGGGGGB#BGPY?JY5PPGGGGP5YY5PGGGGGPPPY7.             \n"
-    "            :7JJYY?!:.:~JYJ?JYY5PPPPPGGGGGPYJJJJ?JYPGGBBBBG5J7!7777JPPPP5J~            \n"
-    "         :^77!777!7J!?YY77??JJ55Y55555P5YJ?JJJYYYY5PGB#BPJJY555J?~:JPP5P5JJ.          \n"
-    "        ^.~J!7????775JY?777!7JJJJYYYJ??7!7?????J5YY5P55Y5B##B###G5:7PGP5PYJY.         \n"
-    "       :.~J7?J???77~YJ!?JYYY7^^^:~JPPPGBBBBBBGPY77~!?~^#&#Y?YG#&&B?7YGGP5PYJ5.        \n"
-    "       ~^?!!??7?77!~Y!^~~~~!!^^.:5GGPP5PGB##&&&&B7:!Y?:5&Y!?YB#&&&!^7YGP55PJYY        \n"
-    "       ~?!!!?7!7!~~?7^~!!~~!!!!::P5!^~~?5P5PB###B!!P5PY~YGP5PB#&&B^?77YP5Y55J5~       \n"
-    "       !Y?~!7!!~~!7!^^~!^~!7?YYY775Y?~~!JG5!?PGBY:7PPGBGJ?PBBGG5JJGB?!7Y5YY5YY7       \n"
-    "      ^7!77?7!!!!~^^~~~^~!?5PGGGGJ~PBGPPGGGGBBBG:^PGBBBBBGP555PGB###B777Y5YY5JY.      \n"
-    "      ?77^:!!~~^:^^^^~^^~7YPGBBBBBPYGGPPGGGPG5J?JGBBBBBGPYJJJJJYPB&##P77?JJJYYY?      \n"
-    "     :?!!^^7!^~^^^:^^^^^!5PBBBBBBBGGGP5YY555PPPGBB#BBBG7^::......:G&#B???J??JYYY.     \n"
-    "     ^7!!^~7!^^:::^^^:::!PGBBBBBBBBBBB####BBBBBBBBBBBB5^...  ....^5##BY?J??7?JJY!     \n"
-    "     ~!!!~^!7~^::^^:^:^^~YGBBBBBBBBBBBBBBBBBBBGGGGGGGGP7^.......:^YGBB5?J?7!7??JY.    \n"
-    "     ~!!!~^!!~^:^^:^^^^^~75GBBBBBBBBBBBBBBBBBGGGGGGGGPPY?!^:...:^!J5GGY?JJ7!!7??Y!    \n"
-    "     ~!!!~^~7~^^~~~~^^^^~~75PGGGGGGGGBGGGGBBBGGGPPPP555Y?!^^^^^^~!?5GGY??J?!!7??J7    \n"
-    "       !!!!~^^7!^~!77!~~~~~~~YPPGGGGGPGGGGGGGGGGGPPPPPP55?~:::...::!5BBGJ7?J?!!??7?!    \n"
-    "    :7!!!~^^!7^~7?p?!!~!~~!JPGGGGGGGPPPGGGGGGPPPPPP5Y?~:...:::^~7YBBBG?7p?!!!7?7??    \n"
-    "    ??~!~!~^^7~^??4?!77!!!~JPGGGGGGGGGGPGGGGP5Y?7!~~~^^~!!!!!!7YGBBBBP77?!~~!7?7?7    \n"
-    "   .5J!!~!!~:^^!?7J?7?J77~~75PGGGGGGGGGPPPGGPP5J777?YYY55YYYYYPGBBGGGJ7?7!~~!777:     \n"
-    "   ~PY?!!~!!^^!?7?J?????7!775GGBGGGGPPPPPPPGGGGGPPPPPPPGGGGGGBBBBGGGY777!!!!77!.      \n"
-    "   JPYY7!~!77777???7!!!7!~!!JPPGGGGGGGGGPPPPPPPPPPPPPPGGGGGGBGBBGGPY!!?!!7777.        \n"
-    "   PP55YJJ3?!!7??7~^^^^~!~~!75PGGGGGBBBGGPPPPPPPPPPPPPGGGGGGGGGBG57!77!!2?!^          \n"
-    "   GGGPPP5J!!7?7~^:::^~~!!!77YPPPGGGGGGGGPPPPPPPPPPPPPPGGGGGGGGGGYJY7~!?BP.           \n"
-    "   GBGGGGP55YYJ7~^^!~^~77!77775PPPPPP55PPPPPPPPPPPPPPGGGGGGGGGGGGGBPJY5PG~            \n"
-    "                    Fve, V'q yvxr gb fcrnx gb gur gur jbjareb znantre                   "
-    << ENDL);
-    MGINFO_MAGENTA(ENDL <<
-    "                                  ██   ██  █████  ██████  ███████ ███    ██   \n"
-    "                                  ██  ██  ██   ██ ██   ██ ██      ████   ██   \n"
-    "                                  █████   ███████ ██████  █████   ██ ██  ██   \n"
-    "                                  ██  ██  ██   ██ ██   ██ ██      ██  ██ ██   \n"
-    "                                  ██   ██ ██   ██ ██   ██ ███████ ██   ████   \n"
-    << ENDL);
       MGINFO_YELLOW(ENDL << "**********************************************************************" << ENDL
         << main_message << ENDL
         << ENDL

src/cryptonote_core/cryptonote_core.h

@@ -510,6 +510,23 @@ namespace cryptonote
      bool get_pool_transaction_hashes(std::vector<crypto::hash>& txs, bool include_sensitive_txes = false) const;
 
      /**
+      * @copydoc tx_memory_pool::get_pool_transactions_info
+      * @param include_sensitive_txes include private transactions
+      *
+      * @note see tx_memory_pool::get_pool_transactions_info
+      */
+     bool get_pool_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& txs, bool include_sensitive_txes = false) const;
+
+     /**
+      * @copydoc tx_memory_pool::get_pool_info
+      * @param include_sensitive_txes include private transactions
+      * @param max_tx_count max allowed added_txs in response
+      *
+      * @note see tx_memory_pool::get_pool_info
+      */
+     bool get_pool_info(time_t start_time, bool include_sensitive_txes, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const;
+
+    /**
       * @copydoc tx_memory_pool::get_transactions
       * @param include_sensitive_txes include private transactions
       *

src/cryptonote_core/cryptonote_tx_utils.cpp

@@ -450,6 +450,8 @@ namespace cryptonote
     if (!sort_tx_extra(tx.extra, tx.extra))
       return false;
 
+    CHECK_AND_ASSERT_MES(tx.extra.size() <= MAX_TX_EXTRA_SIZE, false, "TX extra size (" << tx.extra.size() << ") is greater than max allowed (" << MAX_TX_EXTRA_SIZE << ")");
+
     //check money
     if(summary_outs_money > summary_inputs_money )
     {

src/cryptonote_core/tx_pool.cpp

@@ -133,6 +133,12 @@ namespace cryptonote
     // class code expects unsigned values throughout
     if (m_next_check < time_t(0))
       throw std::runtime_error{"Unexpected time_t (system clock) value"};
+
+    m_added_txs_start_time = (time_t)0;
+    m_removed_txs_start_time = (time_t)0;
+    // We don't set these to "now" already here as we don't know how long it takes from construction
+    // of the pool until it "goes to work". It's safer to set when the first actual txs enter the
+    // corresponding lists.
   }
   //---------------------------------------------------------------------------------
   bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, relay_method tx_relay, bool relayed, uint8_t version)
@@ -207,6 +213,7 @@ namespace cryptonote
     {
       tvc.m_verifivation_failed = true;
       tvc.m_fee_too_low = true;
+      tvc.m_no_drop_offense = true;
       return false;
     }
 
@@ -225,6 +232,7 @@ namespace cryptonote
       LOG_PRINT_L1("transaction tx-extra is too big: " << tx_extra_size << " bytes, the limit is: " << MAX_TX_EXTRA_SIZE);
       tvc.m_verifivation_failed = true;
       tvc.m_tx_extra_too_big = true;
+      tvc.m_no_drop_offense = true;
       return false;
     }
 
@@ -290,7 +298,7 @@ namespace cryptonote
             return false;
 
           m_blockchain.add_txpool_tx(id, blob, meta);
-          m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)(tx_weight ? tx_weight : 1), receive_time), id);
+          add_tx_to_transient_lists(id, fee / (double)(tx_weight ? tx_weight : 1), receive_time);
           lock.commit();
         }
         catch (const std::exception &e)
@@ -361,7 +369,7 @@ namespace cryptonote
 
           m_blockchain.remove_txpool_tx(id);
           m_blockchain.add_txpool_tx(id, blob, meta);
-          m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)(tx_weight ? tx_weight : 1), receive_time), id);
+          add_tx_to_transient_lists(id, meta.fee / (double)(tx_weight ? tx_weight : 1), receive_time);
         }
         lock.commit();
       }
@@ -382,7 +390,7 @@ namespace cryptonote
 
     ++m_cookie;
 
-    MINFO("Transaction added to pool: txid " << id << " weight: " << tx_weight << " fee/byte: " << (fee / (double)(tx_weight ? tx_weight : 1)));
+    MINFO("Transaction added to pool: txid " << id << " weight: " << tx_weight << " fee/byte: " << (fee / (double)(tx_weight ? tx_weight : 1)) << ", count: " << m_added_txs_by_id.size());
 
     prune(m_txpool_max_weight);
 
@@ -473,7 +481,8 @@ namespace cryptonote
         reduce_txpool_weight(meta.weight);
         remove_transaction_keyimages(tx, txid);
         MINFO("Pruned tx " << txid << " from txpool: weight: " << meta.weight << ", fee/byte: " << it->first.first);
-        m_txs_by_fee_and_receive_time.erase(it--);
+        remove_tx_from_transient_lists(it, txid, !meta.matches(relay_category::broadcasted));
+        it--;
         changed = true;
       }
       catch (const std::exception &e)
@@ -555,8 +564,7 @@ namespace cryptonote
     CRITICAL_REGION_LOCAL(m_transactions_lock);
     CRITICAL_REGION_LOCAL1(m_blockchain);
 
-    auto sorted_it = find_tx_in_sorted_container(id);
-
+    bool sensitive = false;
     try
     {
       LockedTXN lock(m_blockchain.get_db());
@@ -587,6 +595,7 @@ namespace cryptonote
       do_not_relay = meta.do_not_relay;
       double_spend_seen = meta.double_spend_seen;
       pruned = meta.pruned;
+      sensitive = !meta.matches(relay_category::broadcasted);
 
       // remove first, in case this throws, so key images aren't removed
       m_blockchain.remove_txpool_tx(id);
@@ -600,13 +609,12 @@ namespace cryptonote
       return false;
     }
 
-    if (sorted_it != m_txs_by_fee_and_receive_time.end())
-      m_txs_by_fee_and_receive_time.erase(sorted_it);
+    remove_tx_from_transient_lists(find_tx_in_sorted_container(id), id, sensitive);
     ++m_cookie;
     return true;
   }
   //---------------------------------------------------------------------------------
-  bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td) const
+  bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data, bool include_blob) const
   {
     PERF_TIMER(get_transaction_info);
     CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -618,7 +626,12 @@ namespace cryptonote
       txpool_tx_meta_t meta;
       if (!m_blockchain.get_txpool_tx_meta(txid, meta))
       {
-        MERROR("Failed to find tx in txpool");
+        LOG_PRINT_L2("Failed to find tx in txpool: " << txid);
+        return false;
+      }
+      if (!include_sensitive_data && !meta.matches(relay_category::broadcasted))
+      {
+        // We don't want sensitive data && the tx is sensitive, so no need to return it
         return false;
       }
       cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid, relay_category::all);
@@ -644,11 +657,13 @@ namespace cryptonote
       td.kept_by_block = meta.kept_by_block;
       td.last_failed_height = meta.last_failed_height;
       td.last_failed_id = meta.last_failed_id;
-      td.receive_time = meta.receive_time;
-      td.last_relayed_time = meta.dandelionpp_stem ? 0 : meta.last_relayed_time;
+      td.receive_time = include_sensitive_data ? meta.receive_time : 0;
+      td.last_relayed_time = (include_sensitive_data && !meta.dandelionpp_stem) ? meta.last_relayed_time : 0;
       td.relayed = meta.relayed;
       td.do_not_relay = meta.do_not_relay;
       td.double_spend_seen = meta.double_spend_seen;
+      if (include_blob)
+        td.tx_blob = std::move(txblob);
     }
     catch (const std::exception &e)
     {
@@ -658,6 +673,25 @@ namespace cryptonote
 
     return true;
   }
+  //------------------------------------------------------------------
+  bool tx_memory_pool::get_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_details>>& txs, bool include_sensitive) const
+  {
+    CRITICAL_REGION_LOCAL(m_transactions_lock);
+    CRITICAL_REGION_LOCAL1(m_blockchain);
+
+    txs.clear();
+
+    for (const auto &it: txids)
+    {
+      tx_details details;
+      bool success = get_transaction_info(it, details, include_sensitive, true/*include_blob*/);
+      if (success)
+      {
+        txs.push_back(std::make_pair(it, std::move(details)));
+      }
+    }
+    return true;
+  }
   //---------------------------------------------------------------------------------
   bool tx_memory_pool::get_complement(const std::vector<crypto::hash> &hashes, std::vector<cryptonote::blobdata> &txes) const
   {
@@ -719,15 +753,7 @@ namespace cryptonote
          (tx_age > CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME && meta.kept_by_block) )
       {
         LOG_PRINT_L1("Tx " << txid << " removed from tx pool due to outdated, age: " << tx_age );
-        auto sorted_it = find_tx_in_sorted_container(txid);
-        if (sorted_it == m_txs_by_fee_and_receive_time.end())
-        {
-          LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!");
-        }
-        else
-        {
-          m_txs_by_fee_and_receive_time.erase(sorted_it);
-        }
+        remove_tx_from_transient_lists(find_tx_in_sorted_container(txid), txid, !meta.matches(relay_category::broadcasted));
         m_timed_out_transactions.insert(txid);
         remove.push_back(std::make_pair(txid, meta.weight));
       }
@@ -881,9 +907,12 @@ namespace cryptonote
             meta.last_relayed_time = std::chrono::system_clock::to_time_t(now);
 
           m_blockchain.update_txpool_tx(hash, meta);
-
           // wait until db update succeeds to ensure tx is visible in the pool
           was_just_broadcasted = !already_broadcasted && meta.matches(relay_category::broadcasted);
+
+          if (was_just_broadcasted)
+            // Make sure the tx gets re-added with an updated time
+            add_tx_to_transient_lists(hash, meta.fee / (double)meta.weight, std::chrono::system_clock::to_time_t(now));
         }
       }
       catch (const std::exception &e)
@@ -936,6 +965,81 @@ namespace cryptonote
     }, false, category);
   }
   //------------------------------------------------------------------
+  bool tx_memory_pool::get_pool_info(time_t start_time, bool include_sensitive, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const
+  {
+    CRITICAL_REGION_LOCAL(m_transactions_lock);
+    CRITICAL_REGION_LOCAL1(m_blockchain);
+
+    incremental = true;
+    if (start_time == (time_t)0)
+    {
+      // Giving no start time means give back whole pool
+      incremental = false;
+    }
+    else if ((m_added_txs_start_time != (time_t)0) && (m_removed_txs_start_time != (time_t)0))
+    {
+      if ((start_time <= m_added_txs_start_time) || (start_time <= m_removed_txs_start_time))
+      {
+        // If either of the two lists do not go back far enough it's not possible to
+        // deliver incremental pool info
+        incremental = false;
+      }
+      // The check uses "<=": We cannot be sure to have ALL txs exactly at start_time, only AFTER that time
+    }
+    else
+    {
+      // Some incremental info still missing completely
+      incremental = false;
+    }
+
+    added_txs.clear();
+    remaining_added_txids.clear();
+    removed_txs.clear();
+
+    std::vector<crypto::hash> txids;
+    if (!incremental)
+    {
+      LOG_PRINT_L2("Giving back the whole pool");
+      // Give back the whole pool in 'added_txs'; because calling 'get_transaction_info' right inside the
+      // anonymous method somehow results in an LMDB error with transactions we have to build a list of
+      // ids first and get the full info afterwards
+      get_transaction_hashes(txids, include_sensitive);
+      if (txids.size() > max_tx_count)
+      {
+        remaining_added_txids = std::vector<crypto::hash>(txids.begin() + max_tx_count, txids.end());
+        txids.erase(txids.begin() + max_tx_count, txids.end());
+      }
+      get_transactions_info(txids, added_txs, include_sensitive);
+      return true;
+    }
+
+    // Give back incrementally, based on time of entry into the map
+    for (const auto &pit : m_added_txs_by_id)
+    {
+      if (pit.second >= start_time)
+        txids.push_back(pit.first);
+    }
+    get_transactions_info(txids, added_txs, include_sensitive);
+    if (added_txs.size() > max_tx_count)
+    {
+      remaining_added_txids.reserve(added_txs.size() - max_tx_count);
+      for (size_t i = max_tx_count; i < added_txs.size(); ++i)
+        remaining_added_txids.push_back(added_txs[i].first);
+      added_txs.erase(added_txs.begin() + max_tx_count, added_txs.end());
+    }
+
+    std::multimap<time_t, removed_tx_info>::const_iterator rit = m_removed_txs_by_time.lower_bound(start_time);
+    while (rit != m_removed_txs_by_time.end())
+    {
+      if (include_sensitive || !rit->second.sensitive)
+      {
+        removed_txs.push_back(rit->second.txid);
+      }
+      ++rit;
+    }
+    return true;
+  }
+  //------------------------------------------------------------------
   void tx_memory_pool::get_transaction_backlog(std::vector<tx_backlog_entry>& backlog, bool include_sensitive) const
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -1640,6 +1744,12 @@ namespace cryptonote
     CRITICAL_REGION_LOCAL(m_transactions_lock);
     CRITICAL_REGION_LOCAL1(m_blockchain);
 
+    // Simply throw away incremental info, too difficult to update
+    m_added_txs_by_id.clear();
+    m_added_txs_start_time = (time_t)0;
+    m_removed_txs_by_time.clear();
+    m_removed_txs_start_time = (time_t)0;
+
     MINFO("Validating txpool contents for v" << (unsigned)version);
 
     LockedTXN lock(m_blockchain.get_db());
@@ -1697,6 +1807,106 @@ namespace cryptonote
     return n_removed;
   }
   //---------------------------------------------------------------------------------
+  void tx_memory_pool::add_tx_to_transient_lists(const crypto::hash& txid, double fee, time_t receive_time)
+  {
+
+    time_t now = time(NULL);
+    const std::unordered_map<crypto::hash, time_t>::iterator it = m_added_txs_by_id.find(txid);
+    if (it == m_added_txs_by_id.end())
+    {
+       m_added_txs_by_id.insert(std::make_pair(txid, now));
+    }
+    else
+    {
+      // This tx was already added to the map earlier, probably because then it was in the "stem"
+      // phase of Dandelion++ and now is in the "fluff" phase i.e. got broadcasted: We have to set
+      // a new time for clients that are not allowed to see sensitive txs to make sure they will
+      // see it now if they query incrementally
+      it->second = now;
+
+      auto sorted_it = find_tx_in_sorted_container(txid);
+      if (sorted_it == m_txs_by_fee_and_receive_time.end())
+      {
+        MERROR("Re-adding tx " << txid << " to tx pool, but it was not found in the sorted txs container");
+      }
+      else
+      {
+        m_txs_by_fee_and_receive_time.erase(sorted_it);
+      }
+    }
+    m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(fee, receive_time), txid);
+
+    // Don't check for "resurrected" txs in case of reorgs i.e. don't check in 'm_removed_txs_by_time'
+    // whether we have that txid there and if yes remove it; this results in possible duplicates
+    // where we return certain txids as deleted AND in the pool at the same time which requires
+    // clients to process deleted ones BEFORE processing pool txs
+    if (m_added_txs_start_time == (time_t)0)
+    {
+      m_added_txs_start_time = now;
+    }
+  }
+  //---------------------------------------------------------------------------------
+  void tx_memory_pool::remove_tx_from_transient_lists(const cryptonote::sorted_tx_container::iterator& sorted_it, const crypto::hash& txid, bool sensitive)
+  {
+    if (sorted_it == m_txs_by_fee_and_receive_time.end())
+    {
+      LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!");
+    }
+    else
+    {
+      m_txs_by_fee_and_receive_time.erase(sorted_it);
+    }
+
+    const std::unordered_map<crypto::hash, time_t>::iterator it = m_added_txs_by_id.find(txid);
+    if (it != m_added_txs_by_id.end())
+    {
+       m_added_txs_by_id.erase(it);
+    }
+    else
+    {
+      MDEBUG("Removing tx " << txid << " from tx pool, but it was not found in the map of added txs");
+    }
+    track_removed_tx(txid, sensitive);
+  }
+  //---------------------------------------------------------------------------------
+  void tx_memory_pool::track_removed_tx(const crypto::hash& txid, bool sensitive)
+  {
+    time_t now = time(NULL);
+    m_removed_txs_by_time.insert(std::make_pair(now, removed_tx_info{txid, sensitive}));
+    MDEBUG("Transaction removed from pool: txid " << txid << ", total entries in removed list now " << m_removed_txs_by_time.size());
+    if (m_removed_txs_start_time == (time_t)0)
+    {
+      m_removed_txs_start_time = now;
+    }
+
+    // Simple system to make sure the list of removed ids does not swell to an unmanageable size: Set
+    // an absolute size limit plus delete entries that are x minutes old (which is ok because clients
+    // will sync with sensible time intervalls and should not ask for incremental info e.g. 1 hour back)
+    const int MAX_REMOVED = 20000;
+    if (m_removed_txs_by_time.size() > MAX_REMOVED)
+    {
+      auto erase_it = m_removed_txs_by_time.begin();
+      std::advance(erase_it, MAX_REMOVED / 4 + 1);
+      m_removed_txs_by_time.erase(m_removed_txs_by_time.begin(), erase_it);
+      m_removed_txs_start_time = m_removed_txs_by_time.begin()->first;
+      MDEBUG("Erased old transactions from big removed list, leaving " << m_removed_txs_by_time.size());
+    }
+    else
+    {
+      time_t earliest = now - (30 * 60);  // 30 minutes
+      std::map<time_t, removed_tx_info>::iterator from, to;
+      from = m_removed_txs_by_time.begin();
+      to = m_removed_txs_by_time.lower_bound(earliest);
+      int distance = std::distance(from, to);
+      if (distance > 0)
+      {
+        m_removed_txs_by_time.erase(from, to);
+        m_removed_txs_start_time = earliest;
+        MDEBUG("Erased " << distance << " old transactions from removed list, leaving " << m_removed_txs_by_time.size());
+      }
+    }
+  }
+  //---------------------------------------------------------------------------------
   bool tx_memory_pool::init(size_t max_txpool_weight, bool mine_stem_txes)
   {
     CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -1704,6 +1914,10 @@ namespace cryptonote
 
     m_txpool_max_weight = max_txpool_weight ? max_txpool_weight : DEFAULT_TXPOOL_MAX_WEIGHT;
     m_txs_by_fee_and_receive_time.clear();
+    m_added_txs_by_id.clear();
+    m_added_txs_start_time = (time_t)0;
+    m_removed_txs_by_time.clear();
+    m_removed_txs_start_time = (time_t)0;
     m_spent_key_images.clear();
     m_txpool_weight = 0;
     std::vector<crypto::hash> remove;
@@ -1728,7 +1942,7 @@ namespace cryptonote
           MFATAL("Failed to insert key images from txpool tx");
           return false;
         }
-        m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(meta.fee / (double)meta.weight, meta.receive_time), txid);
+        add_tx_to_transient_lists(txid, meta.fee / (double)meta.weight, meta.receive_time);
         m_txpool_weight += meta.weight;
         return true;
       }, true, relay_category::all);

src/cryptonote_core/tx_pool.h

@@ -428,6 +428,7 @@ namespace cryptonote
     struct tx_details
     {
       transaction tx;  //!< the transaction
+      cryptonote::blobdata tx_blob; //!< the transaction's binary blob
       size_t blob_size;  //!< the transaction's size
       size_t weight;  //!< the transaction's weight
       uint64_t fee;  //!< the transaction's fee amount
@@ -466,13 +467,25 @@ namespace cryptonote
     /**
      * @brief get infornation about a single transaction
      */
-    bool get_transaction_info(const crypto::hash &txid, tx_details &td) const;
+    bool get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data, bool include_blob = false) const;
+
+    /**
+     * @brief get information about multiple transactions
+     */
+    bool get_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_details>>& txs, bool include_sensitive_data = false) const;
 
     /**
      * @brief get transactions not in the passed set
      */
     bool get_complement(const std::vector<crypto::hash> &hashes, std::vector<cryptonote::blobdata> &txes) const;
 
+    /**
+     * @brief get info necessary for update of pool-related info in a wallet, preferably incremental
+     *
+     * @return true on success, false on error
+     */
+    bool get_pool_info(time_t start_time, bool include_sensitive, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const;
+
   private:
 
     /**
@@ -577,6 +590,10 @@ namespace cryptonote
      */
     void prune(size_t bytes = 0);
 
+    void add_tx_to_transient_lists(const crypto::hash& txid, double fee, time_t receive_time);
+    void remove_tx_from_transient_lists(const cryptonote::sorted_tx_container::iterator& sorted_it, const crypto::hash& txid, bool sensitive);
+    void track_removed_tx(const crypto::hash& txid, bool sensitive);
+
     //TODO: confirm the below comments and investigate whether or not this
     //      is the desired behavior
     //! map key images to transactions which spent them
@@ -609,6 +626,26 @@ private:
 
     std::atomic<uint64_t> m_cookie; //!< incremented at each change
 
+    // Info when transactions entered the pool, accessible by txid
+    std::unordered_map<crypto::hash, time_t> m_added_txs_by_id;
+
+    // Info at what time the pool started to track the adding of transactions
+    time_t m_added_txs_start_time;
+
+    struct removed_tx_info
+    {
+      crypto::hash txid;
+      bool sensitive;
+    };
+
+    // Info about transactions that were removed from the pool, ordered by the time
+    // of deletion
+    std::multimap<time_t, removed_tx_info> m_removed_txs_by_time;
+
+    // Info how far back in time the list of removed tx ids currently reaches
+    // (it gets shorted periodically to prevent overflow)
+    time_t m_removed_txs_start_time;
+
     /**
      * @brief get an iterator to a transaction in the sorted container
      *

src/cryptonote_protocol/cryptonote_protocol_handler.inl

@@ -979,8 +979,18 @@ namespace cryptonote
   int t_cryptonote_protocol_handler<t_core>::handle_notify_new_transactions(int command, NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& context)
   {
     MLOG_P2P_MESSAGE("Received NOTIFY_NEW_TRANSACTIONS (" << arg.txs.size() << " txes)");
+    std::unordered_set<blobdata> seen;
     for (const auto &blob: arg.txs)
+    {
       MLOGIF_P2P_MESSAGE(cryptonote::transaction tx; crypto::hash hash; bool ret = cryptonote::parse_and_validate_tx_from_blob(blob, tx, hash);, ret, "Including transaction " << hash);
+      if (seen.find(blob) != seen.end())
+      {
+        LOG_PRINT_CCONTEXT_L1("Duplicate transaction in notification, dropping connection");
+        drop_connection(context, false, false);
+        return 1;
+      }
+      seen.insert(blob);
+    }
 
     if(context.m_state != cryptonote_connection_context::state_normal)
       return 1;
@@ -1020,7 +1030,7 @@ namespace cryptonote
     for (auto& tx : arg.txs)
     {
       tx_verification_context tvc{};
-      if (!m_core.handle_incoming_tx({tx, crypto::null_hash}, tvc, tx_relay, true))
+      if (!m_core.handle_incoming_tx({tx, crypto::null_hash}, tvc, tx_relay, true) && !tvc.m_no_drop_offense)
       {
         LOG_PRINT_CCONTEXT_L1("Tx verification failed, dropping connection");
         drop_connection(context, false, false);
@@ -1683,22 +1693,8 @@ namespace cryptonote
                 + std::to_string(previous_stripe) + " -> " + std::to_string(current_stripe);
             if (ELPP->vRegistry()->allowed(el::Level::Debug, "sync-info"))
               timing_message += std::string(": ") + m_block_queue.get_overview(current_blockchain_height);
-            uint64_t num = (rand() % 4) + 1;
-            switch (num)
-            {
-                case 1:
-                    MGINFO_MAGENTA("*•.¸♡ ♡¸.•* synced *•.¸♡ ♡¸.•* " << current_blockchain_height << "/" << target_blockchain_height << progress_message << timing_message << " ˚ ༘♡ ⋆｡˚");
-                    break;
-                case 2:
-                    MGINFO_YELLOW("ˏˋ°•*⁀➷ pǝɔuʎs ˏˋ°•*⁀➷ " << current_blockchain_height << "/" << target_blockchain_height << progress_message << timing_message << " ︶︶༉‧₊ ☄. *.⋆");
-                    break;
-                case 3:
-                    MGINFO_BLUE("s ★ y ★ n ★ c ★ e ★ d " << current_blockchain_height << "/" << target_blockchain_height << progress_message << timing_message << " *ೃ༄ *·˚˚ ༘♡ ⋆｡˚");
-                    break;
-                case 4:
-                    MGINFO_GREEN("s ♥ y ♥ n ♥ c ♥ e ♥ d " << current_blockchain_height << "/" << target_blockchain_height << progress_message << timing_message << " ⎯୧◦•◦❥•◦'*•.¸♡ ");
-                    break;
-            }
+            MGINFO_YELLOW("Synced " << current_blockchain_height << "/" << target_blockchain_height
+                << progress_message << timing_message);
             if (previous_stripe != current_stripe)
               notify_new_stripe(context, current_stripe);
           }

src/daemon/main.cpp

@@ -219,6 +219,19 @@ int main(int argc, char const * argv[])
       {
         po::store(po::parse_config_file<char>(config_path.string<std::string>().c_str(), core_settings), vm);
       }
+      catch (const po::unknown_option &e)
+      {
+        std::string unrecognized_option = e.get_option_name();
+        if (all_options.find_nothrow(unrecognized_option, false))
+        {
+          std::cerr << "Option '" << unrecognized_option << "' is not allowed in the config file, please use it as a command line flag." << std::endl;
+        }
+        else
+        {
+          std::cerr << "Unrecognized option '" << unrecognized_option << "' in config file." << std::endl;
+        }
+        return 1;
+      }
       catch (const std::exception &e)
       {
         // log system isn't initialized yet

src/daemonizer/windows_service.cpp

@@ -196,7 +196,7 @@ bool install_service(
       , 0
       //, GENERIC_EXECUTE | GENERIC_READ
       , SERVICE_WIN32_OWN_PROCESS
-      , SERVICE_AUTO_START
+      , SERVICE_DEMAND_START
       , SERVICE_ERROR_NORMAL
       , full_command.c_str()
       , nullptr

src/daemonizer/windows_service_runner.h

@@ -146,6 +146,9 @@ namespace windows {
       m_handler.run();
 
       on_state_change_request_(SERVICE_CONTROL_STOP);
+
+      // Ensure that the service is uninstalled
+      uninstall_service(m_name);
     }
 
     static void WINAPI on_state_change_request(DWORD control_code)

src/hardforks/hardforks.cpp

@@ -47,8 +47,7 @@ const hardfork_t mainnet_hard_forks[] = {
   { 19, 331458, 0, 1624793373 },
   { 20, 514000, 0, 1677222289 },
 };
-// since Wownero starts from v7, added + 6 so that the total number of hard forks = version number
-const size_t num_mainnet_hard_forks = sizeof(mainnet_hard_forks) / sizeof(mainnet_hard_forks[0]) + 6;
+const size_t num_mainnet_hard_forks = sizeof(mainnet_hard_forks) / sizeof(mainnet_hard_forks[0]);
 const uint64_t mainnet_hard_fork_version_1_till = 0;
 
 const hardfork_t testnet_hard_forks[] = {

src/p2p/net_node.inl

@@ -714,18 +714,20 @@ namespace nodetool
     }
     else
     {
-      full_addrs.insert("158.69.60.225:34567");   //  explore.wownero.com
-      full_addrs.insert("159.65.91.59:34567");    //  jw
+      //full_addrs.insert("158.69.60.225:34567");   //  explore.wownero.com
+      //full_addrs.insert("159.65.91.59:34567");    //  jw
       full_addrs.insert("51.161.131.176:34567");  //  node.suchwow.xyz
-      full_addrs.insert("167.114.196.241:34567"); //  wowbux.org
-      full_addrs.insert("142.93.144.79:34567"); // idontwanttogototoronto.wow.fail
-      full_addrs.insert("51.75.76.161:34567"); // eu-west-1.wow.xmr.pm
-      full_addrs.insert("145.239.93.75:34567"); // eu-west-2.wow.xmr.pm
+      //full_addrs.insert("167.114.196.241:34567"); //  wowbux.org
+      //full_addrs.insert("142.93.144.79:34567"); // idontwanttogototoronto.wow.fail
+      //full_addrs.insert("51.75.76.161:34567"); // eu-west-1.wow.xmr.pm
+      //full_addrs.insert("145.239.93.75:34567"); // eu-west-2.wow.xmr.pm
       full_addrs.insert("88.198.199.23:34567");
-      full_addrs.insert("167.114.119.46:34567"); // wownero.stackwallet.com
+      //full_addrs.insert("167.114.119.46:34567"); // wownero.stackwallet.com
       full_addrs.insert("143.198.195.132:34567"); // singapore.muchwow.lol
       full_addrs.insert("134.122.53.193:34567"); // amsterdam.muchwow.lol
       full_addrs.insert("204.48.28.218:34567"); // nyc.muchwow.lol
+      full_addrs.insert("192.99.8.110:34567"); // node.monerodevs.org
+      full_addrs.insert("37.187.74.171:34567"); // node2.monerodevs.org
     }
     return full_addrs;
   }
@@ -859,6 +861,9 @@ namespace nodetool
           "77uase4p6y6jsjdf6z2kdgpxgh7nkvywagvhurzphbm7vrkyj2d2gdid.onion:34566",
           "v2admi6gbeprxnk6i2oscizhgy4v5ixu6iezkhj5udiwbfjjs2w7dnid.onion:34566",
           "ttc6kxud3fikyaypn5voknyyvqje7j3wnoevsb7rfjerolynnisurkqd.onion:34566",
+          "5enbij6tz3n2hw5ixzezfayd5gvckyg4xlktz74gj4l6u7olq7ovr3id.onion:34566",
+          "ukpgpdd5gqvholcctejvaaig5hb266td6zaszt55eivuf7docoox5lid.onion:34566",
+          "iorkdy6t4gdtwzn3iblnhy76nu7lhyz2qcvpj4wrspfxyvdrul7u22qd.onion:34566",
         };
       }
       return {};

src/rpc/core_rpc_server.cpp

@@ -599,88 +599,165 @@ namespace cryptonote
 
     CHECK_PAYMENT(req, res, 1);
 
-    // quick check for noop
-    if (!req.block_ids.empty())
+    res.daemon_time = (uint64_t)time(NULL);
+    // Always set daemon time, and set it early rather than late, as delivering some incremental pool
+    // info twice because of slightly overlapping time intervals is no problem, whereas producing gaps
+    // and never delivering something is
+
+    bool get_blocks = false;
+    bool get_pool = false;
+    switch (req.requested_info)
     {
-      uint64_t last_block_height;
-      crypto::hash last_block_hash;
-      m_core.get_blockchain_top(last_block_height, last_block_hash);
-      if (last_block_hash == req.block_ids.front())
+      case COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_ONLY:
+        // Compatibility value 0: Clients that do not set 'requested_info' want blocks, and only blocks
+        get_blocks = true;
+        break;
+      case COMMAND_RPC_GET_BLOCKS_FAST::BLOCKS_AND_POOL:
+        get_blocks = true;
+        get_pool = true;
+        break;
+      case COMMAND_RPC_GET_BLOCKS_FAST::POOL_ONLY:
+        get_pool = true;
+        break;
+      default:
+        res.status = "Failed, wrong requested info";
+        return true;
+    }
+
+    res.pool_info_extent = COMMAND_RPC_GET_BLOCKS_FAST::NONE;
+
+    if (get_pool)
+    {
+      const bool restricted = m_restricted && ctx;
+      const bool request_has_rpc_origin = ctx != NULL;
+      const bool allow_sensitive = !request_has_rpc_origin || !restricted;
+      const size_t max_tx_count = restricted ? RESTRICTED_TRANSACTIONS_COUNT : std::numeric_limits<size_t>::max();
+
+      bool incremental;
+      std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>> added_pool_txs;
+      bool success = m_core.get_pool_info((time_t)req.pool_info_since, allow_sensitive, max_tx_count, added_pool_txs, res.remaining_added_pool_txids, res.removed_pool_txids, incremental);
+      if (success)
       {
-        res.start_height = 0;
-        res.current_height = m_core.get_current_blockchain_height();
-        res.status = CORE_RPC_STATUS_OK;
+        res.added_pool_txs.clear();
+        if (m_rpc_payment)
+        {
+          CHECK_PAYMENT_SAME_TS(req, res, added_pool_txs.size() * COST_PER_TX + (res.remaining_added_pool_txids.size() + res.removed_pool_txids.size()) * COST_PER_POOL_HASH);
+        }
+        for (const auto &added_pool_tx: added_pool_txs)
+        {
+          COMMAND_RPC_GET_BLOCKS_FAST::pool_tx_info info;
+          info.tx_hash = added_pool_tx.first;
+          std::stringstream oss;
+          binary_archive<true> ar(oss);
+          bool r = req.prune
+            ? const_cast<cryptonote::transaction&>(added_pool_tx.second.tx).serialize_base(ar)
+            : ::serialization::serialize(ar, const_cast<cryptonote::transaction&>(added_pool_tx.second.tx));
+          if (!r)
+          {
+            res.status = "Failed to serialize transaction";
+            return true;
+          }
+          info.tx_blob = oss.str();
+          info.double_spend_seen = added_pool_tx.second.double_spend_seen;
+          res.added_pool_txs.push_back(std::move(info));
+        }
+      }
+      if (success)
+      {
+        res.pool_info_extent = incremental ? COMMAND_RPC_GET_BLOCKS_FAST::INCREMENTAL : COMMAND_RPC_GET_BLOCKS_FAST::FULL;
+      }
+      else
+      {
+        res.status = "Failed to get pool info";
         return true;
       }
     }
 
-    size_t max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT;
-    if (m_rpc_payment)
+    if (get_blocks)
     {
-      max_blocks = res.credits / COST_PER_BLOCK;
-      if (max_blocks > COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT)
-        max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT;
-      if (max_blocks == 0)
+      // quick check for noop
+      if (!req.block_ids.empty())
       {
-        res.status = CORE_RPC_STATUS_PAYMENT_REQUIRED;
+        uint64_t last_block_height;
+        crypto::hash last_block_hash;
+        m_core.get_blockchain_top(last_block_height, last_block_hash);
+        if (last_block_hash == req.block_ids.front())
+        {
+          res.start_height = 0;
+          res.current_height = last_block_height + 1;
+          res.status = CORE_RPC_STATUS_OK;
+          return true;
+        }
+      }
+
+      size_t max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT;
+      if (m_rpc_payment)
+      {
+        max_blocks = res.credits / COST_PER_BLOCK;
+        if (max_blocks > COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT)
+          max_blocks = COMMAND_RPC_GET_BLOCKS_FAST_MAX_BLOCK_COUNT;
+        if (max_blocks == 0)
+        {
+          res.status = CORE_RPC_STATUS_PAYMENT_REQUIRED;
+          return true;
+        }
+      }
+
+      std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > > bs;
+      if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, max_blocks, COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT))
+      {
+        res.status = "Failed";
+        add_host_fail(ctx);
         return true;
       }
-    }
 
-    std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > > bs;
-    if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, max_blocks, COMMAND_RPC_GET_BLOCKS_FAST_MAX_TX_COUNT))
-    {
-      res.status = "Failed";
-      add_host_fail(ctx);
-      return true;
-    }
+      CHECK_PAYMENT_SAME_TS(req, res, bs.size() * COST_PER_BLOCK);
 
-    CHECK_PAYMENT_SAME_TS(req, res, bs.size() * COST_PER_BLOCK);
-
-    size_t size = 0, ntxes = 0;
-    res.blocks.reserve(bs.size());
-    res.output_indices.reserve(bs.size());
-    for(auto& bd: bs)
-    {
-      res.blocks.resize(res.blocks.size()+1);
-      res.blocks.back().pruned = req.prune;
-      res.blocks.back().block = bd.first.first;
-      size += bd.first.first.size();
-      res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices());
-      ntxes += bd.second.size();
-      res.output_indices.back().indices.reserve(1 + bd.second.size());
-      if (req.no_miner_tx)
-        res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices());
-      res.blocks.back().txs.reserve(bd.second.size());
-      for (std::vector<std::pair<crypto::hash, cryptonote::blobdata>>::iterator i = bd.second.begin(); i != bd.second.end(); ++i)
+      size_t size = 0, ntxes = 0;
+      res.blocks.reserve(bs.size());
+      res.output_indices.reserve(bs.size());
+      for(auto& bd: bs)
       {
-        res.blocks.back().txs.push_back({std::move(i->second), crypto::null_hash});
-        i->second.clear();
-        i->second.shrink_to_fit();
-        size += res.blocks.back().txs.back().blob.size();
-      }
+        res.blocks.resize(res.blocks.size()+1);
+        res.blocks.back().pruned = req.prune;
+        res.blocks.back().block = bd.first.first;
+        size += bd.first.first.size();
+        res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices());
+        ntxes += bd.second.size();
+        res.output_indices.back().indices.reserve(1 + bd.second.size());
+        if (req.no_miner_tx)
+          res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices());
+        res.blocks.back().txs.reserve(bd.second.size());
+        for (std::vector<std::pair<crypto::hash, cryptonote::blobdata>>::iterator i = bd.second.begin(); i != bd.second.end(); ++i)
+        {
+          res.blocks.back().txs.push_back({std::move(i->second), crypto::null_hash});
+          i->second.clear();
+          i->second.shrink_to_fit();
+          size += res.blocks.back().txs.back().blob.size();
+        }
 
-      const size_t n_txes_to_lookup = bd.second.size() + (req.no_miner_tx ? 0 : 1);
-      if (n_txes_to_lookup > 0)
-      {
-        std::vector<std::vector<uint64_t>> indices;
-        bool r = m_core.get_tx_outputs_gindexs(req.no_miner_tx ? bd.second.front().first : bd.first.second, n_txes_to_lookup, indices);
-        if (!r)
+        const size_t n_txes_to_lookup = bd.second.size() + (req.no_miner_tx ? 0 : 1);
+        if (n_txes_to_lookup > 0)
         {
-          res.status = "Failed";
-          return true;
+          std::vector<std::vector<uint64_t>> indices;
+          bool r = m_core.get_tx_outputs_gindexs(req.no_miner_tx ? bd.second.front().first : bd.first.second, n_txes_to_lookup, indices);
+          if (!r)
+          {
+            res.status = "Failed";
+            return true;
+          }
+          if (indices.size() != n_txes_to_lookup || res.output_indices.back().indices.size() != (req.no_miner_tx ? 1 : 0))
+          {
+            res.status = "Failed";
+            return true;
+          }
+          for (size_t i = 0; i < indices.size(); ++i)
+            res.output_indices.back().indices.push_back({std::move(indices[i])});
         }
-        if (indices.size() != n_txes_to_lookup || res.output_indices.back().indices.size() != (req.no_miner_tx ? 1 : 0))
-        {
-          res.status = "Failed";
-          return true;
-        }
-        for (size_t i = 0; i < indices.size(); ++i)
-          res.output_indices.back().indices.push_back({std::move(indices[i])});
       }
+      MDEBUG("on_get_blocks: " << bs.size() << " blocks, " << ntxes << " txes, size " << size);
     }
 
-    MDEBUG("on_get_blocks: " << bs.size() << " blocks, " << ntxes << " txes, size " << size);
     res.status = CORE_RPC_STATUS_OK;
     return true;
   }
@@ -920,17 +997,16 @@ namespace cryptonote
     // try the pool for any missing txes
     size_t found_in_pool = 0;
     std::unordered_set<crypto::hash> pool_tx_hashes;
-    std::unordered_map<crypto::hash, tx_info> per_tx_pool_tx_info;
+    std::unordered_map<crypto::hash, tx_memory_pool::tx_details> per_tx_pool_tx_details;
     if (!missed_txs.empty())
     {
-      std::vector<tx_info> pool_tx_info;
-      std::vector<spent_key_image_info> pool_key_image_info;
-      bool r = m_core.get_pool_transactions_and_spent_keys_info(pool_tx_info, pool_key_image_info, !request_has_rpc_origin || !restricted);
+      std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>> pool_txs;
+      bool r = m_core.get_pool_transactions_info(missed_txs, pool_txs, !request_has_rpc_origin || !restricted);
       if(r)
       {
         // sort to match original request
         std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>> sorted_txs;
-        std::vector<tx_info>::const_iterator i;
+        std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>>::const_iterator i;
         unsigned txs_processed = 0;
         for (const crypto::hash &h: vh)
         {
@@ -950,36 +1026,23 @@ namespace cryptonote
             sorted_txs.push_back(std::move(txs[txs_processed]));
             ++txs_processed;
           }
-          else if ((i = std::find_if(pool_tx_info.begin(), pool_tx_info.end(), [h](const tx_info &txi) { return epee::string_tools::pod_to_hex(h) == txi.id_hash; })) != pool_tx_info.end())
+          else if ((i = std::find_if(pool_txs.begin(), pool_txs.end(), [h](const std::pair<crypto::hash, tx_memory_pool::tx_details> &pt) { return h == pt.first; })) != pool_txs.end())
           {
-            cryptonote::transaction tx;
-            if (!cryptonote::parse_and_validate_tx_from_blob(i->tx_blob, tx))
-            {
-              res.status = "Failed to parse and validate tx from blob";
-              return true;
-            }
+            const tx_memory_pool::tx_details &td = i->second;
             std::stringstream ss;
             binary_archive<true> ba(ss);
-            bool r = const_cast<cryptonote::transaction&>(tx).serialize_base(ba);
+            bool r = const_cast<cryptonote::transaction&>(td.tx).serialize_base(ba);
             if (!r)
             {
               res.status = "Failed to serialize transaction base";
               return true;
             }
             const cryptonote::blobdata pruned = ss.str();
-            const crypto::hash prunable_hash = tx.version == 1 ? crypto::null_hash : get_transaction_prunable_hash(tx);
-            sorted_txs.push_back(std::make_tuple(h, pruned, prunable_hash, std::string(i->tx_blob, pruned.size())));
+            const crypto::hash prunable_hash = td.tx.version == 1 ? crypto::null_hash : get_transaction_prunable_hash(td.tx);
+            sorted_txs.push_back(std::make_tuple(h, pruned, prunable_hash, std::string(td.tx_blob, pruned.size())));
             missed_txs.erase(std::find(missed_txs.begin(), missed_txs.end(), h));
             pool_tx_hashes.insert(h);
-            const std::string hash_string = epee::string_tools::pod_to_hex(h);
-            for (const auto &ti: pool_tx_info)
-            {
-              if (ti.id_hash == hash_string)
-              {
-                per_tx_pool_tx_info.insert(std::make_pair(h, ti));
-                break;
-              }
-            }
+            per_tx_pool_tx_details.insert(std::make_pair(h, td));
             ++found_in_pool;
           }
         }
@@ -1010,17 +1073,7 @@ namespace cryptonote
       CHECK_AND_ASSERT_MES(tx_hash == std::get<0>(tx), false, "mismatched tx hash");
       e.tx_hash = *txhi++;
       e.prunable_hash = epee::string_tools::pod_to_hex(std::get<2>(tx));
-
-      // coinbase txes do not have signatures to prune, so they appear to be pruned if looking just at prunable data being empty
-      bool pruned = std::get<3>(tx).empty();
-      if (pruned)
-      {
-        cryptonote::transaction t;
-        if (cryptonote::parse_and_validate_tx_base_from_blob(std::get<1>(tx), t) && is_coinbase(t))
-          pruned = false;
-      }
-
-      if (req.split || req.prune || pruned)
+      if (req.split || req.prune || std::get<3>(tx).empty())
       {
         // use splitted form with pruned and prunable (filled only when prune=false and the daemon has it), leaving as_hex as empty
         e.pruned_as_hex = string_tools::buff_to_hex_nodelimer(std::get<1>(tx));
@@ -1085,8 +1138,8 @@ namespace cryptonote
       {
         e.block_height = e.block_timestamp = std::numeric_limits<uint64_t>::max();
         e.confirmations = 0;
-        auto it = per_tx_pool_tx_info.find(tx_hash);
-        if (it != per_tx_pool_tx_info.end())
+        auto it = per_tx_pool_tx_details.find(tx_hash);
+        if (it != per_tx_pool_tx_details.end())
         {
           e.double_spend_seen = it->second.double_spend_seen;
           e.relayed = it->second.relayed;
@@ -2127,7 +2180,8 @@ namespace cryptonote
     // Fixing of high orphan issue for most pools
     // Thanks Boolberry!
     block b;
-    if(!parse_and_validate_block_from_blob(blockblob, b))
+    crypto::hash blk_id;
+    if(!parse_and_validate_block_from_blob(blockblob, b, blk_id))
     {
       error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB;
       error_resp.message = "Wrong block blob";
@@ -2150,6 +2204,7 @@ namespace cryptonote
       error_resp.message = "Block not accepted";
       return false;
     }
+    res.block_id = epee::string_tools::pod_to_hex(blk_id);
     res.status = CORE_RPC_STATUS_OK;
     return true;
   }

src/rpc/core_rpc_server_commands_defs.h

@@ -88,7 +88,7 @@ namespace cryptonote
 // advance which version they will stop working with
 // Don't go over 32767 for any of these
 #define CORE_RPC_VERSION_MAJOR 3
-#define CORE_RPC_VERSION_MINOR 12
+#define CORE_RPC_VERSION_MINOR 13
 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor))
 #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR)
 
@@ -162,18 +162,29 @@ namespace cryptonote
   struct COMMAND_RPC_GET_BLOCKS_FAST
   {
 
+    enum REQUESTED_INFO
+    {
+      BLOCKS_ONLY = 0,
+      BLOCKS_AND_POOL = 1,
+      POOL_ONLY = 2
+    };
+
     struct request_t: public rpc_access_request_base
     {
+      uint8_t     requested_info;
       std::list<crypto::hash> block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */
       uint64_t    start_height;
       bool        prune;
       bool        no_miner_tx;
+      uint64_t    pool_info_since;
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE_PARENT(rpc_access_request_base)
+        KV_SERIALIZE_OPT(requested_info, (uint8_t)0)
         KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids)
         KV_SERIALIZE(start_height)
         KV_SERIALIZE(prune)
         KV_SERIALIZE_OPT(no_miner_tx, false)
+        KV_SERIALIZE_OPT(pool_info_since, (uint64_t)0)
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<request_t> request;
@@ -196,12 +207,37 @@ namespace cryptonote
       END_KV_SERIALIZE_MAP()
     };
 
+    struct pool_tx_info
+    {
+      crypto::hash tx_hash;
+      blobdata tx_blob;
+      bool double_spend_seen;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE_VAL_POD_AS_BLOB(tx_hash)
+        KV_SERIALIZE(tx_blob)
+        KV_SERIALIZE(double_spend_seen)
+      END_KV_SERIALIZE_MAP()
+    };
+
+    enum POOL_INFO_EXTENT
+    {
+      NONE = 0,
+      INCREMENTAL = 1,
+      FULL = 2
+    };
+
     struct response_t: public rpc_access_response_base
     {
       std::vector<block_complete_entry> blocks;
       uint64_t    start_height;
       uint64_t    current_height;
       std::vector<block_output_indices> output_indices;
+      uint64_t    daemon_time;
+      uint8_t     pool_info_extent;
+      std::vector<pool_tx_info> added_pool_txs;
+      std::vector<crypto::hash> remaining_added_pool_txids;
+      std::vector<crypto::hash> removed_pool_txids;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE_PARENT(rpc_access_response_base)
@@ -209,6 +245,17 @@ namespace cryptonote
         KV_SERIALIZE(start_height)
         KV_SERIALIZE(current_height)
         KV_SERIALIZE(output_indices)
+        KV_SERIALIZE_OPT(daemon_time, (uint64_t) 0)
+        KV_SERIALIZE_OPT(pool_info_extent, (uint8_t) 0)
+        if (pool_info_extent != POOL_INFO_EXTENT::NONE)
+        {
+          KV_SERIALIZE(added_pool_txs)
+          KV_SERIALIZE_CONTAINER_POD_AS_BLOB(remaining_added_pool_txids)
+        }
+        if (pool_info_extent == POOL_INFO_EXTENT::INCREMENTAL)
+        {
+          KV_SERIALIZE_CONTAINER_POD_AS_BLOB(removed_pool_txids)
+        }
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<response_t> response;
@@ -1072,8 +1119,11 @@ namespace cryptonote
     
     struct response_t: public rpc_response_base
     {
+      std::string block_id;
+
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE_PARENT(rpc_response_base)
+        KV_SERIALIZE(block_id)
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<response_t> response;

src/serialization/json_object.cpp

@@ -1146,6 +1146,7 @@ void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::rctSig&
     INSERT_INTO_JSON_OBJECT(dest, bulletproofs, sig.p.bulletproofs);
     INSERT_INTO_JSON_OBJECT(dest, bulletproofs_plus, sig.p.bulletproofs_plus);
     INSERT_INTO_JSON_OBJECT(dest, mlsags, sig.p.MGs);
+    INSERT_INTO_JSON_OBJECT(dest, clsags, sig.p.CLSAGs);
     INSERT_INTO_JSON_OBJECT(dest, pseudo_outs, sig.get_pseudo_outs());
 
     dest.EndObject();
@@ -1181,6 +1182,7 @@ void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig)
     GET_FROM_JSON_OBJECT(prunable->value, sig.p.bulletproofs, bulletproofs);
     GET_FROM_JSON_OBJECT(prunable->value, sig.p.bulletproofs_plus, bulletproofs_plus);
     GET_FROM_JSON_OBJECT(prunable->value, sig.p.MGs, mlsags);
+    GET_FROM_JSON_OBJECT(prunable->value, sig.p.CLSAGs, clsags);
     GET_FROM_JSON_OBJECT(prunable->value, pseudo_outs, pseudo_outs);
 
     sig.get_pseudo_outs() = std::move(pseudo_outs);
@@ -1191,6 +1193,7 @@ void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig)
     sig.p.bulletproofs.clear();
     sig.p.bulletproofs_plus.clear();
     sig.p.MGs.clear();
+    sig.p.CLSAGs.clear();
     sig.get_pseudo_outs().clear();
   }
 }
@@ -1399,6 +1402,29 @@ void fromJsonValue(const rapidjson::Value& val, rct::mgSig& sig)
   GET_FROM_JSON_OBJECT(val, sig.cc, cc);
 }
 
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::clsag& sig)
+{
+  dest.StartObject();
+
+  INSERT_INTO_JSON_OBJECT(dest, s, sig.s);
+  INSERT_INTO_JSON_OBJECT(dest, c1, sig.c1);
+  INSERT_INTO_JSON_OBJECT(dest, D, sig.D);
+
+  dest.EndObject();
+}
+
+void fromJsonValue(const rapidjson::Value& val, rct::clsag& sig)
+{
+  if (!val.IsObject())
+  {
+    throw WRONG_TYPE("key64 (rct::key[64])");
+  }
+
+  GET_FROM_JSON_OBJECT(val, sig.s, s);
+  GET_FROM_JSON_OBJECT(val, sig.c1, c1);
+  GET_FROM_JSON_OBJECT(val, sig.D, D);
+}
+
 void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::DaemonInfo& info)
 {
   dest.StartObject();

src/serialization/json_object.h

@@ -304,6 +304,9 @@ void fromJsonValue(const rapidjson::Value& val, rct::boroSig& sig);
 void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::mgSig& sig);
 void fromJsonValue(const rapidjson::Value& val, rct::mgSig& sig);
 
+void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::clsag& sig);
+void fromJsonValue(const rapidjson::Value& val, rct::clsag& sig);
+
 void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::DaemonInfo& info);
 void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& info);
 

src/simplewallet/simplewallet.cpp

@@ -3228,7 +3228,6 @@ bool simple_wallet::scan_tx(const std::vector<std::string> &args)
     }
     txids.insert(txid);
   }
-  std::vector<crypto::hash> txids_v(txids.begin(), txids.end());
 
   if (!m_wallet->is_trusted_daemon()) {
     message_writer(console_color_red, true) << tr("WARNING: this operation may reveal the txids to the remote node and affect your privacy");
@@ -3241,7 +3240,9 @@ bool simple_wallet::scan_tx(const std::vector<std::string> &args)
   LOCK_IDLE_SCOPE();
   m_in_manual_refresh.store(true);
   try {
-    m_wallet->scan_tx(txids_v);
+    m_wallet->scan_tx(txids);
+  } catch (const tools::error::wont_reprocess_recent_txs_via_untrusted_daemon &e) {
+    fail_msg_writer() << e.what() << ". Either connect to a trusted daemon by passing --trusted-daemon when starting the wallet, or use rescan_bc to rescan the chain.";
   } catch (const std::exception &e) {
     fail_msg_writer() << e.what();
   }
@@ -5913,7 +5914,10 @@ bool simple_wallet::refresh_main(uint64_t start_height, enum ResetType reset, bo
   {
     m_in_manual_refresh.store(true, std::memory_order_relaxed);
     epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);});
-    m_wallet->refresh(m_wallet->is_trusted_daemon(), start_height, fetched_blocks, received_money);
+    // For manual refresh don't allow incremental checking of the pool: Because we did not process the txs
+    // for us in the pool during automatic refresh we could miss some of them if we checked the pool
+    // incrementally here
+    m_wallet->refresh(m_wallet->is_trusted_daemon(), start_height, fetched_blocks, received_money, true, false);
 
     if (reset == ResetSoftKeepKI)
     {

src/version.cpp.in

@@ -1,5 +1,5 @@
 #define DEF_MONERO_VERSION_TAG "@VERSIONTAG@"
-#define DEF_MONERO_VERSION "0.11.0.1"
+#define DEF_MONERO_VERSION "0.11.1.0"
 #define DEF_MONERO_RELEASE_NAME "Kunty Karen"
 #define DEF_MONERO_VERSION_FULL DEF_MONERO_VERSION "-" DEF_MONERO_VERSION_TAG
 #define DEF_MONERO_VERSION_IS_RELEASE @VERSION_IS_RELEASE@

src/wallet/api/coins.cpp

@@ -84,6 +84,7 @@ namespace Monero {
             ci->m_unlocked = m_wallet->m_wallet->is_transfer_unlocked(td);
             ci->m_pubKey = string_tools::pod_to_hex(td.get_public_key());
             ci->m_coinbase = td.m_tx.vin.size() == 1 && td.m_tx.vin[0].type() == typeid(cryptonote::txin_gen);
+            ci->m_description = m_wallet->m_wallet->get_tx_note(td.m_txid);
 
             m_rows.push_back(ci);
         }

src/wallet/api/coins_info.cpp

@@ -113,6 +113,10 @@ namespace Monero {
     bool CoinsInfoImpl::coinbase() const {
         return m_coinbase;
     }
+
+    string CoinsInfoImpl::description() const {
+        return m_description;
+    }
 } // namespace
 
 namespace Bitmonero = Monero;

src/wallet/api/coins_info.h

@@ -35,6 +35,7 @@ namespace Monero {
         virtual bool unlocked() const override;
         virtual std::string pubKey() const override;
         virtual bool coinbase() const override;
+        virtual std::string description() const override;
 
     private:
         uint64_t    m_blockHeight;
@@ -57,6 +58,7 @@ namespace Monero {
         bool        m_unlocked;
         std::string m_pubKey;
         bool        m_coinbase;
+        std::string m_description;
 
         friend class CoinsImpl;
 

src/wallet/api/pending_transaction.cpp

@@ -79,6 +79,22 @@ std::vector<std::string> PendingTransactionImpl::txid() const
     return txid;
 }
 
+std::vector<std::string> PendingTransactionImpl::hex() const
+{
+    std::vector<std::string> hexs;
+    for (const auto &pt: m_pending_tx)
+        hexs.push_back(epee::string_tools::buff_to_hex_nodelimer(cryptonote::tx_to_blob(pt.tx)));
+    return hexs;
+}
+
+std::vector<std::string> PendingTransactionImpl::txKey() const
+{
+    std::vector<std::string> keys;
+    for (const auto& pt: m_pending_tx)
+        keys.push_back(epee::string_tools::pod_to_hex(pt.tx_key));
+    return keys;
+}
+
 bool PendingTransactionImpl::commit(const std::string &filename, bool overwrite)
 {
 

src/wallet/api/pending_transaction.h

@@ -66,6 +66,8 @@ public:
     std::string multisigSignData() override;
     void signMultisigTx() override;
     std::vector<std::string> signersKeys() const override;
+    std::vector<std::string> hex() const override;
+    std::vector<std::string> txKey() const override;
 
 private:
     friend class WalletImpl;

src/wallet/api/wallet.cpp

@@ -434,6 +434,9 @@ WalletImpl::WalletImpl(NetworkType nettype, uint64_t kdf_rounds)
 
     m_refreshIntervalMillis = DEFAULT_REFRESH_INTERVAL_MILLIS;
 
+    m_refreshThread = boost::thread([this] () {
+        this->refreshThreadFunc();
+    });
 }
 
 WalletImpl::~WalletImpl()
@@ -1384,11 +1387,15 @@ bool WalletImpl::scanTransactions(const std::vector<std::string> &txids)
         }
         txids_u.insert(txid);
     }
-    std::vector<crypto::hash> txids_v(txids_u.begin(), txids_u.end());
 
     try
     {
-        m_wallet->scan_tx(txids_v);
+        m_wallet->scan_tx(txids_u);
+    }
+    catch (const tools::error::wont_reprocess_recent_txs_via_untrusted_daemon &e)
+    {
+        setStatusError(e.what());
+        return false;
     }
     catch (const std::exception &e)
     {
@@ -1673,13 +1680,13 @@ PendingTransaction* WalletImpl::restoreMultisigTransaction(const string& signDat
 //    - unconfirmed_transfer_details;
 //    - confirmed_transfer_details)
 
-PendingTransaction *WalletImpl::createTransactionMultDest(const std::vector<string> &dst_addr, const string &payment_id, optional<std::vector<uint64_t>> amount, uint32_t mixin_count, PendingTransaction::Priority priority, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices)
+PendingTransaction *WalletImpl::createTransactionMultDest(const std::vector<string> &dst_addr, const string &payment_id, optional<std::vector<uint64_t>> amount, uint32_t mixin_count, PendingTransaction::Priority priority, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, const std::set<std::string> &preferred_inputs)
 
 {
     clearStatus();
     // Pause refresh thread while creating transaction
     pauseRefresh();
-      
+
     cryptonote::address_parse_info info;
 
     uint32_t adjusted_priority = m_wallet->adjust_priority(static_cast<uint32_t>(priority));
@@ -1739,6 +1746,19 @@ PendingTransaction *WalletImpl::createTransactionMultDest(const std::vector<stri
                 }
             }
         }
+        std::vector<crypto::key_image> preferred_input_list;
+        if (!preferred_inputs.empty()) {
+          for (const auto &public_key : preferred_inputs) {
+            crypto::key_image keyImage;
+            bool r = epee::string_tools::hex_to_pod(public_key, keyImage);
+            if (!r) {
+              error = true;
+              setStatusError(tr("failed to parse key image"));
+              break;
+            }
+            preferred_input_list.push_back(keyImage);
+          }
+        }
         if (error) {
             break;
         }
@@ -1753,11 +1773,11 @@ PendingTransaction *WalletImpl::createTransactionMultDest(const std::vector<stri
             if (amount) {
                 transaction->m_pending_tx = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */,
                                                                             adjusted_priority,
-                                                                            extra, subaddr_account, subaddr_indices);
+                                                                            extra, subaddr_account, subaddr_indices, preferred_input_list);
             } else {
                 transaction->m_pending_tx = m_wallet->create_transactions_all(0, info.address, info.is_subaddress, 1, fake_outs_count, 0 /* unlock_time */,
                                                                               adjusted_priority,
-                                                                              extra, subaddr_account, subaddr_indices);
+                                                                              extra, subaddr_account, subaddr_indices, preferred_input_list);
             }
             pendingTxPostProcess(transaction);
 
@@ -1838,10 +1858,10 @@ PendingTransaction *WalletImpl::createTransactionMultDest(const std::vector<stri
 }
 
 PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const string &payment_id, optional<uint64_t> amount, uint32_t mixin_count,
-                                                  PendingTransaction::Priority priority, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices)
+                                                  PendingTransaction::Priority priority, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, const std::set<std::string> &preferred_inputs)
 
 {
-    return createTransactionMultDest(std::vector<string> {dst_addr}, payment_id, amount ? (std::vector<uint64_t> {*amount}) : (optional<std::vector<uint64_t>>()), mixin_count, priority, subaddr_account, subaddr_indices);
+    return createTransactionMultDest(std::vector<string> {dst_addr}, payment_id, amount ? (std::vector<uint64_t> {*amount}) : (optional<std::vector<uint64_t>>()), mixin_count, priority, subaddr_account, subaddr_indices, preferred_inputs);
 }
 
 PendingTransaction *WalletImpl::createTransactionSingle(const string &key_image, const string &dst_addr,
@@ -2581,7 +2601,7 @@ void WalletImpl::doRefresh()
         LOG_PRINT_L3(__FUNCTION__ << ": doRefresh, rescan = "<<rescan);
         if(rescan)
             m_wallet->rescan_blockchain(false);
-        m_wallet->refresh(trustedDaemon());
+        m_wallet->refresh(true);
         if (!m_synchronized) {
             m_synchronized = true;
         }

src/wallet/api/wallet.h

@@ -164,13 +164,14 @@ public:
                                         optional<std::vector<uint64_t>> amount, uint32_t mixin_count,
                                         PendingTransaction::Priority priority = PendingTransaction::Priority_Low,
                                         uint32_t subaddr_account = 0,
-                                        std::set<uint32_t> subaddr_indices = {}) override;
+                                        std::set<uint32_t> subaddr_indices = {},
+                                        const std::set<std::string> &preferred_inputs = {}) override;
     PendingTransaction * createTransaction(const std::string &dst_addr, const std::string &payment_id,
                                         optional<uint64_t> amount, uint32_t mixin_count,
                                         PendingTransaction::Priority priority = PendingTransaction::Priority_Low,
                                         uint32_t subaddr_account = 0,
-                                        std::set<uint32_t> subaddr_indices = {}) override;
-
+                                        std::set<uint32_t> subaddr_indices = {},
+                                        const std::set<std::string> &preferred_inputs = {}) override;
     PendingTransaction * createTransactionSingle(const std::string &key_image, const std::string &dst_addr,
                                                  size_t outputs = 1, PendingTransaction::Priority priority = PendingTransaction::Priority_Low) override;
 

src/wallet/api/wallet2_api.h

@@ -38,6 +38,7 @@
 #include <ctime>
 #include <iostream>
 #include <stdexcept>
+#include <cstdint>
 
 //  Public interface for libwallet library
 namespace Monero {
@@ -173,6 +174,8 @@ struct PendingTransaction
      * @return vector of base58-encoded signers' public keys
      */
     virtual std::vector<std::string> signersKeys() const = 0;
+    virtual std::vector<std::string> hex() const = 0;
+    virtual std::vector<std::string> txKey() const = 0;
 };
 
 /**
@@ -339,6 +342,7 @@ struct CoinsInfo
     virtual bool unlocked() const = 0;
     virtual std::string pubKey() const = 0;
     virtual bool coinbase() const = 0;
+    virtual std::string description() const = 0;
 };
 
 struct Coins
@@ -931,6 +935,7 @@ struct Wallet
      * \param subaddr_account           subaddress account from which the input funds are taken
      * \param subaddr_indices           set of subaddress indices to use for transfer or sweeping. if set empty, all are chosen when sweeping, and one or more are automatically chosen when transferring. after execution, returns the set of actually used indices
      * \param priority
+     * \param preferred_inputs  optional set of key_image strings from preferred inputs
      * \return                          PendingTransaction object. caller is responsible to check PendingTransaction::status()
      *                                  after object returned
      */
@@ -939,7 +944,7 @@ struct Wallet
                                                    optional<std::vector<uint64_t>> amount, uint32_t mixin_count,
                                                    PendingTransaction::Priority = PendingTransaction::Priority_Low,
                                                    uint32_t subaddr_account = 0,
-                                                   std::set<uint32_t> subaddr_indices = {}) = 0;
+                                                   std::set<uint32_t> subaddr_indices = {}, const std::set<std::string> &preferred_inputs = {}) = 0;
 
     /*!
      * \brief createTransaction creates transaction. if dst_addr is an integrated address, payment_id is ignored
@@ -950,6 +955,7 @@ struct Wallet
      * \param subaddr_account   subaddress account from which the input funds are taken
      * \param subaddr_indices   set of subaddress indices to use for transfer or sweeping. if set empty, all are chosen when sweeping, and one or more are automatically chosen when transferring. after execution, returns the set of actually used indices
      * \param priority
+     * \param preferred_inputs  optional set of key_image strings from preferred inputs
      * \return                  PendingTransaction object. caller is responsible to check PendingTransaction::status()
      *                          after object returned
      */
@@ -958,7 +964,8 @@ struct Wallet
                                                    optional<uint64_t> amount, uint32_t mixin_count,
                                                    PendingTransaction::Priority = PendingTransaction::Priority_Low,
                                                    uint32_t subaddr_account = 0,
-                                                   std::set<uint32_t> subaddr_indices = {}) = 0;
+                                                   std::set<uint32_t> subaddr_indices = {},
+                                                   const std::set<std::string> &preferred_inputs = {}) = 0;
 
     /*!
      * \brief createTransactionSingle creates transaction with single input

src/wallet/node_rpc_proxy.cpp

@@ -392,4 +392,56 @@ boost::optional<std::string> NodeRPCProxy::get_rpc_payment_info(bool mining, boo
   return boost::none;
 }
 
+boost::optional<std::string> NodeRPCProxy::get_transactions(const std::vector<crypto::hash> &txids, const std::function<void(const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request&, const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response&, bool)> &f)
+{
+  const size_t SLICE_SIZE = 100; // RESTRICTED_TRANSACTIONS_COUNT as defined in rpc/core_rpc_server.cpp
+  for (size_t offset = 0; offset < txids.size(); offset += SLICE_SIZE)
+  {
+    cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request req_t = AUTO_VAL_INIT(req_t);
+    cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response resp_t = AUTO_VAL_INIT(resp_t);
+
+    const size_t n_txids = std::min<size_t>(SLICE_SIZE, txids.size() - offset);
+    for (size_t n = offset; n < (offset + n_txids); ++n)
+      req_t.txs_hashes.push_back(epee::string_tools::pod_to_hex(txids[n]));
+    MDEBUG("asking for " << req_t.txs_hashes.size() << " transactions");
+    req_t.decode_as_json = false;
+    req_t.prune = true;
+
+    bool r = false;
+    {
+      const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
+      uint64_t pre_call_credits = m_rpc_payment_state.credits;
+      req_t.client = cryptonote::make_rpc_payment_signature(m_client_id_secret_key);
+      r = net_utils::invoke_http_json("/gettransactions", req_t, resp_t, m_http_client, rpc_timeout);
+      if (r && resp_t.status == CORE_RPC_STATUS_OK)
+        check_rpc_cost(m_rpc_payment_state, "/gettransactions", resp_t.credits, pre_call_credits, resp_t.txs.size() * COST_PER_TX);
+    }
+
+    f(req_t, resp_t, r);
+  }
+  return boost::optional<std::string>();
+}
+
+boost::optional<std::string> NodeRPCProxy::get_block_header_by_height(uint64_t height, cryptonote::block_header_response &block_header)
+{
+  if (m_offline)
+    return boost::optional<std::string>("offline");
+
+  cryptonote::COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request req_t = AUTO_VAL_INIT(req_t);
+  cryptonote::COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response resp_t = AUTO_VAL_INIT(resp_t);
+  req_t.height = height;
+
+  {
+    const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex};
+    uint64_t pre_call_credits = m_rpc_payment_state.credits;
+    req_t.client = cryptonote::make_rpc_payment_signature(m_client_id_secret_key);
+    bool r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheaderbyheight", req_t, resp_t, m_http_client, rpc_timeout);
+    RETURN_ON_RPC_RESPONSE_ERROR(r, epee::json_rpc::error{}, resp_t, "getblockheaderbyheight");
+    check_rpc_cost(m_rpc_payment_state, "getblockheaderbyheight", resp_t.credits, pre_call_credits, COST_PER_BLOCK_HEADER);
+  }
+
+  block_header = std::move(resp_t.block_header);
+  return boost::optional<std::string>();
+}
+
 }

src/wallet/node_rpc_proxy.h

@@ -59,6 +59,8 @@ public:
   boost::optional<std::string> get_dynamic_base_fee_estimate_2021_scaling(uint64_t grace_blocks, std::vector<uint64_t> &fees);
   boost::optional<std::string> get_fee_quantization_mask(uint64_t &fee_quantization_mask);
   boost::optional<std::string> get_rpc_payment_info(bool mining, bool &payment_required, uint64_t &credits, uint64_t &diff, uint64_t &credits_per_hash_found, cryptonote::blobdata &blob, uint64_t &height, uint64_t &seed_height, crypto::hash &seed_hash, crypto::hash &next_seed_hash, uint32_t &cookie);
+  boost::optional<std::string> get_transactions(const std::vector<crypto::hash> &txids, const std::function<void(const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request&, const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response&, bool)> &f);
+  boost::optional<std::string> get_block_header_by_height(uint64_t height, cryptonote::block_header_response &block_header);
 
 private:
   template<typename T> void handle_payment_changes(const T &res, std::true_type) {

src/wallet/wallet2.h

@@ -101,6 +101,7 @@ namespace tools
     uint64_t pick();
     gamma_picker(const std::vector<uint64_t> &rct_offsets);
     gamma_picker(const std::vector<uint64_t> &rct_offsets, double shape, double scale);
+    uint64_t get_num_rct_outs() const { return num_rct_outputs; }
 
   private:
     struct gamma_engine
@@ -475,7 +476,7 @@ private:
       time_t m_sent_time;
       std::vector<cryptonote::tx_destination_entry> m_dests;
       crypto::hash m_payment_id;
-      enum { pending, pending_not_in_pool, failed } m_state;
+      enum { pending, pending_in_pool, failed } m_state;
       uint64_t m_timestamp;
       uint32_t m_subaddr_account;   // subaddress account of your wallet to be used in this transfer
       std::set<uint32_t> m_subaddr_indices;  // set of address indices used as inputs in this transfer
@@ -816,6 +817,30 @@ private:
       bool empty() const { return tx_extra_fields.empty() && primary.empty() && additional.empty(); }
     };
 
+    struct detached_blockchain_data
+    {
+      hashchain detached_blockchain;
+      size_t original_chain_size;
+      std::unordered_set<crypto::hash> detached_tx_hashes;
+      std::unordered_map<crypto::hash, std::vector<cryptonote::tx_destination_entry>> detached_confirmed_txs_dests;
+    };
+
+    struct process_tx_entry_t
+    {
+      cryptonote::COMMAND_RPC_GET_TRANSACTIONS::entry tx_entry;
+      cryptonote::transaction tx;
+      crypto::hash tx_hash;
+    };
+
+    struct tx_entry_data
+    {
+      std::vector<process_tx_entry_t> tx_entries;
+      uint64_t lowest_height;
+      uint64_t highest_height;
+
+      tx_entry_data(): lowest_height((uint64_t)-1), highest_height(0) {}
+    };
+
     /*!
      * \brief  Generates a wallet or restores one. Assumes the multisig setup
       *        has already completed for the provided multisig info.
@@ -915,22 +940,32 @@ private:
     /*!
      * \brief store_to  Stores wallet to another file(s), deleting old ones
      * \param path      Path to the wallet file (keys and address filenames will be generated based on this filename)
-     * \param password  Password to protect new wallet (TODO: probably better save the password in the wallet object?)
+     * \param password  Password that currently locks the wallet
+     * \param force_rewrite_keys if true, always rewrite keys file
+     *
+     * Leave both "path" and "password" blank to restore the cache file to the current position in the disk
+     * (which is the same as calling `store()`). If you want to store the wallet with a new password,
+     * use the method `change_password()`.
+     *
+     * Normally the keys file is not overwritten when storing, except when force_rewrite_keys is true
+     * or when `path` is a new wallet file.
+     *
+     * \throw error::invalid_password If storing keys file and old password is incorrect
      */
-    void store_to(const std::string &path, const epee::wipeable_string &password);
+    void store_to(const std::string &path, const epee::wipeable_string &password, bool force_rewrite_keys = false);
     /*!
      * \brief get_keys_file_data  Get wallet keys data which can be stored to a wallet file.
-     * \param password            Password of the encrypted wallet buffer (TODO: probably better save the password in the wallet object?)
+     * \param password            Password that currently locks the wallet
      * \param watch_only          true to include only view key, false to include both spend and view keys
      * \return                    Encrypted wallet keys data which can be stored to a wallet file
+     * \throw                     error::invalid_password if password does not match current wallet
      */
     boost::optional<wallet2::keys_file_data> get_keys_file_data(const epee::wipeable_string& password, bool watch_only);
     /*!
      * \brief get_cache_file_data   Get wallet cache data which can be stored to a wallet file.
-     * \param password              Password to protect the wallet cache data (TODO: probably better save the password in the wallet object?)
-     * \return                      Encrypted wallet cache data which can be stored to a wallet file
+     * \return                      Encrypted wallet cache data which can be stored to a wallet file (using current password)
      */
-    boost::optional<wallet2::cache_file_data> get_cache_file_data(const epee::wipeable_string& password);
+    boost::optional<wallet2::cache_file_data> get_cache_file_data();
 
     std::string path() const;
 
@@ -1024,7 +1059,7 @@ private:
     bool is_deprecated() const;
     void refresh(bool trusted_daemon);
     void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched);
-    void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money, bool check_pool = true);
+    void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money, bool check_pool = true, bool try_incremental = true, uint64_t max_blocks = std::numeric_limits<uint64_t>::max());
     bool refresh(bool trusted_daemon, uint64_t & blocks_fetched, bool& received_money, bool& ok);
 
     void set_refresh_type(RefreshType refresh_type) { m_refresh_type = refresh_type; }
@@ -1035,7 +1070,7 @@ private:
     bool multisig(bool *ready = NULL, uint32_t *threshold = NULL, uint32_t *total = NULL) const;
     bool has_multisig_partial_key_images() const;
     bool has_unknown_key_images() const;
-    bool get_multisig_seed(epee::wipeable_string& seed, const epee::wipeable_string &passphrase = std::string(), bool raw = true) const;
+    bool get_multisig_seed(epee::wipeable_string& seed, const epee::wipeable_string &passphrase = std::string()) const;
     bool key_on_device() const { return get_device_type() != hw::device::device_type::SOFTWARE; }
     hw::device::device_type get_device_type() const { return m_key_device_type; }
     bool reconnect_device();
@@ -1077,8 +1112,8 @@ private:
     bool parse_unsigned_tx_from_str(const std::string &unsigned_tx_st, unsigned_tx_set &exported_txs) const;
     bool load_tx(const std::string &signed_filename, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set&)> accept_func = NULL);
     bool parse_tx_from_str(const std::string &signed_tx_st, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set &)> accept_func);
-    std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices);     // pass subaddr_indices by value on purpose
-    std::vector<wallet2::pending_tx> create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices);
+    std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, const std::vector<crypto::key_image>& preferred_input_list = {});     // pass subaddr_indices by value on purpose
+    std::vector<wallet2::pending_tx> create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, const std::vector<crypto::key_image>& preferred_input_list = {});
     std::vector<wallet2::pending_tx> create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra);
     std::vector<wallet2::pending_tx> create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra);
     bool sanity_check(const std::vector<wallet2::pending_tx> &ptx_vector, std::vector<cryptonote::tx_destination_entry> dsts) const;
@@ -1383,7 +1418,7 @@ private:
     std::string get_spend_proof(const crypto::hash &txid, const std::string &message);
     bool check_spend_proof(const crypto::hash &txid, const std::string &message, const std::string &sig_str);
 
-    void scan_tx(const std::vector<crypto::hash> &txids);
+    void scan_tx(const std::unordered_set<crypto::hash> &txids);
 
     /*!
      * \brief  Generates a proof that proves the reserve of unspent funds
@@ -1510,9 +1545,9 @@ private:
     crypto::public_key get_tx_pub_key_from_received_outs(const tools::wallet2::transfer_details &td) const;
     void import_tx(const std::string &txid, std::vector<uint64_t> &o_indices, uint64_t height, uint8_t block_version, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen);
 
-    void update_pool_state(std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed = false);
+    void update_pool_state(std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed = false, bool try_incremental = false);
     void process_pool_state(const std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &txs);
-    void remove_obsolete_pool_txs(const std::vector<crypto::hash> &tx_hashes);
+    void remove_obsolete_pool_txs(const std::vector<crypto::hash> &tx_hashes, bool remove_if_found);
 
     std::string encrypt(const char *plaintext, size_t len, const crypto::secret_key &skey, bool authenticated = true) const;
     std::string encrypt(const epee::span<char> &span, const crypto::secret_key &skey, bool authenticated = true) const;
@@ -1648,6 +1683,7 @@ private:
     void thaw(const crypto::key_image &ki);
     bool frozen(const crypto::key_image &ki) const;
     bool frozen(const transfer_details &td) const;
+    bool frozen(const multisig_tx_set& txs) const; // does partially signed txset contain frozen enotes?
 
     bool save_to_file(const std::string& path_to_file, const std::string& binary, bool is_printable = false) const;
     static bool load_from_file(const std::string& path_to_file, std::string& target_str, size_t max_size = 1000000000);
@@ -1725,18 +1761,24 @@ private:
      */
     bool load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password);
     bool load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password, boost::optional<crypto::chacha_key>& keys_to_encrypt);
-    void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint8_t block_version, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL);
+    void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint8_t block_version, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL, bool ignore_callbacks = false);
     bool should_skip_block(const cryptonote::block &b, uint64_t height) const;
     void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector<tx_cache_data> &tx_cache_data, size_t tx_cache_data_offset, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL);
-    void detach_blockchain(uint64_t height, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL);
+    detached_blockchain_data detach_blockchain(uint64_t height, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL);
+    void handle_reorg(uint64_t height, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL);
     void get_short_chain_history(std::list<crypto::hash>& ids, uint64_t granularity = 1) const;
     bool clear();
     void clear_soft(bool keep_key_images=false);
-    void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, uint64_t &current_height);
+    void pull_blocks(bool first, bool try_incremental, uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, uint64_t &current_height);
     void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<crypto::hash> &hashes);
     void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, bool force = false);
-    void pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::vector<cryptonote::block_complete_entry> &prev_blocks, const std::vector<parsed_block> &prev_parsed_blocks, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<parsed_block> &parsed_blocks, bool &last, bool &error, std::exception_ptr &exception);
+    void pull_and_parse_next_blocks(bool first, bool try_incremental, uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::vector<cryptonote::block_complete_entry> &prev_blocks, const std::vector<parsed_block> &prev_parsed_blocks, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<parsed_block> &parsed_blocks, bool &last, bool &error, std::exception_ptr &exception);
     void process_parsed_blocks(uint64_t start_height, const std::vector<cryptonote::block_complete_entry> &blocks, const std::vector<parsed_block> &parsed_blocks, uint64_t& blocks_added, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL);
+    bool accept_pool_tx_for_processing(const crypto::hash &txid);
+    void process_unconfirmed_transfer(bool incremental, const crypto::hash &txid, wallet2::unconfirmed_transfer_details &tx_details, bool seen_in_pool, std::chrono::system_clock::time_point now, bool refreshed);
+    void process_pool_info_extent(const cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::response &res, std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed);
+    void update_pool_state_by_pool_query(std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed = false);
+    void update_pool_state_from_pool_data(bool incremental, const std::vector<crypto::hash> &removed_pool_txids, const std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &added_pool_txs, std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> &process_txs, bool refreshed);
     uint64_t select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers) const;
     bool prepare_file_names(const std::string& file_path);
     void process_unconfirmed(const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t height);
@@ -1755,7 +1797,7 @@ private:
     std::vector<uint64_t> get_unspent_amounts_vector(bool strict);
     uint64_t get_dynamic_base_fee_estimate();
     float get_output_relatedness(const transfer_details &td0, const transfer_details &td1) const;
-    std::vector<size_t> pick_preferred_rct_inputs(uint64_t needed_money, uint32_t subaddr_account, const std::set<uint32_t> &subaddr_indices);
+    std::vector<size_t> pick_preferred_rct_inputs(uint64_t needed_money, uint32_t subaddr_account, const std::set<uint32_t> &subaddr_indices, const std::vector<crypto::key_image>& preferred_input_list);
     void set_spent(size_t idx, uint64_t height);
     void set_unspent(size_t idx);
     bool is_spent(const transfer_details &td, bool strict = true) const;
@@ -1779,6 +1821,9 @@ private:
     crypto::chacha_key get_ringdb_key();
     void setup_keys(const epee::wipeable_string &password);
     size_t get_transfer_details(const crypto::key_image &ki) const;
+    tx_entry_data get_tx_entries(const std::unordered_set<crypto::hash> &txids);
+    void sort_scan_tx_entries(std::vector<process_tx_entry_t> &unsorted_tx_entries);
+    void process_scan_txs(const tx_entry_data &txs_to_scan, const tx_entry_data &txs_to_reprocess, const std::unordered_set<crypto::hash> &tx_hashes_to_reprocess, detached_blockchain_data &dbd);
 
     void register_devices();
     hw::device& lookup_device(const std::string & device_descriptor);
@@ -1871,6 +1916,11 @@ private:
     // If m_refresh_from_block_height is explicitly set to zero we need this to differentiate it from the case that
     // m_refresh_from_block_height was defaulted to zero.*/
     bool m_explicit_refresh_from_block_height;
+    uint64_t m_pool_info_query_time;
+    std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>> m_process_pool_txs;
+    uint64_t m_skip_to_height;
+    // m_skip_to_height is useful when we don't want to modify the wallet's restore height.
+    // m_refresh_from_block_height is also a wallet's restore height which should remain constant unless explicitly modified by the user.
     bool m_confirm_non_default_ring_size;
     AskPasswordType m_ask_password;
     uint64_t m_max_reorg_depth;

src/wallet/wallet_errors.h

@@ -93,6 +93,8 @@ namespace tools
     //         get_output_distribution
     //         payment_required
     //       wallet_files_doesnt_correspond
+    //       scan_tx_error *
+    //         wont_reprocess_recent_txs_via_untrusted_daemon
     //
     // * - class with protected ctor
 
@@ -915,6 +917,23 @@ namespace tools
       }
     };
     //----------------------------------------------------------------------------------------------------
+    struct scan_tx_error : public wallet_logic_error
+    {
+    protected:
+      explicit scan_tx_error(std::string&& loc, const std::string& message)
+        : wallet_logic_error(std::move(loc), message)
+      {
+      }
+    };
+    //----------------------------------------------------------------------------------------------------
+    struct wont_reprocess_recent_txs_via_untrusted_daemon : public scan_tx_error
+    {
+      explicit wont_reprocess_recent_txs_via_untrusted_daemon(std::string&& loc)
+        : scan_tx_error(std::move(loc), "The wallet has already seen 1 or more recent transactions than the scanned tx")
+      {
+      }
+    };
+    //----------------------------------------------------------------------------------------------------
 
 #if !defined(_MSC_VER)
 

src/wallet/wallet_rpc_server.cpp

@@ -60,6 +60,7 @@ using namespace epee;
 #define MONERO_DEFAULT_LOG_CATEGORY "wallet.rpc"
 
 #define DEFAULT_AUTO_REFRESH_PERIOD 20 // seconds
+#define REFRESH_INFICATIVE_BLOCK_CHUNK_SIZE 256    // just to split refresh in separate calls to play nicer with other threads
 
 #define CHECK_MULTISIG_ENABLED() \
   do \
@@ -79,6 +80,7 @@ namespace
   const command_line::arg_descriptor<bool> arg_restricted = {"restricted-rpc", "Restricts to view-only commands", false};
   const command_line::arg_descriptor<std::string> arg_wallet_dir = {"wallet-dir", "Directory for newly created wallets"};
   const command_line::arg_descriptor<bool> arg_prompt_for_password = {"prompt-for-password", "Prompts for password when not provided", false};
+  const command_line::arg_descriptor<bool> arg_no_initial_sync = {"no-initial-sync", "Skips the initial sync before listening for connections", false};
 
   constexpr const char default_rpc_username[] = "wownero";
 
@@ -149,12 +151,17 @@ namespace tools
         return true;
       if (boost::posix_time::microsec_clock::universal_time() < m_last_auto_refresh_time + boost::posix_time::seconds(m_auto_refresh_period))
         return true;
+      uint64_t blocks_fetched = 0;
       try {
-        if (m_wallet) m_wallet->refresh(m_wallet->is_trusted_daemon());
+        bool received_money = false;
+        if (m_wallet) m_wallet->refresh(m_wallet->is_trusted_daemon(), 0, blocks_fetched, received_money, true, true, REFRESH_INFICATIVE_BLOCK_CHUNK_SIZE);
       } catch (const std::exception& ex) {
         LOG_ERROR("Exception at while refreshing, what=" << ex.what());
       }
-      m_last_auto_refresh_time = boost::posix_time::microsec_clock::universal_time();
+      // if we got the max amount of blocks, do not set the last refresh time, we did only part of the refresh and will
+      // continue asap, and only set the last refresh time once the refresh is actually finished
+      if (blocks_fetched < REFRESH_INFICATIVE_BLOCK_CHUNK_SIZE)
+        m_last_auto_refresh_time = boost::posix_time::microsec_clock::universal_time();
       return true;
     }, 1000);
     m_net_server.add_idle_handler([this](){
@@ -3166,7 +3173,7 @@ namespace tools
           return false;
       }
 
-      std::vector<crypto::hash> txids;
+      std::unordered_set<crypto::hash> txids;
       std::list<std::string>::const_iterator i = req.txids.begin();
       while (i != req.txids.end())
       {
@@ -3179,11 +3186,15 @@ namespace tools
           }
 
           crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_blob.data());
-          txids.push_back(txid);
+          txids.insert(txid);
       }
 
       try {
           m_wallet->scan_tx(txids);
+      }  catch (const tools::error::wont_reprocess_recent_txs_via_untrusted_daemon &e) {
+          er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR;
+          er.message = e.what() + std::string(". Either connect to a trusted daemon or rescan the chain.");
+          return false;
       } catch (const std::exception &e) {
           handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR);
           return false;
@@ -3801,7 +3812,7 @@ namespace tools
     std::string old_language;
 
     // check the given seed
-    {
+    if (!req.enable_multisig_experimental) {
       if (!crypto::ElectrumWords::words_to_bytes(req.seed, recovery_key, old_language))
       {
         er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR;
@@ -3824,6 +3835,13 @@ namespace tools
 
     // process seed_offset if given
     {
+      if (req.enable_multisig_experimental && !req.seed_offset.empty())
+      {
+        er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR;
+        er.message = "Multisig seeds are not compatible with seed offsets";
+        return false;
+      }
+
       if (!req.seed_offset.empty())
       {
         recovery_key = cryptonote::decrypt_key(recovery_key, req.seed_offset);
@@ -3887,7 +3905,27 @@ namespace tools
     crypto::secret_key recovery_val;
     try
     {
-      recovery_val = wal->generate(wallet_file, std::move(rc.second).password(), recovery_key, true, false, false);
+      if (req.enable_multisig_experimental)
+      {
+        // Parse multisig seed into raw multisig data
+        epee::wipeable_string multisig_data;
+        multisig_data.resize(req.seed.size() / 2);
+        if (!epee::from_hex::to_buffer(epee::to_mut_byte_span(multisig_data), req.seed))
+        {
+          er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR;
+          er.message = "Multisig seed not represented as hexadecimal string";
+          return false;
+        }
+
+        // Generate multisig wallet
+        wal->generate(wallet_file, std::move(rc.second).password(), multisig_data, false);
+        wal->enable_multisig(true);
+      }
+      else
+      {
+        // Generate normal wallet
+        recovery_val = wal->generate(wallet_file, std::move(rc.second).password(), recovery_key, true, false, false);
+      }
       MINFO("Wallet has been restored.\n");
     }
     catch (const std::exception &e)
@@ -3898,7 +3936,7 @@ namespace tools
 
     // // Convert the secret key back to seed
     epee::wipeable_string electrum_words;
-    if (!crypto::ElectrumWords::bytes_to_words(recovery_val, electrum_words, mnemonic_language))
+    if (!req.enable_multisig_experimental && !crypto::ElectrumWords::bytes_to_words(recovery_val, electrum_words, mnemonic_language))
     {
       er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR;
       er.message = "Failed to encode seed";
@@ -4520,6 +4558,7 @@ public:
       const auto password_file = command_line::get_arg(vm, arg_password_file);
       const auto prompt_for_password = command_line::get_arg(vm, arg_prompt_for_password);
       const auto password_prompt = prompt_for_password ? password_prompter : nullptr;
+      const auto no_initial_sync = command_line::get_arg(vm, arg_no_initial_sync);
 
       if(!wallet_file.empty() && !from_json.empty())
       {
@@ -4588,7 +4627,8 @@ public:
 
       try
       {
-        wal->refresh(wal->is_trusted_daemon());
+        if (!no_initial_sync)
+          wal->refresh(wal->is_trusted_daemon());
       }
       catch (const std::exception& e)
       {
@@ -4699,6 +4739,7 @@ int main(int argc, char** argv) {
   command_line::add_arg(desc_params, arg_wallet_dir);
   command_line::add_arg(desc_params, arg_prompt_for_password);
   command_line::add_arg(desc_params, arg_rpc_client_secret_key);
+  command_line::add_arg(desc_params, arg_no_initial_sync);
 
   daemonizer::init_options(hidden_options, desc_params);
   desc_params.add(hidden_options);

src/wallet/wallet_rpc_server_commands_defs.h

@@ -530,6 +530,33 @@ namespace wallet_rpc
     END_KV_SERIALIZE_MAP()
   };
 
+  struct single_transfer_response
+  {
+    std::string tx_hash;
+    std::string tx_key;
+    uint64_t amount;
+    uint64_t fee;
+    uint64_t weight;
+    std::string tx_blob;
+    std::string tx_metadata;
+    std::string multisig_txset;
+    std::string unsigned_txset;
+    key_image_list spent_key_images;
+
+    BEGIN_KV_SERIALIZE_MAP()
+      KV_SERIALIZE(tx_hash)
+      KV_SERIALIZE(tx_key)
+      KV_SERIALIZE(amount)
+      KV_SERIALIZE(fee)
+      KV_SERIALIZE(weight)
+      KV_SERIALIZE(tx_blob)
+      KV_SERIALIZE(tx_metadata)
+      KV_SERIALIZE(multisig_txset)
+      KV_SERIALIZE(unsigned_txset)
+      KV_SERIALIZE(spent_key_images)
+    END_KV_SERIALIZE_MAP()
+  };
+
   struct COMMAND_RPC_TRANSFER
   {
     struct request_t
@@ -562,35 +589,37 @@ namespace wallet_rpc
     };
     typedef epee::misc_utils::struct_init<request_t> request;
 
-    struct response_t
-    {
-      std::string tx_hash;
-      std::string tx_key;
-      uint64_t amount;
-      uint64_t fee;
-      uint64_t weight;
-      std::string tx_blob;
-      std::string tx_metadata;
-      std::string multisig_txset;
-      std::string unsigned_txset;
-      key_image_list spent_key_images;
-
-      BEGIN_KV_SERIALIZE_MAP()
-        KV_SERIALIZE(tx_hash)
-        KV_SERIALIZE(tx_key)
-        KV_SERIALIZE(amount)
-        KV_SERIALIZE(fee)
-        KV_SERIALIZE(weight)
-        KV_SERIALIZE(tx_blob)
-        KV_SERIALIZE(tx_metadata)
-        KV_SERIALIZE(multisig_txset)
-        KV_SERIALIZE(unsigned_txset)
-        KV_SERIALIZE(spent_key_images)
-      END_KV_SERIALIZE_MAP()
-    };
+    typedef single_transfer_response response_t;
     typedef epee::misc_utils::struct_init<response_t> response;
   };
 
+  struct split_transfer_response
+  {
+    std::list<std::string> tx_hash_list;
+    std::list<std::string> tx_key_list;
+    std::list<uint64_t> amount_list;
+    std::list<uint64_t> fee_list;
+    std::list<uint64_t> weight_list;
+    std::list<std::string> tx_blob_list;
+    std::list<std::string> tx_metadata_list;
+    std::string multisig_txset;
+    std::string unsigned_txset;
+    std::list<key_image_list> spent_key_images_list;
+
+    BEGIN_KV_SERIALIZE_MAP()
+      KV_SERIALIZE(tx_hash_list)
+      KV_SERIALIZE(tx_key_list)
+      KV_SERIALIZE(amount_list)
+      KV_SERIALIZE(fee_list)
+      KV_SERIALIZE(weight_list)
+      KV_SERIALIZE(tx_blob_list)
+      KV_SERIALIZE(tx_metadata_list)
+      KV_SERIALIZE(multisig_txset)
+      KV_SERIALIZE(unsigned_txset)
+      KV_SERIALIZE(spent_key_images_list)
+    END_KV_SERIALIZE_MAP()
+  };
+
   struct COMMAND_RPC_TRANSFER_SPLIT
   {
     struct request_t
@@ -623,41 +652,7 @@ namespace wallet_rpc
     };
     typedef epee::misc_utils::struct_init<request_t> request;
 
-    struct key_list
-    {
-      std::list<std::string> keys;
-
-      BEGIN_KV_SERIALIZE_MAP()
-        KV_SERIALIZE(keys)
-      END_KV_SERIALIZE_MAP()
-    };
-
-    struct response_t
-    {
-      std::list<std::string> tx_hash_list;
-      std::list<std::string> tx_key_list;
-      std::list<uint64_t> amount_list;
-      std::list<uint64_t> fee_list;
-      std::list<uint64_t> weight_list;
-      std::list<std::string> tx_blob_list;
-      std::list<std::string> tx_metadata_list;
-      std::string multisig_txset;
-      std::string unsigned_txset;
-      std::list<key_image_list> spent_key_images_list;
-
-      BEGIN_KV_SERIALIZE_MAP()
-        KV_SERIALIZE(tx_hash_list)
-        KV_SERIALIZE(tx_key_list)
-        KV_SERIALIZE(amount_list)
-        KV_SERIALIZE(fee_list)
-        KV_SERIALIZE(weight_list)
-        KV_SERIALIZE(tx_blob_list)
-        KV_SERIALIZE(tx_metadata_list)
-        KV_SERIALIZE(multisig_txset)
-        KV_SERIALIZE(unsigned_txset)
-        KV_SERIALIZE(spent_key_images_list)
-      END_KV_SERIALIZE_MAP()
-    };
+    typedef split_transfer_response response_t;
     typedef epee::misc_utils::struct_init<response_t> response;
   };
 
@@ -821,41 +816,7 @@ namespace wallet_rpc
     };
     typedef epee::misc_utils::struct_init<request_t> request;
 
-    struct key_list
-    {
-      std::list<std::string> keys;
-
-      BEGIN_KV_SERIALIZE_MAP()
-        KV_SERIALIZE(keys)
-      END_KV_SERIALIZE_MAP()
-    };
-
-    struct response_t
-    {
-      std::list<std::string> tx_hash_list;
-      std::list<std::string> tx_key_list;
-      std::list<uint64_t> amount_list;
-      std::list<uint64_t> fee_list;
-      std::list<uint64_t> weight_list;
-      std::list<std::string> tx_blob_list;
-      std::list<std::string> tx_metadata_list;
-      std::string multisig_txset;
-      std::string unsigned_txset;
-      std::list<key_image_list> spent_key_images_list;
-
-      BEGIN_KV_SERIALIZE_MAP()
-        KV_SERIALIZE(tx_hash_list)
-        KV_SERIALIZE(tx_key_list)
-        KV_SERIALIZE(amount_list)
-        KV_SERIALIZE(fee_list)
-        KV_SERIALIZE(weight_list)
-        KV_SERIALIZE(tx_blob_list)
-        KV_SERIALIZE(tx_metadata_list)
-        KV_SERIALIZE(multisig_txset)
-        KV_SERIALIZE(unsigned_txset)
-        KV_SERIALIZE(spent_key_images_list)
-      END_KV_SERIALIZE_MAP()
-    };
+    typedef split_transfer_response response_t;
     typedef epee::misc_utils::struct_init<response_t> response;
   };
 
@@ -897,41 +858,7 @@ namespace wallet_rpc
     };
     typedef epee::misc_utils::struct_init<request_t> request;
 
-    struct key_list
-    {
-      std::list<std::string> keys;
-
-      BEGIN_KV_SERIALIZE_MAP()
-        KV_SERIALIZE(keys)
-      END_KV_SERIALIZE_MAP()
-    };
-
-    struct response_t
-    {
-      std::list<std::string> tx_hash_list;
-      std::list<std::string> tx_key_list;
-      std::list<uint64_t> amount_list;
-      std::list<uint64_t> fee_list;
-      std::list<uint64_t> weight_list;
-      std::list<std::string> tx_blob_list;
-      std::list<std::string> tx_metadata_list;
-      std::string multisig_txset;
-      std::string unsigned_txset;
-      std::list<key_image_list> spent_key_images_list;
-
-      BEGIN_KV_SERIALIZE_MAP()
-        KV_SERIALIZE(tx_hash_list)
-        KV_SERIALIZE(tx_key_list)
-        KV_SERIALIZE(amount_list)
-        KV_SERIALIZE(fee_list)
-        KV_SERIALIZE(weight_list)
-        KV_SERIALIZE(tx_blob_list)
-        KV_SERIALIZE(tx_metadata_list)
-        KV_SERIALIZE(multisig_txset)
-        KV_SERIALIZE(unsigned_txset)
-        KV_SERIALIZE(spent_key_images_list)
-      END_KV_SERIALIZE_MAP()
-    };
+    typedef split_transfer_response response_t;
     typedef epee::misc_utils::struct_init<response_t> response;
   };
 
@@ -967,32 +894,7 @@ namespace wallet_rpc
     };
     typedef epee::misc_utils::struct_init<request_t> request;
 
-    struct response_t
-    {
-      std::string tx_hash;
-      std::string tx_key;
-      uint64_t amount;
-      uint64_t fee;
-      uint64_t weight;
-      std::string tx_blob;
-      std::string tx_metadata;
-      std::string multisig_txset;
-      std::string unsigned_txset;
-      key_image_list spent_key_images;
-
-      BEGIN_KV_SERIALIZE_MAP()
-        KV_SERIALIZE(tx_hash)
-        KV_SERIALIZE(tx_key)
-        KV_SERIALIZE(amount)
-        KV_SERIALIZE(fee)
-        KV_SERIALIZE(weight)
-        KV_SERIALIZE(tx_blob)
-        KV_SERIALIZE(tx_metadata)
-        KV_SERIALIZE(multisig_txset)
-        KV_SERIALIZE(unsigned_txset)
-        KV_SERIALIZE(spent_key_images)
-      END_KV_SERIALIZE_MAP()
-    };
+    typedef single_transfer_response response_t;
     typedef epee::misc_utils::struct_init<response_t> response;
   };
 
@@ -2360,6 +2262,7 @@ namespace wallet_rpc
       std::string password;
       std::string language;
       bool autosave_current;
+      bool enable_multisig_experimental;
 
       BEGIN_KV_SERIALIZE_MAP()
       KV_SERIALIZE_OPT(restore_height, (uint64_t)0)
@@ -2369,6 +2272,7 @@ namespace wallet_rpc
       KV_SERIALIZE(password)
       KV_SERIALIZE(language)
       KV_SERIALIZE_OPT(autosave_current, true)
+      KV_SERIALIZE_OPT(enable_multisig_experimental, false)
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<request_t> request;

tests/CMakeLists.txt

@@ -72,14 +72,8 @@ else ()
   include_directories(SYSTEM "${CMAKE_CURRENT_SOURCE_DIR}/gtest/include")
 endif (GTest_FOUND)
 
-file(COPY
-  data/wallet_9svHk1.keys
-  data/wallet_9svHk1
-  data/outputs
-  data/unsigned_monero_tx
-  data/signed_monero_tx
-  data/sha256sum
-  DESTINATION data)
+message(STATUS "Copying test data directory...")
+file(COPY data DESTINATION .) # Copy data directory from source root to build root
 
 if (CMAKE_BUILD_TYPE STREQUAL "fuzz" OR OSSFUZZ)
   add_subdirectory(fuzz)

tests/README.md

@@ -54,7 +54,7 @@ Functional tests are located under the `tests/functional_tests` directory.
 
 Building all the tests requires installing the following dependencies:
 ```bash
-pip install requests psutil monotonic zmq
+pip install requests psutil monotonic zmq deepdiff
 ```
 
 First, run a regtest daemon in the offline mode and with a fixed difficulty:

tests/functional_tests/CMakeLists.txt

@@ -67,7 +67,7 @@ target_link_libraries(make_test_signature
 monero_add_minimal_executable(cpu_power_test cpu_power_test.cpp)
 find_program(PYTHON3_FOUND python3 REQUIRED)
 
-execute_process(COMMAND ${PYTHON3_FOUND} "-c" "import requests; import psutil; import monotonic; import zmq; print('OK')" OUTPUT_VARIABLE REQUESTS_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE)
+execute_process(COMMAND ${PYTHON3_FOUND} "-c" "import requests; import psutil; import monotonic; import zmq; import deepdiff; print('OK')" OUTPUT_VARIABLE REQUESTS_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE)
 if (REQUESTS_OUTPUT STREQUAL "OK")
   add_test(
     NAME    functional_tests_rpc
@@ -76,6 +76,6 @@ if (REQUESTS_OUTPUT STREQUAL "OK")
     NAME    check_missing_rpc_methods
     COMMAND ${PYTHON3_FOUND} "${CMAKE_CURRENT_SOURCE_DIR}/check_missing_rpc_methods.py" "${CMAKE_SOURCE_DIR}")
 else()
-  message(WARNING "functional_tests_rpc and check_missing_rpc_methods skipped, needs the 'requests', 'psutil', 'monotonic', and 'zmq' python modules")
+  message(WARNING "functional_tests_rpc and check_missing_rpc_methods skipped, needs the 'requests', 'psutil', 'monotonic', 'zmq', and 'deepdiff' python modules")
   set(CTEST_CUSTOM_TESTS_IGNORE ${CTEST_CUSTOM_TESTS_IGNORE} functional_tests_rpc check_missing_rpc_methods)
 endif()

tests/functional_tests/mining.py

@@ -36,6 +36,7 @@ import math
 import monotonic
 import util_resources
 import multiprocessing
+import string
 
 """Test daemon mining RPC calls
 
@@ -52,6 +53,11 @@ Control the behavior with these environment variables:
 from framework.daemon import Daemon
 from framework.wallet import Wallet
 
+def assert_non_null_hash(s):
+    assert len(s) == 64 # correct length
+    assert all((c in string.hexdigits for c in s)) # is parseable as hex
+    assert s != ('0' * 64) # isn't null hash
+
 class MiningTest():
     def run_test(self):
         self.reset()
@@ -250,6 +256,8 @@ class MiningTest():
             block_hash = hashes[i]
             assert len(block_hash) == 64
             res = daemon.submitblock(blocks[i])
+            submitted_block_id = res.block_id
+            assert_non_null_hash(submitted_block_id)
             res = daemon.get_height()
             assert res.height == height + i + 1
             assert res.hash == block_hash
@@ -346,6 +354,8 @@ class MiningTest():
         t0 = time.time()
         for h in range(len(block_hashes)):
             res = daemon.submitblock(blocks[h])
+            submitted_block_id = res.block_id
+            assert_non_null_hash(submitted_block_id)
         t0 = time.time() - t0
         res = daemon.get_info()
         assert height == res.height

tests/functional_tests/multisig.py

@@ -29,6 +29,7 @@
 # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 from __future__ import print_function
+import random
 
 """Test multisig transfers
 """
@@ -36,47 +37,61 @@ from __future__ import print_function
 from framework.daemon import Daemon
 from framework.wallet import Wallet
 
+TEST_CASES = \
+[
+#  M  N  Primary Address
+  [2, 2, '45J58b7PmKJFSiNPFFrTdtfMcFGnruP7V4CMuRpX7NsH4j3jGHKAjo3YJP2RePX6HMaSkbvTbrWUFhDNcNcHgtNmQ3gr7sG'],
+  [2, 3, '44G2TQNfsiURKkvxp7gbgaJY8WynZvANnhmyMAwv6WeEbAvyAWMfKXRhh3uBXT2UAKhAsUJ7Fg5zjjF2U1iGciFk5duN94i'],
+  [3, 3, '41mro238grj56GnrWkakAKTkBy2yDcXYsUZ2iXCM9pe5Ueajd2RRc6Fhh3uBXT2UAKhAsUJ7Fg5zjjF2U1iGciFk5ief4ZP'],
+  [3, 4, '44vZSprQKJQRFe6t1VHgU4ESvq2dv7TjBLVGE7QscKxMdFSiyyPCEV64NnKUQssFPyWxc2meyt7j63F2S2qtCTRL6dakeff'],
+  [2, 4, '47puypSwsV1gvUDratmX4y58fSwikXVehEiBhVLxJA1gRCxHyrRgTDr4NnKUQssFPyWxc2meyt7j63F2S2qtCTRL6aRPj5U'],
+  [1, 2, '4A8RnBQixry4VXkqeWhmg8L7vWJVDJj4FN9PV4E7Mgad5ZZ6LKQdn8dYJP2RePX6HMaSkbvTbrWUFhDNcNcHgtNmQ4S8RSB']
+]
+
+PUB_ADDRS = [case[2] for case in TEST_CASES]
+
 class MultisigTest():
     def run_test(self):
         self.reset()
-        self.mine('45J58b7PmKJFSiNPFFrTdtfMcFGnruP7V4CMuRpX7NsH4j3jGHKAjo3YJP2RePX6HMaSkbvTbrWUFhDNcNcHgtNmQ3gr7sG', 5)
-        self.mine('44G2TQNfsiURKkvxp7gbgaJY8WynZvANnhmyMAwv6WeEbAvyAWMfKXRhh3uBXT2UAKhAsUJ7Fg5zjjF2U1iGciFk5duN94i', 5)
-        self.mine('41mro238grj56GnrWkakAKTkBy2yDcXYsUZ2iXCM9pe5Ueajd2RRc6Fhh3uBXT2UAKhAsUJ7Fg5zjjF2U1iGciFk5ief4ZP', 5)
-        self.mine('44vZSprQKJQRFe6t1VHgU4ESvq2dv7TjBLVGE7QscKxMdFSiyyPCEV64NnKUQssFPyWxc2meyt7j63F2S2qtCTRL6dakeff', 5)
-        self.mine('47puypSwsV1gvUDratmX4y58fSwikXVehEiBhVLxJA1gRCxHyrRgTDr4NnKUQssFPyWxc2meyt7j63F2S2qtCTRL6aRPj5U', 5)
+        for pub_addr in PUB_ADDRS:
+            self.mine(pub_addr, 4)
         self.mine('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 80)
 
         self.test_states()
 
-        self.create_multisig_wallets(2, 2, '45J58b7PmKJFSiNPFFrTdtfMcFGnruP7V4CMuRpX7NsH4j3jGHKAjo3YJP2RePX6HMaSkbvTbrWUFhDNcNcHgtNmQ3gr7sG')
-        self.import_multisig_info([1, 0], 5)
-        txid = self.transfer([1, 0])
-        self.import_multisig_info([0, 1], 6)
-        self.check_transaction(txid)
+        self.fund_addrs_with_normal_wallet(PUB_ADDRS)
 
-        self.create_multisig_wallets(2, 3, '44G2TQNfsiURKkvxp7gbgaJY8WynZvANnhmyMAwv6WeEbAvyAWMfKXRhh3uBXT2UAKhAsUJ7Fg5zjjF2U1iGciFk5duN94i')
-        self.import_multisig_info([0, 2], 5)
-        txid = self.transfer([0, 2])
-        self.import_multisig_info([0, 1, 2], 6)
-        self.check_transaction(txid)
+        for M, N, pub_addr in TEST_CASES:
+            assert M <= N
+            shuffled_participants = list(range(N))
+            random.shuffle(shuffled_participants)
+            shuffled_signers = shuffled_participants[:M]
 
-        self.create_multisig_wallets(3, 3, '41mro238grj56GnrWkakAKTkBy2yDcXYsUZ2iXCM9pe5Ueajd2RRc6Fhh3uBXT2UAKhAsUJ7Fg5zjjF2U1iGciFk5ief4ZP')
-        self.import_multisig_info([2, 0, 1], 5)
-        txid = self.transfer([2, 1, 0])
-        self.import_multisig_info([0, 2, 1], 6)
-        self.check_transaction(txid)
+            expected_outputs = 5 # each wallet owns four mined outputs & one transferred output
 
-        self.create_multisig_wallets(3, 4, '44vZSprQKJQRFe6t1VHgU4ESvq2dv7TjBLVGE7QscKxMdFSiyyPCEV64NnKUQssFPyWxc2meyt7j63F2S2qtCTRL6dakeff')
-        self.import_multisig_info([0, 2, 3], 5)
-        txid = self.transfer([0, 2, 3])
-        self.import_multisig_info([0, 1, 2, 3], 6)
-        self.check_transaction(txid)
+            # Create multisig wallet and test transferring
+            self.create_multisig_wallets(M, N, pub_addr)
+            self.import_multisig_info(shuffled_signers if M != 1 else shuffled_participants, expected_outputs)
+            txid = self.transfer(shuffled_signers)
+            expected_outputs += 1
+            self.import_multisig_info(shuffled_participants, expected_outputs)
+            self.check_transaction(txid)
 
-        self.create_multisig_wallets(2, 4, '47puypSwsV1gvUDratmX4y58fSwikXVehEiBhVLxJA1gRCxHyrRgTDr4NnKUQssFPyWxc2meyt7j63F2S2qtCTRL6aRPj5U')
-        self.import_multisig_info([1, 2], 5)
-        txid = self.transfer([1, 2])
-        self.import_multisig_info([0, 1, 2, 3], 6)
-        self.check_transaction(txid)
+            # If more than 1 signer, try to freeze key image of one signer, make tx using that key
+            # image on another signer, then have first signer sign multisg_txset. Should fail
+            if M != 1:
+                txid = self.try_transfer_frozen(shuffled_signers)
+                expected_outputs += 1
+                self.import_multisig_info(shuffled_participants, expected_outputs)
+                self.check_transaction(txid)
+
+            # Recreate wallet from multisig seed and test transferring
+            self.remake_some_multisig_wallets_by_multsig_seed(M)
+            self.import_multisig_info(shuffled_signers if M != 1 else shuffled_participants, expected_outputs)
+            txid = self.transfer(shuffled_signers)
+            expected_outputs += 1
+            self.import_multisig_info(shuffled_participants, expected_outputs)
+            self.check_transaction(txid)
 
     def reset(self):
         print('Resetting blockchain')
@@ -90,6 +105,11 @@ class MultisigTest():
         daemon = Daemon()
         daemon.generateblocks(address, blocks)
 
+    # This method sets up N_total wallets with a threshold of M_threshold doing the following steps:
+    #   * restore_deterministic_wallet(w/ hardcoded seeds)
+    #   * prepare_multisig(enable_multisig_experimental = True)
+    #   * make_multisig()
+    #   * exchange_multisig_keys()
     def create_multisig_wallets(self, M_threshold, N_total, expected_address):
       print('Creating ' + str(M_threshold) + '/' + str(N_total) + ' multisig wallet')
       seeds = [
@@ -100,6 +120,8 @@ class MultisigTest():
       ]
       assert M_threshold <= N_total
       assert N_total <= len(seeds)
+
+      # restore_deterministic_wallet() & prepare_multisig()
       self.wallet = [None] * N_total
       info = []
       for i in range(N_total):
@@ -111,10 +133,12 @@ class MultisigTest():
         assert len(res.multisig_info) > 0
         info.append(res.multisig_info)
 
+      # Assert that all wallets are multisig
       for i in range(N_total):
         res = self.wallet[i].is_multisig()
         assert res.multisig == False
 
+      # make_multisig() with each other's info
       addresses = []
       next_stage = []
       for i in range(N_total):
@@ -122,6 +146,7 @@ class MultisigTest():
         addresses.append(res.address)
         next_stage.append(res.multisig_info)
 
+      # Assert multisig paramaters M/N for each wallet
       for i in range(N_total):
         res = self.wallet[i].is_multisig()
         assert res.multisig == True
@@ -129,13 +154,15 @@ class MultisigTest():
         assert res.threshold == M_threshold
         assert res.total == N_total
 
-      while True:
+      # exchange_multisig_keys()
+      num_exchange_multisig_keys_stages = 0
+      while True: # while not all wallets are ready
         n_ready = 0
         for i in range(N_total):
           res = self.wallet[i].is_multisig()
           if res.ready == True:
             n_ready += 1
-        assert n_ready == 0 or n_ready == N_total
+        assert n_ready == 0 or n_ready == N_total # No partial readiness
         if n_ready == N_total:
           break
         info = next_stage
@@ -145,10 +172,17 @@ class MultisigTest():
           res = self.wallet[i].exchange_multisig_keys(info)
           next_stage.append(res.multisig_info)
           addresses.append(res.address)
+        num_exchange_multisig_keys_stages += 1
+
+      # We should only need N - M + 1 key exchange rounds
+      assert num_exchange_multisig_keys_stages == N_total - M_threshold + 1
+
+      # Assert that the all wallets have expected public address
       for i in range(N_total):
-        assert addresses[i] == expected_address
+        assert addresses[i] == expected_address, addresses[i]
       self.wallet_address = expected_address
 
+      # Assert multisig paramaters M/N and "ready" for each wallet
       for i in range(N_total):
         res = self.wallet[i].is_multisig()
         assert res.multisig == True
@@ -156,6 +190,73 @@ class MultisigTest():
         assert res.threshold == M_threshold
         assert res.total == N_total
 
+    # We want to test if multisig wallets can receive normal transfers as well and mining transfers
+    def fund_addrs_with_normal_wallet(self, addrs):
+      print("Funding multisig wallets with normal wallet-to-wallet transfers")
+
+      # Generate normal deterministic wallet
+      normal_seed = 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted'
+      assert not hasattr(self, 'wallet') or not self.wallet
+      self.wallet = [Wallet(idx = 0)]
+      res = self.wallet[0].restore_deterministic_wallet(seed = normal_seed)
+      assert res.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm'
+
+      self.wallet[0].refresh()
+
+      # Check that we own enough spendable enotes
+      res = self.wallet[0].incoming_transfers(transfer_type = 'available')
+      assert 'transfers' in res
+      num_outs_spendable = 0
+      min_out_amount = None
+      for t in res.transfers:
+          if not t.spent:
+            num_outs_spendable += 1
+            min_out_amount = min(min_out_amount, t.amount) if min_out_amount is not None else t.amount
+      assert num_outs_spendable >= 2 * len(addrs)
+
+      # Transfer to addrs and mine to confirm tx
+      dsts = [{'address': addr, 'amount': int(min_out_amount * 0.95)} for addr in addrs]
+      res = self.wallet[0].transfer(dsts, get_tx_metadata = True)
+      tx_hex = res.tx_metadata
+      res = self.wallet[0].relay_tx(tx_hex)
+      self.mine('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 10)
+
+    def remake_some_multisig_wallets_by_multsig_seed(self, threshold):
+      N = len(self.wallet)
+      num_signers_to_remake = random.randint(1, N) # Do at least one
+      signers_to_remake = list(range(N))
+      random.shuffle(signers_to_remake)
+      signers_to_remake = signers_to_remake[:num_signers_to_remake]
+
+      for i in signers_to_remake:
+        print("Remaking {}/{} multsig wallet from multisig seed: #{}".format(threshold, N, i+1))
+
+        otherwise_unused_seed = \
+          'factual wiggle awakened maul sash biscuit pause reinvest fonts sleepless knowledge tossed jewels request gusts dagger gumball onward dotted amended powder cynical strained topic request'
+
+        # Get information about wallet, will compare against later
+        old_viewkey = self.wallet[i].query_key('view_key').key
+        old_spendkey = self.wallet[i].query_key('spend_key').key
+        old_multisig_seed = self.wallet[i].query_key('mnemonic').key
+
+        # Close old wallet and restore w/ random seed so we know that restoring actually did something
+        self.wallet[i].close_wallet()
+        self.wallet[i].restore_deterministic_wallet(seed=otherwise_unused_seed)
+        mid_viewkey = self.wallet[i].query_key('view_key').key
+        assert mid_viewkey != old_viewkey
+
+        # Now restore w/ old multisig seed and check against original
+        self.wallet[i].close_wallet()
+        self.wallet[i].restore_deterministic_wallet(seed=old_multisig_seed, enable_multisig_experimental=True)
+        new_viewkey = self.wallet[i].query_key('view_key').key
+        new_spendkey = self.wallet[i].query_key('spend_key').key
+        new_multisig_seed = self.wallet[i].query_key('mnemonic').key
+        assert new_viewkey == old_viewkey
+        assert new_spendkey == old_spendkey
+        assert new_multisig_seed == old_multisig_seed
+
+        self.wallet[i].refresh()
+
     def test_states(self):
         print('Testing multisig states')
         seeds = [
@@ -248,7 +349,7 @@ class MultisigTest():
           assert res.n_outputs == expected_outputs
 
     def transfer(self, signers):
-        assert len(signers) >= 2
+        assert len(signers) >= 1
 
         daemon = Daemon()
 
@@ -316,6 +417,104 @@ class MultisigTest():
         daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 1)
         return txid
 
+    def try_transfer_frozen(self, signers):
+        assert len(signers) >= 2
+
+        daemon = Daemon()
+
+        print("Creating multisig transaction from wallet " + str(signers[0]))
+
+        dst = {'address': '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 'amount': 1000000000000}
+        res = self.wallet[signers[0]].transfer([dst])
+        assert len(res.tx_hash) == 0 # not known yet
+        txid = res.tx_hash
+        assert len(res.tx_key) == 32*2
+        assert res.amount > 0
+        amount = res.amount
+        assert res.fee > 0
+        fee = res.fee
+        assert len(res.tx_blob) == 0
+        assert len(res.tx_metadata) == 0
+        assert len(res.multisig_txset) > 0
+        assert len(res.unsigned_txset) == 0
+        spent_key_images = res.spent_key_images.key_images
+        multisig_txset = res.multisig_txset
+
+        daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 1)
+        for i in range(len(self.wallet)):
+          self.wallet[i].refresh()
+
+        for i in range(len(signers[1:])):
+          # Check that each signer wallet has key image and it is not frozen
+          for ki in spent_key_images:
+            frozen = self.wallet[signers[i+1]].frozen(ki).frozen
+            assert not frozen
+
+        # Freeze key image involved with initiated transfer
+        assert len(spent_key_images)
+        ki0 = spent_key_images[0]
+        print("Freezing involved key image:", ki0)
+        self.wallet[signers[1]].freeze(ki0)
+        frozen = self.wallet[signers[1]].frozen(ki).frozen
+        assert frozen
+
+        # Try signing multisig (this operation should fail b/c of the frozen key image)
+        print("Attemping to sign with frozen key image. This should fail")
+        try:
+          res = self.wallet[signers[1]].sign_multisig(multisig_txset)
+          raise ValueError('sign_multisig should not have succeeded w/ frozen enotes')
+        except AssertionError:
+          pass
+
+        # Thaw key image and continue transfer as normal
+        print("Thawing key image and continuing transfer as normal")
+        self.wallet[signers[1]].thaw(ki0)
+        frozen = self.wallet[signers[1]].frozen(ki).frozen
+        assert not frozen
+
+        for i in range(len(signers[1:])):
+          print('Signing multisig transaction with wallet ' + str(signers[i+1]))
+          res = self.wallet[signers[i+1]].describe_transfer(multisig_txset = multisig_txset)
+          assert len(res.desc) == 1
+          desc = res.desc[0]
+          assert desc.amount_in >= amount + fee
+          assert desc.amount_out == desc.amount_in - fee
+          assert desc.ring_size == 16
+          assert desc.unlock_time == 0
+          assert not 'payment_id' in desc or desc.payment_id in ['', '0000000000000000']
+          assert desc.change_amount == desc.amount_in - 1000000000000 - fee
+          assert desc.change_address == self.wallet_address
+          assert desc.fee == fee
+          assert len(desc.recipients) == 1
+          rec = desc.recipients[0]
+          assert rec.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm'
+          assert rec.amount == 1000000000000
+
+          res = self.wallet[signers[i+1]].sign_multisig(multisig_txset)
+          multisig_txset = res.tx_data_hex
+          assert len(res.tx_hash_list if 'tx_hash_list' in res else []) == (i == len(signers[1:]) - 1)
+
+          if i < len(signers[1:]) - 1:
+            print('Submitting multisig transaction prematurely with wallet ' + str(signers[-1]))
+            ok = False
+            try: self.wallet[signers[-1]].submit_multisig(multisig_txset)
+            except: ok = True
+            assert ok
+
+        print('Submitting multisig transaction with wallet ' + str(signers[-1]))
+        res = self.wallet[signers[-1]].submit_multisig(multisig_txset)
+        assert len(res.tx_hash_list) == 1
+        txid = res.tx_hash_list[0]
+
+        for i in range(len(self.wallet)):
+          self.wallet[i].refresh()
+          res = self.wallet[i].get_transfers()
+          assert len([x for x in (res['pending'] if 'pending' in res else []) if x.txid == txid]) == (1 if i == signers[-1] else 0)
+          assert len([x for x in (res['out'] if 'out' in res else []) if x.txid == txid]) == 0
+
+        daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 1)
+        return txid
+
     def check_transaction(self, txid):
         for i in range(len(self.wallet)):
           self.wallet[i].refresh()

tests/functional_tests/transfer.py

@@ -30,6 +30,9 @@
 
 from __future__ import print_function
 import json
+import pprint
+from deepdiff import DeepDiff
+pp = pprint.PrettyPrinter(indent=2)
 
 """Test simple transfers
 """
@@ -37,6 +40,12 @@ import json
 from framework.daemon import Daemon
 from framework.wallet import Wallet
 
+seeds = [
+    'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted',
+    'peeled mixture ionic radar utopia puddle buying illness nuns gadget river spout cavernous bounced paradise drunk looking cottage jump tequila melting went winter adjust spout',
+    'dilute gutter certain antics pamphlet macro enjoy left slid guarded bogeys upload nineteen bomb jubilee enhanced irritate turnip eggs swung jukebox loudly reduce sedan slid',
+]
+
 class TransferTest():
     def run_test(self):
         self.reset()
@@ -52,6 +61,7 @@ class TransferTest():
         self.check_tx_notes()
         self.check_rescan()
         self.check_is_key_image_spent()
+        self.check_scan_tx()
 
     def reset(self):
         print('Resetting blockchain')
@@ -62,11 +72,6 @@ class TransferTest():
 
     def create(self):
         print('Creating wallets')
-        seeds = [
-          'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted',
-          'peeled mixture ionic radar utopia puddle buying illness nuns gadget river spout cavernous bounced paradise drunk looking cottage jump tequila melting went winter adjust spout',
-          'dilute gutter certain antics pamphlet macro enjoy left slid guarded bogeys upload nineteen bomb jubilee enhanced irritate turnip eggs swung jukebox loudly reduce sedan slid',
-        ]
         self.wallet = [None] * len(seeds)
         for i in range(len(seeds)):
             self.wallet[i] = Wallet(idx = i)
@@ -829,6 +834,217 @@ class TransferTest():
         res = daemon.is_key_image_spent(ki)
         assert res.spent_status == expected
 
+    def check_scan_tx(self):
+        daemon = Daemon()
+
+        print('Testing scan_tx')
+
+        def diff_transfers(actual_transfers, expected_transfers):
+            diff = DeepDiff(actual_transfers, expected_transfers)
+            if diff != {}:
+                pp.pprint(diff)
+            assert diff == {}
+
+        # set up sender_wallet
+        sender_wallet = self.wallet[0]
+        try: sender_wallet.close_wallet()
+        except: pass
+        sender_wallet.restore_deterministic_wallet(seed = seeds[0])
+        sender_wallet.auto_refresh(enable = False)
+        sender_wallet.refresh()
+        res = sender_wallet.get_transfers()
+        out_len = 0 if 'out' not in res else len(res.out)
+        sender_starting_balance = sender_wallet.get_balance().balance
+        amount = 1000000000000
+        assert sender_starting_balance > amount
+
+        # set up receiver_wallet
+        receiver_wallet = self.wallet[1]
+        try: receiver_wallet.close_wallet()
+        except: pass
+        receiver_wallet.restore_deterministic_wallet(seed = seeds[1])
+        receiver_wallet.auto_refresh(enable = False)
+        receiver_wallet.refresh()
+        res = receiver_wallet.get_transfers()
+        in_len = 0 if 'in' not in res else len(res['in'])
+        receiver_starting_balance = receiver_wallet.get_balance().balance
+
+        # transfer from sender_wallet to receiver_wallet
+        dst = {'address': '44Kbx4sJ7JDRDV5aAhLJzQCjDz2ViLRduE3ijDZu3osWKBjMGkV1XPk4pfDUMqt1Aiezvephdqm6YD19GKFD9ZcXVUTp6BW', 'amount': amount}
+        res = sender_wallet.transfer([dst])
+        assert len(res.tx_hash) == 32*2
+        txid = res.tx_hash
+        assert res.amount == amount
+        assert res.fee > 0
+        fee = res.fee
+
+        expected_sender_balance = sender_starting_balance - (amount + fee)
+        expected_receiver_balance = receiver_starting_balance + amount
+
+        test = 'Checking scan_tx on outgoing pool tx'
+        for attempt in range(2): # test re-scanning
+            print(test + ' (' + ('first attempt' if attempt == 0 else 're-scanning tx') + ')')
+            sender_wallet.scan_tx([txid])
+            res = sender_wallet.get_transfers()
+            assert 'pool' not in res or len(res.pool) == 0
+            if out_len == 0:
+                assert 'out' not in res
+            else:
+                assert len(res.out) == out_len
+            assert len(res.pending) == 1
+            tx = [x for x in res.pending if x.txid == txid]
+            assert len(tx) == 1
+            tx = tx[0]
+            assert tx.amount == amount
+            assert tx.fee == fee
+            assert len(tx.destinations) == 1
+            assert tx.destinations[0].amount == amount
+            assert tx.destinations[0].address == dst['address']
+            assert sender_wallet.get_balance().balance == expected_sender_balance
+
+        test = 'Checking scan_tx on incoming pool tx'
+        for attempt in range(2): # test re-scanning
+            print(test + ' (' + ('first attempt' if attempt == 0 else 're-scanning tx') + ')')
+            receiver_wallet.scan_tx([txid])
+            res = receiver_wallet.get_transfers()
+            assert 'pending' not in res or len(res.pending) == 0
+            if in_len == 0:
+                assert 'in' not in res
+            else:
+                assert len(res['in']) == in_len
+            assert 'pool' in res and len(res.pool) == 1
+            tx = [x for x in res.pool if x.txid == txid]
+            assert len(tx) == 1
+            tx = tx[0]
+            assert tx.amount == amount
+            assert tx.fee == fee
+            assert receiver_wallet.get_balance().balance == expected_receiver_balance
+
+        # mine the tx
+        height = daemon.generateblocks(dst['address'], 1).height
+        block_header = daemon.getblockheaderbyheight(height = height).block_header
+        miner_txid = block_header.miner_tx_hash
+        expected_receiver_balance += block_header.reward
+
+        print('Checking scan_tx on outgoing tx before refresh')
+        sender_wallet.scan_tx([txid])
+        res = sender_wallet.get_transfers()
+        assert 'pending' not in res or len(res.pending) == 0
+        assert 'pool' not in res or len (res.pool) == 0
+        assert len(res.out) == out_len + 1
+        tx = [x for x in res.out if x.txid == txid]
+        assert len(tx) == 1
+        tx = tx[0]
+        assert tx.amount == amount
+        assert tx.fee == fee
+        assert len(tx.destinations) == 1
+        assert tx.destinations[0].amount == amount
+        assert tx.destinations[0].address == dst['address']
+        assert sender_wallet.get_balance().balance == expected_sender_balance
+
+        print('Checking scan_tx on outgoing tx after refresh')
+        sender_wallet.refresh()
+        sender_wallet.scan_tx([txid])
+        diff_transfers(sender_wallet.get_transfers(), res)
+        assert sender_wallet.get_balance().balance == expected_sender_balance
+
+        print("Checking scan_tx on outgoing wallet's earliest tx")
+        earliest_height = height
+        earliest_txid = txid
+        for x in res['in']:
+            if x.height < earliest_height:
+                earliest_height = x.height
+                earliest_txid = x.txid
+        sender_wallet.scan_tx([earliest_txid])
+        diff_transfers(sender_wallet.get_transfers(), res)
+        assert sender_wallet.get_balance().balance == expected_sender_balance
+
+        test = 'Checking scan_tx on outgoing wallet restored at current height'
+        for i, out_tx in enumerate(res.out):
+            if 'destinations' in out_tx:
+                del res.out[i]['destinations'] # destinations are not expected after wallet restore
+        out_txids = [x.txid for x in res.out]
+        in_txids = [x.txid for x in res['in']]
+        all_txs = out_txids + in_txids
+        for test_type in ["all txs", "incoming first", "duplicates within", "duplicates across"]:
+            print(test + ' (' + test_type + ')')
+            sender_wallet.close_wallet()
+            sender_wallet.restore_deterministic_wallet(seed = seeds[0], restore_height = height)
+            assert sender_wallet.get_transfers() == {}
+            if test_type == "all txs":
+                sender_wallet.scan_tx(all_txs)
+            elif test_type == "incoming first":
+                sender_wallet.scan_tx(in_txids)
+                sender_wallet.scan_tx(out_txids)
+            # TODO: test_type == "outgoing first"
+            elif test_type == "duplicates within":
+                sender_wallet.scan_tx(all_txs + all_txs)
+            elif test_type == "duplicates across":
+                sender_wallet.scan_tx(all_txs)
+                sender_wallet.scan_tx(all_txs)
+            else:
+                assert True == False
+            diff_transfers(sender_wallet.get_transfers(), res)
+            assert sender_wallet.get_balance().balance == expected_sender_balance
+
+        print('Sanity check against outgoing wallet restored at height 0')
+        sender_wallet.close_wallet()
+        sender_wallet.restore_deterministic_wallet(seed = seeds[0], restore_height = 0)
+        sender_wallet.refresh()
+        diff_transfers(sender_wallet.get_transfers(), res)
+        assert sender_wallet.get_balance().balance == expected_sender_balance
+
+        print('Checking scan_tx on incoming txs before refresh')
+        receiver_wallet.scan_tx([txid, miner_txid])
+        res = receiver_wallet.get_transfers()
+        assert 'pending' not in res or len(res.pending) == 0
+        assert 'pool' not in res or len (res.pool) == 0
+        assert len(res['in']) == in_len + 2
+        tx = [x for x in res['in'] if x.txid == txid]
+        assert len(tx) == 1
+        tx = tx[0]
+        assert tx.amount == amount
+        assert tx.fee == fee
+        assert receiver_wallet.get_balance().balance == expected_receiver_balance
+
+        print('Checking scan_tx on incoming txs after refresh')
+        receiver_wallet.refresh()
+        receiver_wallet.scan_tx([txid, miner_txid])
+        diff_transfers(receiver_wallet.get_transfers(), res)
+        assert receiver_wallet.get_balance().balance == expected_receiver_balance
+
+        print("Checking scan_tx on incoming wallet's earliest tx")
+        earliest_height = height
+        earliest_txid = txid
+        for x in res['in']:
+            if x.height < earliest_height:
+                earliest_height = x.height
+                earliest_txid = x.txid
+        receiver_wallet.scan_tx([earliest_txid])
+        diff_transfers(receiver_wallet.get_transfers(), res)
+        assert receiver_wallet.get_balance().balance == expected_receiver_balance
+
+        print('Checking scan_tx on incoming wallet restored at current height')
+        txids = [x.txid for x in res['in']]
+        if 'out' in res:
+            txids = txids + [x.txid for x in res.out]
+        receiver_wallet.close_wallet()
+        receiver_wallet.restore_deterministic_wallet(seed = seeds[1], restore_height = height)
+        assert receiver_wallet.get_transfers() == {}
+        receiver_wallet.scan_tx(txids)
+        if 'out' in res:
+            for i, out_tx in enumerate(res.out):
+                if 'destinations' in out_tx:
+                    del res.out[i]['destinations'] # destinations are not expected after wallet restore
+        diff_transfers(receiver_wallet.get_transfers(), res)
+        assert receiver_wallet.get_balance().balance == expected_receiver_balance
+
+        print('Sanity check against incoming wallet restored at height 0')
+        receiver_wallet.close_wallet()
+        receiver_wallet.restore_deterministic_wallet(seed = seeds[1], restore_height = 0)
+        receiver_wallet.refresh()
+        diff_transfers(receiver_wallet.get_transfers(), res)
+        assert receiver_wallet.get_balance().balance == expected_receiver_balance
 
 if __name__ == '__main__':
     TransferTest().run_test()

tests/unit_tests/CMakeLists.txt

@@ -90,12 +90,14 @@ set(unit_tests_sources
   hardfork.cpp
   unbound.cpp
   uri.cpp
+  util.cpp
   varint.cpp
   ver_rct_non_semantics_simple_cached.cpp
   ringct.cpp
   output_selection.cpp
   vercmp.cpp
   ringdb.cpp
+  wallet_storage.cpp
   wipeable_string.cpp
   is_hdd.cpp
   aligned.cpp

tests/unit_tests/long_term_block_weight.cpp

@@ -407,3 +407,38 @@ TEST(long_term_block_weight, long_growth_spike_and_drop)
   ASSERT_GT(long_term_effective_median_block_weight, 300000 * 1.07);
   ASSERT_LT(long_term_effective_median_block_weight, 300000 * 1.09);
 }
+
+TEST(long_term_block_weight, cache_matches_true_value)
+{
+  PREFIX(16);
+
+  // Add big blocks to increase the block weight limit 
+  for (uint64_t h = 0; h <= 2000; ++h)
+  {
+    size_t w = bc->get_current_cumulative_block_weight_limit();
+    uint64_t ltw = bc->get_next_long_term_block_weight(w);
+    bc->get_db().add_block(std::make_pair(cryptonote::block(), ""), w, ltw, h, h, {});
+    bc->update_next_cumulative_weight_limit();
+  }
+
+  ASSERT_GT(bc->get_current_cumulative_block_weight_limit() * 10/17 , 300000);
+
+  // Add small blocks to the top of the chain
+  for (uint64_t h = 2000; h <= 5001; ++h)
+  {
+    size_t w = (bc->get_current_cumulative_block_weight_median() * 10/17) - 1000;
+    uint64_t ltw = bc->get_next_long_term_block_weight(w);
+    bc->get_db().add_block(std::make_pair(cryptonote::block(), ""), w, ltw, h, h, {});
+    bc->update_next_cumulative_weight_limit();
+  }
+ 
+  // get the weight limit
+  uint64_t weight_limit = bc->get_current_cumulative_block_weight_limit();
+  // refresh the cache
+  bc->m_long_term_block_weights_cache_rolling_median.clear();
+  bc->get_long_term_block_weight_median(bc->get_db().height() - TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW, TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW);
+  bc->update_next_cumulative_weight_limit();
+
+  // make sure the weight limit is the same
+  ASSERT_EQ(weight_limit, bc->get_current_cumulative_block_weight_limit());
+}

tests/unit_tests/util.cpp

@@ -0,0 +1,50 @@
+// Copyright (c) 2023-2023, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "gtest/gtest.h"
+
+#include "common/util.h"
+
+TEST(LocalAddress, localhost) { ASSERT_TRUE(tools::is_local_address("localhost")); }
+TEST(LocalAddress, localhost_port) { ASSERT_TRUE(tools::is_local_address("localhost:18081")); }
+TEST(LocalAddress, localhost_suffix) { ASSERT_TRUE(tools::is_local_address("test.localhost")); }
+TEST(LocalAddress, loopback) { ASSERT_TRUE(tools::is_local_address("127.0.0.1")); }
+TEST(LocalAddress, loopback_port) { ASSERT_TRUE(tools::is_local_address("127.0.0.1:18081")); }
+TEST(LocalAddress, loopback_protocol) { ASSERT_TRUE(tools::is_local_address("http://127.0.0.1")); }
+TEST(LocalAddress, loopback_hi) { ASSERT_TRUE(tools::is_local_address("127.255.255.255")); }
+TEST(LocalAddress, loopback_lo) { ASSERT_TRUE(tools::is_local_address("127.0.0.0")); }
+TEST(LocalAddress, loopback_ipv6) { ASSERT_TRUE(tools::is_local_address("[0:0:0:0:0:0:0:1]")); }
+
+TEST(LocalAddress, onion) { ASSERT_FALSE(tools::is_local_address("vww6ybal4bd7szmgncyruucpgfkqahzddi37ktceo3ah7ngmcopnpyyd.onion")); }
+TEST(LocalAddress, i2p) { ASSERT_FALSE(tools::is_local_address("xmrto2bturnore26xmrto2bturnore26xmrto2bturnore26xmr2.b32.i2p")); }
+TEST(LocalAddress, valid_ip) { ASSERT_FALSE(tools::is_local_address("1.2.3.4")); }
+TEST(LocalAddress, valid_ipv6) { ASSERT_FALSE(tools::is_local_address("[0:0:0:0:0:0:0:2]")); }
+TEST(LocalAddress, valid_domain) { ASSERT_FALSE(tools::is_local_address("getmonero.org")); }
+TEST(LocalAddress, local_prefix) { ASSERT_FALSE(tools::is_local_address("localhost.com")); }
+TEST(LocalAddress, invalid) { ASSERT_FALSE(tools::is_local_address("test")); }
+TEST(LocalAddress, empty) { ASSERT_FALSE(tools::is_local_address("")); }

tests/unit_tests/wallet_storage.cpp

@@ -0,0 +1,266 @@
+// Copyright (c) 2023, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "unit_tests_utils.h"
+#include "gtest/gtest.h"
+
+#include "file_io_utils.h"
+#include "wallet/wallet2.h"
+
+using namespace boost::filesystem;
+using namespace epee::file_io_utils;
+
+static constexpr const char WALLET_00fd416a_PRIMARY_ADDRESS[] =
+    "45p2SngJAPSJbqSiUvYfS3BfhEdxZmv8pDt25oW1LzxrZv9Uq6ARagiFViMGUE3gJk5VPWingCXVf1p2tyAy6SUeSHPhbve";
+
+TEST(wallet_storage, store_to_file2file)
+{
+    const path source_wallet_file = unit_test::data_dir / "wallet_00fd416a";
+    const path interm_wallet_file = unit_test::data_dir / "wallet_00fd416a_copy_file2file";
+    const path target_wallet_file = unit_test::data_dir / "wallet_00fd416a_new_file2file";
+
+    ASSERT_TRUE(is_file_exist(source_wallet_file.string()));
+    ASSERT_TRUE(is_file_exist(source_wallet_file.string() + ".keys"));
+
+    copy_file(source_wallet_file, interm_wallet_file, copy_option::overwrite_if_exists);
+    copy_file(source_wallet_file.string() + ".keys", interm_wallet_file.string() + ".keys", copy_option::overwrite_if_exists);
+
+    ASSERT_TRUE(is_file_exist(interm_wallet_file.string()));
+    ASSERT_TRUE(is_file_exist(interm_wallet_file.string() + ".keys"));
+
+    if (is_file_exist(target_wallet_file.string()))
+        remove(target_wallet_file);
+    if (is_file_exist(target_wallet_file.string() + ".keys"))
+        remove(target_wallet_file.string() + ".keys");
+    ASSERT_FALSE(is_file_exist(target_wallet_file.string()));
+    ASSERT_FALSE(is_file_exist(target_wallet_file.string() + ".keys"));
+
+    epee::wipeable_string password("beepbeep");
+
+    const auto files_are_expected = [&]()
+    {
+        EXPECT_FALSE(is_file_exist(interm_wallet_file.string()));
+        EXPECT_FALSE(is_file_exist(interm_wallet_file.string() + ".keys"));
+        EXPECT_TRUE(is_file_exist(target_wallet_file.string()));
+        EXPECT_TRUE(is_file_exist(target_wallet_file.string() + ".keys"));
+    };
+
+    {
+        tools::wallet2 w;
+        w.load(interm_wallet_file.string(), password);
+        const std::string primary_address = w.get_address_as_str();
+        EXPECT_EQ(WALLET_00fd416a_PRIMARY_ADDRESS, primary_address);
+        w.store_to(target_wallet_file.string(), password);
+        files_are_expected();
+    }
+
+    files_are_expected();
+
+    {
+        tools::wallet2 w;
+        w.load(target_wallet_file.string(), password);
+        const std::string primary_address = w.get_address_as_str();
+        EXPECT_EQ(WALLET_00fd416a_PRIMARY_ADDRESS, primary_address);
+        w.store_to("", "");
+        files_are_expected();
+    }
+
+    files_are_expected();
+}
+
+TEST(wallet_storage, store_to_mem2file)
+{
+    const path target_wallet_file = unit_test::data_dir / "wallet_mem2file";
+
+    if (is_file_exist(target_wallet_file.string()))
+        remove(target_wallet_file);
+    if (is_file_exist(target_wallet_file.string() + ".keys"))
+        remove(target_wallet_file.string() + ".keys");
+    ASSERT_FALSE(is_file_exist(target_wallet_file.string()));
+    ASSERT_FALSE(is_file_exist(target_wallet_file.string() + ".keys"));
+
+    epee::wipeable_string password("beepbeep2");
+
+    {
+        tools::wallet2 w;
+        w.generate("", password);
+        w.store_to(target_wallet_file.string(), password);
+
+        EXPECT_TRUE(is_file_exist(target_wallet_file.string()));
+        EXPECT_TRUE(is_file_exist(target_wallet_file.string() + ".keys"));
+    }
+
+    EXPECT_TRUE(is_file_exist(target_wallet_file.string()));
+    EXPECT_TRUE(is_file_exist(target_wallet_file.string() + ".keys"));
+
+    {
+        tools::wallet2 w;
+        w.load(target_wallet_file.string(), password);
+
+        EXPECT_TRUE(is_file_exist(target_wallet_file.string()));
+        EXPECT_TRUE(is_file_exist(target_wallet_file.string() + ".keys"));
+    }
+
+    EXPECT_TRUE(is_file_exist(target_wallet_file.string()));
+    EXPECT_TRUE(is_file_exist(target_wallet_file.string() + ".keys"));
+}
+
+TEST(wallet_storage, change_password_same_file)
+{
+    const path source_wallet_file = unit_test::data_dir / "wallet_00fd416a";
+    const path interm_wallet_file = unit_test::data_dir / "wallet_00fd416a_copy_change_password_same";
+
+    ASSERT_TRUE(is_file_exist(source_wallet_file.string()));
+    ASSERT_TRUE(is_file_exist(source_wallet_file.string() + ".keys"));
+
+    copy_file(source_wallet_file, interm_wallet_file, copy_option::overwrite_if_exists);
+    copy_file(source_wallet_file.string() + ".keys", interm_wallet_file.string() + ".keys", copy_option::overwrite_if_exists);
+
+    ASSERT_TRUE(is_file_exist(interm_wallet_file.string()));
+    ASSERT_TRUE(is_file_exist(interm_wallet_file.string() + ".keys"));
+
+    epee::wipeable_string old_password("beepbeep");
+    epee::wipeable_string new_password("meepmeep");
+
+    {
+        tools::wallet2 w;
+        w.load(interm_wallet_file.string(), old_password);
+        const std::string primary_address = w.get_address_as_str();
+        EXPECT_EQ(WALLET_00fd416a_PRIMARY_ADDRESS, primary_address);
+        w.change_password(w.get_wallet_file(), old_password, new_password);
+    }
+
+    {
+        tools::wallet2 w;
+        w.load(interm_wallet_file.string(), new_password);
+        const std::string primary_address = w.get_address_as_str();
+        EXPECT_EQ(WALLET_00fd416a_PRIMARY_ADDRESS, primary_address);
+    }
+
+    {
+        tools::wallet2 w;
+        EXPECT_THROW(w.load(interm_wallet_file.string(), old_password), tools::error::invalid_password);
+    }
+}
+
+TEST(wallet_storage, change_password_different_file)
+{
+    const path source_wallet_file = unit_test::data_dir / "wallet_00fd416a";
+    const path interm_wallet_file = unit_test::data_dir / "wallet_00fd416a_copy_change_password_diff";
+    const path target_wallet_file = unit_test::data_dir / "wallet_00fd416a_new_change_password_diff";
+
+    ASSERT_TRUE(is_file_exist(source_wallet_file.string()));
+    ASSERT_TRUE(is_file_exist(source_wallet_file.string() + ".keys"));
+
+    copy_file(source_wallet_file, interm_wallet_file, copy_option::overwrite_if_exists);
+    copy_file(source_wallet_file.string() + ".keys", interm_wallet_file.string() + ".keys", copy_option::overwrite_if_exists);
+
+    ASSERT_TRUE(is_file_exist(interm_wallet_file.string()));
+    ASSERT_TRUE(is_file_exist(interm_wallet_file.string() + ".keys"));
+
+    if (is_file_exist(target_wallet_file.string()))
+        remove(target_wallet_file);
+    if (is_file_exist(target_wallet_file.string() + ".keys"))
+        remove(target_wallet_file.string() + ".keys");
+    ASSERT_FALSE(is_file_exist(target_wallet_file.string()));
+    ASSERT_FALSE(is_file_exist(target_wallet_file.string() + ".keys"));
+
+    epee::wipeable_string old_password("beepbeep");
+    epee::wipeable_string new_password("meepmeep");
+
+    {
+        tools::wallet2 w;
+        w.load(interm_wallet_file.string(), old_password);
+        const std::string primary_address = w.get_address_as_str();
+        EXPECT_EQ(WALLET_00fd416a_PRIMARY_ADDRESS, primary_address);
+        w.change_password(target_wallet_file.string(), old_password, new_password);
+    }
+
+    EXPECT_FALSE(is_file_exist(interm_wallet_file.string()));
+    EXPECT_FALSE(is_file_exist(interm_wallet_file.string() + ".keys"));
+    EXPECT_TRUE(is_file_exist(target_wallet_file.string()));
+    EXPECT_TRUE(is_file_exist(target_wallet_file.string() + ".keys"));
+
+    {
+        tools::wallet2 w;
+        w.load(target_wallet_file.string(), new_password);
+        const std::string primary_address = w.get_address_as_str();
+        EXPECT_EQ(WALLET_00fd416a_PRIMARY_ADDRESS, primary_address);
+    }
+}
+
+TEST(wallet_storage, change_password_in_memory)
+{
+    const epee::wipeable_string password1("monero");
+    const epee::wipeable_string password2("means money");
+    const epee::wipeable_string password_wrong("is traceable");
+
+    tools::wallet2 w;
+    w.generate("", password1);
+    const std::string primary_address_1 = w.get_address_as_str();
+    w.change_password("", password1, password2);
+    const std::string primary_address_2 = w.get_address_as_str();
+    EXPECT_EQ(primary_address_1, primary_address_2);
+
+    EXPECT_THROW(w.change_password("", password_wrong, password1), tools::error::invalid_password);
+}
+
+TEST(wallet_storage, change_password_mem2file)
+{
+    const path target_wallet_file = unit_test::data_dir / "wallet_change_password_mem2file";
+
+    if (is_file_exist(target_wallet_file.string()))
+        remove(target_wallet_file);
+    if (is_file_exist(target_wallet_file.string() + ".keys"))
+        remove(target_wallet_file.string() + ".keys");
+    ASSERT_FALSE(is_file_exist(target_wallet_file.string()));
+    ASSERT_FALSE(is_file_exist(target_wallet_file.string() + ".keys"));
+
+    const epee::wipeable_string password1("https://safecurves.cr.yp.to/rigid.html");
+    const epee::wipeable_string password2(
+        "https://csrc.nist.gov/csrc/media/projects/crypto-standards-development-process/documents/dualec_in_x982_and_sp800-90.pdf");
+    
+    std::string primary_address_1, primary_address_2;
+    {
+        tools::wallet2 w;
+        w.generate("", password1);
+        primary_address_1 = w.get_address_as_str();
+        w.change_password(target_wallet_file.string(), password1, password2);
+    }
+
+    EXPECT_TRUE(is_file_exist(target_wallet_file.string()));
+    EXPECT_TRUE(is_file_exist(target_wallet_file.string() + ".keys"));
+
+    {
+        tools::wallet2 w;
+        w.load(target_wallet_file.string(), password2);
+        primary_address_2 = w.get_address_as_str();
+    }
+
+    EXPECT_EQ(primary_address_1, primary_address_2);
+}

utils/gpg_keys/wowario.asc

@@ -1,45 +1,52 @@
 -----BEGIN PGP PUBLIC KEY BLOCK-----
 
-mQINBFrF9ZEBEACU6C2YnUT2BFwC9e8S05lTrgggOoyjx34AxJd/QE5mY4fmRutB
-EwIrvZgti+t0qdsYis5QA/BXjn4WVjZzyjpQIV3C+eB3hsodkx7UzzPAYK4pmBZP
-dMNMdvvQjsuJbmb5GeRV0g97MZtXbZ+BPc8CdE0pWwCPuqVNjH5LcQbKaz8ZgGMh
-MR1JfpkrIT/pq6fzOjPBsyJOQ0fbMMwI7tiI5cCnnnz04VkoYm+mPyUEx+euGQ4j
-lYOOieYWu57IkpzXbwhVrfwMk5J/NhEc+647Mnzs/+6riJz9pU+/Pn3qSJFID5WL
-79ToI5W3k9FjXgnIiAlXAhQneXKjcdaeUjJj6pMg35VaOX8r2iativQyzEuXNZ2S
-reOaM8aKTGmY3Ice+UHmEuoJziAX22nZXNJVjIUKa20lCMBHOWbaPOHTWJ4K443L
-epHNhmLVxvLnINsA5PdaskryP858B1NiGjXi9i4+fG9MtjeU8xxcvOJ2bDelDR+1
-mlruNN27KdBNpIXwYcIr7q3rFekiwfForrNUrXc3jLrlMFH8FKOSTn2OLwbrhHxz
-EISIxHwfs6YjYIdy0d0kZiQhWQ4mYDJRcSRpieu9P1HOwYTV77uBem7e4aSlx+x2
-8izX6wsnCQbxx024VvL03EEkfLV1GqtEsQbec3opIaQ/oUGB/QFbeM/MpQARAQAB
-tCBXb3dhcmlvIDx3b3dhcmlvQHByb3Rvbm1haWwuY29tPokCNwQTAQgAIQUCWsX1
-kQIbAwULCQgHAgYVCAkKCwIEFgIDAQIeAQIXgAAKCRAk3L52LenBEV9GEACJrjBe
-GbJTGkjczsblQieK3KIRmE9erUPy0AmywS2a1YhCO4U0cPC0mHnuIltPnSaA9k14
-OkPKiKoGrxl9iE27r8bwpH/cs6KFjrnjrgxeTOdWemDs17I+JPpNqdNyIxc9HMZ/
-tvIHK7iU2312DgB2AiCsTOMcNng1ysF+tLna9sbTlqSE9DkJSgM1CRZKoLEO2pS8
-M56EQt5yD6qpV8wUs23q0z/VGs8ckONQ4TG5FXl9frFgJr1bgWwNrJiweFX7oZKg
-bp3xepDwVZ0rYtdKciqlKceC3Om8zzFuqRSDNzadsGdwqXbvT3WS3XD8LMhc9wyx
-ZeKFQY4fctJF6yBpqvV1Qo992WNEwSjxMitypeSKTkieSkVICjGjtuVqAlmgNUrF
-4bK0qTxmkA7c+mbBtURl+oLPyy37uNo8XbL02nT+vZa5sAO76+gObFx4NgopFOlW
-OveF1d7Ve/cLO3/7+VarVq4ZtObZ8LP2NInauqWu1HNZ7gQlEuuvBOVf4nX7J+TY
-6c+xju5ELHkEY4HRs5KoWUVjxe6zvcuIu43Oth49S76gKRwZ0EPIjOGzr0GT0DR+
-V0Nso74+wTl8ia70fhAltcCNg4/llnnpCfBZojHNTE/TPje7BNm3oEwNQ7g+MMqf
-LQ8WtsdX3gIzt3B1lNR2+DHlaK1sUNJkghkWK7kBDQRaxfWRAQgA3uathYQ1pKlY
-shJw/iXbkhoKHnEE9Indm1tNCh2djr2pnzBmWkbcTx8KYwXPVPV+WtLRoyl9woei
-5giA5xGfHm8/PhmNrzbzDCucz/Pl7VKjbBEWooIhTkvY3OhUJqzPM1+AknKjCeEt
-CU8ZEfIU+NsMzMgsvfZ51simgkUWHR3Zs295E7UTEDrEW8INgQIAXotFHyaBTvIc
-AN2YsU3OZpK6Krq1Us6KDvNurKSaO8VCSkVfCPoVzs1E2JLeA5r7MFxSobGWjibI
-W9KZr6taqgg8KAYGL3WuXjZ5J+8JC+GJkWVTh8ioeiUJOYt1JaedQxbn7dtKIUmZ
-1GpK7UCvjQARAQABiQIfBBgBCAAJBQJaxfWRAhsMAAoJECTcvnYt6cER5V4P/2k3
-x6c1vPVIr9PqedCFNuG2io1Hwe9JH0xxEDq2nKVlmsn70BgY0/2LjTI/HfuTUpmH
-PHIBmP11FwZz79x3at5Sj/W4TbGmsodZmmdPs8cyD2CLKiA1kuBPmO//5sN51B+R
-gRnDJegGBXUsibIR0rGJeIK88ayZlcoXEEijtUl+SFGGuHe91ehwJVh/Nmqq591Z
-xOXkkoiR87EDiK41wVIkNSyEZY6JQQKQGddYwwAU4jjYbdRhkxq8UUO16ySu8k8D
-wi67OfnxbeiPAqmaZM0M0fWbf0rjKW4dJGry4OeoTgi+uRrjyaHvAt1vQh6L1GmT
-NqyFZrIOQ/xPJK88p1US4EA67rxgdQTKtD8FmM5pCGlTj+kGMN4Zl1xh15ImAfp4
-5XRC1Je9aQ7waFaCxEYOdxPNcwINFEGmw8B58b1+w4m2eXx75hI+Z7cEs5vDa/Ou
-0Izgv0gLZ7b/C6haHGDCfNNlppYMk6EtsgW6sdR6ipNIqcSUdMAbsabm4xJtnvT5
-nI/UU9L6v1T0e034VEjPXQsULxOuelpMHLblahHcgOwUMgZt4sR8/VT/gkhtRooP
-sYvglcR0Y3nrPZseajpkK9vKcwv6zDuGBoTznE/0t1euEUluORjnWJrEy+ikYaqs
-wJkDQ4U/HgXfkHyQE38C5yqQvl9OGEBNP2oVHlj1
-=/hgc
+mQINBGROehcBEACXWWc6dHqCos1PmKI32iHi0jP3mYM3jU57YxbjwT78QEtEwSqf
+YklpXkgTYq7jexx2JElfegM6w1sPYarq1y051RjnCgzl32da5I506SMvcJTmXumV
+Rw6erPeDxAO74PflDSlALgtGOgbKhwwWRudbWgT5hKGkl62qy0mI6GStul0rbT+3
+gq77DCGyURfe1PG1pymhO5XVz3WGtOa12NvRA+3wGIcqIji2MbtXuOhGMg//kVI5
+m2vcfHyMMuQ01xUXRu57WxRujYaJ1RB4p86JCbDX3YU2XlzTxGAhqChDLuJGqo54
+AZMUWDceftXsAoOqH8Hwmm5gFkYSpMt86ZT+umvWygmxohD5k85MuRj4AGagFj/u
+CMcQjI/SN1UU/Qozg6VL/5FO8aH9IybDzX7eE3j0V/jTweStw1CIUajYgfemWOWl
+whLPBDflRz/8EEqTN0CaSSaiYiULZUiawBO/bRIiCO2Q6QrAi3KpPUhCwiw/Yecd
+rAMLH7bytpECDdbNonQ/VMxWwtWJQ87qBtWvHFQxXBKjyuANsKL9X7v3KcYOUdd2
+fSt7eqE9GDT4DbK6sTmuTpq2TgHXET0cA39+N2zxTh5xFupI/pi2iAHJ6hgIiQnn
+662TngjGOSFvrTV/51Ua0Vx8OCMJJOcRdOVaYzuzg9DsjVcJin3aRqUh4wARAQAB
+tCBXb3dhcmlvIDx3b3dhcmlvQHByb3Rvbm1haWwuY29tPokCVAQTAQgAPhYhBKs6
+L3JYGPz/J5SEHHk1BLRJxpIgBQJkTnoXAhsDBQkHhh77BQsJCAcCBhUKCQgLAgQW
+AgMBAh4BAheAAAoJEHk1BLRJxpIgwgEP/109vw1WXRh9EaRr3Y1+sBi+/PRQ5TCx
+UEcP9ru5sQPJ0BsZK8RYw0BNIfDQX9OB1k/AoiBelL+0EoDKvjXmwz9fPUmSVk5r
+3RzfClXTnxn4HXPKkSGMt4WBUnvohTexK7CPkb9xy+K0Jtx8XF1XiQLDFg2a9lBj
+IIX2H6aHn4VjdUBv7TrTCAI2Vg0cQUpeJUwyHH+lk0r2WM3zAxzS3Iy2yDDstNT8
+audXEX4BtJhyEU1m57jwgscrbTtgwYOAsaRLcnUaAFWhbov3IiGInk7N1fkMsuW5
+HE5RcegSZRS3X4o6O/nmwdSjCEB9weydOCPrtfdbvfvuTiMg/jZBikOk/Sj7FM/D
+eZKghSHpLbT/V3S76FyIcc/xFkUmR+2fGvCNjJ1Qn2lXTS8xcbyzqR4LZPeUGppV
+hvriilLnXSjyc60wuD3kmCCo1Zw4tNL8pr09BtVmScUy6eiwca8LLzvbbivqxF1g
+Mrkkv8yQE0ZwO1kgNSn+PSzUPbwAoklcyN5Rhr5DxZh0UudiH5Jt5WWYeE8O2Uc1
+si13X575kymGkkeiUcp9WtBkh2uial+RVmTrUTDUTIR2HzT6MAR84/DHlC5dsW8a
+h4uDUhzeG2cTxuIfZC881UHKL+xT/I3PPuFdLbU5uoWJpXYpxKYulYWd7LA/k4bi
+JWBrQo7VDvvPuQINBGROehcBEADBaRUFe9zHcU3Q3fhvyF/XgWV+99W6N+UWq83T
+SDHgXCyuPNV62Uym2XCmwzKLuBpsBiVf6cCsJtFTbbL2Mv9CClYNSvIh7KKPpbQ0
+adE79DjfdbAEn5Hg3rlCvByshCa0uSr3AWNMRxKORVjGILqw1DqQiQRhUy9AE175
+wnUTsqgPSocGI+sYHObDzReN4uJka6yLOvyD86QX3TAf8SHUVaEN/OxMJ6DsYUGg
+Z5/QZkncG/JuX+bQ7Mho+csyTnDleDCALVFzs0/XDktiokxzgi2F5Z+g/4Ehwlp0
+IDMNGVzCbBcN+2yy7OFez4nvsPRngjNyKBYQCKGheTIY87vjzS5Ccq5cHvaNP7Ty
+TjsrNRzDvjWsebvUX2C2nTfROcV55R6AO6GJdGpVxXQi8FCuMhKbd8jEIJiszke7
+fK4PuheSNanq9SoU6pQRgl5shPIrKCwWebjfbzEvEAY1mmTnWEZwf9+wbMg586HN
+qxYbbaWGBTQrxdjNnI1pY3It101DS0sgZZy8yTS0GtRU4fRaIpC1ihj37AYoWWIT
+ShIbvd7LDiZBdOuS4EfogyBg/Ge/x11IM5qiF4O045jCO+nMXZCOiGbF8Ugms2/9
+8RiAcIAB5KknfDmd6f6r6l5GxeHaxvuiourfjrYff2cS2uydsFvPHEQLOwSJrx7H
+E8ibYwARAQABiQI8BBgBCAAmFiEEqzovclgY/P8nlIQceTUEtEnGkiAFAmROehcC
+GwwFCQeGHvsACgkQeTUEtEnGkiDFDQ/7Bvpxi88UPhq/j6iw5o7NxNpj1IFY90XJ
+Iyl0HkM0dQ52kwbTSj64pHJtZCO3yYTgOOz+fu1f7HDpYDS9M4219+8Tbu/bEZqA
+xB/chubIxrRrLVn5skSgtRXyon3FhtL5FPzEDmGqUiqkKXC42vtBVkB7cuBshRDx
+gdR9aU4yBpAOF0fnCiSxtQfvamiKn1V7GNwOu8u/lMmnWQ8dsAF1kFv9fha7z9Ps
+B4lGDoTURGcBc4rAQhkZOIym4Zaeks/An2hqSlonOL1QP1GodWGPPDCfKlvdFH6S
+z8E7vAIKT8hN9z9sM7sLMvKoohIGUTbhQsC+NxOfFIO8x2lSKHpNUqRsExulxoa8
+5ejslwNGNbxIofjUw991bfI8sh5+5vJJ+Jo/yhEd+5D7oa6t7Bm3APulCGCFsDMW
+mD8pI/33AoXx+ejA+jV9CkKKbiYWUIWXmA6prbCc3naOtRneAeyHntHGs4EgGMO6
+p+A0FmRoMuJcKL6esfsaaP7NkVZZVyTR1h71bcWgoOQLczHqvs8SMeARoGTDCSiD
+LS1oveqJbhYxUXysAh7nL07nwAWaVQMi0zYM5peKm2vZtamtOwlKBAYjUNPamUJD
+F3+OFgL8dyWGvnsfN/gbDOtJyh+zDKEEZCwalqR/v8SJ1mu8JM2/KAuyXCm2Dl1v
+/Q6sM4xf2Y0=
+=QKTj
 -----END PGP PUBLIC KEY BLOCK-----

utils/python-rpc/framework/wallet.py

@@ -297,7 +297,7 @@ class Wallet(object):
         }
         return self.rpc.send_json_rpc_request(query_key)
 
-    def restore_deterministic_wallet(self, seed = '', seed_offset = '', filename = '', restore_height = 0, password = '', language = '', autosave_current = True):
+    def restore_deterministic_wallet(self, seed = '', seed_offset = '', filename = '', restore_height = 0, password = '', language = '', autosave_current = True, enable_multisig_experimental = False):
         restore_deterministic_wallet = {
             'method': 'restore_deterministic_wallet',
             'params' : {
@@ -308,6 +308,7 @@ class Wallet(object):
                 'password': password,
                 'language': language,
                 'autosave_current': autosave_current,
+                'enable_multisig_experimental': enable_multisig_experimental
             },
             'jsonrpc': '2.0', 
             'id': '0'