diff --git a/.gitmodules b/.gitmodules index 03d1c8767..89ae03351 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,6 +10,6 @@ branch = monero [submodule "external/randomwow"] path = external/randomwow - url = https://codeberg.org/wownero/RandomWOW + url = https://github.com/wownero/RandomWOW branch = 1.2.1-wow diff --git a/CMakeLists.txt b/CMakeLists.txt index c6fe04aed..901c9a458 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,6 +100,13 @@ set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) +if(APPLE) + # C++17 aligned new/delete (std::align_val_t) require macOS deployment target >= 10.14. The depends + # macOS toolchain targets older (and Apple Silicon aarch64 cross-builds hit this on boost/any.hpp). + # Disable aligned allocation so wownero builds under C++17 on all macOS targets. + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-aligned-allocation") +endif() + # When built as Feather/wowlet's wallet submodule (it sets BUILD_GUI_DEPS=ON before # add_subdirectory(monero)), match Feather's anti-AV namespace rename so this library's symbols line # up with the app, which Feather compiles with the same -Dcryptonote=MWYSUTPAXIBT define. Gated so it @@ -616,7 +623,6 @@ find_package(HIDAPI) add_definition_if_library_exists(c memset_s "string.h" HAVE_MEMSET_S) add_definition_if_library_exists(c explicit_bzero "strings.h" HAVE_EXPLICIT_BZERO) -add_definition_if_function_found(strptime HAVE_STRPTIME) add_definitions(-DAUTO_INITIALIZE_EASYLOGGINGPP) @@ -808,9 +814,8 @@ else() # mingw doesn't support LTO (multiple definition errors at link time) set(USE_LTO_DEFAULT false) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--stack,10485760") - if(NOT BUILD_64) - add_definitions(-DWINVER=0x0600 -D_WIN32_WINNT=0x0600) - endif() + # TryAcquireSRWLock* requires Windows 7 or later. + add_definitions(-DWINVER=0x0601 -D_WIN32_WINNT=0x0601) endif() set(C_WARNINGS "-Waggregate-return -Wnested-externs -Wold-style-definition -Wstrict-prototypes") set(CXX_WARNINGS "-Wno-reorder -Wno-missing-field-initializers") @@ -913,6 +918,11 @@ else() add_c_flag_if_supported( -Werror=return-type C_SECURITY_FLAGS) add_cxx_flag_if_supported(-Werror=return-type CXX_SECURITY_FLAGS) + # A missing WinAPI prototype (e.g. TryAcquireSRWLock*) must fail the build + if(MINGW) + add_c_flag_if_supported(-Werror=implicit-function-declaration C_SECURITY_FLAGS) + endif() + message(STATUS "Using C security hardening flags: ${C_SECURITY_FLAGS}") message(STATUS "Using C++ security hardening flags: ${CXX_SECURITY_FLAGS}") message(STATUS "Using linker security hardening flags: ${LD_SECURITY_FLAGS}") @@ -1172,7 +1182,7 @@ list(APPEND EXTRA_LIBRARIES ${CMAKE_DL_LIBS}) if (HIDAPI_FOUND OR LibUSB_COMPILE_TEST_PASSED) if (APPLE) if(DEPENDS) - list(APPEND EXTRA_LIBRARIES "-framework Foundation -framework AppKit -framework IOKit") + list(APPEND EXTRA_LIBRARIES "-framework Foundation -framework AppKit -framework IOKit -framework Security") else() find_library(COREFOUNDATION CoreFoundation) find_library(APPKIT AppKit) diff --git a/README.md b/README.md index 02b0bda09..06c6d007a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ~~Mo~~Wownero - Such privacy! Many coins! Wow! - + ## Introduction @@ -15,7 +15,7 @@ Unlike Opposing Projects. - Twitter: [@w0wn3r0](https://twitter.com/w0wn3r0) - Reddit: [/r/wownero](https://www.reddit.com/r/wownero) - Mail: [wownero@wownero.org](mailto:wownero@wownero.org) -- Git: [codeberg.org/wownero/wownero](https://codeberg.org/wownero/wownero) +- Git: [github.com/wownero/wownero](https://github.com/wownero/wownero) - Matrix General Chat Room: [#wownero-gen:wowne.ro](https://matrix.to/#/#wownero-gen:wowne.ro) - IRC: [OFTC #wownero](https://webchat.oftc.net/?channels=wownero) - Discord: [discord.gg/q2rmxeYvWy](https://discord.gg/q2rmxeYvWy) @@ -31,7 +31,7 @@ Unlike Opposing Projects. ## Wallets -- Wonero CLI Wallet: [codeberg.org/wownero/wownero](https://codeberg.org/wownero/wownero/releases) +- Wonero CLI Wallet: [github.com/wownero/wownero](https://github.com/wownero/wownero/releases) - Stack Wallet iOS & Android Mobile Wallet: [stackwallet.com](https://stackwallet.com) ## Blockchain Explorers @@ -62,7 +62,7 @@ BTC: `bc1qcw9zglp3fxyl25zswemw7jczlqryms2lsmu464` If you have a fix or code change, feel free to submit it as a pull request. Ahead of a scheduled software upgrade, a development branch will be created with the new release version tag. Pull requests that address bugs should be made to Master. Pull requests that require review and testing (generally, optimizations and new features) should be made to the development branch. All pull requests will be considered safe until the US dollar valuation of 1 Wownero equals $1000. After this valuation has been reached, more research will be needed to introduce experimental cryptography and/or code into the codebase. -Things to Do, Work in Progress, and Help Wanted tasks are tracked in the [Meta](https://codeberg.org/wownero/meta/issues) repo. +Things to Do, Work in Progress, and Help Wanted tasks are tracked in the [Meta](https://github.com/wownero/meta/issues) repo. Join `#wownero` on IRC OFTC to participate in development conversation. @@ -83,7 +83,7 @@ Dates are provided in the format YYYY-MM-DD. | - | 2020-06-28 | Hallucinogenic Hypnotoad | v0.8.0.0 | v0.8.0.2 | Dandelion++ support | 253,999 | 2020-10-09 | Illiterate Illuminati | v0.9.0.0 | v0.9.3.3 | Dynamic coinbase unlock (up to 1 mo.), Deterministic unlock times, Enforce maximum coinbase amount, show_qr_code wallet command, CLSAG | 331,170 | 2021-07-04 | Junkie Jeff | v0.10.0.0 | v0.10.2.0 | Bulletproofs+, Miner Block Header Signing, Vote by Block, Change coinbase unlock time to 1 day, Reset difficulty and switch back to Monero's difficulty algorithm -| 514,000 | 2023-04-01 | Kunty Karen | v0.11.0.0 | v0.11.4.0 | View tags, fee changes, adjusted dynamic block weight algorithm, multisig security fixes, RPC broadcast node donation sub-address, Limit tx_extra max size to ~1kb, 12-hour difficulty adjustment window +| 514,000 | 2023-04-01 | Kunty Karen | v0.11.0.0 | v0.11.5.0 | View tags, fee changes, adjusted dynamic block weight algorithm, multisig security fixes, RPC broadcast node donation sub-address, Limit tx_extra max size to ~1kb, 12-hour difficulty adjustment window X's indicate that these details have not been determined as of commit date. @@ -110,7 +110,7 @@ Packaging for your favorite distribution would be a welcome contribution! sudo pacman -Syu --needed base-devel cmake boost openssl zeromq libpgm unbound libsodium libunwind xz readline expat gtest python3 ccache doxygen graphviz qt5-tools hidapi libusb protobuf systemd - git clone https://codeberg.org/wownero/wownero && cd wownero + git clone https://github.com/wownero/wownero && cd wownero make @@ -118,7 +118,7 @@ Packaging for your favorite distribution would be a welcome contribution! sudo apt update && sudo apt install build-essential cmake pkg-config libssl-dev libzmq3-dev libunbound-dev libsodium-dev libunwind8-dev liblzma-dev libreadline6-dev libexpat1-dev libpgm-dev qttools5-dev-tools libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler libudev-dev libboost-chrono-dev libboost-date-time-dev libboost-filesystem-dev libboost-locale-dev libboost-program-options-dev libboost-regex-dev libboost-serialization-dev libboost-system-dev libboost-thread-dev python3 ccache doxygen graphviz - git clone https://codeberg.org/wownero/wownero && cd wownero + git clone https://github.com/wownero/wownero && cd wownero make diff --git a/contrib/depends/funcs.mk b/contrib/depends/funcs.mk index e08669c98..adb9d2788 100644 --- a/contrib/depends/funcs.mk +++ b/contrib/depends/funcs.mk @@ -143,7 +143,7 @@ $(1)_config_env+=PKG_CONFIG_PATH=$($($(1)_type)_prefix)/share/pkgconfig $(1)_config_env+=PATH="$(build_prefix)/bin:$(PATH)" $(1)_build_env+=PATH="$(build_prefix)/bin:$(PATH)" $(1)_stage_env+=PATH="$(build_prefix)/bin:$(PATH)" -$(1)_autoconf=./configure --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)" +$(1)_autoconf=./configure --build=$(BUILD) --host=$($($(1)_type)_host) --prefix=$($($(1)_type)_prefix) $$($(1)_config_opts) CC="$$($(1)_cc)" CXX="$$($(1)_cxx)" ifeq ($(filter $(1),libusb unbound),) $(1)_autoconf += --disable-dependency-tracking diff --git a/contrib/depends/hosts/darwin.mk b/contrib/depends/hosts/darwin.mk index 79d449054..34001bb3f 100644 --- a/contrib/depends/hosts/darwin.mk +++ b/contrib/depends/hosts/darwin.mk @@ -6,7 +6,7 @@ else CC_target=$(host) endif darwin_CC=clang -target $(CC_target) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(host_prefix)/native/SDK/ -mlinker-version=$(LD64_VERSION) -B$(host_prefix)/native/bin/$(host)- -darwin_CXX=clang++ -target $(CC_target) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(host_prefix)/native/SDK/ -mlinker-version=$(LD64_VERSION) -stdlib=libc++ -B$(host_prefix)/native/bin/$(host)- +darwin_CXX=clang++ -target $(CC_target) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(host_prefix)/native/SDK/ -mlinker-version=$(LD64_VERSION) -stdlib=libc++ -fno-aligned-allocation -B$(host_prefix)/native/bin/$(host)- darwin_CFLAGS=-pipe darwin_CXXFLAGS=$(darwin_CFLAGS) diff --git a/contrib/depends/packages/libusb.mk b/contrib/depends/packages/libusb.mk index c1d9fe6a9..c6767bba0 100644 --- a/contrib/depends/packages/libusb.mk +++ b/contrib/depends/packages/libusb.mk @@ -1,8 +1,8 @@ package=libusb -$(package)_version=1.0.26 +$(package)_version=1.0.30 $(package)_download_path=https://github.com/libusb/libusb/releases/download/v$($(package)_version) $(package)_file_name=$(package)-$($(package)_version).tar.bz2 -$(package)_sha256_hash=12ce7a61fc9854d1d2a1ffe095f7b5fac19ddba095c259e6067a46500381b5a5 +$(package)_sha256_hash=fea36f34f9156400209595e300840767ab1a385ede1dc7ee893015aea9c6dbaf define $(package)_preprocess_cmds autoreconf -i @@ -15,6 +15,12 @@ define $(package)_set_vars $(package)_config_opts_darwin=--disable-udev endef +# Remove blobs +define $(package)_preprocess_cmds + rm -rf tests/fuzz/corpus && \ + rm doc/libusb.png +endef + ifneq ($(host_os),darwin) define $(package)_config_cmds cp -f $(BASEDIR)/config.guess config.guess &&\ diff --git a/contrib/depends/packages/openssl.mk b/contrib/depends/packages/openssl.mk index 77d4b2d11..0fc1f6501 100644 --- a/contrib/depends/packages/openssl.mk +++ b/contrib/depends/packages/openssl.mk @@ -1,8 +1,8 @@ package=openssl -$(package)_version=3.0.19 -$(package)_download_path=https://www.openssl.org/source +$(package)_version=3.0.22 +$(package)_download_path=https://github.com/openssl/openssl/releases/download/openssl-$($(package)_version) $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=fa5a4143b8aae18be53ef2f3caf29a2e0747430b8bc74d32d88335b94ab63072 +$(package)_sha256_hash=67ebca7e50d17383028045486653492195b83db95f8558709701bb47b5c1ef81 define $(package)_set_vars $(package)_config_env=AR="$($(package)_ar)" ARFLAGS=$($(package)_arflags) RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" diff --git a/contrib/depends/packages/unbound.mk b/contrib/depends/packages/unbound.mk index d9ef0076a..ba7b39d46 100644 --- a/contrib/depends/packages/unbound.mk +++ b/contrib/depends/packages/unbound.mk @@ -1,10 +1,10 @@ package=unbound -$(package)_version=1.19.1 +$(package)_version=1.26.1 $(package)_download_path=https://www.nlnetlabs.nl/downloads/$(package)/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=bc1d576f3dd846a0739adc41ffaa702404c6767d2b6082deb9f2f97cbb24a3a9 +$(package)_sha256_hash=35a6dc0e425a9282c3426d9a3043144011bf0534aed4b73ab62c52aee0af1503 $(package)_dependencies=openssl expat -$(package)_patches=disable-glibc-reallocarray.patch +$(package)_patches=disable-glibc-reallocarray.patch hardcode_config_date.patch define $(package)_set_vars @@ -20,7 +20,10 @@ endef define $(package)_preprocess_cmds patch -p1 < $($(package)_patch_dir)/disable-glibc-reallocarray.patch &&\ - autoconf + patch -p1 < $($(package)_patch_dir)/hardcode_config_date.patch &&\ + autoconf && \ + rm doc/*.odp doc/*.pdf contrib/*.tar.gz contrib/*.tar.bz2 &&\ + rm -rf testdata dnscrypt/testdata endef define $(package)_config_cmds diff --git a/contrib/depends/patches/unbound/hardcode_config_date.patch b/contrib/depends/patches/unbound/hardcode_config_date.patch new file mode 100644 index 000000000..fd075da29 --- /dev/null +++ b/contrib/depends/patches/unbound/hardcode_config_date.patch @@ -0,0 +1,22 @@ +diff --git a/configure.ac b/configure.ac +index f1c5b04..13f4ce6 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -994,7 +994,7 @@ if test "`uname`" = "Linux"; then + GCC_DOCKER_LINTFLAGS='-syntax' + AC_SUBST(GCC_DOCKER_LINTFLAGS) + fi +-AX_BUILD_DATE_EPOCH(CONFIG_DATE, [%Y%m%d]) ++CONFIG_DATE=20000101 + AC_ARG_VAR(SOURCE_DATE_EPOCH, [If it is set, it uses the value of that variable instead of the current time as the build timestamp. The format is a unix timestamp. This enables reproducible build output.]) + AC_SUBST(CONFIG_DATE) + +@@ -2559,7 +2559,7 @@ char *unbound_stat_strdup_log(const char *s, const char* file, int line, + dnl if we build from source tree, the man pages need @date@ and @version@ + dnl if this is a distro tarball, that was already done by makedist.sh + AC_SUBST(version, [VERSION_MAJOR.VERSION_MINOR.VERSION_MICRO]) +-AX_BUILD_DATE_EPOCH(date, [[%b %e, %Y]]) ++date='Jan 1, 2000' + AC_SUBST(date) + + AC_CONFIG_FILES([Makefile doc/example.conf doc/libunbound.3 doc/unbound.8 doc/unbound-anchor.8 doc/unbound-checkconf.8 doc/unbound.conf.5 doc/unbound-control.8 doc/unbound-host.1 smallapp/unbound-control-setup.sh dnstap/dnstap_config.h dnscrypt/dnscrypt_config.h contrib/libunbound.pc contrib/unbound.socket contrib/unbound.service contrib/unbound_portable.service]) diff --git a/contrib/epee/include/net/abstract_http_client.h b/contrib/epee/include/net/abstract_http_client.h index 487466883..f1eddc298 100644 --- a/contrib/epee/include/net/abstract_http_client.h +++ b/contrib/epee/include/net/abstract_http_client.h @@ -69,6 +69,7 @@ namespace http virtual void set_auto_connect(bool auto_connect) = 0; virtual bool connect(std::chrono::milliseconds timeout) = 0; virtual bool disconnect() = 0; + virtual bool shutdown() { return disconnect(); } // tear down: the client need not be usable afterwards; overrides should interrupt blocked calls and be callable from another thread (the default plain disconnect() gives neither guarantee) virtual bool cancel_read() = 0; virtual bool is_connected(bool *ssl = NULL) = 0; virtual bool invoke(const boost::string_ref uri, const boost::string_ref method, const boost::string_ref body, std::chrono::milliseconds timeout, const http_response_info** ppresponse_info = NULL, const fields_list& additional_params = fields_list()) = 0; diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h index fe201f155..7fcd86629 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.h +++ b/contrib/epee/include/net/abstract_tcp_server2.h @@ -100,6 +100,14 @@ namespace net_utils { public: typedef typename t_protocol_handler::connection_context t_connection_context; + + enum status_t { + TERMINATED, + RUNNING, + INTERRUPTED, + TERMINATING, + WASTED, + }; private: using connection_t = connection; using connection_ptr = boost::shared_ptr; @@ -149,14 +157,6 @@ namespace net_utils boost::optional real_remote ); - enum status_t { - TERMINATED, - RUNNING, - INTERRUPTED, - TERMINATING, - WASTED, - }; - struct state_t { struct stat_t { struct { @@ -320,7 +320,9 @@ namespace net_utils bool speed_limit_is_enabled() const; ///< tells us should we be sleeping here (e.g. do not sleep on RPC connections) - bool cancel(); + bool cancel(bool wait_for_shutdown = false); + + status_t get_status() const noexcept { return m_state.status; } private: //----------------- i_service_endpoint --------------------- @@ -378,8 +380,21 @@ namespace net_utils /// wait for service workers stop bool timed_wait_server_stop(uint64_t wait_mseconds); + /// Mark the server as stopping without closing connections or stopping the io_context. + bool mark_stop_signal_sent(); + + /// Close boosted_tcp_server-owned connections, including ones not yet registered with the protocol handler. + void close_server_connections(); + + /// Stop the server io_context. + void stop_io_context(); + /// Stop the server. - void send_stop_signal(std::function close_all_connections = [](){}); + /// + /// Warning: Do NOT call this if the io_context is shared for connections + /// managed outside the boosted_tcp_server. See p2p net_node shutdown for + /// the correct staged shutdown in that case. + void send_stop_signal(); bool is_stop_signal_sent() const noexcept { return m_stop_signal_sent; }; diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index e202d9152..acd69cedc 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -824,7 +824,8 @@ namespace net_utils return false; // Wait for the write queue to fall below the max. If it doesn't after a - // randomized delay, drop the connection. + // randomized delay, drop the connection. P2P senders fail fast instead of + // parking an io_context worker thread here. auto wait_consume = [this] { auto random_delay = []{ using engine = std::mt19937; @@ -848,6 +849,12 @@ namespace net_utils if (m_state.data.write.queue.size() <= ABSTRACT_SERVER_SEND_QUE_MAX_COUNT && m_state.data.write.total_bytes <= static_cast(connection_basic::get_state()).response_soft_limit) return true; + + if (m_connection_type == e_connection_type_P2P) { + MWARNING("Connection " << m_conn_context.m_connection_id << " tripped write limit, terminating"); + terminate_async(); + return false; + } m_state.data.write.wait_consume = true; bool success = m_state.condition.wait_for( m_state.lock, @@ -886,7 +893,11 @@ namespace net_utils }; if (!wait_sender()) return false; - constexpr size_t CHUNK_SIZE = 32 * 1024; + /* CHUNK_SIZE indirectly caps outgoing to 128 * 1024 * 1000 + (ABSTRACT_SERVER_SEND_QUE_MAX_COUNT). The "soft" limit total is currently + 100 MiB (ABSTRACT_SERVER_SEND_QUE_MAX_BYTES_DEFAULT). These values will + need to be re-visited alongside block limit increases. */ + constexpr size_t CHUNK_SIZE = 128 * 1024; if (m_connection_type == e_connection_type_RPC || message.size() <= 2 * CHUNK_SIZE ) { @@ -898,13 +909,19 @@ namespace net_utils start_write(); } else { + std::size_t soft_limit = 0; + const epee::misc_utils::auto_scope_leave_caller scope_exit_handler = + epee::misc_utils::create_scope_leave_handler([&soft_limit, this] { + m_state.data.write.total_bytes += soft_limit; + }); + while (!message.empty()) { if (!wait_consume()) return false; m_state.data.write.queue.emplace_front( message.take_slice(CHUNK_SIZE) ); - m_state.data.write.total_bytes += m_state.data.write.queue.front().size(); + soft_limit += m_state.data.write.queue.front().size(); start_write(); } } @@ -960,14 +977,16 @@ namespace net_utils ec_t ec; #if !defined(_WIN32) || !defined(__i686) - connection_basic::socket_.next_layer().set_option( - boost::asio::detail::socket_option::integer{ - connection_basic::get_tos_flag() - }, - ec - ); - if (ec.value()) - return false; + if (real_remote->get_type_id() == ipv4_network_address::get_type_id()) { + connection_basic::socket_.next_layer().set_option( + boost::asio::detail::socket_option::integer{ + connection_basic::get_tos_flag() + }, + ec + ); + if (ec.value()) + return false; + } #endif connection_basic::socket_.next_layer().set_option( boost::asio::ip::tcp::no_delay{false}, @@ -1117,9 +1136,9 @@ namespace net_utils } template - bool connection::cancel() + bool connection::cancel(const bool wait_for_shutdown) { - return close(false); + return close(wait_for_shutdown); } template @@ -1138,7 +1157,9 @@ namespace net_utils bool connection::close(const bool wait_for_shutdown) { std::lock_guard guard(m_state.lock); - if (m_state.status != status_t::RUNNING) + if (m_state.status == status_t::TERMINATED || m_state.status == status_t::WASTED) + return true; + if (!wait_for_shutdown && m_state.status != status_t::RUNNING) return false; terminate_async(); @@ -1215,16 +1236,13 @@ namespace net_utils template bool connection::add_ref() { - try { - auto self = connection::shared_from_this(); - std::lock_guard guard(m_state.lock); - this->self = std::move(self); - ++m_state.protocol.reference_counter; - return true; - } - catch (boost::bad_weak_ptr &exception) { + auto self = connection::weak_from_this().lock(); + if (!self) return false; - } + std::lock_guard guard(m_state.lock); + this->self = std::move(self); + ++m_state.protocol.reference_counter; + return true; } template @@ -1284,7 +1302,7 @@ namespace net_utils template boosted_tcp_server::~boosted_tcp_server() { - this->send_stop_signal(); + send_stop_signal(); timed_wait_server_stop(10000); } //--------------------------------------------------------------------------------- @@ -1575,26 +1593,55 @@ namespace net_utils } //--------------------------------------------------------------------------------- template - void boosted_tcp_server::send_stop_signal(std::function close_all_connections) + bool boosted_tcp_server::mark_stop_signal_sent() { - m_stop_signal_sent = true; + if (m_stop_signal_sent.exchange(true)) + { + MDEBUG("Stop signal already sent"); + return false; + } typename connection::shared_state *state = static_cast::shared_state*>(m_state.get()); state->stop_signal_sent = true; - TRY_ENTRY(); - connections_mutex.lock(); - for (auto &c: connections_) + return true; + } + //--------------------------------------------------------------------------------- + template + void boosted_tcp_server::close_server_connections() + { + decltype(connections_) connections; { - c->cancel(); + boost::unique_lock lock(connections_mutex); + connections.swap(connections_); } - connections_.clear(); - connections_mutex.unlock(); - // Since we shut down connections in the strand, we want to make sure to complete the shutdown sequence before - // stopping the io_context. We let the caller handle closing because the caller is the one keeping track of all - // connections (connections_ is only a subset of all connections). - close_all_connections(); + for (auto &c: connections) + { + c->cancel(true/*wait_for_shutdown*/); + } + } + //--------------------------------------------------------------------------------- + template + void boosted_tcp_server::stop_io_context() + { + { + boost::unique_lock lock(connections_mutex); + if (!connections_.empty()) + { + MERROR("Stopping io_context with " << connections_.size() << " server-owned connections still open"); + } + } + MDEBUG("Stopping io_context"); io_context_.stop(); - MDEBUG("Done with send_stop_signal"); + } + //--------------------------------------------------------------------------------- + template + void boosted_tcp_server::send_stop_signal() + { + TRY_ENTRY(); + if (!mark_stop_signal_sent()) + return; + close_server_connections(); + stop_io_context(); CATCH_ENTRY_L0("boosted_tcp_server::send_stop_signal()", void()); } //--------------------------------------------------------------------------------- diff --git a/contrib/epee/include/net/http_base.h b/contrib/epee/include/net/http_base.h index f32fdd9ae..aec0689a5 100644 --- a/contrib/epee/include/net/http_base.h +++ b/contrib/epee/include/net/http_base.h @@ -29,7 +29,7 @@ #pragma once #include "memwipe.h" -#include +#include #include #include @@ -70,7 +70,7 @@ namespace net_utils std::string get_value_from_uri_line(const std::string& param_name, const std::string& uri); - static inline void add_field(std::string& out, const boost::string_ref name, const boost::string_ref value) + static inline void add_field(std::string& out, boost::string_view name, boost::string_view value) { out.append(name.data(), name.size()).append(": "); out.append(value.data(), value.size()).append("\r\n"); @@ -80,6 +80,42 @@ namespace net_utils add_field(out, field.first, field.second); } + namespace detail + { + inline bool parse_header_line(boost::string_view line, boost::string_view& name, boost::string_view& value) + { + if(!line.empty() && line.back() == '\r') + line.remove_suffix(1); + if(line.empty()) + return false; + if(line.front() == ' ' || line.front() == '\t') + return false; + + const size_t colon = line.find(':'); + if(colon == boost::string_view::npos || colon == 0) + return false; + + name = line.substr(0, colon); + value = line.substr(colon + 1); + while(!value.empty() && (value.front() == ' ' || value.front() == '\t')) + value.remove_prefix(1); + while(!value.empty() && (value.back() == ' ' || value.back() == '\t')) + value.remove_suffix(1); + + for(char c : name) + { + const bool alnum = (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'); + const bool allowed = alnum || + c == '!' || c == '#' || c == '$' || c == '%' || c == '&' || + c == '\'' || c == '*' || c == '+' || c == '-' || c == '.' || + c == '^' || c == '_' || c == '`' || c == '|' || c == '~'; + if(!allowed) + return false; + } + + return true; + } + } struct http_header_info { diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index 944e0c911..82beecdd2 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -207,6 +207,11 @@ namespace net_utils m_reconnect = false; return m_net_client.cancel_read(); } + //--------------------------------------------------------------------------- + bool shutdown() override + { + return m_net_client.shutdown(); // no m_lock; interrupts an invoke blocked while holding it + } //--------------------------------------------------------------------------- bool is_connected(bool *ssl = NULL) override { @@ -426,7 +431,11 @@ namespace net_utils recv_buff.assign(m_header_cache.begin()+pos+4, m_header_cache.end()); m_header_cache.erase(m_header_cache.begin()+pos+4, m_header_cache.end()); - analize_cached_header_and_invoke_state(); + if(!analize_cached_header_and_invoke_state()) + { + m_state = reciev_machine_state_error; + return false; + } if (!on_header(m_response_info)) { MDEBUG("Connection cancelled by on_header"); @@ -597,7 +606,7 @@ namespace net_utils } if(!get_chunk_head(m_chunked_cache, m_len_in_remain, is_matched)) { - LOG_ERROR("http_stream_filter::handle_chunked(*) Failed to get length from chunked head:" << m_chunked_cache); + LOG_ERROR("http_stream_filter::handle_chunked(*) Failed to get length from chunked head (" << m_chunked_cache.size() << " bytes)"); m_state = reciev_machine_state_error; return false; } @@ -659,64 +668,46 @@ namespace net_utils { MTRACE("http_stream_filter::parse_cached_header(*)"); - const char *ptr = m_cache_to_process.c_str(); - while (ptr[0] != '\r' || ptr[1] != '\n') + size_t cur = 0; + while(cur < m_cache_to_process.size()) { - // optional \n - if (*ptr == '\n') - ++ptr; - // an identifier composed of letters or - - const char *key_pos = ptr; - while (isalnum(*ptr) || *ptr == '_' || *ptr == '-') - ++ptr; - const char *key_end = ptr; - // optional space (not in RFC, but in previous code) - if (*ptr == ' ') - ++ptr; - CHECK_AND_ASSERT_MES(*ptr == ':', true, "http_stream_filter::parse_cached_header() invalid header in: " << m_cache_to_process); - ++ptr; - // optional whitespace, but not newlines - line folding is obsolete, let's ignore it - while (isblank(*ptr)) - ++ptr; - const char *value_pos = ptr; - while (*ptr != '\r' && *ptr != '\n') - ++ptr; - const char *value_end = ptr; - // optional trailing whitespace - while (value_end > value_pos && isblank(*(value_end-1))) - --value_end; - if (*ptr == '\r') - ++ptr; - CHECK_AND_ASSERT_MES(*ptr == '\n', true, "http_stream_filter::parse_cached_header() invalid header in: " << m_cache_to_process); - ++ptr; + const size_t line_end = m_cache_to_process.find('\n', cur); + CHECK_AND_ASSERT_MES(line_end != std::string::npos, false, "http_stream_filter::parse_cached_header() invalid header (" << m_cache_to_process.size() << " bytes)"); - const std::string key = std::string(key_pos, key_end - key_pos); - const std::string value = std::string(value_pos, value_end - value_pos); - if (!key.empty()) - { - if (!string_tools::compare_no_case(key, "Connection")) - body_info.m_connection = value; - else if(!string_tools::compare_no_case(key, "Referrer")) - body_info.m_referer = value; - else if(!string_tools::compare_no_case(key, "Content-Length")) - body_info.m_content_length = value; - else if(!string_tools::compare_no_case(key, "Content-Type")) - body_info.m_content_type = value; - else if(!string_tools::compare_no_case(key, "Transfer-Encoding")) - body_info.m_transfer_encoding = value; - else if(!string_tools::compare_no_case(key, "Content-Encoding")) - body_info.m_content_encoding = value; - else if(!string_tools::compare_no_case(key, "Host")) - body_info.m_host = value; - else if(!string_tools::compare_no_case(key, "Cookie")) - body_info.m_cookie = value; - else if(!string_tools::compare_no_case(key, "User-Agent")) - body_info.m_user_agent = value; - else if(!string_tools::compare_no_case(key, "Origin")) - body_info.m_origin = value; - else - body_info.m_etc_fields.emplace_back(key, value); - } + boost::string_view line(m_cache_to_process.data() + cur, line_end - cur); + cur = line_end + 1; + + if(line == "\r" || line.empty()) + break; + + boost::string_view name; + boost::string_view value; + CHECK_AND_ASSERT_MES(detail::parse_header_line(line, name, value), false, "http_stream_filter::parse_cached_header() invalid header (" << m_cache_to_process.size() << " bytes)"); + + std::string key(name.data(), name.size()); + std::string val(value.data(), value.size()); + if (!string_tools::compare_no_case(key, "Connection")) + body_info.m_connection = std::move(val); + else if(!string_tools::compare_no_case(key, "Referer")) + body_info.m_referer = std::move(val); + else if(!string_tools::compare_no_case(key, "Content-Length")) + body_info.m_content_length = std::move(val); + else if(!string_tools::compare_no_case(key, "Content-Type")) + body_info.m_content_type = std::move(val); + else if(!string_tools::compare_no_case(key, "Transfer-Encoding")) + body_info.m_transfer_encoding = std::move(val); + else if(!string_tools::compare_no_case(key, "Content-Encoding")) + body_info.m_content_encoding = std::move(val); + else if(!string_tools::compare_no_case(key, "Host")) + body_info.m_host = std::move(val); + else if(!string_tools::compare_no_case(key, "Cookie")) + body_info.m_cookie = std::move(val); + else if(!string_tools::compare_no_case(key, "User-Agent")) + body_info.m_user_agent = std::move(val); + else if(!string_tools::compare_no_case(key, "Origin")) + body_info.m_origin = std::move(val); + else + body_info.m_etc_fields.emplace_back(std::move(key), std::move(val)); } return true; } @@ -725,25 +716,25 @@ namespace net_utils { //First line response, look like this: "HTTP/1.1 200 OK" const char *ptr = m_header_cache.c_str(); - CHECK_AND_ASSERT_MES(!memcmp(ptr, "HTTP/", 5), false, "Invalid first response line: " + m_header_cache); + CHECK_AND_ASSERT_MES(!memcmp(ptr, "HTTP/", 5), false, "Invalid first response line (" << m_header_cache.size() << " header bytes)"); ptr += 5; - CHECK_AND_ASSERT_MES(epee::misc_utils::parse::isdigit(*ptr), false, "Invalid first response line: " + m_header_cache); + CHECK_AND_ASSERT_MES(epee::misc_utils::parse::isdigit(*ptr), false, "Invalid first response line (" << m_header_cache.size() << " header bytes)"); unsigned long ul; char *end; ul = strtoul(ptr, &end, 10); - CHECK_AND_ASSERT_MES(ul <= INT_MAX && *end =='.', false, "Invalid first response line: " + m_header_cache); + CHECK_AND_ASSERT_MES(ul <= INT_MAX && *end =='.', false, "Invalid first response line (" << m_header_cache.size() << " header bytes)"); m_response_info.m_http_ver_hi = ul; ptr = end + 1; - CHECK_AND_ASSERT_MES(epee::misc_utils::parse::isdigit(*ptr), false, "Invalid first response line: " + m_header_cache + ", ptr: " << ptr); + CHECK_AND_ASSERT_MES(epee::misc_utils::parse::isdigit(*ptr), false, "Invalid first response line (" << m_header_cache.size() << " header bytes)"); ul = strtoul(ptr, &end, 10); - CHECK_AND_ASSERT_MES(ul <= INT_MAX && isblank(*end), false, "Invalid first response line: " + m_header_cache + ", ptr: " << ptr); + CHECK_AND_ASSERT_MES(ul <= INT_MAX && isblank(*end), false, "Invalid first response line (" << m_header_cache.size() << " header bytes)"); m_response_info.m_http_ver_lo = ul; ptr = end + 1; while (isblank(*ptr)) ++ptr; - CHECK_AND_ASSERT_MES(epee::misc_utils::parse::isdigit(*ptr), false, "Invalid first response line: " + m_header_cache); + CHECK_AND_ASSERT_MES(epee::misc_utils::parse::isdigit(*ptr), false, "Invalid first response line (" << m_header_cache.size() << " header bytes)"); ul = strtoul(ptr, &end, 10); - CHECK_AND_ASSERT_MES(ul >= 100 && ul <= 999 && isspace(*end), false, "Invalid first response line: " + m_header_cache); + CHECK_AND_ASSERT_MES(ul >= 100 && ul <= 999 && isspace(*end), false, "Invalid first response line (" << m_header_cache.size() << " header bytes)"); m_response_info.m_response_code = ul; ptr = end; // ignore the optional text, till the end @@ -751,7 +742,7 @@ namespace net_utils ++ptr; if (*ptr == '\r') ++ptr; - CHECK_AND_ASSERT_MES(*ptr == '\n', false, "Invalid first response line: " << m_header_cache); + CHECK_AND_ASSERT_MES(*ptr == '\n', false, "Invalid first response line (" << m_header_cache.size() << " header bytes)"); ++ptr; m_header_cache.erase(0, ptr - m_header_cache.c_str()); @@ -783,7 +774,7 @@ namespace net_utils std::string fake_str; //gcc error workaround bool res = parse_header(m_response_info.m_header_info, m_header_cache); - CHECK_AND_ASSERT_MES(res, false, "http_stream_filter::analize_cached_reply_header_and_invoke_state(): failed to anilize reply header: " << m_header_cache); + CHECK_AND_ASSERT_MES(res, false, "http_stream_filter::analize_cached_reply_header_and_invoke_state(): failed to anilize reply header (" << m_header_cache.size() << " bytes)"); set_reply_content_encoder(); @@ -805,7 +796,7 @@ namespace net_utils string_tools::trim(m_response_info.m_header_info.m_transfer_encoding); if(string_tools::compare_no_case(m_response_info.m_header_info.m_transfer_encoding, "chunked")) { - LOG_ERROR("Wrong Transfer-Encoding:" << m_response_info.m_header_info.m_transfer_encoding); + LOG_ERROR("Wrong Transfer-Encoding (" << m_response_info.m_header_info.m_transfer_encoding.size() << " bytes)"); m_state = reciev_machine_state_error; return false; } @@ -818,7 +809,7 @@ namespace net_utils //In the response header the length was specified if(!content_len_valid) { - LOG_ERROR("http_stream_filter::analize_cached_reply_header_and_invoke_state(): Failed to get_len_from_content_lenght();, m_query_info.m_content_length="< +#include #include +#include #include "http_protocol_handler.h" #include "reg_exp_definer.h" #include "string_tools.h" @@ -100,7 +101,7 @@ namespace net_utils entry.m_etc_header_fields.push_back(std::pair(result[field_etc_name], result[field_val])); else { - LOG_ERROR("simple_http_connection_handler::parse_header() not matched last entry in:"<second == 1 || elem->second == 0) @@ -242,7 +243,7 @@ namespace net_utils bool simple_http_connection_handler::after_init_connection() { CRITICAL_REGION_LOCAL(m_config.m_lock); - ++m_config.m_connections[m_conn_context.m_remote_address.host_str()]; + ++m_config.m_connections[get_rpc_connection_limit_key(m_conn_context.m_remote_address)]; ++m_config.m_connection_count; m_initialized = true; return true; @@ -429,7 +430,7 @@ namespace net_utils }else { m_state = http_state_error; - LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler::handle_invoke_query_line(): Failed to match first line: " << m_cache); + LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler::handle_invoke_query_line(): Failed to match first line (" << m_cache.size() << " bytes)"); return false; } @@ -453,14 +454,14 @@ namespace net_utils template bool simple_http_connection_handler::analize_cached_request_header_and_invoke_state(size_t pos) { - LOG_PRINT_L3("HTTP HEAD:\r\n" << m_cache.substr(0, pos)); + LOG_PRINT_L3("HTTP HEAD: " << pos << " bytes"); m_query_info.m_full_request_buf_size = pos; m_query_info.m_request_head.assign(m_cache.begin(), m_cache.begin()+pos); if(!parse_cached_header(m_query_info.m_header_info, m_cache, pos)) { - LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler::analize_cached_request_header_and_invoke_state(): failed to anilize request header: " << m_cache); + LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler::analize_cached_request_header_and_invoke_state(): failed to anilize request header (" << pos << " bytes)"); m_state = http_state_error; return false; } @@ -476,7 +477,7 @@ namespace net_utils m_body_transfer_type = http_body_transfer_measure; if(!get_len_from_content_lenght(m_query_info.m_header_info.m_content_length, m_len_summary)) { - LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler::analize_cached_request_header_and_invoke_state(): Failed to get_len_from_content_lenght();, m_query_info.m_content_length="<::analize_cached_request_header_and_invoke_state(): Failed to get_len_from_content_lenght() (" << m_query_info.m_header_info.m_content_length.size() << " bytes)"); m_state = http_state_error; return false; } @@ -546,54 +547,52 @@ namespace net_utils template bool simple_http_connection_handler::parse_cached_header(http_header_info& body_info, const std::string& m_cache_to_process, size_t pos) { - STATIC_REGEXP_EXPR_1(rexp_mach_field, - "\n?((Connection)|(Referer)|(Content-Length)|(Content-Type)|(Transfer-Encoding)|(Content-Encoding)|(Host)|(Cookie)|(User-Agent)|(Origin)" - // 12 3 4 5 6 7 8 9 10 11 - "|([\\w-]+?)) ?: ?((.*?)(\r?\n))[^\t ]", - //11 1213 14 - boost::regex::icase | boost::regex::normal); - - boost::smatch result; - std::string::const_iterator it_current_bound = m_cache_to_process.begin(); - std::string::const_iterator it_end_bound = m_cache_to_process.begin()+pos; - body_info.clear(); + if(pos > m_cache_to_process.size() || pos > HTTP_MAX_HEADER_LEN) + return false; - //lookup all fields and fill well-known fields - while( boost::regex_search( it_current_bound, it_end_bound, result, rexp_mach_field, boost::match_default) && result[0].matched) + size_t cur = 0; + + while(cur < pos) { - const size_t field_val = 14; - const size_t field_etc_name = 12; + const size_t line_end = m_cache_to_process.find('\n', cur); + if(line_end == std::string::npos || line_end >= pos) + break; - int i = 2; //start position = 2 - if(result[i++].matched)//"Connection" - body_info.m_connection = result[field_val]; - else if(result[i++].matched)//"Referer" - body_info.m_referer = result[field_val]; - else if(result[i++].matched)//"Content-Length" - body_info.m_content_length = result[field_val]; - else if(result[i++].matched)//"Content-Type" - body_info.m_content_type = result[field_val]; - else if(result[i++].matched)//"Transfer-Encoding" - body_info.m_transfer_encoding = result[field_val]; - else if(result[i++].matched)//"Content-Encoding" - body_info.m_content_encoding = result[field_val]; - else if(result[i++].matched)//"Host" - body_info.m_host = result[field_val]; - else if(result[i++].matched)//"Cookie" - body_info.m_cookie = result[field_val]; - else if(result[i++].matched)//"User-Agent" - body_info.m_user_agent = result[field_val]; - else if(result[i++].matched)//"Origin" - body_info.m_origin = result[field_val]; - else if(result[i++].matched)//e.t.c (HAVE TO BE MATCHED!) - body_info.m_etc_fields.push_back(std::pair(result[field_etc_name], result[field_val])); + boost::string_view line(m_cache_to_process.data() + cur, line_end - cur); + cur = line_end + 1; + + // End of header block. + if(line == "\r" || line.empty()) + break; + + boost::string_view name; + boost::string_view value; + if(!detail::parse_header_line(line, name, value)) + return false; + + if(boost::iequals(name, "Connection")) + body_info.m_connection = std::string(value.data(), value.size()); + else if(boost::iequals(name, "Referer")) + body_info.m_referer = std::string(value.data(), value.size()); + else if(boost::iequals(name, "Content-Length")) + body_info.m_content_length = std::string(value.data(), value.size()); + else if(boost::iequals(name, "Content-Type")) + body_info.m_content_type = std::string(value.data(), value.size()); + else if(boost::iequals(name, "Transfer-Encoding")) + body_info.m_transfer_encoding = std::string(value.data(), value.size()); + else if(boost::iequals(name, "Content-Encoding")) + body_info.m_content_encoding = std::string(value.data(), value.size()); + else if(boost::iequals(name, "Host")) + body_info.m_host = std::string(value.data(), value.size()); + else if(boost::iequals(name, "Cookie")) + body_info.m_cookie = std::string(value.data(), value.size()); + else if(boost::iequals(name, "User-Agent")) + body_info.m_user_agent = std::string(value.data(), value.size()); + else if(boost::iequals(name, "Origin")) + body_info.m_origin = std::string(value.data(), value.size()); else - { - LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler::parse_cached_header() not matched last entry in:" << m_cache_to_process); - } - - it_current_bound = result[(int)result.size()-1]. first; + body_info.m_etc_fields.push_back(std::make_pair(std::string(name.data(), name.size()), std::string(value.data(), value.size()))); } return true; } diff --git a/contrib/epee/include/net/http_server_handlers_map2.h b/contrib/epee/include/net/http_server_handlers_map2.h index 8e7db53c0..8ed0a3ccc 100644 --- a/contrib/epee/include/net/http_server_handlers_map2.h +++ b/contrib/epee/include/net/http_server_handlers_map2.h @@ -76,7 +76,7 @@ bool parse_res = epee::serialization::load_t_from_json(static_cast(req), query_info.m_body); \ if (!parse_res) \ { \ - MERROR("Failed to parse json: \r\n" << query_info.m_body); \ + MERROR("Failed to parse JSON request"); \ response_info.m_response_code = 400; \ response_info.m_response_comment = "Bad request"; \ return true; \ diff --git a/contrib/epee/include/net/http_server_impl_base.h b/contrib/epee/include/net/http_server_impl_base.h index e24391674..da65db95c 100644 --- a/contrib/epee/include/net/http_server_impl_base.h +++ b/contrib/epee/include/net/http_server_impl_base.h @@ -155,7 +155,7 @@ namespace epee return true; const bool is_private = na.is_loopback() || na.is_local(); - const auto elem = config.m_connections.find(na.host_str()); + const auto elem = config.m_connections.find(net_utils::http::get_rpc_connection_limit_key(na)); if (elem != config.m_connections.end()) { if (is_private) diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index 73d46fd8b..e7122cdf4 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -106,7 +106,7 @@ public: template int invoke_async(int command, message_writer in_msg, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED); - int send(epee::byte_slice message, const boost::uuids::uuid& connection_id); + bool send(epee::byte_slice message, const boost::uuids::uuid& connection_id); bool close(boost::uuids::uuid connection_id, const bool wait_for_shutdown); bool update_connection_context(const t_connection_context& contxt); bool request_callback(boost::uuids::uuid connection_id); @@ -263,6 +263,8 @@ public: } virtual void reset_timer() { + if (m_command == connection_context::handshake_command()) + return; boost::system::error_code ignored_ec; if (!m_cancel_timer_called && m_timer.cancel(ignored_ec) > 0) { @@ -562,7 +564,13 @@ public: return false; } else - m_config.m_pcommands_handler->notify(m_current_head.m_command, buff_to_invoke, m_connection_context); + { + const int notify_result = m_config.m_pcommands_handler->notify( + m_current_head.m_command, buff_to_invoke, m_connection_context + ); + if(notify_result < 0) + return false; + } } // reuse small buffer if (!temp.empty() && temp.capacity() <= 64 * 1024) @@ -576,32 +584,34 @@ public: { if(m_cache_in_buffer.size() < sizeof(bucket_head2)) { - if(m_cache_in_buffer.size() >= sizeof(uint64_t) && *((uint64_t*)m_cache_in_buffer.span(8).data()) != SWAP64LE(LEVIN_SIGNATURE)) + if(m_cache_in_buffer.size() >= sizeof(uint64_t)) { - MWARNING(m_connection_context << "Signature mismatch, connection will be closed"); - return false; + std::uint64_t levin_sig; + memcpy(&levin_sig, m_cache_in_buffer.span(sizeof(std::uint64_t)).data(), sizeof(std::uint64_t)); + if (SWAP64LE(LEVIN_SIGNATURE) != levin_sig) + { + MWARNING(m_connection_context << "Signature mismatch, connection will be closed"); + return false; + } } is_continue = false; break; } -#if BYTE_ORDER == LITTLE_ENDIAN - bucket_head2& phead = *(bucket_head2*)m_cache_in_buffer.span(sizeof(bucket_head2)).data(); -#else - bucket_head2 phead = *(bucket_head2*)m_cache_in_buffer.span(sizeof(bucket_head2)).data(); - phead.m_signature = SWAP64LE(phead.m_signature); - phead.m_cb = SWAP64LE(phead.m_cb); - phead.m_command = SWAP32LE(phead.m_command); - phead.m_return_code = SWAP32LE(phead.m_return_code); - phead.m_flags = SWAP32LE(phead.m_flags); - phead.m_protocol_version = SWAP32LE(phead.m_protocol_version); + memcpy(&m_current_head, m_cache_in_buffer.span(sizeof(bucket_head2)).data(), sizeof(bucket_head2)); +#if BYTE_ORDER != LITTLE_ENDIAN + m_current_head.m_signature = SWAP64LE(m_current_head.m_signature); + m_current_head.m_cb = SWAP64LE(m_current_head.m_cb); + m_current_head.m_command = SWAP32LE(m_current_head.m_command); + m_current_head.m_return_code = SWAP32LE(m_current_head.m_return_code); + m_current_head.m_flags = SWAP32LE(m_current_head.m_flags); + m_current_head.m_protocol_version = SWAP32LE(m_current_head.m_protocol_version); #endif - if(LEVIN_SIGNATURE != phead.m_signature) + if(LEVIN_SIGNATURE != m_current_head.m_signature) { LOG_ERROR_CC(m_connection_context, "Signature mismatch, connection will be closed"); return false; } - m_current_head = phead; m_cache_in_buffer.erase(sizeof(bucket_head2)); m_state = stream_state_body; @@ -735,8 +745,8 @@ public: `message_writer::finalize_notify`. See additional instructions for `make_fragmented_notify`. - \return 1 on success */ - int send(byte_slice message) + \return true on success */ + bool send(byte_slice message) { const misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( boost::bind(&async_protocol_handler::finish_outer_call, this) @@ -745,9 +755,9 @@ public: if (!send_message(std::move(message))) { LOG_ERROR_CC(m_connection_context, "Failed to send message, dropping it"); - return -1; + return false; } - return 1; + return true; } //------------------------------------------------------------------------------------------ boost::uuids::uuid get_connection_id() {return m_connection_context.m_connection_id;} @@ -927,11 +937,11 @@ void async_protocol_handler_config::set_handler(levin_comm } //------------------------------------------------------------------------------------------ template -int async_protocol_handler_config::send(byte_slice message, const boost::uuids::uuid& connection_id) +bool async_protocol_handler_config::send(byte_slice message, const boost::uuids::uuid& connection_id) { async_protocol_handler* aph; int r = find_and_lock_connection(connection_id, aph); - return LEVIN_OK == r ? aph->send(std::move(message)) : 0; + return LEVIN_OK == r ? aph->send(std::move(message)) : false; } //------------------------------------------------------------------------------------------ template diff --git a/contrib/epee/include/net/net_helper.h b/contrib/epee/include/net/net_helper.h index 366ca9b61..a1c4cfb87 100644 --- a/contrib/epee/include/net/net_helper.h +++ b/contrib/epee/include/net/net_helper.h @@ -32,10 +32,12 @@ //#include //#include #include +#include #include #include #include #include +#include #include #include #include @@ -45,6 +47,13 @@ #include #include #include +#ifdef BOOST_ASIO_HAS_POSIX_STREAM_DESCRIPTOR +#include +#include +#include +#include +#include +#endif #include "net/net_utils_base.h" #include "net/net_ssl.h" #include "misc_language.h" @@ -109,12 +118,40 @@ namespace net_utils m_initialized(true), m_connected(false), m_deadline(m_io_service, std::chrono::steady_clock::time_point::max()), - m_shutdowned(false), +#ifdef BOOST_ASIO_HAS_POSIX_STREAM_DESCRIPTOR + m_wake_reader(m_io_service), + m_wake_writer(-1), +#endif + m_aborted(false), m_cancel_read(false), m_bytes_sent(0), m_bytes_received(0) { check_deadline(); +#ifdef BOOST_ASIO_HAS_POSIX_STREAM_DESCRIPTOR + // self-pipe so shutdown() can wake a blocked run_one() with an async-signal-safe + // write(); without it aborts would wait out the op deadline, so refuse to construct + int fds[2]; + if (0 != ::pipe(fds)) + throw std::runtime_error("failed to create shutdown wake pipe"); + bool flags_set = true; + for (const int fd : fds) + { + if (-1 == ::fcntl(fd, F_SETFL, O_NONBLOCK) || -1 == ::fcntl(fd, F_SETFD, FD_CLOEXEC)) + flags_set = false; + } + boost::system::error_code ec; + if (flags_set) + m_wake_reader.assign(fds[0], ec); + if (!flags_set || ec) + { + ::close(fds[0]); + ::close(fds[1]); + throw std::runtime_error("failed to set up shutdown wake pipe"); + } + m_wake_writer = fds[1]; + wait_wake(); +#endif } /*! The first/second parameters are host/port respectively. The third @@ -137,8 +174,12 @@ namespace net_utils ~blocked_mode_client() { //profile_tools::local_coast lc("~blocked_mode_client()", 3); - try { shutdown(); } + try { disconnect(); } catch(...) { /* ignore */ } +#ifdef BOOST_ASIO_HAS_POSIX_STREAM_DESCRIPTOR + if (m_wake_writer != -1) + ::close(m_wake_writer); // read end is owned and closed by m_wake_reader +#endif } inline void set_ssl(ssl_options_t ssl_options) @@ -159,10 +200,16 @@ namespace net_utils inline try_connect_result_t try_connect(const std::string& addr, const std::string& port, std::chrono::milliseconds timeout) { + const auto deadline = std::chrono::steady_clock::now() + timeout; m_deadline.expires_after(timeout); boost::unique_future connection = m_connector(addr, port, m_deadline); for (;;) { + // aborted or timed out: abandon the attempt; its handlers own their state + // and their socket dies with them, at completion or client destruction + if (shutdown_requested() || std::chrono::steady_clock::now() >= deadline) + return CONNECT_FAILURE; + m_io_service.restart(); m_io_service.run_one(); @@ -172,6 +219,12 @@ namespace net_utils m_ssl_socket->next_layer() = connection.get(); m_deadline.cancel(); + if (shutdown_requested()) + { + boost::system::error_code ignored_ec; + m_ssl_socket->next_layer().close(ignored_ec); + return CONNECT_FAILURE; // aborted by shutdown() from another thread + } if (m_ssl_socket->next_layer().is_open()) { m_connected = true; @@ -179,8 +232,13 @@ namespace net_utils // SSL Options if (m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_enabled || m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) { - if (!m_ssl_options.handshake(m_io_service, *m_ssl_socket, boost::asio::ssl::stream_base::client, {}, addr, timeout)) + if (!m_ssl_options.handshake(m_io_service, *m_ssl_socket, boost::asio::ssl::stream_base::client, {}, addr, timeout, [this] { return shutdown_requested(); })) { + if (shutdown_requested()) + { + m_connected = false; + return CONNECT_FAILURE; // aborted: says nothing about the server's TLS support + } if (m_ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_autodetect) { boost::system::error_code ignored_ec; @@ -197,6 +255,15 @@ namespace net_utils } } } + // recheck after publishing m_connected: shutdown() between the check above + // and the publish must not leave a live connection behind + if (shutdown_requested()) + { + m_connected = false; + boost::system::error_code ignored_ec; + m_ssl_socket->next_layer().close(ignored_ec); + return CONNECT_FAILURE; + } return CONNECT_SUCCESS; }else { @@ -209,10 +276,13 @@ namespace net_utils inline bool connect(const std::string& addr, const std::string& port, std::chrono::milliseconds timeout) { + if (shutdown_requested()) + return false; // torn down: the client never reconnects after shutdown() m_connected = false; try { m_ssl_socket->next_layer().close(); + m_pending_read.reset(); // Set SSL options // disable sslv2 @@ -256,14 +326,18 @@ namespace net_utils inline bool disconnect() { + m_connected = false; try { - if(m_connected) + if(!shutdown_requested() && m_ssl_socket->next_layer().is_open()) { - m_connected = false; if(m_ssl_options) shutdown_ssl(); - m_ssl_socket->next_layer().shutdown(boost::asio::ip::tcp::socket::shutdown_both); + // The peer or the TLS shutdown deadline may already have closed the connection. + boost::system::error_code ignored_ec; + m_ssl_socket->next_layer().shutdown(boost::asio::ip::tcp::socket::shutdown_both, ignored_ec); + // shutdown() alone leaves is_open() true and repeats TLS shutdown on later calls. + m_ssl_socket->next_layer().close(); } } catch(const boost::system::system_error& er) @@ -283,7 +357,6 @@ namespace net_utils inline bool send(const boost::string_ref buff, std::chrono::milliseconds timeout) { - try { m_deadline.expires_after(timeout); @@ -302,12 +375,25 @@ namespace net_utils async_write(buff.data(), buff.size(), ec); // Block until the asynchronous operation has completed. - while (ec == boost::asio::error::would_block) + while (ec == boost::asio::error::would_block && !shutdown_requested()) { m_io_service.restart(); m_io_service.run_one(); } + // aborted by shutdown(): the pending handler references this frame, so + // close the socket and pump until it completes before returning + if (ec == boost::asio::error::would_block) + { + boost::system::error_code ignored_ec; + m_ssl_socket->next_layer().close(ignored_ec); + while (ec == boost::asio::error::would_block) + { + m_io_service.restart(); + m_io_service.run_one(); + } + } + if (ec) { LOG_PRINT_L3("Problems at write: " << ec.message()); @@ -396,15 +482,66 @@ namespace net_utils { if (!m_connected || !m_ssl_socket->next_layer().is_open()) return false; + + boost::system::error_code ec; + if (m_ssl_options) + { + // let Asio process TLS alerts and retain a byte of application data for recv + if (!m_pending_read) + { + const auto pending = std::make_shared(); + // reconnect may replace the stream before this handler completes + const auto socket = m_ssl_socket; + socket->async_read_some(boost::asio::buffer(&pending->byte, 1), + [socket, pending](const boost::system::error_code& error, size_t) { + pending->error = error; + if (pending->handler) + { + auto handler = std::move(pending->handler); + handler(error, pending->byte); + } + }); + m_pending_read = pending; + } + m_io_service.restart(); + m_io_service.poll(); + ec = m_pending_read->error; + } + else + { + // peek for EOF without consuming application data or changing the socket mode + auto& socket = m_ssl_socket->next_layer(); + const bool non_blocking = socket.non_blocking(); + socket.non_blocking(true, ec); + if (!ec) + { + char byte; + socket.receive(boost::asio::buffer(&byte, 1), boost::asio::ip::tcp::socket::message_peek, ec); + boost::system::error_code restore_error; + socket.non_blocking(non_blocking, restore_error); + if (restore_error) + { + m_connected = false; + return false; + } + } + } + if (ec && ec != boost::asio::error::would_block && + ec != boost::asio::error::try_again && ec != boost::asio::error::interrupted) + { + MDEBUG("Peer closed idle connection, marking disconnected: " << ec.message()); + m_connected = false; + return false; + } + if (ssl) *ssl = m_ssl_options.support != ssl_support_t::e_ssl_support_disabled; - return true; + return m_connected; } inline bool recv(std::string& buff, std::chrono::milliseconds timeout) { - try { // Set a deadline for the asynchronous operation. Since this function uses @@ -435,12 +572,24 @@ namespace net_utils async_read(&buff[0], max_size, boost::asio::transfer_at_least(1), hndlr); // Block until the asynchronous operation has completed. - while (ec == boost::asio::error::would_block && !m_shutdowned) + while (ec == boost::asio::error::would_block && !shutdown_requested()) { m_io_service.restart(); m_io_service.run_one(); } + // aborted by shutdown(): the pending handler references this frame, so + // close the socket and pump until it completes before returning + if (ec == boost::asio::error::would_block) + { + boost::system::error_code ignored_ec; + m_ssl_socket->next_layer().close(ignored_ec); + while (ec == boost::asio::error::would_block) + { + m_io_service.restart(); + m_io_service.run_one(); + } + } if (m_cancel_read) { LOG_ERROR("Cancelling read, returning false"); @@ -450,6 +599,7 @@ namespace net_utils if (ec) { + m_connected = false; MTRACE("READ ENDS: Connection err_code " << ec.value()); if(ec == boost::asio::error::eof) { @@ -460,7 +610,6 @@ namespace net_utils } MDEBUG("Problems at read: " << ec.message()); - m_connected = false; return false; }else { @@ -525,11 +674,24 @@ namespace net_utils async_read((char*)buff.data(), buff.size(), boost::asio::transfer_at_least(buff.size()), hndlr); // Block until the asynchronous operation has completed. - while (ec == boost::asio::error::would_block && !m_shutdowned) + while (ec == boost::asio::error::would_block && !shutdown_requested()) { m_io_service.run_one(); } + // aborted by shutdown(): the pending handler references this frame, so + // close the socket and pump until it completes before returning + if (ec == boost::asio::error::would_block) + { + boost::system::error_code ignored_ec; + m_ssl_socket->next_layer().close(ignored_ec); + while (ec == boost::asio::error::would_block) + { + m_io_service.restart(); + m_io_service.run_one(); + } + } + if (ec) { LOG_PRINT_L3("Problems at read: " << ec.message()); @@ -569,21 +731,20 @@ namespace net_utils bool shutdown() { - m_deadline.cancel(); - boost::system::error_code ec; - if(m_ssl_options) - shutdown_ssl(); - m_ssl_socket->next_layer().cancel(ec); - if(ec) - MDEBUG("Problems at cancel: " << ec.message()); - m_ssl_socket->next_layer().shutdown(boost::asio::ip::tcp::socket::shutdown_both, ec); - if(ec) - MDEBUG("Problems at shutdown: " << ec.message()); - m_ssl_socket->next_layer().close(ec); - if(ec) - MDEBUG("Problems at close: " << ec.message()); - m_shutdowned = true; - m_connected = false; + // callable from any thread, on POSIX even a signal handler: only atomics and a self-pipe + // write(); a blocked call closes the socket on abort, an idle client's closes at destruction + m_aborted = true; + m_connected = false; +#ifdef BOOST_ASIO_HAS_POSIX_STREAM_DESCRIPTOR + // construction guarantees the pipe exists + const int saved_errno = errno; + const char wake = 0; + while (::write(m_wake_writer, &wake, 1) == -1 && errno == EINTR) {} + errno = saved_errno; // a full pipe already holds a pending wake +#else + // Windows console handlers run on their own thread: post() a wake + boost::asio::post(m_io_service, []{}); +#endif return true; } @@ -616,6 +777,27 @@ namespace net_utils private: + struct pending_read + { + char byte = 0; + boost::system::error_code error = boost::asio::error::would_block; + std::function handler; + }; + + bool shutdown_requested() const + { + return m_aborted.load(); + } + +#ifdef BOOST_ASIO_HAS_POSIX_STREAM_DESCRIPTOR + // perpetual pipe read so a wake byte written by shutdown() unblocks run_one() + void wait_wake() + { + m_wake_reader.async_read_some(boost::asio::buffer(m_wake_buf), + [this] (const boost::system::error_code& ec, std::size_t) { if (!ec) wait_wake(); }); + } +#endif + void check_deadline() { // Check whether the deadline has passed. We compare the deadline against @@ -682,6 +864,34 @@ namespace net_utils void async_read(char* buff, size_t sz, boost::asio::detail::transfer_at_least_t transfer_at_least, handler_obj& hndlr) { + // finish the probe before starting another TLS read + if (sz && m_pending_read) + { + const auto pending = m_pending_read; + auto on_read = [socket = m_ssl_socket, buff, sz, transfer_at_least, hndlr] + (const boost::system::error_code& error, char byte) mutable { + if (error) + { + hndlr(error, 0); + return; + } + buff[0] = byte; + if (!transfer_at_least(error, 1) || sz == 1) + hndlr(error, 1); + else + boost::asio::async_read(*socket, boost::asio::buffer(buff + 1, sz - 1), + [transfer_at_least](const boost::system::error_code& ec, size_t bytes) mutable { + return transfer_at_least(ec, bytes + 1); + }, + [hndlr](const boost::system::error_code& ec, size_t bytes) mutable { hndlr(ec, bytes + 1); }); + }; + if (pending->error == boost::asio::error::would_block) + pending->handler = std::move(on_read); + m_pending_read.reset(); + if (pending->error != boost::asio::error::would_block) + on_read(pending->error, pending->byte); + return; + } if(m_ssl_options.support == ssl_support_t::e_ssl_support_disabled) boost::asio::async_read(m_ssl_socket->next_layer(), boost::asio::buffer(buff, sz), transfer_at_least, hndlr); else @@ -693,13 +903,22 @@ namespace net_utils boost::asio::io_context m_io_service; boost::asio::ssl::context m_ctx; std::shared_ptr> m_ssl_socket; + std::shared_ptr m_pending_read; std::function m_connector; ssl_options_t m_ssl_options; bool m_initialized; - bool m_connected; + std::atomic m_connected; boost::asio::steady_timer m_deadline; - std::atomic m_shutdowned; - std::atomic m_cancel_read; +#ifdef BOOST_ASIO_HAS_POSIX_STREAM_DESCRIPTOR + // self-pipe: shutdown() write()s the pipe to wake run_one() signal-safely + boost::asio::posix::stream_descriptor m_wake_reader; + int m_wake_writer; + char m_wake_buf[8]; +#endif + // sticky: set once by shutdown(), never cleared; the client is being torn down + std::atomic m_aborted; + // set by cancel_read() (wallet2::stop() in GUI builds): the next read returns false + std::atomic m_cancel_read; std::atomic m_bytes_sent; std::atomic m_bytes_received; }; diff --git a/contrib/epee/include/net/net_ssl.h b/contrib/epee/include/net/net_ssl.h index f6376c72c..1fcaf3660 100644 --- a/contrib/epee/include/net/net_ssl.h +++ b/contrib/epee/include/net/net_ssl.h @@ -30,6 +30,7 @@ #define _NET_SSL_H #include +#include #include #include #include @@ -134,6 +135,8 @@ namespace net_utils situations where multiple hostnames are being handled by a server. If `verification == system_ca` the client also does a rfc2818 check to ensure that the server certificate is to the provided hostname. + \param abort_requested Polled while waiting; when it returns true the + handshake is cancelled and fails instead of running to `timeout`. \return True if the SSL handshake completes with peer verification settings. */ @@ -143,7 +146,8 @@ namespace net_utils boost::asio::ssl::stream_base::handshake_type type, boost::asio::const_buffer buffer = {}, const std::string& host = {}, - std::chrono::milliseconds timeout = std::chrono::seconds(15)) const; + std::chrono::milliseconds timeout = std::chrono::seconds(15), + const std::function& abort_requested = nullptr) const; }; // https://security.stackexchange.com/questions/34780/checking-client-hello-for-https-classification diff --git a/contrib/epee/include/net/net_utils_base.h b/contrib/epee/include/net/net_utils_base.h index 88b50f602..75ae983d8 100644 --- a/contrib/epee/include/net/net_utils_base.h +++ b/contrib/epee/include/net/net_utils_base.h @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include "byte_slice.h" @@ -208,6 +209,9 @@ namespace net_utils END_KV_SERIALIZE_MAP() }; + bool should_group_ipv6_by_prefix(const boost::asio::ip::address_v6& ip); + boost::asio::ip::address_v6 get_ipv6_subnet_address(const boost::asio::ip::address_v6& ip, const std::size_t prefix_bits); + inline bool operator==(const ipv6_network_address& lhs, const ipv6_network_address& rhs) noexcept { return lhs.equal(rhs); } inline bool operator!=(const ipv6_network_address& lhs, const ipv6_network_address& rhs) noexcept diff --git a/contrib/epee/include/serialization/keyvalue_serialization_overloads.h b/contrib/epee/include/serialization/keyvalue_serialization_overloads.h index a120c1d8f..abbd9b5a1 100644 --- a/contrib/epee/include/serialization/keyvalue_serialization_overloads.h +++ b/contrib/epee/include/serialization/keyvalue_serialization_overloads.h @@ -103,7 +103,7 @@ namespace epee static bool unserialize_stl_container_t_val(stl_container& container, t_storage& stg, typename t_storage::hsection hparent_section, const char* pname) { container.clear(); - typename stl_container::value_type exchange_val; + typename stl_container::value_type exchange_val{}; typename t_storage::harray hval_array = stg.get_first_value(pname, exchange_val, hparent_section); if(!hval_array) return false; container.insert(container.end(), std::move(exchange_val)); diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h index 7fd786a53..6697fe69c 100644 --- a/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -138,10 +138,10 @@ namespace epee levin::message_writer to_send; stg.store_to_binary(to_send.buffer); - int res = transport.send(to_send.finalize_notify(command), conn_id); - if(res <=0 ) + bool res = transport.send(to_send.finalize_notify(command), conn_id); + if(!res) { - MERROR("Failed to notify command " << command << " return code " << res); + MERROR("Failed to notify command " << command); return false; } return true; @@ -285,7 +285,7 @@ namespace epee #define END_INVOKE_MAP2() \ LOG_ERROR("Unknown command:" << command); \ on_levin_traffic(context, false, false, true, in_buff.size(), "invalid-command"); \ - return LEVIN_ERROR_CONNECTION_HANDLER_NOT_DEFINED; \ + return is_notify ? LEVIN_OK : LEVIN_ERROR_CONNECTION_HANDLER_NOT_DEFINED; \ } \ catch (const std::exception &e) { \ MERROR("Error in handle_invoke_map: " << e.what()); \ diff --git a/contrib/epee/include/storages/portable_storage_from_json.h b/contrib/epee/include/storages/portable_storage_from_json.h index f21a89a92..a460ef3a5 100644 --- a/contrib/epee/include/storages/portable_storage_from_json.h +++ b/contrib/epee/include/storages/portable_storage_from_json.h @@ -43,7 +43,7 @@ namespace epee { namespace json { -#define CHECK_ISSPACE() if(!epee::misc_utils::parse::isspace(*it)){ ASSERT_MES_AND_THROW("Wrong JSON character at: " << std::string(it, buf_end));} +#define CHECK_ISSPACE() if(!epee::misc_utils::parse::isspace(*it)){ ASSERT_MES_AND_THROW("Unexpected character in JSON data");} /*inline void parse_error() { @@ -168,7 +168,7 @@ namespace epee { //sub section here typename t_storage::hsection new_sec = stg.open_section(name, current_section, true); - CHECK_AND_ASSERT_THROW_MES(new_sec, "Failed to insert new section in json: " << std::string(it, buf_end)); + CHECK_AND_ASSERT_THROW_MES(new_sec, "Failed to insert new section in JSON data"); run_handler(new_sec, it, buf_end, stg, recursion + 1); state = match_state_wonder_after_value; }else if(*it == '[') @@ -403,7 +403,7 @@ namespace epee } catch(const std::exception& ex) { - MERROR("Failed to parse json, what: " << ex.what()); + MERROR("Failed to parse json, what: " << misc_utils::parse::transform_to_escape_sequence(ex.what())); return false; } catch(...) diff --git a/contrib/epee/include/storages/portable_storage_val_converters.h b/contrib/epee/include/storages/portable_storage_val_converters.h index 5eb9acffd..bd3707385 100644 --- a/contrib/epee/include/storages/portable_storage_val_converters.h +++ b/contrib/epee/include/storages/portable_storage_val_converters.h @@ -28,18 +28,15 @@ #pragma once -#include - #include "misc_language.h" #include "portable_storage_base.h" -#include "parserse_base_utils.h" #include "warnings.h" #include "misc_log_ex.h" -#include #include +#include +#include #include -#include namespace epee { @@ -140,36 +137,6 @@ POP_WARNINGS } }; - // For MyMonero/OpenMonero backend compatibility - // MyMonero backend sends amount, fees and timestamp values as strings. - // Until MM backend is updated, this is needed for compatibility between OpenMonero and MyMonero. - template<> - struct convert_to_integral - { - static void convert(const std::string& from, uint64_t& to) - { - MTRACE("Converting std::string to uint64_t. Source: " << from); - // String only contains digits - if(std::all_of(from.begin(), from.end(), epee::misc_utils::parse::isdigit)) - to = boost::lexical_cast(from); - // MyMonero ISO 8061 timestamp (2017-05-06T16:27:06Z) - else if (boost::regex_match (from, boost::regex("\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\dZ"))) - { - // Convert to unix timestamp -#ifdef HAVE_STRPTIME - struct tm tm; - if (strptime(from.c_str(), "%Y-%m-%dT%H:%M:%S", &tm)) -#else - std::tm tm = {}; - std::istringstream ss(from); - if (ss >> std::get_time(&tm, "%Y-%m-%dT%H:%M:%S")) -#endif - to = std::mktime(&tm); - } else - ASSERT_AND_THROW_WRONG_CONVERSION(); - } - }; - template struct is_convertable: std::integral_constant::value && diff --git a/contrib/epee/src/CMakeLists.txt b/contrib/epee/src/CMakeLists.txt index 808b9f09e..4fc204da9 100644 --- a/contrib/epee/src/CMakeLists.txt +++ b/contrib/epee/src/CMakeLists.txt @@ -31,7 +31,7 @@ set(EPEE_INCLUDE_DIR_BASE "${CMAKE_CURRENT_SOURCE_DIR}/../include") # Add headers to the file list, to be able to search for them and autosave in IDEs. monero_find_all_headers(EPEE_HEADERS_PUBLIC "${EPEE_INCLUDE_DIR_BASE}") -monero_add_library(epee byte_slice.cpp byte_stream.cpp hex.cpp abstract_http_client.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp parserse_base_utils.cpp +monero_add_library(epee byte_slice.cpp byte_stream.cpp hex.cpp abstract_http_client.cpp http_auth.cpp http_protocol_handler.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp parserse_base_utils.cpp wipeable_string.cpp levin_base.cpp memwipe.c connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp int-util.cpp portable_storage.cpp misc_language.cpp diff --git a/contrib/epee/src/http_protocol_handler.cpp b/contrib/epee/src/http_protocol_handler.cpp new file mode 100644 index 000000000..1fd3c3c56 --- /dev/null +++ b/contrib/epee/src/http_protocol_handler.cpp @@ -0,0 +1,44 @@ +// Copyright (c) 2026, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "net/http_protocol_handler.h" + +namespace epee { namespace net_utils { namespace http +{ + std::string get_rpc_connection_limit_key(const net_utils::network_address& address) + { + if (address.get_type_id() == net_utils::ipv6_network_address::get_type_id()) + { + const boost::asio::ip::address_v6 ip = address.as().ip(); + if (net_utils::should_group_ipv6_by_prefix(ip)) + return net_utils::get_ipv6_subnet_address(ip, 64).to_string() + "/64"; + } + + return address.host_str(); + } +}}} diff --git a/contrib/epee/src/net_ssl.cpp b/contrib/epee/src/net_ssl.cpp index aef37844e..8ebc05c5f 100644 --- a/contrib/epee/src/net_ssl.cpp +++ b/contrib/epee/src/net_ssl.cpp @@ -197,8 +197,7 @@ bool create_rsa_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert) X509_free(cert); return false; } - X509_NAME *name = X509_get_subject_name(cert); - X509_set_issuer_name(cert, name); + X509_set_issuer_name(cert, X509_get_subject_name(cert)); if (X509_sign(cert, pkey, EVP_sha256()) == 0) { @@ -533,8 +532,16 @@ void ssl_options_t::configure( { // preverified means it passed system or user CA check. System CA is never loaded // when fingerprints are whitelisted. + // + // The RFC 2818 hostname check must run whenever trust comes from a CA + // chain -- system_ca OR user_ca -- otherwise a CA-issued certificate for + // any host is accepted for this host (defeats the point of the chain). + // It is intentionally skipped for user_certificates (exact pinning) and + // none. + const bool chain_ca = verification == ssl_verification_t::system_ca || + verification == ssl_verification_t::user_ca; const bool verified = preverified && - (verification != ssl_verification_t::system_ca || host.empty() || MONERO_HOSTNAME_VERIFY(host)(preverified, ctx)); + (!chain_ca || host.empty() || MONERO_HOSTNAME_VERIFY(host)(preverified, ctx)); if (!verified && !has_fingerprint(ctx)) { @@ -557,7 +564,8 @@ bool ssl_options_t::handshake( boost::asio::ssl::stream_base::handshake_type type, boost::asio::const_buffer buffer, const std::string& host, - std::chrono::milliseconds timeout) const + std::chrono::milliseconds timeout, + const std::function& abort_requested) const { configure(socket, type, host); @@ -611,6 +619,15 @@ bool ssl_options_t::handshake( boost::asio::post( strand, [&]{ + std::lock_guard guard(state.lock); + if (state.cancel_handshake) { + // cancelled before the socket operation could start: finish here so + // the cancelling side cannot leave a handshake running unattended + state.wait_handshake = false; + state.result = boost::asio::error::operation_aborted; + state.condition.notify_all(); + return; + } socket.async_handshake( type, boost::asio::buffer(buffer), @@ -619,10 +636,28 @@ bool ssl_options_t::handshake( } ); + bool aborted = false; while (!io_context.stopped()) { io_context.poll_one(); std::lock_guard guard(state.lock); + if (!aborted && abort_requested && abort_requested()) + { + // abort from another thread: close the socket so the handshake cannot + // stall again, then keep pumping until both handlers have finished + aborted = true; + if (state.wait_handshake && !state.cancel_handshake) + { + state.cancel_handshake = true; + ec_t ec; + socket.next_layer().close(ec); + } + if (state.wait_timer && !state.cancel_timer) + { + state.cancel_timer = true; + deadline.cancel(); + } + } state.condition.wait_for( state.lock, std::chrono::milliseconds(30), diff --git a/contrib/epee/src/net_utils_base.cpp b/contrib/epee/src/net_utils_base.cpp index a2ca3d3d6..9ba61aa87 100644 --- a/contrib/epee/src/net_utils_base.cpp +++ b/contrib/epee/src/net_utils_base.cpp @@ -1,6 +1,8 @@ #include "net/net_utils_base.h" +#include + #include #include "string_tools.h" @@ -45,6 +47,34 @@ namespace epee { namespace net_utils bool ipv6_network_address::is_loopback() const { return m_address.is_loopback(); } bool ipv6_network_address::is_local() const { return m_address.is_link_local(); } + static bool is_ipv6_unique_local(const boost::asio::ip::address_v6& ip) + { + const boost::asio::ip::address_v6::bytes_type bytes = ip.to_bytes(); + return (bytes[0] & 0xfe) == 0xfc; + } + + bool should_group_ipv6_by_prefix(const boost::asio::ip::address_v6& ip) + { + return !ip.is_unspecified() && !ip.is_loopback() && !ip.is_multicast() && + !ip.is_link_local() && !ip.is_site_local() && !is_ipv6_unique_local(ip) && + !ip.is_v4_mapped(); + } + + boost::asio::ip::address_v6 get_ipv6_subnet_address(const boost::asio::ip::address_v6& ip, const std::size_t prefix_bits) + { + boost::asio::ip::address_v6::bytes_type bytes = ip.to_bytes(); + const std::size_t bits = std::min(prefix_bits, std::size_t{128}); + if (bits < 128) + { + const std::size_t full_bytes = bits / 8; + const std::size_t remainder_bits = bits % 8; + if (remainder_bits != 0) + bytes[full_bytes] &= uint8_t(0xffu << (8 - remainder_bits)); + std::fill(bytes.begin() + full_bytes + (remainder_bits != 0), bytes.end(), 0); + } + return boost::asio::ip::address_v6(bytes); + } + bool ipv4_network_subnet::equal(const ipv4_network_subnet& other) const noexcept { return is_same_host(other) && m_mask == other.m_mask; } diff --git a/contrib/epee/src/parserse_base_utils.cpp b/contrib/epee/src/parserse_base_utils.cpp index e154a75f8..7568cada1 100644 --- a/contrib/epee/src/parserse_base_utils.cpp +++ b/contrib/epee/src/parserse_base_utils.cpp @@ -129,7 +129,7 @@ namespace misc_utils case '/': //Slash character val.push_back('/');break; case 'u': //Unicode code point - if (buf_end - it < 4) + if (buf_end - it < 5) { ASSERT_MES_AND_THROW("Invalid Unicode escape sequence"); } @@ -165,8 +165,7 @@ namespace misc_utils } break; default: - val.push_back(*it); - LOG_PRINT_L0("Unknown escape sequence :\"\\" << *it << "\""); + ASSERT_MES_AND_THROW("Invalid escape sequence in JSON string"); } escape_mode = false; }else if(*it == '"') @@ -176,13 +175,13 @@ namespace misc_utils }else if(*it == '\\') { escape_mode = true; - } + } else { val.push_back(*it); } } - ASSERT_MES_AND_THROW("Failed to match string in json entry: " << std::string(star_end_string, buf_end)); + ASSERT_MES_AND_THROW("Unterminated JSON string"); } void match_number2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val, bool& is_float_val, bool& is_signed_val) { @@ -215,10 +214,10 @@ namespace misc_utils return; } else - ASSERT_MES_AND_THROW("wrong number in json entry: " << std::string(star_end_string, buf_end)); + ASSERT_MES_AND_THROW("Invalid number in JSON entry"); } } - ASSERT_MES_AND_THROW("wrong number in json entry: " << std::string(star_end_string, buf_end)); + ASSERT_MES_AND_THROW("Invalid number in JSON entry"); } void match_word2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val) { @@ -233,11 +232,11 @@ namespace misc_utils { star_end_string = --it; return; - }else - ASSERT_MES_AND_THROW("failed to match word number in json entry: " << std::string(star_end_string, buf_end)); + }else + ASSERT_MES_AND_THROW("Invalid word in JSON entry"); } } - ASSERT_MES_AND_THROW("failed to match word number in json entry: " << std::string(star_end_string, buf_end)); + ASSERT_MES_AND_THROW("Invalid word in JSON entry"); } } } diff --git a/contrib/epee/src/readline_buffer.cpp b/contrib/epee/src/readline_buffer.cpp index ac68d1fdb..76ac9c8b5 100644 --- a/contrib/epee/src/readline_buffer.cpp +++ b/contrib/epee/src/readline_buffer.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -201,7 +202,7 @@ static void handle_line(char* line) static bool same_as_last_line(const std::string& test_line) { // Note that state->offset == state->length, when a new line was entered. - HISTORY_STATE* state = history_get_history_state(); + auto state = std::unique_ptr{history_get_history_state(), free}; return state->length > 0 && test_line.compare(state->entries[state->length-1]->line) == 0; } diff --git a/contrib/gitian/dockrun.sh b/contrib/gitian/dockrun.sh index 816454bf0..371187292 100755 --- a/contrib/gitian/dockrun.sh +++ b/contrib/gitian/dockrun.sh @@ -9,12 +9,18 @@ VERSION=$1 DOCKER=`command -v docker` CACHER=`command -v apt-cacher-ng` -if [ -z "$DOCKER" -o -z "$CACHER" ]; then - echo "$0: you must first install docker.io and apt-cacher-ng" - echo " e.g. sudo apt-get install docker.io apt-cacher-ng" +if [ -z "$DOCKER" ]; then + echo "$0: you must first install docker.io" + echo " e.g. sudo apt-get install docker.io" exit 1 fi +# only use APT cacher if package is present +DOCKER_CACHE_LINE="" +if [ "$CACHER" ]; then + DOCKER_CACHE_LINE="RUN echo 'Acquire::http { Proxy \"http://172.17.0.1:3142\"; };' > /etc/apt/apt.conf.d/50cacher" +fi + GH_USER=${GH_USER-$USER} TAG=gitrun-bionic @@ -33,7 +39,7 @@ cat < ${TAG}.Dockerfile FROM ubuntu:bionic ENV DEBIAN_FRONTEND=noninteractive -RUN echo 'Acquire::http { Proxy "http://172.17.0.1:3142"; };' > /etc/apt/apt.conf.d/50cacher +$DOCKER_CACHE_LINE RUN echo "$GID" >> /etc/group RUN apt-get update && apt-get --no-install-recommends -y install lsb-release ruby git make wget docker.io python3 curl @@ -41,10 +47,10 @@ RUN useradd -ms /bin/bash -U ubuntu -G docker USER ubuntu:docker WORKDIR $WORKDIR -RUN git clone https://codeberg.org/wownero/gitian.sigs.git sigs; \ +RUN git clone https://github.com/wownero/gitian.sigs.git sigs; \ git clone https://github.com/devrandom/gitian-builder.git builder; \ cd builder; git checkout c0f77ca018cb5332bfd595e0aff0468f77542c23; mkdir -p inputs var; cd inputs; \ - git clone https://codeberg.org/wownero/wownero + git clone https://github.com/wownero/wownero CMD ["sleep", "infinity"] EOF @@ -66,7 +72,7 @@ cat < ${TAG2}.Dockerfile FROM ubuntu:bionic ENV DEBIAN_FRONTEND=noninteractive -RUN echo 'Acquire::http { Proxy "http://172.17.0.1:3142"; };' > /etc/apt/apt.conf.d/50cacher +$DOCKER_CACHE_LINE RUN apt-get update && apt-get --no-install-recommends -y install build-essential git language-pack-en \ wget lsb-release curl gcc-7 g++-7 gcc g++ binutils-gold pkg-config autoconf libtool automake faketime \ bsdmainutils ca-certificates python cmake gperf @@ -109,7 +115,7 @@ if [ "$check" != "sign" ]; then fi if [ ! -d sigs ]; then - git clone https://codeberg.org/wownero/gitian.sigs.git sigs + git clone https://github.com/wownero/gitian.sigs.git sigs cd sigs git remote add $GH_USER git@github.com:$GH_USER/gitian.sigs.git cd .. diff --git a/contrib/gitian/gitian-android.yml b/contrib/gitian/gitian-android.yml index ae3e15a24..606d86ed0 100644 --- a/contrib/gitian/gitian-android.yml +++ b/contrib/gitian/gitian-android.yml @@ -26,7 +26,7 @@ packages: - "cmake" - "unzip" remotes: -- "url": "https://codeberg.org/wownero/wownero.git" +- "url": "https://github.com/wownero/wownero.git" "dir": "wownero" files: [] script: | diff --git a/contrib/gitian/gitian-build.py b/contrib/gitian/gitian-build.py index b02285ebc..ffdec3144 100755 --- a/contrib/gitian/gitian-build.py +++ b/contrib/gitian/gitian-build.py @@ -5,7 +5,7 @@ import os import subprocess import sys -gsigs = 'https://codeberg.org/wownero/gitian.sigs.git' +gsigs = 'https://github.com/wownero/gitian.sigs.git' gbrepo = 'https://github.com/devrandom/gitian-builder.git' platforms = {'l': ['Linux', 'linux', 'tar.bz2'], @@ -113,7 +113,7 @@ def main(): parser = argparse.ArgumentParser(description='Script for running full Gitian builds.', usage='%(prog)s [options] signer version') parser.add_argument('-c', '--commit', action='store_true', dest='commit', help='Indicate that the version argument is for a commit or branch') parser.add_argument('-p', '--pull', action='store_true', dest='pull', help='Indicate that the version argument is the number of a github repository pull request') - parser.add_argument('-u', '--url', dest='url', default='https://codeberg.org/wownero/wownero', help='Specify the URL of the repository. Default is %(default)s') + parser.add_argument('-u', '--url', dest='url', default='https://github.com/wownero/wownero', help='Specify the URL of the repository. Default is %(default)s') parser.add_argument('-v', '--verify', action='store_true', dest='verify', help='Verify the Gitian build') parser.add_argument('-b', '--build', action='store_true', dest='build', help='Do a Gitian build') parser.add_argument('-B', '--buildsign', action='store_true', dest='buildsign', help='Build both signed and unsigned binaries') diff --git a/contrib/gitian/gitian-freebsd.yml b/contrib/gitian/gitian-freebsd.yml index d5b27fb97..59805dd70 100644 --- a/contrib/gitian/gitian-freebsd.yml +++ b/contrib/gitian/gitian-freebsd.yml @@ -26,7 +26,7 @@ packages: - "python" - "cmake" remotes: -- "url": "https://codeberg.org/wownero/wownero.git" +- "url": "https://github.com/wownero/wownero.git" "dir": "wownero" files: [] script: | diff --git a/contrib/gitian/gitian-linux.yml b/contrib/gitian/gitian-linux.yml index 5c9583e6b..1f3f08e4f 100644 --- a/contrib/gitian/gitian-linux.yml +++ b/contrib/gitian/gitian-linux.yml @@ -38,7 +38,7 @@ packages: - "python" - "cmake" remotes: -- "url": "https://codeberg.org/wownero/wownero.git" +- "url": "https://github.com/wownero/wownero.git" "dir": "wownero" files: [] script: | diff --git a/contrib/gitian/gitian-osx.yml b/contrib/gitian/gitian-osx.yml index 06496cb4c..a546e9172 100644 --- a/contrib/gitian/gitian-osx.yml +++ b/contrib/gitian/gitian-osx.yml @@ -24,7 +24,7 @@ packages: - "python-dev" - "python-setuptools" remotes: -- "url": "https://codeberg.org/wownero/wownero.git" +- "url": "https://github.com/wownero/wownero.git" "dir": "wownero" files: [] script: | diff --git a/contrib/gitian/gitian-win.yml b/contrib/gitian/gitian-win.yml index 0ca697fdf..9ab8c6de2 100644 --- a/contrib/gitian/gitian-win.yml +++ b/contrib/gitian/gitian-win.yml @@ -35,7 +35,7 @@ alternatives: package: "x86_64-w64-mingw32-gcc" path: "/usr/bin/x86_64-w64-mingw32-gcc-posix" remotes: -- "url": "https://codeberg.org/wownero/wownero.git" +- "url": "https://github.com/wownero/wownero.git" "dir": "wownero" files: [] script: | diff --git a/external/easylogging++/easylogging++.cc b/external/easylogging++/easylogging++.cc index d60b8e268..1a3bc7a34 100644 --- a/external/easylogging++/easylogging++.cc +++ b/external/easylogging++/easylogging++.cc @@ -16,6 +16,7 @@ #define EASYLOGGING_CC #include "easylogging++.h" +#include #include #include @@ -2502,11 +2503,11 @@ void DefaultLogDispatchCallback::handle(const LogDispatchData* data) { template -static inline void utf8canonical(std::string &s, Transform t = [](wint_t c)->wint_t { return c; }) +static inline void utf8canonical(std::string &s, Transform t = [](std::uint32_t c)->std::uint32_t { return c; }) { size_t avail = s.size(); const char *ptr = s.data(); - wint_t cp = 0; + std::uint32_t cp = 0; int rbytes = 1, bytes = -1; char wbuf[8], *wptr; size_t w_offset = 0; @@ -2585,7 +2586,7 @@ static inline void utf8canonical(std::string &s, Transform t = [](wint_t c)->win void sanitize(std::string &s) { - utf8canonical(s, [](wint_t c)->wint_t { + utf8canonical(s, [](std::uint32_t c)->std::uint32_t { if (c == 9 || c == 10 || c == 13) return c; if (c < 0x20) diff --git a/external/easylogging++/easylogging++.h b/external/easylogging++/easylogging++.h index be7fb713a..fc5c11322 100644 --- a/external/easylogging++/easylogging++.h +++ b/external/easylogging++/easylogging++.h @@ -2791,6 +2791,7 @@ class Storage : base::NoCopy, public base::threading::ThreadSafe { } }; extern ELPP_EXPORT base::type::StoragePointer elStorage; +ELPP_EXPORT void sanitize(std::string& s); #define ELPP el::base::Storage::getELPP() class DefaultLogDispatchCallback : public LogDispatchCallback { protected: diff --git a/external/randomwow b/external/randomwow index 27b099b6d..ff4747966 160000 --- a/external/randomwow +++ b/external/randomwow @@ -1 +1 @@ -Subproject commit 27b099b6dd6fef6e17f58c6dfe00009e9c5df587 +Subproject commit ff47479662593fa18295642752b4df8d04c21b66 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c08dd9d82..3786710f7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -32,6 +32,11 @@ if (WIN32 OR STATIC) add_definitions(-DSTATICLIB) endif () +set(STACK_TRACE_USE_WRAP OFF) +if (STACK_TRACE AND NOT STATIC AND CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(STACK_TRACE_USE_WRAP ON) +endif() + function (monero_private_headers group) source_group("${group}\\Private" FILES diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 846bc0e3a..54bc085ab 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -356,6 +356,7 @@ typedef struct outtx { } outtx; std::atomic mdb_txn_safe::num_active_txns{0}; +thread_local uint64_t mdb_txn_safe::num_active_txns_per_thread = 0; std::atomic_flag mdb_txn_safe::creation_gate = ATOMIC_FLAG_INIT; mdb_threadinfo::~mdb_threadinfo() @@ -373,9 +374,12 @@ mdb_txn_safe::mdb_txn_safe(const bool check) : m_txn(NULL), m_tinfo(NULL), m_che { if (check) { - while (creation_gate.test_and_set()); - num_active_txns++; - creation_gate.clear(); + const bool nested = num_active_txns_per_thread != 0; + if (!nested) + while (creation_gate.test_and_set()); + increment_txns(1); + if (!nested) + creation_gate.clear(); } } @@ -406,12 +410,12 @@ mdb_txn_safe::~mdb_txn_safe() } mdb_txn_abort(m_txn); } - num_active_txns--; + increment_txns(-1); } void mdb_txn_safe::uncheck() { - num_active_txns--; + increment_txns(-1); m_check = false; } @@ -466,7 +470,8 @@ void mdb_txn_safe::allow_new_txns() void mdb_txn_safe::increment_txns(int i) { - num_active_txns += i; + num_active_txns_per_thread += i; + num_active_txns += i; } #define TXN_PREFIX(flags); \ @@ -644,18 +649,13 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const float resize_percent = RESIZE_PERCENT; MDEBUG(boost::format("Percent used: %.04f Percent threshold: %.04f") % (100.*size_used/mei.me_mapsize) % (100.*resize_percent)); - if (threshold_size > 0) + if (threshold_size > 0 && mei.me_mapsize - size_used <= threshold_size) { - if (mei.me_mapsize - size_used < threshold_size) - { - MINFO("Threshold met (size-based)"); - return true; - } - else - return false; + MINFO("Threshold met (size-based)"); + return true; } - if ((double)size_used / mei.me_mapsize > resize_percent) + if ((double)size_used / mei.me_mapsize >= resize_percent) { MINFO("Threshold met (percent-based)"); return true; @@ -766,7 +766,7 @@ estim: MDEBUG("estimated average block size for batch: " << avg_block_size); // bigger safety margin on smaller block sizes - if (batch_fudge_factor < 5000.0) + if (!batch_bytes && batch_fudge_factor < 5000.0) batch_fudge_factor = 5000.0; threshold_size = avg_block_size * db_expand_factor * batch_fudge_factor; return threshold_size; diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index bdae44948..7db237bc8 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -161,6 +161,7 @@ struct mdb_txn_safe bool m_batch_txn = false; bool m_check; static std::atomic num_active_txns; + static thread_local uint64_t num_active_txns_per_thread; // could use a mutex here, but this should be sufficient. static std::atomic_flag creation_gate; diff --git a/src/blockchain_utilities/blockchain_prune.cpp b/src/blockchain_utilities/blockchain_prune.cpp index 0a4fa43ae..d5b0146c3 100644 --- a/src/blockchain_utilities/blockchain_prune.cpp +++ b/src/blockchain_utilities/blockchain_prune.cpp @@ -185,7 +185,7 @@ static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned if (dbr) throw std::runtime_error("Failed to commit txn: " + std::string(mdb_strerror(dbr))); tx_active1 = false; MDB_stat stats; - dbr = mdb_env_stat(env0, &stats); + dbr = mdb_stat(txn0, dbi0, &stats); if (dbr) throw std::runtime_error("Failed to stat " + std::string(table) + " LMDB table: " + std::string(mdb_strerror(dbr))); check_resize(env1, (stats.ms_branch_pages + stats.ms_overflow_pages + stats.ms_leaf_pages) * stats.ms_psize); dbr = mdb_txn_begin(env1, NULL, 0, &txn1); diff --git a/src/checkpoints/checkpoints.cpp b/src/checkpoints/checkpoints.cpp index 2902bafb2..a17fc9da3 100644 --- a/src/checkpoints/checkpoints.cpp +++ b/src/checkpoints/checkpoints.cpp @@ -158,6 +158,19 @@ namespace cryptonote return m_points.rbegin()->first; } //--------------------------------------------------------------------------- + uint64_t checkpoints::get_nearest_checkpoint_height(uint64_t block_height) const + { + if (m_points.empty()) + return 0; + + auto it = m_points.lower_bound(block_height); + if (it == m_points.begin()) + return 0; + + --it; + return it->first; + } + //--------------------------------------------------------------------------- const std::map& checkpoints::get_points() const { return m_points; diff --git a/src/checkpoints/checkpoints.h b/src/checkpoints/checkpoints.h index 07daeb4c0..d8de54eff 100644 --- a/src/checkpoints/checkpoints.h +++ b/src/checkpoints/checkpoints.h @@ -129,6 +129,14 @@ namespace cryptonote */ uint64_t get_max_height() const; + /** + * @brief gets the highest checkpoint height less than the given block height + * + * @param block_height the reference block height + * @return the nearest checkpoint height below block_height, or 0 if none + */ + uint64_t get_nearest_checkpoint_height(uint64_t block_height) const; + /** * @brief gets the checkpoints container * diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index b712ee6b1..ee5878646 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -67,6 +67,11 @@ monero_add_library(common ${common_headers} ${common_private_headers} DEPENDS generate_translations_header) + +if (STACK_TRACE_USE_WRAP) + target_compile_definitions(obj_common PRIVATE STACK_TRACE_USE_WRAP) +endif() + target_link_libraries(common PUBLIC cncrypto @@ -82,5 +87,13 @@ target_link_libraries(common ${OPENSSL_LIBRARIES} ${EXTRA_LIBRARIES}) +if (STACK_TRACE_USE_WRAP) + set_property(TARGET common APPEND PROPERTY LINK_FLAGS + "-Wl,--wrap=__cxa_throw") + set_property(TARGET common APPEND PROPERTY INTERFACE_LINK_OPTIONS + "-Wl,--wrap=__cxa_throw" + "-Wl,-u,__wrap___cxa_throw") +endif() + #monero_install_headers(common # ${common_headers}) diff --git a/src/common/scoped_message_writer.h b/src/common/scoped_message_writer.h index cf4e6855f..91716693e 100644 --- a/src/common/scoped_message_writer.h +++ b/src/common/scoped_message_writer.h @@ -99,17 +99,20 @@ public: { m_flush = false; - MCLOG_FILE(m_log_level, "msgwriter", m_oss.str()); + std::string message = m_oss.str(); + MCLOG_FILE(m_log_level, "msgwriter", message); + try { el::base::sanitize(message); } + catch (const std::exception&) { message = ""; } PAUSE_READLINE(); if (epee::console_color_default == m_color) { - std::cout << m_oss.str(); + std::cout << message; } else { set_console_color(m_color, m_bright); - std::cout << m_oss.str(); + std::cout << message; epee::reset_console_color(); } std::cout << std::endl; diff --git a/src/common/stack_trace.cpp b/src/common/stack_trace.cpp index 1f946ddef..c6c12058a 100644 --- a/src/common/stack_trace.cpp +++ b/src/common/stack_trace.cpp @@ -40,7 +40,7 @@ #include #include #endif -#ifndef STATICLIB +#if !defined(STATICLIB) && !defined(STACK_TRACE_USE_WRAP) #include #endif #include @@ -71,12 +71,12 @@ #define CXA_THROW_INFO_T void #endif // !__clang__ -#ifdef STATICLIB +#if defined(STATICLIB) || defined(STACK_TRACE_USE_WRAP) #define CXA_THROW __wrap___cxa_throw extern "C" __attribute__((noreturn)) void __real___cxa_throw(void *ex, CXA_THROW_INFO_T *info, void (*dest)(void*)); -#else // !STATICLIB +#else // STATICLIB || STACK_TRACE_USE_WRAP #define CXA_THROW __cxa_throw extern "C" typedef @@ -84,7 +84,7 @@ typedef __attribute__((noreturn)) #endif // __clang__ void (cxa_throw_t)(void *ex, CXA_THROW_INFO_T *info, void (*dest)(void*)); -#endif // !STATICLIB +#endif // STATICLIB || STACK_TRACE_USE_WRAP extern "C" __attribute__((noreturn)) @@ -96,12 +96,12 @@ void CXA_THROW(void *ex, CXA_THROW_INFO_T *info, void (*dest)(void*)) tools::log_stack_trace((std::string("Exception: ")+((!status && dsym) ? dsym : (const char*)info)).c_str()); free(dsym); -#ifndef STATICLIB +#if !defined(STATICLIB) && !defined(STACK_TRACE_USE_WRAP) #ifndef __clang__ // for GCC the attr can't be applied in typedef like for clang __attribute__((noreturn)) #endif // !__clang__ cxa_throw_t *__real___cxa_throw = (cxa_throw_t*)dlsym(RTLD_NEXT, "__cxa_throw"); -#endif // !STATICLIB +#endif // !STATICLIB && !STACK_TRACE_USE_WRAP __real___cxa_throw(ex, info, dest); } diff --git a/src/crypto/c_threads.h b/src/crypto/c_threads.h index 3457738b3..d84f7977e 100644 --- a/src/crypto/c_threads.h +++ b/src/crypto/c_threads.h @@ -65,6 +65,6 @@ #define CTHR_THREAD_RETURN return NULL #define CTHR_THREAD_CREATE(thr, func, arg) (pthread_create(&thr, NULL, func, arg) == 0) #define CTHR_THREAD_JOIN(thr) pthread_join(thr, NULL) -#define CTHR_THREAD_CLOSE(thr) +#define CTHR_THREAD_CLOSE(thr) pthread_detach(thr) #endif diff --git a/src/crypto/crypto-ops-data.c b/src/crypto/crypto-ops-data.c index 1a85de60d..bbdd021c7 100644 --- a/src/crypto/crypto-ops-data.c +++ b/src/crypto/crypto-ops-data.c @@ -877,3 +877,5 @@ const ge_p3 ge_p3_H = { {1, 0, 0, 0, 0, 0, 0, 0, 0, 0}, {23443568, -5110398, -8776029, -4345135, 6889568, -14710814, 7474843, 3279062, 14550766, -7453428} }; + +const unsigned char sc_inv_eight[32] = {0x79, 0x2f, 0xdc, 0xe2, 0x29, 0xe5, 0x06, 0x61, 0xd0, 0xda, 0x1c, 0x7d, 0xb3, 0x9d, 0xd3, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06}; diff --git a/src/crypto/crypto-ops.c b/src/crypto/crypto-ops.c index 4b392d472..8694c80ad 100644 --- a/src/crypto/crypto-ops.c +++ b/src/crypto/crypto-ops.c @@ -2307,6 +2307,17 @@ void ge_mul8(ge_p1p1 *r, const ge_p2 *t) { ge_p2_dbl(r, &u); } +void ge_clear_torsion_vartime(unsigned char *out, const ge_p3 *point) { + // mul by inv 8, then mul by 8 + ge_p2 point_inv_8; + ge_scalarmult(&point_inv_8, sc_inv_eight, point); + ge_p1p1 point_inv_8_mul_8; + ge_mul8(&point_inv_8_mul_8, &point_inv_8); + ge_p3 torsion_cleared_point; + ge_p1p1_to_p3(&torsion_cleared_point, &point_inv_8_mul_8); + ge_p3_tobytes(out, &torsion_cleared_point); +} + void ge_fromfe_frombytes_vartime(ge_p2 *r, const unsigned char *s) { fe u, v, w, x, y, z; unsigned char sign; diff --git a/src/crypto/crypto-ops.h b/src/crypto/crypto-ops.h index e4901e080..07c529a05 100644 --- a/src/crypto/crypto-ops.h +++ b/src/crypto/crypto-ops.h @@ -135,6 +135,7 @@ void ge_triple_scalarmult_precomp_vartime(ge_p2 *, const unsigned char *, const void ge_double_scalarmult_precomp_vartime2(ge_p2 *, const unsigned char *, const ge_dsmp, const unsigned char *, const ge_dsmp); void ge_double_scalarmult_precomp_vartime2_p3(ge_p3 *, const unsigned char *, const ge_dsmp, const unsigned char *, const ge_dsmp); void ge_mul8(ge_p1p1 *, const ge_p2 *); +void ge_clear_torsion_vartime(unsigned char *, const ge_p3 *); extern const fe fe_ma2; extern const fe fe_ma; extern const fe fe_fffb1; @@ -143,6 +144,7 @@ extern const fe fe_fffb3; extern const fe fe_fffb4; extern const ge_p3 ge_p3_identity; extern const ge_p3 ge_p3_H; +extern const unsigned char sc_inv_eight[32]; void ge_fromfe_frombytes_vartime(ge_p2 *, const unsigned char *); void sc_0(unsigned char *); void sc_reduce32(unsigned char *); diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index 77a36069a..3db7f8d43 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -177,6 +177,23 @@ namespace crypto { return ge_frombytes_vartime(&point, &key) == 0; } + bool get_valid_torsion_cleared_point_vartime(const ec_point &point, ec_point &torsion_cleared_out) { + ge_p3 p3; + if (ge_frombytes_vartime(&p3, &point) != 0) + return false; + ge_clear_torsion_vartime(&torsion_cleared_out, &p3); + if (memcmp(&torsion_cleared_out, &EC_I, sizeof(EC_I)) == 0) + return false; + return true; + } + + public_key pubkey_clear_torsion(const public_key &pubkey) { + public_key result; + if (!get_valid_torsion_cleared_point_vartime(pubkey, result)) + static_cast(result) = EC_I; + return result; + } + bool crypto_ops::secret_key_to_public_key(const secret_key &sec, public_key &pub) { ge_p3 point; if (sc_check(&unwrap(sec)) != 0) { diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index ee1cac04a..9121d2bc5 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -113,6 +113,8 @@ namespace crypto { sizeof(key_derivation) == 32 && sizeof(key_image) == 32 && sizeof(signature) == 64 && sizeof(view_tag) == 1, "Invalid structure size"); + static const ec_point EC_I = {1}; + class crypto_ops { crypto_ops(); crypto_ops(const crypto_ops &); @@ -217,6 +219,10 @@ namespace crypto { return crypto_ops::check_key(key); } + bool get_valid_torsion_cleared_point_vartime(const ec_point &point, ec_point &torsion_cleared_out); + // Return a canonical key in the prime-order subgroup, or identity on decode failure. + public_key pubkey_clear_torsion(const public_key &pubkey); + /* Checks a private key and computes the corresponding public key. */ inline bool secret_key_to_public_key(const secret_key &sec, public_key &pub) { diff --git a/src/crypto/generic-ops.h b/src/crypto/generic-ops.h index 3ff3619fe..8d103aa18 100644 --- a/src/crypto/generic-ops.h +++ b/src/crypto/generic-ops.h @@ -32,9 +32,13 @@ #include #include +#include #include #include #include +#include + +#include "random.h" #define CRYPTO_MAKE_COMPARABLE(type) \ namespace crypto { \ @@ -57,22 +61,29 @@ namespace crypto { \ } \ } +namespace crypto { + inline std::size_t siphash_to_size_t(const void *data, std::size_t length) { + static_assert(sizeof(crypto_siphash_key) == crypto_shorthash_siphash24_KEYBYTES, + "crypto_siphash_key size must match the SipHash-2-4 key length"); + static_assert(sizeof(std::uint64_t) == crypto_shorthash_siphash24_BYTES, + "std::uint64_t size must match the SipHash-2-4 digest length"); + std::uint64_t h; + crypto_shorthash_siphash24(reinterpret_cast(&h), static_cast(data), length, crypto_siphash_key); + return h; + } +} + #define CRYPTO_DEFINE_HASH_FUNCTIONS(type) \ namespace crypto { \ - static_assert(sizeof(std::size_t) <= sizeof(type), "Size of " #type " must be at least that of size_t"); \ inline std::size_t hash_value(const type &_v) { \ - std::size_t h; \ - memcpy(&h, std::addressof(_v), sizeof(h)); \ - return h; \ + return siphash_to_size_t(std::addressof(_v), sizeof(_v)); \ } \ } \ namespace std { \ template<> \ struct hash { \ std::size_t operator()(const crypto::type &_v) const { \ - std::size_t h; \ - memcpy(&h, std::addressof(_v), sizeof(h)); \ - return h; \ + return ::crypto::siphash_to_size_t(std::addressof(_v), sizeof(_v)); \ } \ }; \ } diff --git a/src/crypto/random.c b/src/crypto/random.c index cfb637fb4..0f6f16130 100644 --- a/src/crypto/random.c +++ b/src/crypto/random.c @@ -96,6 +96,7 @@ static void generate_system_random_bytes(size_t n, void *result) { #endif static union hash_state state; +unsigned char crypto_siphash_key[16]; #if !defined(NDEBUG) static volatile int curstate; /* To catch thread safety problems. */ @@ -107,10 +108,12 @@ FINALIZER(deinit_random) { curstate = 0; #endif memset(&state, 0, sizeof(union hash_state)); + memset(crypto_siphash_key, 0, sizeof(crypto_siphash_key)); } INITIALIZER(init_random) { generate_system_random_bytes(32, &state); + generate_system_random_bytes(sizeof(crypto_siphash_key), crypto_siphash_key); REGISTER_FINALIZER(deinit_random); #if !defined(NDEBUG) assert(curstate == 0); diff --git a/src/crypto/random.h b/src/crypto/random.h index d50f29430..935f76353 100644 --- a/src/crypto/random.h +++ b/src/crypto/random.h @@ -32,5 +32,15 @@ #include +#ifdef __cplusplus +extern "C" { +#endif + void generate_random_bytes_not_thread_safe(size_t n, void *result); void add_extra_entropy_not_thread_safe(const void *ptr, size_t bytes); + +extern unsigned char crypto_siphash_key[16]; + +#ifdef __cplusplus +} +#endif diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 8986b2139..9f9f7045a 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -1162,12 +1162,17 @@ static const int rcon[] = { 0x01,0x01,0x01,0x01, 0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d,0x0c0f0e0d, // rotate-n-splat 0x1b,0x1b,0x1b,0x1b }; +const uint8_t *key_ptr = key; +uint8_t *expanded_key_ptr = expandedKey; +const int *rcon_ptr = rcon; +int count; + __asm__( " eor v0.16b,v0.16b,v0.16b\n" " ld1 {v3.16b},[%0],#16\n" " ld1 {v1.4s,v2.4s},[%2],#32\n" " ld1 {v4.16b},[%0]\n" -" mov w2,#5\n" +" mov %w3,#5\n" " st1 {v3.4s},[%1],#16\n" "\n" "1:\n" @@ -1175,7 +1180,7 @@ __asm__( " ext v5.16b,v0.16b,v3.16b,#12\n" " st1 {v4.4s},[%1],#16\n" " aese v6.16b,v0.16b\n" -" subs w2,w2,#1\n" +" subs %w3,%w3,#1\n" "\n" " eor v3.16b,v3.16b,v5.16b\n" " ext v5.16b,v0.16b,v5.16b,#12\n" @@ -1201,7 +1206,15 @@ __asm__( " eor v4.16b,v4.16b,v6.16b\n" " b 1b\n" "\n" -"2:\n" : : "r"(key), "r"(expandedKey), "r"(rcon)); +"2:\n" + : "+&r"(key_ptr), + "+&r"(expanded_key_ptr), + "+&r"(rcon_ptr), + "=&r"(count), + "=m"(*(uint8_t (*)[176]) expandedKey) + : "m"(*(const int (*)[8]) rcon), + "m"(*(const uint8_t (*)[32]) key) + : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "cc"); } /* An ordinary AES round is a sequence of SubBytes, ShiftRows, MixColumns, AddRoundKey. There diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h index 215903a0c..31df4f63a 100644 --- a/src/cryptonote_basic/cryptonote_basic.h +++ b/src/cryptonote_basic/cryptonote_basic.h @@ -53,8 +53,12 @@ namespace cryptonote { - typedef std::vector ring_signature; - + // These are guaranteed ceilings for the entire chain. Some fork rules may limit some further. + constexpr size_t COINBASE_VIN_COUNT = 1; + constexpr size_t MAX_VIN_COUNT = CRYPTONOTE_MAX_TX_SIZE / sizeof(crypto::key_image); + constexpr size_t MAX_NON_COINBASE_VOUT_COUNT = CRYPTONOTE_MAX_TX_SIZE / sizeof(crypto::public_key); + constexpr size_t MAX_COINBASE_VOUT_COUNT = /*LEVIN_DEFAULT_MAX_PACKET_SIZE=*/100000000 / sizeof(crypto::public_key); + constexpr size_t MAX_TOTAL_KEY_OFFSETS = CRYPTONOTE_MAX_TX_SIZE / sizeof(crypto::public_key); /* outputs */ @@ -143,7 +147,7 @@ namespace cryptonote BEGIN_SERIALIZE_OBJECT() VARINT_FIELD(amount) - FIELD(key_offsets) + CONTAINER_FIELD_CAPPED(key_offsets, MAX_TOTAL_KEY_OFFSETS) FIELD(k_image) END_SERIALIZE() }; @@ -184,11 +188,109 @@ namespace cryptonote VARINT_FIELD(version) if(version == 0 || CURRENT_TRANSACTION_VERSION < version) return false; VARINT_FIELD(unlock_time) - FIELD(vin) - FIELD(vout) + if (typename Archive::is_saving()) + { + // Serializing... + FIELD(vin) + FIELD(vout) + } + else + { + // De-serializing... + // vin + if (!this->deserialize_vin(ar)) + { + ar.set_fail(); + return false; + } + + // vout + const size_t MAX_VOUT_COUNT = (this->is_coinbase() || vin.empty()) ? MAX_COINBASE_VOUT_COUNT : MAX_NON_COINBASE_VOUT_COUNT; + CONTAINER_FIELD_CAPPED(vout, MAX_VOUT_COUNT) + } FIELD(extra) END_SERIALIZE() + //----------------------------------------------------------------------- + bool is_coinbase() const + { + if (vin.size() != COINBASE_VIN_COUNT) + return false; + + if (vin.at(0).type() != typeid(txin_gen)) + return false; + + return true; + } + //----------------------------------------------------------------------- + private: + // Do some validation on vin before doing allocations and reading the archive: + // 1. Can't exceed MAX_VIN_COUNT. + // 2. If coinbase, only 1 txin_gen. + // 3. If not coinbase: type txin_to_key (see check_inputs_types_supported). + // 4. If not coinbase: key offsets don't exceed max allowed for whole tx. + // Note: can't enforce non-empty because wallets may store empty vin. + template