From a2ff89bc2491d8798bcc0e9e6059499163e4c32e Mon Sep 17 00:00:00 2001 From: Lee Clagett Date: Wed, 19 Aug 2020 18:29:32 -0400 Subject: [PATCH] Initial working base separated from top-level monero project --- CMakeLists.txt | 172 ++++ src/CMakeLists.txt | 77 ++ src/admin_main.cpp | 423 ++++++++ src/config.cpp | 9 + src/config.h | 11 + src/db/CMakeLists.txt | 34 + src/db/account.cpp | 179 ++++ src/db/account.h | 114 +++ src/db/data.cpp | 225 +++++ src/db/data.h | 275 ++++++ src/db/fwd.h | 56 ++ src/db/storage.cpp | 1797 ++++++++++++++++++++++++++++++++++ src/db/storage.h | 239 +++++ src/db/string.cpp | 62 ++ src/db/string.h | 55 ++ src/error.cpp | 131 +++ src/error.h | 79 ++ src/fwd.h | 35 + src/options.h | 72 ++ src/rest_server.cpp | 863 ++++++++++++++++ src/rest_server.h | 69 ++ src/rpc/CMakeLists.txt | 33 + src/rpc/client.cpp | 325 ++++++ src/rpc/client.h | 219 +++++ src/rpc/daemon_zmq.cpp | 169 ++++ src/rpc/daemon_zmq.h | 75 ++ src/rpc/fwd.h | 33 + src/rpc/json.h | 96 ++ src/rpc/light_wallet.cpp | 338 +++++++ src/rpc/light_wallet.h | 198 ++++ src/rpc/rates.cpp | 78 ++ src/rpc/rates.h | 74 ++ src/scanner.cpp | 780 +++++++++++++++ src/scanner.h | 59 ++ src/server_main.cpp | 241 +++++ src/util/CMakeLists.txt | 33 + src/util/fwd.h | 35 + src/util/gamma_picker.cpp | 112 +++ src/util/gamma_picker.h | 88 ++ src/util/http_server.h | 124 +++ src/util/random_outputs.cpp | 309 ++++++ src/util/random_outputs.h | 92 ++ src/util/transactions.cpp | 61 ++ src/util/transactions.h | 45 + src/wire.h | 77 ++ src/wire/CMakeLists.txt | 36 + src/wire/crypto.h | 72 ++ src/wire/error.cpp | 93 ++ src/wire/error.h | 135 +++ src/wire/field.h | 280 ++++++ src/wire/filters.h | 73 ++ src/wire/fwd.h | 68 ++ src/wire/json.h | 54 + src/wire/json/CMakeLists.txt | 33 + src/wire/json/base.h | 50 + src/wire/json/error.cpp | 108 ++ src/wire/json/error.h | 53 + src/wire/json/fwd.h | 45 + src/wire/json/read.cpp | 413 ++++++++ src/wire/json/read.h | 134 +++ src/wire/json/write.cpp | 167 ++++ src/wire/json/write.h | 184 ++++ src/wire/read.cpp | 61 ++ src/wire/read.h | 471 +++++++++ src/wire/traits.h | 46 + src/wire/vector.h | 41 + src/wire/write.cpp | 31 + src/wire/write.h | 224 +++++ 68 files changed, 11543 insertions(+) create mode 100644 CMakeLists.txt create mode 100644 src/CMakeLists.txt create mode 100644 src/admin_main.cpp create mode 100644 src/config.cpp create mode 100644 src/config.h create mode 100644 src/db/CMakeLists.txt create mode 100644 src/db/account.cpp create mode 100644 src/db/account.h create mode 100644 src/db/data.cpp create mode 100644 src/db/data.h create mode 100644 src/db/fwd.h create mode 100644 src/db/storage.cpp create mode 100644 src/db/storage.h create mode 100644 src/db/string.cpp create mode 100644 src/db/string.h create mode 100644 src/error.cpp create mode 100644 src/error.h create mode 100644 src/fwd.h create mode 100644 src/options.h create mode 100644 src/rest_server.cpp create mode 100644 src/rest_server.h create mode 100644 src/rpc/CMakeLists.txt create mode 100644 src/rpc/client.cpp create mode 100644 src/rpc/client.h create mode 100644 src/rpc/daemon_zmq.cpp create mode 100644 src/rpc/daemon_zmq.h create mode 100644 src/rpc/fwd.h create mode 100644 src/rpc/json.h create mode 100644 src/rpc/light_wallet.cpp create mode 100644 src/rpc/light_wallet.h create mode 100644 src/rpc/rates.cpp create mode 100644 src/rpc/rates.h create mode 100644 src/scanner.cpp create mode 100644 src/scanner.h create mode 100644 src/server_main.cpp create mode 100644 src/util/CMakeLists.txt create mode 100644 src/util/fwd.h create mode 100644 src/util/gamma_picker.cpp create mode 100644 src/util/gamma_picker.h create mode 100644 src/util/http_server.h create mode 100644 src/util/random_outputs.cpp create mode 100644 src/util/random_outputs.h create mode 100644 src/util/transactions.cpp create mode 100644 src/util/transactions.h create mode 100644 src/wire.h create mode 100644 src/wire/CMakeLists.txt create mode 100644 src/wire/crypto.h create mode 100644 src/wire/error.cpp create mode 100644 src/wire/error.h create mode 100644 src/wire/field.h create mode 100644 src/wire/filters.h create mode 100644 src/wire/fwd.h create mode 100644 src/wire/json.h create mode 100644 src/wire/json/CMakeLists.txt create mode 100644 src/wire/json/base.h create mode 100644 src/wire/json/error.cpp create mode 100644 src/wire/json/error.h create mode 100644 src/wire/json/fwd.h create mode 100644 src/wire/json/read.cpp create mode 100644 src/wire/json/read.h create mode 100644 src/wire/json/write.cpp create mode 100644 src/wire/json/write.h create mode 100644 src/wire/read.cpp create mode 100644 src/wire/read.h create mode 100644 src/wire/traits.h create mode 100644 src/wire/vector.h create mode 100644 src/wire/write.cpp create mode 100644 src/wire/write.h diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..4a11175 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,172 @@ +# Copyright (c) 2020, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +cmake_minimum_required(VERSION 3.1.0) +project(monero-lws) + +set(MONERO_LIBRARIES + daemon_messages + serialization + lmdb_lib + net + cryptonote_core + cryptonote_basic + ringct + ringct_basic + multisig + hardforks + checkpoints + blockchain_db + common + lmdb + device + cncrypto + randomx + epee + easylogging + version + wallet-crypto +) + +set(MONERO_OPTIONAL wallet-crypto) + +set(MONERO_SEARCH_PATHS + "/contrib/epee/src" + "/external/db_drivers/liblmdb" + "/external/easylogging++" + "/src" + "/src/crypto" + "/src/crypto/wallet" + "/src/lmdb" + "/src/ringct" + "/src/rpc" +) + + +# +# Pull some information from monero build +# + +# Needed due to "bug" in monero CMake - the `project` function is used twice! +if (NOT MONERO_SOURCE_DIR) + message(FATAL_ERROR "The argument -DMONERO_SOURCE_DIR must specify a location of a monero source tree") +endif() + +if (NOT MONERO_BUILD_DIR) + message(FATAL_ERROR "The argument -DMONERO_BUILD_DIR must specify a location of an existing monero build") +endif() + +load_cache(${MONERO_BUILD_DIR} READ_WITH_PREFIX monero_ + Boost_THREAD_LIBRARY_RELEASE + CMAKE_CXX_COMPILER + EXTRA_LIBRARIES + HIDAPI_LIBRARY + LMDB_INCLUDE + monero_SOURCE_DIR + OPENSSL_CRYPTO_LIBRARY + OPENSSL_SSL_LIBRARY + SODIUM_LIBRARY + UNBOUND_LIBRARIES + ZMQ_INCLUDE_PATH + ZMQ_LIB +) + +if (NOT (monero_monero_SOURCE_DIR MATCHES "${MONERO_SOURCE_DIR}(/src/cryptonote_protocol)")) + message(FATAL_ERROR "Invalid Monero source dir - does not appear to match source used for build directory") +endif() + +if (NOT (CMAKE_CXX_COMPILER STREQUAL monero_CMAKE_CXX_COMPILER)) + message(FATAL_ERROR "Compiler for monero build differs from this project") +endif() + +# +# Dependencies specific to monero-lws +# + +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) + +find_package(Boost 1.58 QUIET REQUIRED COMPONENTS chrono filesystem program_options regex serialization thread) + +if (NOT (Boost_THREAD_LIBRARY STREQUAL monero_Boost_THREAD_LIBRARY_RELEASE)) + message(FATAL_ERROR "Boost libraries for monero build differs from this project") +endif() + +foreach (LIB ${MONERO_LIBRARIES}) + find_library(LIB_PATH NAMES "${LIB}" PATHS ${MONERO_BUILD_DIR} PATH_SUFFIXES "/src/${LIB}" "external/${LIB}" ${MONERO_SEARCH_PATHS} REQUIRED NO_DEFAULT_PATH) + + list(FIND MONERO_OPTIONAL "${LIB}" LIB_OPTIONAL) + if (NOT LIB_PATH AND NOT LIB_OPTIONAL) + message(FATAL_ERROR "Unable to find required Monero library ${LIB}") + endif() + + set(LIB_NAME "monero::${LIB}") + add_library(${LIB_NAME} STATIC IMPORTED) + set_target_properties(${LIB_NAME} PROPERTIES IMPORTED_LOCATION ${LIB_PATH}) + + list(APPEND IMPORTED_MONERO_LIBRARIES "${LIB_NAME}") + unset(LIB_PATH CACHE) +endforeach() + +add_library(monero::libraries INTERFACE IMPORTED) +target_include_directories(monero::libraries SYSTEM + INTERFACE + ${Boost_INCLUDE_DIR} + "${MONERO_BUILD_DIR}/generated_include" + "${MONERO_SOURCE_DIR}/contrib/epee/include" + "${MONERO_SOURCE_DIR}/external/easylogging++" + "${MONERO_SOURCE_DIR}/external/rapidjson/include" + "${MONERO_SOURCE_DIR}/src" +) +target_link_libraries(monero::libraries + INTERFACE + ${CMAKE_DL_LIBS} + ${Boost_CHRONO_LIBRARY} + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_REGEX_LIBRARY} + ${Boost_SERIALIZATION_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${monero_HIDAPI_LIBRARY} + ${monero_OPENSSL_CRYPTO_LIBRARY} + ${monero_OPENSSL_SSL_LIBRARY} + ${monero_SODIUM_LIBRARY} + ${monero_UNBOUND_LIBRARIES} + ${IMPORTED_MONERO_LIBRARIES} +) + +set(LMDB_INCLUDE "${monero_LMDB_INCLUDE}") +set(LMDB_LIB_PATH "monero::lmdb") +set(ZMQ_LIB "${monero_ZMQ_LIB}") +set(ZMQ_INCLUDE_PATH "${monero_ZMQ_INCLUDE_PATH}") + + +# +# Build monero-lws code +# + +add_subdirectory(src) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 0000000..c59e80d --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,77 @@ +# Copyright (c) 2018-2020, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +include_directories(.) + +add_subdirectory(db) +add_subdirectory(rpc) +add_subdirectory(util) +add_subdirectory(wire) + +# For both the server and admin utility. +set(monero-lws-common_sources config.cpp error.cpp) +set(monero-lws-common_headers config.h error.h fwd.h) + +add_library(monero-lws-common ${monero-lws-common_sources} ${monero-lws-common_headers}) +target_link_libraries(monero-lws-common monero::libraries) + + +add_executable(monero-lws-daemon server_main.cpp rest_server.cpp scanner.cpp) +target_include_directories(monero-lws-daemon PUBLIC ${ZMQ_INCLUDE_PATH}) +target_link_libraries(monero-lws-daemon + PRIVATE + monero::libraries + ${MONERO_lmdb} + monero-lws-common + monero-lws-db + monero-lws-rpc + monero-lws-util + monero-lws-wire-json + ${Boost_CHRONO_LIBRARY} + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_PROGRAM_OPTIONS_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES} + ${ZMQ_LIB} + Threads::Threads +) + +add_executable(monero-lws-admin admin_main.cpp) +target_link_libraries(monero-lws-admin + PRIVATE + monero::libraries + monero-lws-common + monero-lws-db + monero-lws-wire-json + ${Boost_PROGRAM_OPTIONS_LIBRARY} + Threads::Threads +) + +install(TARGETS monero-lws-daemon DESTINATION bin) +install(TARGETS monero-lws-admin DESTINATION bin) diff --git a/src/admin_main.cpp b/src/admin_main.cpp new file mode 100644 index 0000000..a2812b1 --- /dev/null +++ b/src/admin_main.cpp @@ -0,0 +1,423 @@ +// Copyright (c) 2018-2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/command_line.h" // monero/src +#include "common/expect.h" // monero/src +#include "config.h" +#include "error.h" +#include "db/storage.h" +#include "db/string.h" +#include "options.h" +#include "misc_log_ex.h" // monero/contrib/epee/include +#include "span.h" // monero/contrib/epee/include +#include "string_tools.h" // monero/contrib/epee/include +#include "wire/filters.h" +#include "wire/json/write.h" + +namespace +{ + // Do not output "full" debug data provided by `db::data.h` header; truncate output + template + struct truncated + { + T value; + }; + + void write_bytes(wire::json_writer& dest, const truncated& self) + { + wire::object(dest, + wire::field("address", lws::db::address_string(self.value.address)), + wire::field("scan_height", self.value.scan_height), + wire::field("access_time", self.value.access) + ); + }; + + void write_bytes(wire::json_writer& dest, const truncated& self) + { + wire::object(dest, + wire::field("address", lws::db::address_string(self.value.address)), + wire::field("start_height", self.value.start_height) + ); + } + + template + void write_bytes(wire::json_writer& dest, const truncated>> self) + { + const auto truncate = [] (V src) { return truncated{std::move(src)}; }; + wire::array(dest, std::move(self.value), truncate); + } + + template + void stream_json_object(std::ostream& dest, boost::iterator_range> self) + { + using value_range = boost::iterator_range>; + const auto truncate = [] (value_range src) -> truncated + { + return {std::move(src)}; + }; + + wire::json_stream_writer json{dest}; + wire::dynamic_object(json, std::move(self), wire::enum_as_string, truncate); + json.finish(); + } + + void write_json_addresses(std::ostream& dest, epee::span self) + { + // writes an array of monero base58 address strings + wire::json_stream_writer stream{dest}; + wire::object(stream, wire::field("updated", wire::as_array(self, lws::db::address_string))); + stream.finish(); + } + + struct options : lws::options + { + const command_line::arg_descriptor show_sensitive; + const command_line::arg_descriptor command; + const command_line::arg_descriptor> arguments; + + options() + : lws::options() + , show_sensitive{"show-sensitive", "Show view keys", false} + , command{"command", "Admin command to execute", ""} + , arguments{"arguments", "Arguments to command"} + {} + + void prepare(boost::program_options::options_description& description) const + { + lws::options::prepare(description); + command_line::add_arg(description, show_sensitive); + command_line::add_arg(description, command); + command_line::add_arg(description, arguments); + } + }; + + struct program + { + lws::db::storage disk; + std::vector arguments; + bool show_sensitive; + }; + + crypto::secret_key get_key(std::string const& hex) + { + crypto::secret_key out{}; + if (!epee::string_tools::hex_to_pod(hex, out)) + MONERO_THROW(lws::error::bad_view_key, "View key has invalid hex"); + return out; + } + + std::vector get_addresses(epee::span arguments) + { + // first entry is currently always some other option + assert(!arguments.empty()); + arguments.remove_prefix(1); + + std::vector addresses{}; + for (std::string const& address : arguments) + addresses.push_back(lws::db::address_string(address).value()); + return addresses; + } + + void accept_requests(program prog, std::ostream& out) + { + if (prog.arguments.size() < 2) + throw std::runtime_error{"accept_requests requires 2 or more arguments"}; + + const lws::db::request req = + MONERO_UNWRAP(lws::db::request_from_string(prog.arguments[0])); + std::vector addresses = + get_addresses(epee::to_span(prog.arguments)); + + const std::vector updated = + prog.disk.accept_requests(req, epee::to_span(addresses)).value(); + + write_json_addresses(out, epee::to_span(updated)); + } + + void add_account(program prog, std::ostream& out) + { + if (prog.arguments.size() != 2) + throw std::runtime_error{"add_account needs exactly two arguments"}; + + const lws::db::account_address address[1] = { + lws::db::address_string(prog.arguments[0]).value() + }; + const crypto::secret_key key{get_key(prog.arguments[1])}; + + MONERO_UNWRAP(prog.disk.add_account(address[0], key)); + write_json_addresses(out, address); + } + + void debug_database(program prog, std::ostream& out) + { + if (!prog.arguments.empty()) + throw std::runtime_error{"debug_database takes zero arguments"}; + + auto reader = prog.disk.start_read().value(); + reader.json_debug(out, prog.show_sensitive); + } + + void list_accounts(program prog, std::ostream& out) + { + if (!prog.arguments.empty()) + throw std::runtime_error{"list_accounts takes zero arguments"}; + + auto reader = prog.disk.start_read().value(); + auto stream = reader.get_accounts().value(); + stream_json_object(out, stream.make_range()); + } + + void list_requests(program prog, std::ostream& out) + { + if (!prog.arguments.empty()) + throw std::runtime_error{"list_requests takes zero arguments"}; + + auto reader = prog.disk.start_read().value(); + auto stream = reader.get_requests().value(); + stream_json_object(out, stream.make_range()); + } + + void modify_account(program prog, std::ostream& out) + { + if (prog.arguments.size() < 2) + throw std::runtime_error{"modify_account_status requires 2 or more arguments"}; + + const lws::db::account_status status = + lws::db::account_status_from_string(prog.arguments[0]).value(); + std::vector addresses = + get_addresses(epee::to_span(prog.arguments)); + + const std::vector updated = + prog.disk.change_status(status, epee::to_span(addresses)).value(); + + write_json_addresses(out, epee::to_span(updated)); + } + + void reject_requests(program prog, std::ostream& out) + { + if (prog.arguments.size() < 2) + MONERO_THROW(common_error::kInvalidArgument, "reject_requests requires 2 or more arguments"); + + const lws::db::request req = + lws::db::request_from_string(prog.arguments[0]).value(); + std::vector addresses = + get_addresses(epee::to_span(prog.arguments)); + + MONERO_UNWRAP(prog.disk.reject_requests(req, epee::to_span(addresses))); + } + + void rescan(program prog, std::ostream& out) + { + if (prog.arguments.size() < 2) + throw std::runtime_error{"rescan requires 2 or more arguments"}; + + const auto height = lws::db::block_id(std::stoull(prog.arguments[0])); + const std::vector addresses = + get_addresses(epee::to_span(prog.arguments)); + + const std::vector updated = + prog.disk.rescan(height, epee::to_span(addresses)).value(); + + write_json_addresses(out, epee::to_span(updated)); + } + + void rollback(program prog, std::ostream& out) + { + if (prog.arguments.size() != 1) + throw std::runtime_error{"rollback requires 1 argument"}; + + const auto height = lws::db::block_id(std::stoull(prog.arguments[0])); + MONERO_UNWRAP(prog.disk.rollback(height)); + + wire::json_stream_writer json{out}; + wire::object(json, wire::field("new_height", height)); + json.finish(); + } + + struct command + { + char const* const name; + void (*const handler)(program, std::ostream&); + char const* const parameters; + }; + + static constexpr const command commands[] = + { + {"accept_requests", &accept_requests, "<\"create\"|\"import\"> [base 58 address]..."}, + {"add_account", &add_account, " "}, + {"debug_database", &debug_database, ""}, + {"list_accounts", &list_accounts, ""}, + {"list_requests", &list_requests, ""}, + {"modify_account_status", &modify_account, "<\"active\"|\"inactive\"|\"hidden\"> [base 58 address]..."}, + {"reject_requests", &reject_requests, "<\"create\"|\"import\"> [base 58 address]..."}, + {"rescan", &rescan, " [base 58 address]..."}, + {"rollback", &rollback, ""} + }; + + void print_help(std::ostream& out) + { + boost::program_options::options_description description{"Options"}; + options{}.prepare(description); + + out << "Usage: [options] [command] [arguments]" << std::endl; + out << description << std::endl; + out << "Commands:" << std::endl; + for (command cmd : commands) + { + out << " " << cmd.name << "\t\t" << cmd.parameters << std::endl; + } + } + + boost::optional> get_program(int argc, char** argv) + { + namespace po = boost::program_options; + + const options opts{}; + po::variables_map args{}; + { + po::options_description description{"Options"}; + opts.prepare(description); + + po::positional_options_description positional{}; + positional.add(opts.command.name, 1); + positional.add(opts.arguments.name, -1); + + po::store( + po::command_line_parser(argc, argv) + .options(description).positional(positional).run() + , args + ); + po::notify(args); + } + + if (command_line::get_arg(args, command_line::arg_help)) + { + print_help(std::cout); + return boost::none; + } + + opts.set_network(args); // do this first, sets global variable :/ + + program prog{ + lws::db::storage::open(command_line::get_arg(args, opts.db_path).c_str(), 0) + }; + + prog.show_sensitive = command_line::get_arg(args, opts.show_sensitive); + auto cmd = args[opts.command.name]; + if (cmd.empty()) + throw std::runtime_error{"No command given"}; + + prog.arguments = command_line::get_arg(args, opts.arguments); + return {{cmd.as(), std::move(prog)}}; + } + + void run(boost::string_ref name, program prog, std::ostream& out) + { + struct by_name + { + bool operator()(command const& left, command const& right) const noexcept + { + assert(left.name && right.name); + return std::strcmp(left.name, right.name) < 0; + } + bool operator()(boost::string_ref left, command const& right) const noexcept + { + assert(right.name); + return left < right.name; + } + bool operator()(command const& left, boost::string_ref right) const noexcept + { + assert(left.name); + return left.name < right; + } + }; + + assert(std::is_sorted(std::begin(commands), std::end(commands), by_name{})); + const auto found = std::lower_bound( + std::begin(commands), std::end(commands), name, by_name{} + ); + if (found == std::end(commands) || found->name != name) + throw std::runtime_error{"No such command"}; + + assert(found->handler != nullptr); + found->handler(std::move(prog), out); + + if (out.bad()) + MONERO_THROW(std::io_errc::stream, "Writing to stdout failed"); + + out << std::endl; + } +} // anonymous + +int main (int argc, char** argv) +{ + try + { + mlog_configure("", false, 0, 0); // disable logging + + boost::optional> prog; + + try + { + prog = get_program(argc, argv); + } + catch (std::exception const& e) + { + std::cerr << e.what() << std::endl << std::endl; + print_help(std::cerr); + return EXIT_FAILURE; + } + + if (prog) + run(prog->first, std::move(prog->second), std::cout); + } + catch (std::exception const& e) + { + std::cerr << e.what() << std::endl; + return EXIT_FAILURE; + } + catch (...) + { + std::cerr << "Unknown exception" << std::endl; + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} diff --git a/src/config.cpp b/src/config.cpp new file mode 100644 index 0000000..dc20479 --- /dev/null +++ b/src/config.cpp @@ -0,0 +1,9 @@ +#include "config.h" + +namespace lws +{ +namespace config +{ + cryptonote::network_type network = cryptonote::MAINNET; +} +} diff --git a/src/config.h b/src/config.h new file mode 100644 index 0000000..b0429a5 --- /dev/null +++ b/src/config.h @@ -0,0 +1,11 @@ +#pragma once + +#include "cryptonote_config.h" + +namespace lws +{ +namespace config +{ + extern cryptonote::network_type network; +} +} diff --git a/src/db/CMakeLists.txt b/src/db/CMakeLists.txt new file mode 100644 index 0000000..50d5300 --- /dev/null +++ b/src/db/CMakeLists.txt @@ -0,0 +1,34 @@ +# Copyright (c) 2018-2020, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(monero-lws-db_sources account.cpp data.cpp storage.cpp string.cpp) +set(monero-lws-db_headers account.h data.h fwd.h storage.h string.h) + +add_library(monero-lws-db ${monero-lws-db_sources} ${monero-lws-db_headers}) +target_include_directories(monero-lws-db PUBLIC "${LMDB_INCLUDE}") +target_link_libraries(monero-lws-db monero::libraries ${LMDB_LIB_PATH}) diff --git a/src/db/account.cpp b/src/db/account.cpp new file mode 100644 index 0000000..40f9908 --- /dev/null +++ b/src/db/account.cpp @@ -0,0 +1,179 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "account.h" + +#include +#include + +#include "common/error.h" +#include "common/expect.h" +#include "db/data.h" +#include "db/string.h" + +namespace lws +{ + namespace + { + // update if `crypto::public_key` gets `operator<` + struct sort_pubs + { + bool operator()(crypto::public_key const& lhs, crypto::public_key const& rhs) const noexcept + { + return std::memcmp(std::addressof(lhs), std::addressof(rhs), sizeof(lhs)) < 0; + } + }; + } + + struct account::internal + { + explicit internal(db::account const& source) + : address(db::address_string(source.address)), id(source.id), pubs(source.address), view_key() + { + using inner_type = + std::remove_reference::type; + + static_assert(std::is_standard_layout(), "need standard layout source"); + static_assert(std::is_pod(), "need pod target"); + static_assert(sizeof(view_key) == sizeof(source.key), "different size keys"); + std::memcpy( + std::addressof(tools::unwrap(view_key)), + std::addressof(source.key), + sizeof(source.key) + ); + } + + std::string address; + db::account_id id; + db::account_address pubs; + crypto::secret_key view_key; + }; + + account::account(std::shared_ptr immutable, db::block_id height, std::vector spendable, std::vector pubs) noexcept + : immutable_(std::move(immutable)) + , spendable_(std::move(spendable)) + , pubs_(std::move(pubs)) + , spends_() + , outputs_() + , height_(height) + {} + + void account::null_check() const + { + if (!immutable_) + MONERO_THROW(::common_error::kInvalidArgument, "using moved from account"); + } + + account::account(db::account const& source, std::vector spendable, std::vector pubs) + : account(std::make_shared(source), source.scan_height, std::move(spendable), std::move(pubs)) + { + std::sort(spendable_.begin(), spendable_.end()); + std::sort(pubs_.begin(), pubs_.end(), sort_pubs{}); + } + + account::~account() noexcept + {} + + account account::clone() const + { + account result{immutable_, height_, spendable_, pubs_}; + result.outputs_ = outputs_; + result.spends_ = spends_; + return result; + } + + void account::updated(db::block_id new_height) noexcept + { + height_ = new_height; + spends_.clear(); + spends_.shrink_to_fit(); + outputs_.clear(); + outputs_.shrink_to_fit(); + } + + db::account_id account::id() const noexcept + { + if (immutable_) + return immutable_->id; + return db::account_id::invalid; + } + + std::string const& account::address() const + { + null_check(); + return immutable_->address; + } + + db::account_address const& account::db_address() const + { + null_check(); + return immutable_->pubs; + } + + crypto::public_key const& account::view_public() const + { + null_check(); + return immutable_->pubs.view_public; + } + + crypto::public_key const& account::spend_public() const + { + null_check(); + return immutable_->pubs.spend_public; + } + + crypto::secret_key const& account::view_key() const + { + null_check(); + return immutable_->view_key; + } + + bool account::has_spendable(db::output_id const& id) const noexcept + { + return std::binary_search(spendable_.begin(), spendable_.end(), id); + } + + bool account::add_out(db::output const& out) + { + auto existing_pub = std::lower_bound(pubs_.begin(), pubs_.end(), out.pub, sort_pubs{}); + if (existing_pub != pubs_.end() && *existing_pub == out.pub) + return false; + + pubs_.insert(existing_pub, out.pub); + spendable_.insert( + std::lower_bound(spendable_.begin(), spendable_.end(), out.spend_meta.id), + out.spend_meta.id + ); + outputs_.push_back(out); + return true; + } + + void account::add_spend(db::spend const& spend) + { + spends_.push_back(spend); + } +} // lws + diff --git a/src/db/account.h b/src/db/account.h new file mode 100644 index 0000000..9a18d95 --- /dev/null +++ b/src/db/account.h @@ -0,0 +1,114 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include +#include +#include + +#include "crypto/crypto.h" +#include "fwd.h" +#include "db/fwd.h" + +namespace lws +{ + //! Tracks a subset of DB account info for scanning/updating. + class account + { + struct internal; + + std::shared_ptr immutable_; + std::vector spendable_; + std::vector pubs_; + std::vector spends_; + std::vector outputs_; + db::block_id height_; + + explicit account(std::shared_ptr immutable, db::block_id height, std::vector spendable, std::vector pubs) noexcept; + void null_check() const; + + public: + + //! Construct an account from `source` and current `spendable` outputs. + explicit account(db::account const& source, std::vector spendable, std::vector pubs); + + /*! + \return False if this is a "moved-from" account (i.e. the internal memory + has been moved to another object). + */ + explicit operator bool() const noexcept { return immutable_ != nullptr; } + + account(const account&) = delete; + account(account&&) = default; + ~account() noexcept; + account& operator=(const account&) = delete; + account& operator=(account&&) = default; + + //! \return A copy of `this`. + account clone() const; + + //! \return A copy of `this` with a new height and `outputs().empty()`. + void updated(db::block_id new_height) noexcept; + + //! \return Unique ID from the account database, possibly `db::account_id::kInvalid`. + db::account_id id() const noexcept; + + //! \return Monero base58 string for account. + std::string const& address() const; + + //! \return Object used for lookup in LMDB. + db::account_address const& db_address() const; + + //! \return Extracted view public key from `address()` + crypto::public_key const& view_public() const; + + //! \return Extracted spend public key from `address()`. + crypto::public_key const& spend_public() const; + + //! \return Secret view key for the account. + crypto::secret_key const& view_key() const; + + //! \return Current scan height of `this`. + db::block_id scan_height() const noexcept { return height_; } + + //! \return True iff `id` is spendable by `this`. + bool has_spendable(db::output_id const& id) const noexcept; + + //! \return Outputs matched during the latest scan. + std::vector const& outputs() const noexcept { return outputs_; } + + //! \return Spends matched during the latest scan. + std::vector const& spends() const noexcept { return spends_; } + + //! Track a newly received `out`, \return `false` if `out.pub` is duplicated. + bool add_out(db::output const& out); + + //! Track a possible `spend`. + void add_spend(db::spend const& spend); + }; +} // lws diff --git a/src/db/data.cpp b/src/db/data.cpp new file mode 100644 index 0000000..42edfb8 --- /dev/null +++ b/src/db/data.cpp @@ -0,0 +1,225 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "data.h" + +#include +#include + +#include "wire/crypto.h" +#include "wire.h" + +namespace lws +{ +namespace db +{ + namespace + { + template + void map_output_id(F& format, T& self) + { + wire::object(format, WIRE_FIELD(high), WIRE_FIELD(low)); + } + } + WIRE_DEFINE_OBJECT(output_id, map_output_id); + + namespace + { + constexpr const char* map_account_status[] = {"active", "inactive", "hidden"}; + constexpr const char* map_request[] = {"create", "import"}; + } + WIRE_DEFINE_ENUM(account_status, map_account_status); + WIRE_DEFINE_ENUM(request, map_request); + + namespace + { + template + void map_account_address(F& format, T& self) + { + wire::object(format, WIRE_FIELD(spend_public), WIRE_FIELD(view_public)); + } + } + WIRE_DEFINE_OBJECT(account_address, map_account_address); + + void write_bytes(wire::writer& dest, const account& self, const bool show_key) + { + view_key const* const key = + show_key ? std::addressof(self.key) : nullptr; + const bool admin = (self.flags & admin_account); + const bool generated_locally = (self.flags & account_generated_locally); + + wire::object(dest, + WIRE_FIELD(id), + wire::field("access_time", self.access), + WIRE_FIELD(address), + wire::optional_field("view_key", key), + WIRE_FIELD(scan_height), + WIRE_FIELD(start_height), + wire::field("creation_time", self.creation), + wire::field("admin", admin), + wire::field("generated_locally", generated_locally) + ); + } + + namespace + { + template + void map_block_info(F& format, T& self) + { + wire::object(format, WIRE_FIELD(id), WIRE_FIELD(hash)); + } + } + WIRE_DEFINE_OBJECT(block_info, map_block_info); + + namespace + { + template + void map_transaction_link(F& format, T& self) + { + wire::object(format, WIRE_FIELD(height), WIRE_FIELD(tx_hash)); + } + } + WIRE_DEFINE_OBJECT(transaction_link, map_transaction_link); + + void write_bytes(wire::writer& dest, const output& self) + { + const std::pair unpacked = + db::unpack(self.extra); + + const bool coinbase = (unpacked.first & lws::db::coinbase_output); + const bool rct = (unpacked.first & lws::db::ringct_output); + + const auto rct_mask = rct ? std::addressof(self.ringct_mask) : nullptr; + + epee::span payment_bytes{}; + if (unpacked.second == 32) + payment_bytes = epee::as_byte_span(self.payment_id.long_); + else if (unpacked.second == 8) + payment_bytes = epee::as_byte_span(self.payment_id.short_); + + const auto payment_id = payment_bytes.empty() ? + nullptr : std::addressof(payment_bytes); + + wire::object(dest, + wire::field("id", std::cref(self.spend_meta.id)), + wire::field("block", self.link.height), + wire::field("index", self.spend_meta.index), + wire::field("amount", self.spend_meta.amount), + wire::field("timestamp", self.timestamp), + wire::field("tx_hash", std::cref(self.link.tx_hash)), + wire::field("tx_prefix_hash", std::cref(self.tx_prefix_hash)), + wire::field("tx_public", std::cref(self.spend_meta.tx_public)), + wire::optional_field("rct_mask", rct_mask), + wire::optional_field("payment_id", payment_id), + wire::field("unlock_time", self.unlock_time), + wire::field("mixin_count", self.spend_meta.mixin_count), + wire::field("coinbase", coinbase) + ); + } + + namespace + { + template + void map_spend(F& format, T1& self, T2& payment_id) + { + wire::object(format, + wire::field("height", self.link.height), + wire::field("tx_hash", std::ref(self.link.tx_hash)), + WIRE_FIELD(image), + WIRE_FIELD(source), + WIRE_FIELD(timestamp), + WIRE_FIELD(unlock_time), + WIRE_FIELD(mixin_count), + wire::optional_field("payment_id", payment_id) + ); + } + } + void read_bytes(wire::reader& source, spend& dest) + { + boost::optional payment_id; + map_spend(source, dest, payment_id); + + if (payment_id) + { + dest.length = sizeof(dest.payment_id); + dest.payment_id = std::move(*payment_id); + } + else + dest.length = 0; + } + void write_bytes(wire::writer& dest, const spend& source) + { + crypto::hash const* const payment_id = + (source.length == sizeof(source.payment_id) ? + std::addressof(source.payment_id) : nullptr); + return map_spend(dest, source, payment_id); + } + + namespace + { + template + void map_key_image(F& format, T& self) + { + wire::object(format, + wire::field("key_image", std::ref(self.value)), + wire::field("tx_hash", std::ref(self.link.tx_hash)), + wire::field("height", self.link.height) + ); + } + } + WIRE_DEFINE_OBJECT(key_image, map_key_image); + + void write_bytes(wire::writer& dest, const request_info& self, const bool show_key) + { + db::view_key const* const key = + show_key ? std::addressof(self.key) : nullptr; + const bool generated = (self.creation_flags & lws::db::account_generated_locally); + + wire::object(dest, + WIRE_FIELD(address), + wire::optional_field("view_key", key), + WIRE_FIELD(start_height), + wire::field("generated_locally", generated) + ); + } + + /*! TODO consider making an `operator<` for `crypto::tx_hash`. Not known to be + needed elsewhere yet. */ + + bool operator<(transaction_link const& left, transaction_link const& right) noexcept + { + return left.height == right.height ? + std::memcmp(std::addressof(left.tx_hash), std::addressof(right.tx_hash), sizeof(left.tx_hash)) < 0 : + left.height < right.height; + } + bool operator<=(transaction_link const& left, transaction_link const& right) noexcept + { + return right.height == left.height ? + std::memcmp(std::addressof(left.tx_hash), std::addressof(right.tx_hash), sizeof(left.tx_hash)) <= 0 : + left.height < right.height; + } +} // db +} // lws diff --git a/src/db/data.h b/src/db/data.h new file mode 100644 index 0000000..e5cc2a2 --- /dev/null +++ b/src/db/data.h @@ -0,0 +1,275 @@ +// Copyright (c) 2018-2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include +#include +#include + +#include "crypto/crypto.h" +#include "lmdb/util.h" +#include "ringct/rctTypes.h" //! \TODO brings in lots of includes, try to remove +#include "wire/fwd.h" +#include "wire/traits.h" + +namespace lws +{ +namespace db +{ + /* + Enum classes are used because they generate identical code to native integer + types, but are not implicitly convertible to each other or any integer types. + They also have comparison but not arithmetic operators defined. + */ + + //! References an account stored in the database, faster than by address + enum class account_id : std::uint32_t + { + invalid = std::uint32_t(-1) //!< Always represents _not an_ account id. + }; + WIRE_AS_INTEGER(account_id); + + //! Number of seconds since UNIX epoch. + enum class account_time : std::uint32_t {}; + WIRE_AS_INTEGER(account_time); + + //! References a block height + enum class block_id : std::uint64_t {}; + WIRE_AS_INTEGER(block_id); + + //! References a global output number, as determined by the public chain + struct output_id + { + std::uint64_t high; //!< Amount on public chain; rct outputs are `0` + std::uint64_t low; //!< Offset within `amount` on the public chain + }; + WIRE_DECLARE_OBJECT(output_id); + + enum class account_status : std::uint8_t + { + active = 0, //!< Actively being scanned and reported by API + inactive, //!< Not being scanned, but still reported by API + hidden //!< Not being scanned or reported by API + }; + WIRE_DECLARE_ENUM(account_status); + + enum account_flags : std::uint8_t + { + default_account = 0, + admin_account = 1, //!< Not currently used, for future extensions + account_generated_locally = 2 //!< Flag sent by client on initial login request + }; + + enum class request : std::uint8_t + { + create = 0, //!< Add a new account + import_scan //!< Set account start and scan height to zero. + }; + WIRE_DECLARE_ENUM(request); + + /*! + DB does not use `crypto::secret_key` because it is not POD (UB to copy over + entire struct). LMDB is keeping a copy in process memory anyway (row + encryption not currently used). The roadmap recommends process isolation + per-connection by default as a defense against viewkey leaks due to bug. */ + + struct view_key : crypto::ec_scalar {}; + // wire::is_blob trait below + + //! The public keys of a monero address + struct account_address + { + crypto::public_key spend_public; //!< Must be first for LMDB optimizations. + crypto::public_key view_public; + }; + static_assert(sizeof(account_address) == 64, "padding in account_address"); + WIRE_DECLARE_OBJECT(account_address); + + struct account + { + account_id id; //!< Must be first for LMDB optimizations + account_time access; //!< Last time `get_address_info` was called. + account_address address; + view_key key; //!< Doubles as authorization handle for REST API. + block_id scan_height; //!< Last block scanned; check-ins are always by block + block_id start_height; //!< Account started scanning at this block height + account_time creation; //!< Time account first appeared in database. + account_flags flags; //!< Additional account info bitmask. + char reserved[3]; + }; + static_assert(sizeof(account) == (4 * 2) + 64 + 32 + (8 * 2) + (4 * 2), "padding in account"); + void write_bytes(wire::writer&, const account&, bool show_key = false); + + struct block_info + { + block_id id; //!< Must be first for LMDB optimizations + crypto::hash hash; + }; + static_assert(sizeof(block_info) == 8 + 32, "padding in block_info"); + WIRE_DECLARE_OBJECT(block_info); + + //! `output`s and `spend`s are sorted by these fields to make merging easier. + struct transaction_link + { + block_id height; //!< Block height containing transaction + crypto::hash tx_hash; //!< Hash of the transaction + }; + + //! Additional flags stored in `output`s. + enum extra : std::uint8_t + { + coinbase_output = 1, + ringct_output = 2 + }; + + //! Packed information stored in `output`s. + enum class extra_and_length : std::uint8_t {}; + + //! \return `val` and `length` packed into a single byte. + inline extra_and_length pack(extra val, std::uint8_t length) noexcept + { + assert(length <= 32); + return extra_and_length((std::uint8_t(val) << 6) | (length & 0x3f)); + } + + //! \return `extra` and length unpacked from a single byte. + inline std::pair unpack(extra_and_length val) noexcept + { + const std::uint8_t real_val = std::uint8_t(val); + return {extra(real_val >> 6), std::uint8_t(real_val & 0x3f)}; + } + + //! Information for an output that has been received by an `account`. + struct output + { + transaction_link link; //! Orders and links `output` to `spend`s. + + //! Data that a linked `spend` needs in some REST endpoints. + struct spend_meta_ + { + output_id id; //!< Unique id for output within monero + // `link` and `id` must be in this order for LMDB optimizations + std::uint64_t amount; + std::uint32_t mixin_count;//!< Ring-size of TX + std::uint32_t index; //!< Offset within a tx + crypto::public_key tx_public; + } spend_meta; + + std::uint64_t timestamp; + std::uint64_t unlock_time; //!< Not always a timestamp; mirrors chain value. + crypto::hash tx_prefix_hash; + crypto::public_key pub; //!< One-time spendable public key. + rct::key ringct_mask; //!< Unencrypted CT mask + char reserved[7]; + extra_and_length extra; //!< Extra info + length of payment id + union payment_id_ + { + crypto::hash8 short_; //!< Decrypted short payment id + crypto::hash long_; //!< Long version of payment id (always decrypted) + } payment_id; + }; + static_assert( + sizeof(output) == 8 + 32 + (8 * 3) + (4 * 2) + 32 + (8 * 2) + (32 * 3) + 7 + 1 + 32, + "padding in output" + ); + void write_bytes(wire::writer&, const output&); + + //! Information about a possible spend of a received `output`. + struct spend + { + transaction_link link; //!< Orders and links `spend` to `output`. + crypto::key_image image; //!< Unique ID for the spend + // `link` and `image` must in this order for LMDB optimizations + output_id source; //!< The output being spent + std::uint64_t timestamp; //!< Timestamp of spend + std::uint64_t unlock_time;//!< Unlock time of spend + std::uint32_t mixin_count;//!< Ring-size of TX output + char reserved[3]; + std::uint8_t length; //!< Length of `payment_id` field (0..32). + crypto::hash payment_id; //!< Unencrypted only, can't decrypt spend + }; + static_assert(sizeof(spend) == 8 + 32 * 2 + 8 * 4 + 4 + 3 + 1 + 32, "padding in spend"); + WIRE_DECLARE_OBJECT(spend); + + //! Key image and info needed to retrieve primary `spend` data. + struct key_image + { + crypto::key_image value; //!< Actual key image value + // The above field needs to be first for LMDB optimizations + transaction_link link; //!< Link to `spend` and `output`. + }; + WIRE_DECLARE_OBJECT(key_image); + + struct request_info + { + account_address address;//!< Must be first for LMDB optimizations + view_key key; + block_id start_height; + account_time creation; //!< Time the request was created. + account_flags creation_flags; //!< Generated locally? + char reserved[3]; + }; + static_assert(sizeof(request_info) == 64 + 32 + 8 + (4 * 2), "padding in request_info"); + void write_bytes(wire::writer& dest, const request_info& self, bool show_key = false); + + inline constexpr bool operator==(output_id left, output_id right) noexcept + { + return left.high == right.high && left.low == right.low; + } + inline constexpr bool operator!=(output_id left, output_id right) noexcept + { + return left.high != right.high || left.low != right.low; + } + inline constexpr bool operator<(output_id left, output_id right) noexcept + { + return left.high == right.high ? + left.low < right.low : left.high < right.high; + } + inline constexpr bool operator<=(output_id left, output_id right) noexcept + { + return left.high == right.high ? + left.low <= right.low : left.high < right.high; + } + + bool operator<(transaction_link const& left, transaction_link const& right) noexcept; + bool operator<=(transaction_link const& left, transaction_link const& right) noexcept; + + /*! + Write `address` to `out` in base58 format using `lws::config::network` to + determine tag. */ + std::ostream& operator<<(std::ostream& out, account_address const& address); +} // db +} // lws + +namespace wire +{ + template<> + struct is_blob + : std::true_type + {}; +} diff --git a/src/db/fwd.h b/src/db/fwd.h new file mode 100644 index 0000000..f5eaa93 --- /dev/null +++ b/src/db/fwd.h @@ -0,0 +1,56 @@ +// Copyright (c) 2018-2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include + +namespace lws +{ +namespace db +{ + enum account_flags : std::uint8_t; + enum class account_id : std::uint32_t; + enum class account_status : std::uint8_t; + enum class block_id : std::uint64_t; + enum extra : std::uint8_t; + enum class extra_and_length : std::uint8_t; + enum class request : std::uint8_t; + + struct account; + struct account_address; + struct block_info; + struct key_image; + struct output; + struct output_id; + struct request_info; + struct spend; + class storage; + struct transaction_link; + struct view_key; +} // db +} // lws diff --git a/src/db/storage.cpp b/src/db/storage.cpp new file mode 100644 index 0000000..d7663f6 --- /dev/null +++ b/src/db/storage.cpp @@ -0,0 +1,1797 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "storage.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "checkpoints/checkpoints.h" +#include "config.h" +#include "crypto/crypto.h" +#include "cryptonote_basic/cryptonote_basic.h" +#include "cryptonote_core/cryptonote_tx_utils.h" +#include "db/account.h" +#include "db/string.h" +#include "error.h" +#include "hex.h" +#include "lmdb/database.h" +#include "lmdb/error.h" +#include "lmdb/key_stream.h" +#include "lmdb/table.h" +#include "lmdb/util.h" +#include "lmdb/value_stream.h" +#include "span.h" +#include "wire/filters.h" +#include "wire/json.h" + +namespace lws +{ +namespace db +{ + namespace + { + //! Used for finding `account` instances by other indexes. + struct account_lookup + { + account_id id; + account_status status; + char reserved[3]; + }; + static_assert(sizeof(account_lookup) == 4 + 1 + 3, "padding in account_lookup"); + + //! Used for looking up accounts by their public address. + struct account_by_address + { + account_address address; //!< Must be first for LMDB optimizations + account_lookup lookup; + }; + static_assert(sizeof(account_by_address) == 64 + 4 + 1 + 3, "padding in account_by_address"); + + constexpr const unsigned blocks_version = 0; + constexpr const unsigned by_address_version = 0; + + template + int less(epee::span left, epee::span right) noexcept + { + if (left.size() < sizeof(T)) + { + assert(left.empty()); + return -1; + } + if (right.size() < sizeof(T)) + { + assert(right.empty()); + return 1; + } + + T left_val; + T right_val; + std::memcpy(std::addressof(left_val), left.data(), sizeof(T)); + std::memcpy(std::addressof(right_val), right.data(), sizeof(T)); + + return (left_val < right_val) ? -1 : int(right_val < left_val); + } + + int compare_32bytes(epee::span left, epee::span right) noexcept + { + if (left.size() < 32) + { + assert(left.empty()); + return -1; + } + if (right.size() < 32) + { + assert(right.empty()); + return 1; + } + + return std::memcmp(left.data(), right.data(), 32); + } + + int output_compare(MDB_val const* left, MDB_val const* right) noexcept + { + if (left == nullptr || right == nullptr) + { + assert("MDB_val nullptr" == 0); + return -1; + } + + auto left_bytes = lmdb::to_byte_span(*left); + auto right_bytes = lmdb::to_byte_span(*right); + + int diff = less>(left_bytes, right_bytes); + if (diff) + return diff; + + left_bytes.remove_prefix(sizeof(block_id)); + right_bytes.remove_prefix(sizeof(block_id)); + + static_assert(sizeof(crypto::hash) == 32, "bad memcmp below"); + diff = compare_32bytes(left_bytes, right_bytes); + if (diff) + return diff; + + left_bytes.remove_prefix(sizeof(crypto::hash)); + right_bytes.remove_prefix(sizeof(crypto::hash)); + return less(left_bytes, right_bytes); + } + + int spend_compare(MDB_val const* left, MDB_val const* right) noexcept + { + if (left == nullptr || right == nullptr) + { + assert("MDB_val nullptr" == 0); + return -1; + } + + auto left_bytes = lmdb::to_byte_span(*left); + auto right_bytes = lmdb::to_byte_span(*right); + + int diff = less>(left_bytes, right_bytes); + if (diff) + return diff; + + left_bytes.remove_prefix(sizeof(block_id)); + right_bytes.remove_prefix(sizeof(block_id)); + + static_assert(sizeof(crypto::hash) == 32, "bad memcmp below"); + diff = compare_32bytes(left_bytes, right_bytes); + if (diff) + return diff; + + left_bytes.remove_prefix(sizeof(crypto::hash)); + right_bytes.remove_prefix(sizeof(crypto::hash)); + + static_assert(sizeof(crypto::key_image) == 32, "bad memcmp below"); + return compare_32bytes(left_bytes, right_bytes); + } + + constexpr const lmdb::basic_table blocks{ + "blocks_by_id", (MDB_CREATE | MDB_DUPSORT), MONERO_SORT_BY(block_info, id) + }; + constexpr const lmdb::basic_table accounts{ + "accounts_by_status,id", (MDB_CREATE | MDB_DUPSORT), MONERO_SORT_BY(account, id) + }; + constexpr const lmdb::basic_table accounts_by_address( + "accounts_by_address", (MDB_CREATE | MDB_DUPSORT), MONERO_COMPARE(account_by_address, address.spend_public) + ); + constexpr const lmdb::basic_table accounts_by_height( + "accounts_by_height,id", (MDB_CREATE | MDB_DUPSORT), MONERO_SORT_BY(account_lookup, id) + ); + constexpr const lmdb::basic_table outputs{ + "outputs_by_account_id,block_id,tx_hash,output_id", (MDB_CREATE | MDB_DUPSORT), &output_compare + }; + constexpr const lmdb::basic_table spends{ + "spends_by_account_id,block_id,tx_hash,image", (MDB_CREATE | MDB_DUPSORT), &spend_compare + }; + constexpr const lmdb::basic_table images{ + "key_images_by_output_id,image", (MDB_CREATE | MDB_DUPSORT), MONERO_COMPARE(db::key_image, value) + }; + constexpr const lmdb::basic_table requests{ + "requests_by_type,address", (MDB_CREATE | MDB_DUPSORT), MONERO_COMPARE(request_info, address.spend_public) + }; + + template + expect check_cursor(MDB_txn& txn, MDB_dbi tbl, std::unique_ptr& cur) noexcept + { + if (cur) + { + MONERO_LMDB_CHECK(mdb_cursor_renew(&txn, cur.get())); + } + else + { + auto new_cur = lmdb::open_cursor(txn, tbl); + if (!new_cur) + return new_cur.error(); + cur = std::move(*new_cur); + } + return success(); + } + + template + expect bulk_insert(MDB_cursor& cur, K const& key, epee::span values) noexcept + { + while (!values.empty()) + { + void const* const data = reinterpret_cast(values.data()); + MDB_val key_bytes = lmdb::to_val(key); + MDB_val value_bytes[2] = { + MDB_val{sizeof(V), const_cast(data)}, MDB_val{values.size(), nullptr} + }; + + int err = mdb_cursor_put( + &cur, &key_bytes, value_bytes, (MDB_NODUPDATA | MDB_MULTIPLE) + ); + if (err && err != MDB_KEYEXIST) + return {lmdb::error(err)}; + + values.remove_prefix(value_bytes[1].mv_size + (err == MDB_KEYEXIST ? 1 : 0)); + } + return success(); + } + + //! \return a single instance of compiled-in checkpoints for lws + cryptonote::checkpoints const& get_checkpoints() + { + struct initializer + { + cryptonote::checkpoints data; + + initializer() + : data() + { + data.init_default_checkpoints(lws::config::network); + + std::string const* genesis_tx = nullptr; + std::uint32_t genesis_nonce = 0; + + switch (lws::config::network) + { + case cryptonote::TESTNET: + genesis_tx = std::addressof(::config::testnet::GENESIS_TX); + genesis_nonce = ::config::testnet::GENESIS_NONCE; + break; + + case cryptonote::STAGENET: + genesis_tx = std::addressof(::config::stagenet::GENESIS_TX); + genesis_nonce = ::config::stagenet::GENESIS_NONCE; + break; + + case cryptonote::MAINNET: + genesis_tx = std::addressof(::config::GENESIS_TX); + genesis_nonce = ::config::GENESIS_NONCE; + break; + + default: + MONERO_THROW(lws::error::bad_blockchain, "Unsupported net type"); + } + cryptonote::block b; + cryptonote::generate_genesis_block(b, *genesis_tx, genesis_nonce); + crypto::hash block_hash = cryptonote::get_block_hash(b); + if (!data.add_checkpoint(0, epee::to_hex::string(epee::as_byte_span(block_hash)))) + MONERO_THROW(lws::error::bad_blockchain, "Genesis tx and checkpoints file mismatch"); + } + }; + static const initializer instance; + return instance.data; + } + + //! \return Current block hash at `id` using `cur`. + expect get_block_hash(MDB_cursor& cur, block_id id) noexcept + { + MDB_val key = lmdb::to_val(blocks_version); + MDB_val value = lmdb::to_val(id); + MONERO_LMDB_CHECK(mdb_cursor_get(&cur, &key, &value, MDB_GET_BOTH)); + return blocks.get_value(value); + } + + void check_blockchain(MDB_txn& txn, MDB_dbi tbl) + { + cursor::blocks cur = MONERO_UNWRAP(lmdb::open_cursor(txn, tbl)); + + std::map const& points = + get_checkpoints().get_points(); + + if (points.empty() || points.begin()->first != 0) + MONERO_THROW(lws::error::bad_blockchain, "Checkpoints are empty/expected genesis hash"); + + MDB_val key = lmdb::to_val(blocks_version); + int err = mdb_cursor_get(cur.get(), &key, nullptr, MDB_SET); + if (err) + { + if (err != MDB_NOTFOUND) + MONERO_THROW(lmdb::error(err), "Unable to retrieve blockchain hashes"); + + // new database + block_info checkpoint{ + block_id(points.begin()->first), points.begin()->second + }; + + MDB_val value = lmdb::to_val(checkpoint); + err = mdb_cursor_put(cur.get(), &key, &value, MDB_NODUPDATA); + if (err) + MONERO_THROW(lmdb::error(err), "Unable to add hash to local blockchain"); + + if (1 < points.size()) + { + checkpoint = block_info{ + block_id(points.rbegin()->first), points.rbegin()->second + }; + + value = lmdb::to_val(checkpoint); + err = mdb_cursor_put(cur.get(), &key, &value, MDB_NODUPDATA); + if (err) + MONERO_THROW(lmdb::error(err), "Unable to add hash to local blockchain"); + } + } + else // inspect existing database + { + /// + /// TODO Trim blockchain after a checkpoint has been reached + /// + const crypto::hash genesis = MONERO_UNWRAP(get_block_hash(*cur, block_id(0))); + if (genesis != points.begin()->second) + { + MONERO_THROW( + lws::error::bad_blockchain, "Genesis hash mismatch" + ); + } + } + } + + template + expect get_blocks(MDB_cursor& cur, std::size_t max_internal) + { + T out{}; + + max_internal = std::min(std::size_t(64), max_internal); + out.reserve(12 + max_internal); + + MDB_val key = lmdb::to_val(blocks_version); + MDB_val value{}; + MONERO_LMDB_CHECK(mdb_cursor_get(&cur, &key, &value, MDB_SET)); + MONERO_LMDB_CHECK(mdb_cursor_get(&cur, &key, &value, MDB_LAST_DUP)); + for (unsigned i = 0; i < 10; ++i) + { + expect next = blocks.get_value(value); + if (!next) + return next.error(); + + out.push_back(std::move(*next)); + + const int err = mdb_cursor_get(&cur, &key, &value, MDB_PREV_DUP); + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + if (out.back().id != block_id(0)) + return {lws::error::bad_blockchain}; + return out; + } + } + + const auto add_block = [&cur, &out] (std::uint64_t id) -> expect + { + expect next = get_block_hash(cur, block_id(id)); + if (!next) + return next.error(); + out.push_back(block_info{block_id(id), std::move(*next)}); + return success(); + }; + + const std::uint64_t checkpoint = get_checkpoints().get_max_height(); + const std::uint64_t anchor = lmdb::to_native(out.back().id); + + for (unsigned i = 1; i <= max_internal; ++i) + { + const std::uint64_t offset = 2 << i; + if (anchor < offset || anchor - offset < checkpoint) + break; + MONERO_CHECK(add_block(anchor - offset)); + } + + if (block_id(checkpoint) < out.back().id) + MONERO_CHECK(add_block(checkpoint)); + if (out.back().id != block_id(0)) + MONERO_CHECK(add_block(0)); + return out; + } + + expect find_last_id(MDB_cursor& cur) noexcept + { + account_id best = account_id(0); + + MDB_val key{}; + MDB_val value{}; + + int err = mdb_cursor_get(&cur, &key, &value, MDB_FIRST); + if (err == MDB_NOTFOUND) + return best; + if (err) + return {lmdb::error(err)}; + + do + { + MONERO_LMDB_CHECK(mdb_cursor_get(&cur, &key, &value, MDB_LAST_DUP)); + const expect current = + accounts.get_value(value); + if (!current) + return current.error(); + + + best = std::max(best, *current); + err = mdb_cursor_get(&cur, &key, &value, MDB_NEXT_NODUP); + if (err == MDB_NOTFOUND) + return best; + } while (err == 0); + return {lmdb::error(err)}; + } + } // anonymous + + struct storage_internal : lmdb::database + { + struct tables_ + { + MDB_dbi blocks; + MDB_dbi accounts; + MDB_dbi accounts_ba; + MDB_dbi accounts_bh; + MDB_dbi outputs; + MDB_dbi spends; + MDB_dbi images; + MDB_dbi requests; + } tables; + + const unsigned create_queue_max; + + explicit storage_internal(lmdb::environment env, unsigned create_queue_max) + : lmdb::database(std::move(env)), tables{}, create_queue_max(create_queue_max) + { + lmdb::write_txn txn = this->create_write_txn().value(); + assert(txn != nullptr); + + tables.blocks = blocks.open(*txn).value(); + tables.accounts = accounts.open(*txn).value(); + tables.accounts_ba = accounts_by_address.open(*txn).value(); + tables.accounts_bh = accounts_by_height.open(*txn).value(); + tables.outputs = outputs.open(*txn).value(); + tables.spends = spends.open(*txn).value(); + tables.images = images.open(*txn).value(); + tables.requests = requests.open(*txn).value(); + + check_blockchain(*txn, tables.blocks); + + MONERO_UNWRAP(this->commit(std::move(txn))); + } + }; + + storage_reader::~storage_reader() noexcept + {} + + expect storage_reader::get_last_block() noexcept + { + MONERO_PRECOND(txn != nullptr); + assert(db != nullptr); + MONERO_CHECK(check_cursor(*txn, db->tables.blocks, curs.blocks_cur)); + + MDB_val key = lmdb::to_val(blocks_version); + MDB_val value{}; + MONERO_LMDB_CHECK(mdb_cursor_get(curs.blocks_cur.get(), &key, &value, MDB_SET)); + MONERO_LMDB_CHECK(mdb_cursor_get(curs.blocks_cur.get(), &key, &value, MDB_LAST_DUP)); + + return blocks.get_value(value); + } + + expect> storage_reader::get_chain_sync() + { + MONERO_PRECOND(txn != nullptr); + assert(db != nullptr); + MONERO_CHECK(check_cursor(*txn, db->tables.blocks, curs.blocks_cur)); + auto blocks = get_blocks>(*curs.blocks_cur, 64); + if (!blocks) + return blocks.error(); + + std::list out{}; + for (block_info const& block : *blocks) + out.push_back(block.hash); + return out; + } + + expect> + storage_reader::get_accounts(cursor::accounts cur) noexcept + { + MONERO_PRECOND(txn != nullptr); + assert(db != nullptr); // both are moved in pairs + MONERO_CHECK(check_cursor(*txn, db->tables.accounts, cur)); + return accounts.get_key_stream(std::move(cur)); + } + + expect> + storage_reader::get_accounts(account_status status, cursor::accounts cur) noexcept + { + MONERO_PRECOND(txn != nullptr); + assert(db != nullptr); // both are moved in pairs + MONERO_CHECK(check_cursor(*txn, db->tables.accounts, cur)); + return accounts.get_value_stream(status, std::move(cur)); + } + + expect> + storage_reader::get_account(account_address const& address, cursor::accounts& cur) noexcept + { + MONERO_PRECOND(txn != nullptr); + assert(db != nullptr); + + MONERO_CHECK(check_cursor(*txn, db->tables.accounts_ba, curs.accounts_ba_cur)); + + MDB_val key = lmdb::to_val(by_address_version); + MDB_val value = lmdb::to_val(address); + const int err = mdb_cursor_get(curs.accounts_ba_cur.get(), &key, &value, MDB_GET_BOTH); + if (err) + { + if (err == MDB_NOTFOUND) + return {lws::error::account_not_found}; + return {lmdb::error(err)}; + } + + const expect lookup = + accounts_by_address.get_value(value); + if (!lookup) + return lookup.error(); + + MONERO_CHECK(check_cursor(*txn, db->tables.accounts, cur)); + assert(cur != nullptr); + + key = lmdb::to_val(lookup->status); + value = lmdb::to_val(lookup->id); + MONERO_LMDB_CHECK(mdb_cursor_get(cur.get(), &key, &value, MDB_GET_BOTH)); + + const expect user = accounts.get_value(value); + if (!user) + return user.error(); + return {{lookup->status, *user}}; + } + + expect> + storage_reader::get_outputs(account_id id, cursor::outputs cur) noexcept + { + MONERO_PRECOND(txn != nullptr); + assert(db != nullptr); + MONERO_CHECK(check_cursor(*txn, db->tables.outputs, cur)); + return outputs.get_value_stream(id, std::move(cur)); + } + + expect> + storage_reader::get_spends(account_id id, cursor::spends cur) noexcept + { + MONERO_PRECOND(txn != nullptr); + assert(db != nullptr); + MONERO_CHECK(check_cursor(*txn, db->tables.spends, cur)); + return spends.get_value_stream(id, std::move(cur)); + } + + expect> + storage_reader::get_images(output_id id, cursor::images cur) noexcept + { + MONERO_PRECOND(txn != nullptr); + assert(db != nullptr); + MONERO_CHECK(check_cursor(*txn, db->tables.images, cur)); + return images.get_value_stream(id, std::move(cur)); + } + + expect> + storage_reader::get_requests(cursor::requests cur) noexcept + { + MONERO_PRECOND(txn != nullptr); + assert(db != nullptr); + MONERO_CHECK(check_cursor(*txn, db->tables.requests, cur)); + return requests.get_key_stream(std::move(cur)); + } + + expect + storage_reader::get_request(request type, account_address const& address, cursor::requests cur) noexcept + { + MONERO_PRECOND(txn != nullptr); + assert(db != nullptr); + + MONERO_CHECK(check_cursor(*txn, db->tables.requests, cur)); + + MDB_val key = lmdb::to_val(type); + MDB_val value = lmdb::to_val(address); + MONERO_LMDB_CHECK(mdb_cursor_get(cur.get(), &key, &value, MDB_GET_BOTH)); + return requests.get_value(value); + } + + namespace + { + //! `write_bytes` implementation will forward a third argument for `show_keys`. + template + struct show_keys_wrapper + { + T value; + bool show_keys; + }; + + //! Filter that will instruct type to `show_keys` (or not). + struct toggle_key_output + { + const bool show_keys; + + template + show_keys_wrapper operator()(T value) const noexcept + { + return {std::move(value), show_keys}; + } + }; + + struct output_id_key + { + std::string operator()(const output_id id) const + { + return std::to_string(id.high) + ":" + std::to_string(id.low); + } + }; + + template + void write_bytes(wire::json_writer& dest, show_keys_wrapper self) + { + lws::db::write_bytes(dest, self.value, self.show_keys); + } + void write_bytes(wire::json_writer& dest, const account_lookup self) + { + wire::object(dest, WIRE_FIELD_COPY(id), WIRE_FIELD_COPY(status)); + } + } + + // accounts_by_height is output as a sorted array of objects + static void write_bytes(wire::json_writer& dest, std::pair>> self) + { + wire::object(dest, + wire::field("scan_height", self.first), + wire::field("accounts", wire::as_array(std::move(self.second))) + ); + } + + expect storage_reader::json_debug(std::ostream& out, bool show_keys) + { + using boost::adaptors::reverse; + using boost::adaptors::transform; + + MONERO_PRECOND(txn != nullptr); + assert(db != nullptr); + + const auto address_as_key = [](account_by_address const& src) + { + return std::make_pair(address_string(src.address), src.lookup); + }; + + cursor::accounts accounts_cur; + cursor::outputs outputs_cur; + cursor::spends spends_cur; + cursor::images images_cur; + cursor::requests requests_cur; + + MONERO_CHECK(check_cursor(*txn, db->tables.blocks, curs.blocks_cur)); + MONERO_CHECK(check_cursor(*txn, db->tables.accounts, accounts_cur)); + MONERO_CHECK(check_cursor(*txn, db->tables.accounts_ba, curs.accounts_ba_cur)); + MONERO_CHECK(check_cursor(*txn, db->tables.accounts_bh, curs.accounts_bh_cur)); + MONERO_CHECK(check_cursor(*txn, db->tables.outputs, outputs_cur)); + MONERO_CHECK(check_cursor(*txn, db->tables.spends, spends_cur)); + MONERO_CHECK(check_cursor(*txn, db->tables.images, images_cur)); + MONERO_CHECK(check_cursor(*txn, db->tables.requests, requests_cur)); + + auto blocks_partial = + get_blocks>(*curs.blocks_cur, 0); + if (!blocks_partial) + return blocks_partial.error(); + + auto accounts_stream = accounts.get_key_stream(std::move(accounts_cur)); + if (!accounts_stream) + return accounts_stream.error(); + + auto accounts_ba_stream = accounts_by_address.get_value_stream( + by_address_version, std::move(curs.accounts_ba_cur) + ); + if (!accounts_ba_stream) + return accounts_ba_stream.error(); + + auto accounts_bh_stream = accounts_by_height.get_key_stream( + std::move(curs.accounts_bh_cur) + ); + if (!accounts_bh_stream) + return accounts_bh_stream.error(); + + auto outputs_stream = outputs.get_key_stream(std::move(outputs_cur)); + if (!outputs_stream) + return outputs_stream.error(); + + auto spends_stream = spends.get_key_stream(std::move(spends_cur)); + if (!spends_stream) + return spends_stream.error(); + + auto images_stream = images.get_key_stream(std::move(images_cur)); + if (!images_stream) + return images_stream.error(); + + auto requests_stream = requests.get_key_stream(std::move(requests_cur)); + if (!requests_stream) + return requests_stream.error(); + + const wire::as_array_filter toggle_keys_filter{{show_keys}}; + wire::json_stream_writer json_stream{out}; + wire::object(json_stream, + wire::field(blocks.name, wire::as_array(reverse(*blocks_partial))), + wire::field(accounts.name, wire::as_object(accounts_stream->make_range(), wire::enum_as_string, toggle_keys_filter)), + wire::field(accounts_by_address.name, wire::as_object(transform(accounts_ba_stream->make_range(), address_as_key))), + wire::field(accounts_by_height.name, wire::as_array(accounts_bh_stream->make_range())), + wire::field(outputs.name, wire::as_object(outputs_stream->make_range(), wire::as_integer, wire::as_array)), + wire::field(spends.name, wire::as_object(spends_stream->make_range(), wire::as_integer, wire::as_array)), + wire::field(images.name, wire::as_object(images_stream->make_range(), output_id_key{}, wire::as_array)), + wire::field(requests.name, wire::as_object(requests_stream->make_range(), wire::enum_as_string, toggle_keys_filter)) + ); + json_stream.finish(); + + curs.accounts_ba_cur = accounts_ba_stream->give_cursor(); + curs.accounts_bh_cur = accounts_bh_stream->give_cursor(); + + if (!out.good()) + return {std::io_errc::stream}; + return success(); + } + + lmdb::suspended_txn storage_reader::finish_read() noexcept + { + if (txn != nullptr) + { + assert(db != nullptr); + auto suspended = db->reset_txn(std::move(txn)); + if (suspended) // errors not currently logged + return {std::move(*suspended)}; + } + return nullptr; + } + + storage storage::open(const char* path, unsigned create_queue_max) + { + return { + std::make_shared( + MONERO_UNWRAP(lmdb::open_environment(path, 20)), create_queue_max + ) + }; + } + + storage::~storage() noexcept + {} + + storage storage::clone() const noexcept + { + return storage{db}; + } + + expect storage::start_read(lmdb::suspended_txn txn) const + { + MONERO_PRECOND(db != nullptr); + + expect reader = db->create_read_txn(std::move(txn)); + if (!reader) + return reader.error(); + + assert(*reader != nullptr); + return storage_reader{db, std::move(*reader)}; + } + + namespace // sub functions for `sync_chain(...)` + { + expect + rollback_spends(account_id user, block_id height, MDB_cursor& spends_cur, MDB_cursor& images_cur) noexcept + { + MDB_val key = lmdb::to_val(user); + MDB_val value = lmdb::to_val(height); + + const int err = mdb_cursor_get(&spends_cur, &key, &value, MDB_GET_BOTH_RANGE); + if (err == MDB_NOTFOUND) + return success(); + if (err) + return {lmdb::error(err)}; + + for (;;) + { + const expect out = spends.get_value(value); + if (!out) + return out.error(); + + const expect image = + spends.get_value(value); + if (!image) + return image.error(); + + key = lmdb::to_val(*out); + value = lmdb::to_val(*image); + MONERO_LMDB_CHECK(mdb_cursor_get(&images_cur, &key, &value, MDB_GET_BOTH)); + MONERO_LMDB_CHECK(mdb_cursor_del(&images_cur, 0)); + + MONERO_LMDB_CHECK(mdb_cursor_del(&spends_cur, 0)); + const int err = mdb_cursor_get(&spends_cur, &key, &value, MDB_NEXT_DUP); + if (err == MDB_NOTFOUND) + break; + if (err) + return {lmdb::error(err)}; + } + return success(); + } + + expect + rollback_outputs(account_id user, block_id height, MDB_cursor& outputs_cur) noexcept + { + MDB_val key = lmdb::to_val(user); + MDB_val value = lmdb::to_val(height); + + const int err = mdb_cursor_get(&outputs_cur, &key, &value, MDB_GET_BOTH_RANGE); + if (err == MDB_NOTFOUND) + return success(); + if (err) + return {lmdb::error(err)}; + + for (;;) + { + MONERO_LMDB_CHECK(mdb_cursor_del(&outputs_cur, 0)); + const int err = mdb_cursor_get(&outputs_cur, &key, &value, MDB_NEXT_DUP); + if (err == MDB_NOTFOUND) + break; + if (err) + return {lmdb::error(err)}; + } + return success(); + } + + expect rollback_accounts(storage_internal::tables_ const& tables, MDB_txn& txn, block_id height) + { + cursor::accounts_by_height accounts_bh_cur; + MONERO_CHECK(check_cursor(txn, tables.accounts_bh, accounts_bh_cur)); + + MDB_val key = lmdb::to_val(height); + MDB_val value{}; + const int err = mdb_cursor_get(accounts_bh_cur.get(), &key, &value, MDB_SET_RANGE); + if (err == MDB_NOTFOUND) + return success(); + if (err) + return {lmdb::error(err)}; + + std::vector new_by_heights{}; + + cursor::accounts accounts_cur; + cursor::outputs outputs_cur; + cursor::spends spends_cur; + cursor::images images_cur; + + MONERO_CHECK(check_cursor(txn, tables.accounts, accounts_cur)); + MONERO_CHECK(check_cursor(txn, tables.outputs, outputs_cur)); + MONERO_CHECK(check_cursor(txn, tables.spends, spends_cur)); + MONERO_CHECK(check_cursor(txn, tables.images, images_cur)); + + const std::uint64_t new_height = std::uint64_t(std::max(height, block_id(1))) - 1; + + // rollback accounts + for (;;) + { + const expect lookup = + accounts_by_height.get_value(value); + if (!lookup) + return lookup.error(); + + key = lmdb::to_val(lookup->status); + value = lmdb::to_val(lookup->id); + + MONERO_LMDB_CHECK(mdb_cursor_get(accounts_cur.get(), &key, &value, MDB_GET_BOTH)); + expect user = accounts.get_value(value); + if (!user) + return user.error(); + + user->scan_height = block_id(new_height); + user->start_height = std::min(user->scan_height, user->start_height); + + value = lmdb::to_val(*user); + MONERO_LMDB_CHECK(mdb_cursor_put(accounts_cur.get(), &key, &value, MDB_CURRENT)); + + new_by_heights.push_back(account_lookup{user->id, lookup->status}); + MONERO_CHECK(rollback_outputs(user->id, height, *outputs_cur)); + MONERO_CHECK(rollback_spends(user->id, height, *spends_cur, *images_cur)); + + MONERO_LMDB_CHECK(mdb_cursor_del(accounts_bh_cur.get(), 0)); + int err = mdb_cursor_get(accounts_bh_cur.get(), &key, &value, MDB_NEXT_DUP); + if (err == MDB_NOTFOUND) + { + err = mdb_cursor_get(accounts_bh_cur.get(), &key, &value, MDB_NEXT_NODUP); + if (err == MDB_NOTFOUND) + break; + } + if (err) + return {lmdb::error(err)}; + } + + return bulk_insert(*accounts_bh_cur, new_height, epee::to_span(new_by_heights)); + } + + expect rollback_chain(storage_internal::tables_ const& tables, MDB_txn& txn, MDB_cursor& cur, block_id height) + { + MDB_val key; + MDB_val value; + + // rollback chain + int err = 0; + do + { + MONERO_LMDB_CHECK(mdb_cursor_del(&cur, 0)); + err = mdb_cursor_get(&cur, &key, &value, MDB_NEXT_DUP); + } while (err == 0); + + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + + return rollback_accounts(tables, txn, height); + } + + template + expect append_block_hashes(MDB_cursor& cur, db::block_id first, T const& chain) + { + std::uint64_t height = std::uint64_t(first); + boost::container::static_vector hashes{}; + static_assert(sizeof(hashes) <= 1024, "using more stack space than expected"); + + for (auto current = chain.begin() ;; ++current) + { + if (current == chain.end() || hashes.size() == hashes.capacity()) + { + MONERO_CHECK(bulk_insert(cur, blocks_version, epee::to_span(hashes))); + if (current == chain.end()) + return success(); + hashes.clear(); + } + + hashes.push_back(block_info{db::block_id(height), *current}); + ++height; + } + } + } // anonymous + + expect storage::rollback(block_id height) + { + MONERO_PRECOND(db != nullptr); + + return db->try_write([this, height] (MDB_txn& txn) -> expect + { + cursor::blocks blocks_cur; + MONERO_CHECK(check_cursor(txn, this->db->tables.blocks, blocks_cur)); + + MDB_val key = lmdb::to_val(blocks_version); + MDB_val value = lmdb::to_val(height); + const int err = mdb_cursor_get(blocks_cur.get(), &key, &value, MDB_GET_BOTH); + if (err == MDB_NOTFOUND) + return success(); + if (err) + return {lmdb::error(err)}; + + return rollback_chain(this->db->tables, txn, *blocks_cur, height); + }); + } + + expect storage::sync_chain(block_id height, epee::span hashes) + { + MONERO_PRECOND(!hashes.empty()); + MONERO_PRECOND(db != nullptr); + + return db->try_write([this, height, hashes] (MDB_txn& txn) -> expect + { + cursor::blocks blocks_cur; + MONERO_CHECK(check_cursor(txn, this->db->tables.blocks, blocks_cur)); + + expect hash = get_block_hash(*blocks_cur, height); + if (!hash) + return hash.error(); + + // the first entry should always match on in the DB + if (*hash != *(hashes.begin())) + return {lws::error::bad_blockchain}; + + MDB_val key{}; + MDB_val value{}; + + std::uint64_t current = std::uint64_t(height) + 1; + auto first = hashes.begin(); + auto chain = boost::make_iterator_range(++first, hashes.end()); + for ( ; !chain.empty(); chain.advance_begin(1), ++current) + { + const int err = mdb_cursor_get(blocks_cur.get(), &key, &value, MDB_NEXT_DUP); + if (err == MDB_NOTFOUND) + break; + if (err) + return {lmdb::error(err)}; + + hash = blocks.get_value(value); + if (!hash) + return hash.error(); + + if (*hash != chain.front()) + { + MONERO_CHECK(rollback_chain(this->db->tables, txn, *blocks_cur, db::block_id(current))); + break; + } + } + return append_block_hashes(*blocks_cur, db::block_id(current), chain); + }); + } + + namespace + { + expect get_account_time() noexcept + { + const auto time = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch() + ); + + if (time.count() < 0) + return {lws::error::system_clock_invalid_range}; + if (std::numeric_limits>::max() < time.count()) + return {lws::error::system_clock_invalid_range}; + return db::account_time(time.count()); + } + } + + expect storage::update_access_time(account_address const& address) noexcept + { + MONERO_PRECOND(db != nullptr); + return db->try_write([this, &address] (MDB_txn& txn) -> expect + { + const expect current_time = get_account_time(); + if (!current_time) + return current_time.error(); + + cursor::accounts accounts_cur; + cursor::accounts accounts_ba_cur; + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts, accounts_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts_ba, accounts_ba_cur)); + + MDB_val key = lmdb::to_val(by_address_version); + MDB_val value = lmdb::to_val(address); + const int err = mdb_cursor_get(accounts_ba_cur.get(), &key, &value, MDB_GET_BOTH); + + if (err == MDB_NOTFOUND) + return {lws::error::account_not_found}; + if (err) + return {lmdb::error(err)}; + + const expect lookup = + accounts_by_address.get_value(value); + if (!lookup) + return lookup.error(); + + key = lmdb::to_val(lookup->status); + value = lmdb::to_val(lookup->id); + MONERO_LMDB_CHECK(mdb_cursor_get(accounts_cur.get(), &key, &value, MDB_GET_BOTH)); + + expect user = accounts.get_value(value); + if (!user) + return user.error(); + + user->access = *current_time; + value = lmdb::to_val(*user); + MONERO_LMDB_CHECK(mdb_cursor_put(accounts_cur.get(), &key, &value, MDB_CURRENT)); + return success(); + }); + } + + expect> + storage::change_status(account_status status , epee::span addresses) + { + MONERO_PRECOND(db != nullptr); + return db->try_write([this, status, addresses] (MDB_txn& txn) -> expect> + { + std::vector changed{}; + changed.reserve(addresses.size()); + + cursor::accounts accounts_cur; + cursor::accounts accounts_ba_cur; + cursor::accounts accounts_bh_cur; + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts, accounts_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts_ba, accounts_ba_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts_bh, accounts_bh_cur)); + + for (account_address const& address : addresses) + { + MDB_val key = lmdb::to_val(by_address_version); + MDB_val value = lmdb::to_val(address); + const int err = mdb_cursor_get(accounts_ba_cur.get(), &key, &value, MDB_GET_BOTH); + + if (err == MDB_NOTFOUND) + continue; + if (err) + return {lmdb::error(err)}; + + expect by_address = + accounts_by_address.get_value(value); + if (!by_address) + return by_address.error(); + + const account_status current = by_address->lookup.status; + if (current != status) + { + by_address->lookup.status = status; + + value = lmdb::to_val(*by_address); + MONERO_LMDB_CHECK(mdb_cursor_put(accounts_ba_cur.get(), &key, &value, MDB_CURRENT)); + + key = lmdb::to_val(current); + value = lmdb::to_val(by_address->lookup.id); + MONERO_LMDB_CHECK(mdb_cursor_get(accounts_cur.get(), &key, &value, MDB_GET_BOTH)); + + expect user = accounts.get_value(value); + if (!user) + return user.error(); + + MONERO_LMDB_CHECK(mdb_cursor_del(accounts_cur.get(), 0)); + + key = lmdb::to_val(status); + value = lmdb::to_val(*user); + MONERO_LMDB_CHECK(mdb_cursor_put(accounts_cur.get(), &key, &value, MDB_NODUPDATA)); + + key = lmdb::to_val(user->scan_height); + value = lmdb::to_val(user->id); + MONERO_LMDB_CHECK(mdb_cursor_get(accounts_bh_cur.get(), &key, &value, MDB_GET_BOTH)); + + value = lmdb::to_val(by_address->lookup); + MONERO_LMDB_CHECK(mdb_cursor_put(accounts_bh_cur.get(), &key, &value, MDB_CURRENT)); + } + + changed.push_back(address); + } + + return changed; + }); + } + + namespace + { + expect do_add_account(MDB_cursor& accounts_cur, MDB_cursor& accounts_ba_cur, MDB_cursor& accounts_bh_cur, account const& user) noexcept + { + { + crypto::secret_key copy{}; + crypto::public_key verify{}; + static_assert(sizeof(copy) == sizeof(user.key), "bad memcpy"); + std::memcpy( + std::addressof(unwrap(copy)), std::addressof(user.key), sizeof(copy) + ); + + if (!crypto::secret_key_to_public_key(copy, verify)) + return {lws::error::bad_view_key}; + + if (verify != user.address.view_public) + return {lws::error::bad_view_key}; + } + + const account_by_address by_address{ + user.address, {user.id, account_status::active} + }; + + MDB_val key = lmdb::to_val(by_address_version); + MDB_val value = lmdb::to_val(by_address); + const int err = mdb_cursor_put(&accounts_ba_cur, &key, &value, MDB_NODUPDATA); + + if (err == MDB_KEYEXIST) + return {lws::error::account_exists}; + if (err) + return {lmdb::error(err)}; + + key = lmdb::to_val(user.scan_height); + value = lmdb::to_val(by_address.lookup); + MONERO_LMDB_CHECK( + mdb_cursor_put(&accounts_bh_cur, &key, &value, MDB_NODUPDATA) + ); + + key = lmdb::to_val(by_address.lookup.status); + value = lmdb::to_val(user); + MONERO_LMDB_CHECK( + mdb_cursor_put(&accounts_cur, &key, &value, MDB_NODUPDATA) + ); + return success(); + } + } // anonymous + + expect storage::add_account(account_address const& address, crypto::secret_key const& key) noexcept + { + MONERO_PRECOND(db != nullptr); + return db->try_write([this, &address, &key] (MDB_txn& txn) -> expect + { + const expect current_time = get_account_time(); + if (!current_time) + return current_time.error(); + + cursor::blocks blocks_cur; + cursor::accounts accounts_cur; + cursor::accounts_by_address accounts_ba_cur; + cursor::accounts_by_height accounts_bh_cur; + + MONERO_CHECK(check_cursor(txn, this->db->tables.blocks, blocks_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts, accounts_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts_ba, accounts_ba_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts_bh, accounts_bh_cur)); + + const expect last_id = find_last_id(*accounts_cur); + if (!last_id) + return last_id.error(); + + MDB_val keyv = lmdb::to_val(blocks_version); + MDB_val value{}; + + MONERO_LMDB_CHECK(mdb_cursor_get(blocks_cur.get(), &keyv, &value, MDB_SET)); + MONERO_LMDB_CHECK(mdb_cursor_get(blocks_cur.get(), &keyv, &value, MDB_LAST_DUP)); + + const expect height = + blocks.get_value(value); + if (!height) + return height.error(); + + const account_id next_id = account_id(lmdb::to_native(*last_id) + 1); + account user{}; + user.id = next_id; + user.address = address; + static_assert(sizeof(user.key) == sizeof(key), "bad memcpy"); + std::memcpy(std::addressof(user.key), std::addressof(key), sizeof(key)); + user.start_height = *height; + user.scan_height = *height; + user.access = *current_time; + user.creation = *current_time; + // ... leave flags set to zero ... + + return do_add_account( + *accounts_cur, *accounts_ba_cur, *accounts_bh_cur, user + ); + }); + } + + namespace + { + //! \return Success, even if `address` was not found (designed for + expect + change_height(MDB_cursor& accounts_cur, MDB_cursor& accounts_ba_cur, MDB_cursor& accounts_bh_cur, block_id height, account_address const& address) + { + MDB_val key = lmdb::to_val(by_address_version); + MDB_val value = lmdb::to_val(address); + const int err = mdb_cursor_get(&accounts_ba_cur, &key, &value, MDB_GET_BOTH); + if (err == MDB_NOTFOUND) + return {lws::error::account_not_found}; + if (err) + return {lmdb::error(err)}; + + const expect lookup = + accounts_by_address.get_value(value); + if (!lookup) + return lookup.error(); + + key = lmdb::to_val(lookup->status); + value = lmdb::to_val(lookup->id); + MONERO_LMDB_CHECK( + mdb_cursor_get(&accounts_cur, &key, &value, MDB_GET_BOTH) + ); + + expect user = accounts.get_value(value); + if (!user) + return user.error(); + + const block_id current_height = user->scan_height; + user->scan_height = std::min(height, user->scan_height); + user->start_height = std::min(height, user->start_height); + + value = lmdb::to_val(*user); + MONERO_LMDB_CHECK( + mdb_cursor_put(&accounts_cur, &key, &value, MDB_CURRENT) + ); + + key = lmdb::to_val(current_height); + MONERO_LMDB_CHECK( + mdb_cursor_get(&accounts_bh_cur, &key, &value, MDB_GET_BOTH) + ); + MONERO_LMDB_CHECK(mdb_cursor_del(&accounts_bh_cur, 0)); + + key = lmdb::to_val(height); + value = lmdb::to_val(*lookup); + MONERO_LMDB_CHECK( + mdb_cursor_put(&accounts_bh_cur, &key, &value, MDB_NODUPDATA) + ); + + return success(); + } + } + + expect> + storage::rescan(db::block_id height, epee::span addresses) + { + MONERO_PRECOND(db != nullptr); + return db->try_write([this, height, addresses] (MDB_txn& txn) -> expect> + { + std::vector updated{}; + updated.reserve(addresses.size()); + + cursor::accounts accounts_cur; + cursor::accounts_by_address accounts_ba_cur; + cursor::accounts_by_height accounts_bh_cur; + + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts, accounts_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts_ba, accounts_ba_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts_bh, accounts_bh_cur)); + + for (account_address const& address : addresses) + { + const expect changed = change_height( + *accounts_cur, *accounts_ba_cur, *accounts_bh_cur, height, address + ); + if (changed) + updated.push_back(address); + else if (changed != lws::error::account_not_found) + return changed.error(); + } + return updated; + }); + } + + expect storage::creation_request(account_address const& address, crypto::secret_key const& key, account_flags flags) noexcept + { + MONERO_PRECOND(db != nullptr); + + if (!db->create_queue_max) + return {lws::error::create_queue_max}; + + return db->try_write([this, &address, &key, flags] (MDB_txn& txn) -> expect + { + const expect current_time = get_account_time(); + if (!current_time) + return current_time.error(); + + cursor::accounts_by_address accounts_ba_cur; + cursor::blocks blocks_cur; + cursor::accounts requests_cur; + + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts_ba, accounts_ba_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.blocks, blocks_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.requests, requests_cur)); + + MDB_val keyv = lmdb::to_val(by_address_version); + MDB_val value = lmdb::to_val(address); + + int err = mdb_cursor_get(accounts_ba_cur.get(), &keyv, &value, MDB_GET_BOTH); + if (err != MDB_NOTFOUND) + { + if (err) + return {lmdb::error(err)}; + return {lws::error::account_exists}; + } + + const request req = request::create; + keyv = lmdb::to_val(req); + value = MDB_val{}; + err = mdb_cursor_get(requests_cur.get(), &keyv, &value, MDB_SET); + if (!err) + { + mdb_size_t count = 0; + MONERO_LMDB_CHECK(mdb_cursor_count(requests_cur.get(), &count)); + if (this->db->create_queue_max <= count) + return {lws::error::create_queue_max}; + } + else if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + + keyv = lmdb::to_val(blocks_version); + value = MDB_val{}; + + MONERO_LMDB_CHECK(mdb_cursor_get(blocks_cur.get(), &keyv, &value, MDB_SET)); + MONERO_LMDB_CHECK(mdb_cursor_get(blocks_cur.get(), &keyv, &value, MDB_LAST_DUP)); + + const expect height = + blocks.get_value(value); + if (!height) + return height.error(); + + request_info info{}; + info.address = address; + static_assert(sizeof(info.key) == sizeof(key), "bad memcpy"); + std::memcpy(std::addressof(info.key), std::addressof(key), sizeof(key)); + info.creation = *current_time; + info.start_height = *height; + info.creation_flags = flags; + + keyv = lmdb::to_val(req); + value = lmdb::to_val(info); + + err = mdb_cursor_put(requests_cur.get(), &keyv, &value, MDB_NODUPDATA); + if (err == MDB_KEYEXIST) + return {lws::error::duplicate_request}; + if (err) + return {lmdb::error(err)}; + + return success(); + }); + } + + expect storage::import_request(account_address const& address, block_id height) noexcept + { + MONERO_PRECOND(db != nullptr); + return db->try_write([this, &address, height] (MDB_txn& txn) -> expect + { + const expect current_time = get_account_time(); + if (!current_time) + return current_time.error(); + + cursor::blocks accounts_ba_cur; + cursor::requests requests_cur; + + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts_ba, accounts_ba_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.requests, requests_cur)); + + MDB_val key = lmdb::to_val(by_address_version); + MDB_val value = lmdb::to_val(address); + + int err = mdb_cursor_get(accounts_ba_cur.get(), &key, &value, MDB_GET_BOTH); + if (err == MDB_NOTFOUND) + return {lws::error::account_not_found}; + if (err) + return {lmdb::error(err)}; + + request_info info{}; + info.address = address; + info.start_height = height; + + const request req = request::import_scan; + key = lmdb::to_val(req); + value = lmdb::to_val(info); + + err = mdb_cursor_put(requests_cur.get(), &key, &value, MDB_NODUPDATA); + if (err == MDB_KEYEXIST) + return {lws::error::duplicate_request}; + if (err) + return {lmdb::error(err)}; + + return success(); + }); + } + + namespace + { + expect> + create_accounts(MDB_txn& txn, storage_internal::tables_ const& tables, epee::span addresses) + { + std::vector stored{}; + stored.reserve(addresses.size()); + + const expect current_time = get_account_time(); + if (!current_time) + return current_time.error(); + + cursor::accounts accounts_cur; + cursor::accounts_by_address accounts_ba_cur; + cursor::accounts_by_height accounts_bh_cur; + cursor::requests requests_cur; + + MONERO_CHECK(check_cursor(txn, tables.accounts, accounts_cur)); + MONERO_CHECK(check_cursor(txn, tables.accounts_ba, accounts_ba_cur)); + MONERO_CHECK(check_cursor(txn, tables.accounts_bh, accounts_bh_cur)); + MONERO_CHECK(check_cursor(txn, tables.requests, requests_cur)); + + expect last_id = find_last_id(*accounts_cur); + if (!last_id) + return last_id.error(); + + const request req = request::create; + for (account_address const& address : addresses) + { + MDB_val keyv = lmdb::to_val(req); + MDB_val value = lmdb::to_val(address); + int err = mdb_cursor_get(requests_cur.get(), &keyv, &value, MDB_GET_BOTH); + if (err == MDB_NOTFOUND) + continue; + if (err) + return {lmdb::error(err)}; + + const expect info = requests.get_value(value); + if (!info) + return info.error(); + + MONERO_LMDB_CHECK(mdb_cursor_del(requests_cur.get(), 0)); + + const account_id next_id = account_id(lmdb::to_native(*last_id) + 1); + if (next_id == account_id::invalid) + return {lws::error::account_max}; + + account user{}; + user.id = next_id; + user.address = address; + user.key = info->key; + user.start_height = info->start_height; + user.scan_height = info->start_height; + user.access = *current_time; + user.creation = info->creation; + user.flags = info->creation_flags; + + const expect added = + do_add_account(*accounts_cur, *accounts_ba_cur, *accounts_bh_cur, user); + + if (!added) + { + if (added == lws::error::account_exists || added == lws::error::bad_view_key) + continue; + return added.error(); + } + + *last_id = next_id; + stored.push_back(address); + } + return stored; + } + + expect> + import_accounts(MDB_txn& txn, storage_internal::tables_ const& tables, epee::span addresses) + { + std::vector updated{}; + updated.reserve(addresses.size()); + + cursor::accounts accounts_cur; + cursor::accounts accounts_ba_cur; + cursor::accounts accounts_bh_cur; + cursor::requests requests_cur; + + MONERO_CHECK(check_cursor(txn, tables.accounts, accounts_cur)); + MONERO_CHECK(check_cursor(txn, tables.accounts_ba, accounts_ba_cur)); + MONERO_CHECK(check_cursor(txn, tables.accounts_bh, accounts_bh_cur)); + MONERO_CHECK(check_cursor(txn, tables.requests, requests_cur)); + + const request req = request::import_scan; + for (account_address const& address : addresses) + { + MDB_val key = lmdb::to_val(req); + MDB_val value = lmdb::to_val(address); + const int err = mdb_cursor_get(requests_cur.get(), &key, &value, MDB_GET_BOTH); + if (err == MDB_NOTFOUND) + continue; + if (err) + return {lmdb::error(err)}; + + const expect new_height = + requests.get_value(value); + MONERO_LMDB_CHECK(mdb_cursor_del(requests_cur.get(), 0)); + if (!new_height) + return new_height.error(); + + const expect changed = change_height( + *accounts_cur, *accounts_ba_cur, *accounts_bh_cur, *new_height, address + ); + if (changed) + updated.push_back(address); + else if (changed != lws::error::account_not_found) + return changed.error(); + } + return updated; + } + } // anonymous + + expect> + storage::accept_requests(request req, epee::span addresses) + { + if (addresses.empty()) + return std::vector{}; + + MONERO_PRECOND(db != nullptr); + return db->try_write([this, req, addresses] (MDB_txn& txn) -> expect> + { + switch (req) + { + case request::create: + return create_accounts(txn, this->db->tables, addresses); + case request::import_scan: + return import_accounts(txn, this->db->tables, addresses); + default: + break; + } + return {common_error::kInvalidArgument}; + }); + } + + expect> + storage::reject_requests(request req, epee::span addresses) + { + if (addresses.empty()) + return std::vector{}; + + MONERO_PRECOND(db != nullptr); + return db->try_write([this, req, addresses] (MDB_txn& txn) -> expect> + { + std::vector rejected{}; + + cursor::requests requests_cur; + MONERO_CHECK(check_cursor(txn, this->db->tables.requests, requests_cur)); + + MDB_val key = lmdb::to_val(req); + for (account_address const& address : addresses) + { + MDB_val value = lmdb::to_val(address); + const int err = mdb_cursor_get(requests_cur.get(), &key, &value, MDB_GET_BOTH); + if (err && err != MDB_NOTFOUND) + return {lmdb::error(err)}; + + if (!err) + { + MONERO_LMDB_CHECK(mdb_cursor_del(requests_cur.get(), 0)); + rejected.push_back(address); + } + } + + return rejected; + }); + } + + namespace + { + expect + add_spends(MDB_cursor& spends_cur, MDB_cursor& images_cur, account_id user, epee::span spends) noexcept + { + MONERO_CHECK(bulk_insert(spends_cur, user, spends)); + for (auto const& entry : spends) + { + const db::key_image image{entry.image, entry.link}; + + MDB_val key = lmdb::to_val(entry.source); + MDB_val value = lmdb::to_val(image); + const int err = mdb_cursor_put(&images_cur, &key, &value, MDB_NODUPDATA); + if (err && err != MDB_KEYEXIST) + return {lmdb::error(err)}; + } + return success(); + } + } // anonymous + + expect storage::update(block_id height, epee::span chain, epee::span users) + { + if (users.empty() && chain.empty()) + return 0; + + MONERO_PRECOND(!chain.empty()); + MONERO_PRECOND(db != nullptr); + + return db->try_write([this, height, chain, users] (MDB_txn& txn) -> expect + { + epee::span chain_copy{chain}; + const std::uint64_t last_update = + lmdb::to_native(height) + chain.size() - 1; + + if (get_checkpoints().get_max_height() <= last_update) + { + cursor::blocks blocks_cur; + MONERO_CHECK(check_cursor(txn, this->db->tables.blocks, blocks_cur)); + + MDB_val key = lmdb::to_val(blocks_version); + MDB_val value; + MONERO_LMDB_CHECK(mdb_cursor_get(blocks_cur.get(), &key, &value, MDB_SET)); + MONERO_LMDB_CHECK(mdb_cursor_get(blocks_cur.get(), &key, &value, MDB_LAST_DUP)); + + const expect last_block = blocks.get_value(value); + if (!last_block) + return last_block.error(); + if (last_block->id < height) + return {lws::error::bad_blockchain}; + + const std::uint64_t last_same = + std::min(lmdb::to_native(last_block->id), last_update); + + const expect hash_check = + get_block_hash(*blocks_cur, block_id(last_same)); + if (!hash_check) + return hash_check.error(); + + const std::uint64_t offset = last_same - lmdb::to_native(height); + if (*hash_check != *(chain_copy.begin() + offset)) + return {lws::error::blockchain_reorg}; + + chain_copy.remove_prefix(offset + 1); + MONERO_CHECK( + append_block_hashes( + *blocks_cur, block_id(lmdb::to_native(height) + offset + 1), chain_copy + ) + ); + } + + cursor::accounts accounts_cur; + cursor::accounts_by_address accounts_ba_cur; + cursor::accounts_by_height accounts_bh_cur; + cursor::outputs outputs_cur; + cursor::spends spends_cur; + cursor::images images_cur; + + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts, accounts_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts_bh, accounts_bh_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.outputs, outputs_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.spends, spends_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.images, images_cur)); + + // for bulk inserts + boost::container::static_vector heights{}; + static_assert(sizeof(heights) <= 1024, "stack vector is large"); + + std::size_t updated = 0; + for (auto user = users.begin() ;; ++user) + { + if (heights.size() == heights.capacity() || user == users.end()) + { + // bulk update account height index + MONERO_CHECK( + bulk_insert(*accounts_bh_cur, last_update, epee::to_span(heights)) + ); + if (user == users.end()) + break; + heights.clear(); + } + + // faster to assume that account is still active + account_status status_key = account_status::active; + const account_id user_id = user->id(); + MDB_val key = lmdb::to_val(status_key); + MDB_val value = lmdb::to_val(user_id); + int err = mdb_cursor_get(accounts_cur.get(), &key, &value, MDB_GET_BOTH); + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + if (accounts_ba_cur == nullptr) + MONERO_CHECK(check_cursor(txn, this->db->tables.accounts_ba, accounts_ba_cur)); + + MDB_val temp_key = lmdb::to_val(by_address_version); + MDB_val temp_value = lmdb::to_val(user->db_address()); + err = mdb_cursor_get(accounts_ba_cur.get(), &temp_key, &temp_value, MDB_GET_BOTH); + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + continue; // to next account + } + + const expect lookup = + accounts_by_address.get_value(temp_value); + if (!lookup) + return lookup.error(); + + status_key = lookup->status; + MONERO_LMDB_CHECK(mdb_cursor_get(accounts_cur.get(), &key, &value, MDB_GET_BOTH)); + } + expect existing = accounts.get_value(value); + if (!existing || existing->scan_height != user->scan_height()) + continue; // to next account + + const block_id existing_height = existing->scan_height; + + existing->scan_height = block_id(last_update); + value = lmdb::to_val(*existing); + MONERO_LMDB_CHECK(mdb_cursor_put(accounts_cur.get(), &key, &value, MDB_CURRENT)); + + heights.push_back(account_lookup{user->id(), status_key}); + + key = lmdb::to_val(existing_height); + value = lmdb::to_val(user_id); + MONERO_LMDB_CHECK(mdb_cursor_get(accounts_bh_cur.get(), &key, &value, MDB_GET_BOTH)); + MONERO_LMDB_CHECK(mdb_cursor_del(accounts_bh_cur.get(), 0)); + + MONERO_CHECK(bulk_insert(*outputs_cur, user->id(), epee::to_span(user->outputs()))); + MONERO_CHECK(add_spends(*spends_cur, *images_cur, user->id(), epee::to_span(user->spends()))); + + ++updated; + } // ... for every account being updated ... + return updated; + }); + } +} // db +} // lws diff --git a/src/db/storage.h b/src/db/storage.h new file mode 100644 index 0000000..7df4362 --- /dev/null +++ b/src/db/storage.h @@ -0,0 +1,239 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include +#include +#include +#include + +#include "common/expect.h" +#include "crypto/crypto.h" +#include "db/account.h" +#include "db/data.h" +#include "fwd.h" +#include "lmdb/transaction.h" +#include "lmdb/key_stream.h" +#include "lmdb/value_stream.h" + +namespace lws +{ +namespace db +{ + namespace cursor + { + MONERO_CURSOR(accounts); + MONERO_CURSOR(outputs); + MONERO_CURSOR(spends); + MONERO_CURSOR(images); + MONERO_CURSOR(requests); + + MONERO_CURSOR(blocks); + MONERO_CURSOR(accounts_by_address); + MONERO_CURSOR(accounts_by_height); + } + + struct storage_internal; + struct reader_internal + { + cursor::blocks blocks_cur; + cursor::accounts_by_address accounts_ba_cur; + cursor::accounts_by_height accounts_bh_cur; + }; + + //! Wrapper for LMDB read access to on-disk storage of light-weight server data. + class storage_reader + { + std::shared_ptr db; + lmdb::read_txn txn; + reader_internal curs; + + public: + storage_reader(std::shared_ptr db, lmdb::read_txn txn) noexcept + : db(std::move(db)), txn(std::move(txn)), curs{} + {} + + storage_reader(storage_reader&&) = default; + storage_reader(storage_reader const&) = delete; + + ~storage_reader() noexcept; + + storage_reader& operator=(storage_reader&&) = default; + storage_reader& operator=(storage_reader const&) = delete; + + //! \return Last known block. + expect get_last_block() noexcept; + + //! \return List for `GetHashesFast` to sync blockchain with daemon. + expect> get_chain_sync(); + + //! \return All registered `account`s. + expect> + get_accounts(cursor::accounts cur = nullptr) noexcept; + + //! \return All `account`s currently in `status` or `lmdb::error(MDB_NOT_FOUND)`. + expect> + get_accounts(account_status status, cursor::accounts cur = nullptr) noexcept; + + //! \return Info related to `address` or `lmdb::error(MDB_NOT_FOUND)`. + expect> + get_account(account_address const& address, cursor::accounts& cur) noexcept; + + expect> + get_account(account_address const& address) noexcept + { + cursor::accounts cur; + return get_account(address, cur); + } + + //! \return All outputs received by `id`. + expect> + get_outputs(account_id id, cursor::outputs cur = nullptr) noexcept; + + //! \return All potential spends by `id`. + expect> + get_spends(account_id id, cursor::spends cur = nullptr) noexcept; + + //! \return All key images associated with `id`. + expect> + get_images(output_id id, cursor::images cur = nullptr) noexcept; + + //! \return All `request_info`s. + expect> + get_requests(cursor::requests cur = nullptr) noexcept; + + //! \return A specific request from `address` of `type`. + expect + get_request(request type, account_address const& address, cursor::requests cur = nullptr) noexcept; + + //! Dump the contents of the database in JSON format to `out`. + expect json_debug(std::ostream& out, bool show_keys); + + //! \return Read txn that can be re-used via `storage::start_read`. + lmdb::suspended_txn finish_read() noexcept; + }; + + //! Wrapper for LMDB on-disk storage of light-weight server data. + class storage + { + std::shared_ptr db; + + storage(std::shared_ptr db) noexcept + : db(std::move(db)) + {} + + public: + /*! + Open a light_wallet_server LDMB database. + + \param path Directory for LMDB storage + \param create_queue_max Maximum number of create account requests allowed. + + \throw std::system_error on any LMDB error (all treated as fatal). + \throw std::bad_alloc If `std::shared_ptr` fails to allocate. + + \return A ready light-wallet server database. + */ + static storage open(const char* path, unsigned create_queue_max); + + storage(storage&&) = default; + storage(storage const&) = delete; + + ~storage() noexcept; + + storage& operator=(storage&&) = default; + storage& operator=(storage const&) = delete; + + //! \return A copy of the LMDB environment, but not reusable txn/cursors. + storage clone() const noexcept; + + //! Rollback chain and accounts to `height`. + expect rollback(block_id height); + + /*! + Sync the local blockchain with a remote version. Pops user txes if reorg + detected. + + \param height The height of the element in `hashes` + \param hashes List of blockchain hashes starting at `height`. + + \return True if the local blockchain is correctly synced. + */ + expect sync_chain(block_id height, epee::span hashes); + + //! Bump the last access time of `address` to the current time. + expect update_access_time(account_address const& address) noexcept; + + //! Change state of `address` to `status`. \return Updated `addresses`. + expect> + change_status(account_status status, epee::span addresses); + + + //! Add an account, for immediate inclusion in the active list. + expect add_account(account_address const& address, crypto::secret_key const& key) noexcept; + + //! Reset `addresses` to `height` for scanning. + expect> + rescan(block_id height, epee::span addresses); + + //! Add an account for later approval. For use with the login endpoint. + expect creation_request(account_address const& address, crypto::secret_key const& key, account_flags flags) noexcept; + + /*! + Request lock height of an existing account. No effect if the `start_height` + is already older. + */ + expect import_request(account_address const& address, block_id height) noexcept; + + //! Accept requests by `addresses` of type `req`. \return Accepted addresses. + expect> + accept_requests(request req, epee::span addresses); + + //! Reject requests by `addresses` of type `req`. \return Rejected addresses. + expect> + reject_requests(request req, epee::span addresses); + + /*! + Updates the status of user accounts, even if inactive or hidden. Duplicate + receives or spends provided in `accts` are silently ignored. If a gap in + `height` vs the stored account record is detected, the entire update will + fail. + + \param height The first hash in `chain` is at this height. + \param chain List of block hashes that `accts` were scanned against. + \param accts Updated to `height + chain.size()` scan height. + + \return True iff LMDB successfully committed the update. + */ + expect update(block_id height, epee::span chain, epee::span accts); + + //! `txn` must have come from a previous call on the same thread. + expect start_read(lmdb::suspended_txn txn = nullptr) const; + }; +} // db +} // lws diff --git a/src/db/string.cpp b/src/db/string.cpp new file mode 100644 index 0000000..3094ef0 --- /dev/null +++ b/src/db/string.cpp @@ -0,0 +1,62 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "string.h" + +#include "config.h" +#include "cryptonote_basic/cryptonote_basic_impl.h" +#include "db/data.h" +#include "error.h" + +namespace lws +{ +namespace db +{ + std::string address_string_::operator()(account_address const& address) const + { + const cryptonote::account_public_address address_{ + address.spend_public, address.view_public + }; + return cryptonote::get_account_address_as_str( + lws::config::network, false, address_ + ); + } + expect + address_string_::operator()(boost::string_ref address) const noexcept + { + cryptonote::address_parse_info info{}; + + if (!cryptonote::get_account_address_from_str(info, lws::config::network, std::string{address})) + return {lws::error::bad_address}; + if (info.is_subaddress || info.has_payment_id) + return {lws::error::bad_address}; + + return account_address{ + info.address.m_spend_public_key, info.address.m_view_public_key + }; + } +} // db +} // lws diff --git a/src/db/string.h b/src/db/string.h new file mode 100644 index 0000000..3591900 --- /dev/null +++ b/src/db/string.h @@ -0,0 +1,55 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include + +#include "common/expect.h" +#include "db/fwd.h" + +namespace lws +{ +namespace db +{ + //! Callable for converting `account_address` to/from monero base58 public address. + struct address_string_ + { + /*! + \return `address` as a monero base58 public address, using + `lws::config::network` for the tag. + */ + std::string operator()(account_address const& address) const; + /*! + \return `address`, as base58 public address, using `lws::config::network` + for the tag. + */ + expect operator()(boost::string_ref address) const noexcept; + }; + constexpr const address_string_ address_string{}; +} // db +} // lws diff --git a/src/error.cpp b/src/error.cpp new file mode 100644 index 0000000..4ea69d0 --- /dev/null +++ b/src/error.cpp @@ -0,0 +1,131 @@ +// Copyright (c) 2018-2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "error.h" + +#include + +namespace lws +{ + struct category final : std::error_category + { + virtual const char* name() const noexcept override final + { + return "lws::error_category()"; + } + + virtual std::string message(int value) const override final + { + switch (lws::error(value)) + { + case error::account_exists: + return "Account with specified address already exists"; + case error::account_max: + return "Account limit has been reached"; + case error::account_not_found: + return "No account with the specified address exists"; + case error::bad_address: + return "Invalid base58 public address - wrong --network ?"; + case error::bad_view_key: + return "Address/viewkey mismatch"; + case error::bad_blockchain: + return "Unable to sync blockchain - wrong --network ?"; + case error::bad_client_tx: + return "Received invalid transaction from REST client"; + case error::bad_daemon_response: + return "Response from monerod daemon was bad/unexpected"; + case error::blockchain_reorg: + return "A blockchain reorg has been detected"; + case error::configuration: + return "Invalid process configuration"; + case error::create_queue_max: + return "Exceeded maxmimum number of pending account requests"; + case error::crypto_failure: + return "A cryptographic function failed"; + case error::daemon_timeout: + return "Connection failed with daemon"; + case error::duplicate_request: + return "A request of this type for this address has already been made"; + case error::exceeded_blockchain_buffer: + return "Exceeded internal buffer for blockchain hashes"; + case error::exceeded_rest_request_limit: + return "Request from client via REST exceeded enforced limits"; + case error::exchange_rates_disabled: + return "Exchange rates feature is disabled"; + case error::exchange_rates_fetch: + return "Unspecified error when retrieving exchange rates"; + case error::http_server: + return "HTTP server failed"; + case error::exchange_rates_old: + return "Exchange rates are older than cache interval"; + case error::not_enough_mixin: + return "Not enough outputs to meet requested mixin count"; + case error::signal_abort_process: + return "An in-process message was received to abort the process"; + case error::signal_abort_scan: + return "An in-process message was received to abort account scanning"; + case error::signal_unknown: + return "An unknown in-process message was received"; + case error::system_clock_invalid_range: + return "System clock is out of range for account storage format"; + case error::tx_relay_failed: + return "The daemon failed to relay transaction from REST client"; + default: + break; + } + return "Unknown lws::error_category() value"; + } + + virtual std::error_condition default_error_condition(int value) const noexcept override final + { + switch (lws::error(value)) + { + case error::bad_address: + case error::bad_view_key: + return std::errc::bad_address; + case error::daemon_timeout: + return std::errc::timed_out; + case error::exceeded_blockchain_buffer: + return std::errc::no_buffer_space; + case error::signal_abort_process: + case error::signal_abort_scan: + case error::signal_unknown: + return std::errc::interrupted; + case error::system_clock_invalid_range: + return std::errc::result_out_of_range; + default: + break; // map to unmatchable category + } + return std::error_condition{value, *this}; + } + }; + + std::error_category const& error_category() noexcept + { + static const category instance{}; + return instance; + } +} // lws diff --git a/src/error.h b/src/error.h new file mode 100644 index 0000000..ca1fe58 --- /dev/null +++ b/src/error.h @@ -0,0 +1,79 @@ +// Copyright (c) 2018-2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include + +namespace lws +{ + enum class error : int + { + // 0 is reserved for no error, as per expect + account_exists = 1, //!< Tried to create an account that already exists + account_max, //!< Maximum number of accounts have been created + account_not_found, //!< Account address is not in database. + bad_address, //!< Invalid base58 public address + bad_view_key, //!< Account has address/viewkey mismatch + bad_blockchain, //!< Blockchain is invalid or wrong network type + bad_client_tx, //!< REST client submitted invalid transaction + bad_daemon_response, //!< RPC Response from daemon was invalid + blockchain_reorg, //!< Blockchain reorg after fetching/scanning block(s) + configuration, //!< Process configuration invalid + crypto_failure, //!< Cryptographic function failed + create_queue_max, //!< Reached maximum pending account requests + daemon_timeout, //!< ZMQ send/receive timeout + duplicate_request, //!< Account already has a request of this type pending + exceeded_blockchain_buffer, //!< Out buffer for blockchain is too small + exceeded_rest_request_limit,//!< Exceeded enforced size limits for request + exchange_rates_disabled, //!< Exchange rates fetching is disabled + exchange_rates_fetch, //!< Exchange rates fetching failed + exchange_rates_old, //!< Exchange rates are older than cache interval + http_server, //!< HTTP server failure (init or run) + not_enough_mixin, //!< Not enough outputs to meet mixin count + signal_abort_process, //!< In process ZMQ PUB to abort the process was received + signal_abort_scan, //!< In process ZMQ PUB to abort the scan was received + signal_unknown, //!< An unknown in process ZMQ PUB was received + system_clock_invalid_range, //!< System clock is out of range for storage format + tx_relay_failed //!< Daemon failed to relayed tx from REST client + }; + + std::error_category const& error_category() noexcept; + + inline std::error_code make_error_code(lws::error value) noexcept + { + return std::error_code{int(value), error_category()}; + } +} + +namespace std +{ + template<> + struct is_error_code_enum<::lws::error> + : true_type + {}; +} diff --git a/src/fwd.h b/src/fwd.h new file mode 100644 index 0000000..b161fa3 --- /dev/null +++ b/src/fwd.h @@ -0,0 +1,35 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +namespace lws +{ + class account; + class rest_server; + class scanner; +} diff --git a/src/options.h b/src/options.h new file mode 100644 index 0000000..7451a0b --- /dev/null +++ b/src/options.h @@ -0,0 +1,72 @@ +// Copyright (c) 2018-2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include +#include +#include + +#include "common/command_line.h" // monero/src +#include "common/util.h" // monero/src + +namespace lws +{ + constexpr const char default_db_subdir[] = "/light_wallet_server"; + + struct options + { + const command_line::arg_descriptor db_path; + const command_line::arg_descriptor network; + + options() + : db_path{"db-path", "Folder for LMDB files", tools::get_default_data_dir() + default_db_subdir} + , network{"network", "<\"main\"|\"stage\"|\"test\"> - Blockchain net type", "main"} + {} + + void prepare(boost::program_options::options_description& description) const + { + command_line::add_arg(description, db_path); + command_line::add_arg(description, network); + command_line::add_arg(description, command_line::arg_help); + } + + void set_network(boost::program_options::variables_map const& args) const + { + const std::string net = command_line::get_arg(args, network); + if (net == "main") + lws::config::network = cryptonote::MAINNET; + else if (net == "stage") + lws::config::network = cryptonote::STAGENET; + else if (net == "test") + lws::config::network = cryptonote::TESTNET; + else + throw std::runtime_error{"Bad --network value"}; + } + }; +} diff --git a/src/rest_server.cpp b/src/rest_server.cpp new file mode 100644 index 0000000..8d2a89b --- /dev/null +++ b/src/rest_server.cpp @@ -0,0 +1,863 @@ +// Copyright (c) 2018-2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "rest_server.h" + +#include +#include +#include +#include +#include +#include + +#include "common/error.h" // monero/src +#include "common/expect.h" // monero/src +#include "crypto/crypto.h" // monero/src +#include "cryptonote_config.h" // monero/src +#include "db/data.h" +#include "db/storage.h" +#include "error.h" +#include "lmdb/util.h" // monero/src +#include "net/http_base.h" // monero/contrib/epee/include +#include "net/net_parse_helpers.h" // monero/contrib/epee/include +#include "rpc/daemon_messages.h" // monero/src +#include "rpc/light_wallet.h" +#include "rpc/rates.h" +#include "util/http_server.h" +#include "util/gamma_picker.h" +#include "util/random_outputs.h" +#include "wire/json.h" + +namespace lws +{ + namespace + { + namespace http = epee::net_utils::http; + + struct context : epee::net_utils::connection_context_base + { + context() + : epee::net_utils::connection_context_base() + {} + }; + + bool is_locked(std::uint64_t unlock_time, db::block_id last) noexcept + { + if (unlock_time > CRYPTONOTE_MAX_BLOCK_NUMBER) + return std::chrono::seconds{unlock_time} > std::chrono::system_clock::now().time_since_epoch(); + return db::block_id(unlock_time) > last; + } + + std::vector::const_iterator + find_metadata(std::vector const& metas, db::output_id id) + { + struct by_output_id + { + bool operator()(db::output::spend_meta_ const& left, db::output_id right) const noexcept + { + return left.id < right; + } + bool operator()(db::output_id left, db::output::spend_meta_ const& right) const noexcept + { + return left < right.id; + } + }; + return std::lower_bound(metas.begin(), metas.end(), id, by_output_id{}); + } + + bool is_hidden(db::account_status status) noexcept + { + switch (status) + { + case db::account_status::active: + case db::account_status::inactive: + return false; + default: + case db::account_status::hidden: + break; + } + return true; + } + + bool key_check(const rpc::account_credentials& creds) + { + crypto::public_key verify{}; + if (!crypto::secret_key_to_public_key(creds.key, verify)) + return false; + if (verify != creds.address.view_public) + return false; + return true; + } + + //! \return Account info from the DB, iff key matches address AND address is NOT hidden. + expect> open_account(const rpc::account_credentials& creds, db::storage disk) + { + if (!key_check(creds)) + return {lws::error::bad_view_key}; + + auto reader = disk.start_read(); + if (!reader) + return reader.error(); + + const auto user = reader->get_account(creds.address); + if (!user) + return user.error(); + if (is_hidden(user->first)) + return {lws::error::account_not_found}; + return {std::make_pair(user->second, std::move(*reader))}; + } + + struct get_address_info + { + using request = rpc::account_credentials; + using response = rpc::get_address_info_response; + + static expect handle(const request& req, db::storage disk, rpc::client const& client) + { + auto user = open_account(req, std::move(disk)); + if (!user) + return user.error(); + + response resp{.rates = {common_error::kInvalidArgument}}; + + auto outputs = user->second.get_outputs(user->first.id); + if (!outputs) + return outputs.error(); + + auto spends = user->second.get_spends(user->first.id); + if (!spends) + return spends.error(); + + const expect last = user->second.get_last_block(); + if (!last) + return last.error(); + + resp.blockchain_height = std::uint64_t(last->id); + resp.transaction_height = resp.blockchain_height; + resp.scanned_height = std::uint64_t(user->first.scan_height); + resp.scanned_block_height = resp.scanned_height; + resp.start_height = std::uint64_t(user->first.start_height); + + std::vector metas{}; + metas.reserve(outputs->count()); + + for (auto output = outputs->make_iterator(); !output.is_end(); ++output) + { + const db::output::spend_meta_ meta = + output.get_value(); + + // these outputs will usually be in correct order post ringct + if (metas.empty() || metas.back().id < meta.id) + metas.push_back(meta); + else + metas.insert(find_metadata(metas, meta.id), meta); + + resp.total_received = rpc::safe_uint64(std::uint64_t(resp.total_received) + meta.amount); + if (is_locked(output.get_value(), last->id)) + resp.locked_funds = rpc::safe_uint64(std::uint64_t(resp.locked_funds) + meta.amount); + } + + resp.spent_outputs.reserve(spends->count()); + for (auto const& spend : spends->make_range()) + { + const auto meta = find_metadata(metas, spend.source); + if (meta == metas.end() || meta->id != spend.source) + { + throw std::logic_error{ + "Serious database error, no receive for spend" + }; + } + + resp.spent_outputs.push_back({*meta, spend}); + resp.total_sent = rpc::safe_uint64(std::uint64_t(resp.total_sent) + meta->amount); + } + + resp.rates = client.get_rates(); + if (!resp.rates) + MWARNING("Unable to retrieve exchange rates: " << resp.rates.error().message()); + + return resp; + } + }; + + struct get_address_txs + { + using request = rpc::account_credentials; + using response = rpc::get_address_txs_response; + + static expect handle(const request& req, db::storage disk, rpc::client const&) + { + auto user = open_account(req, std::move(disk)); + if (!user) + return user.error(); + + auto outputs = user->second.get_outputs(user->first.id); + if (!outputs) + return outputs.error(); + + auto spends = user->second.get_spends(user->first.id); + if (!spends) + return spends.error(); + + const expect last = user->second.get_last_block(); + if (!last) + return last.error(); + + response resp{}; + resp.scanned_height = std::uint64_t(user->first.scan_height); + resp.scanned_block_height = resp.scanned_height; + resp.start_height = std::uint64_t(user->first.start_height); + resp.blockchain_height = std::uint64_t(last->id); + resp.transaction_height = resp.blockchain_height; + + // merge input and output info into a single set of txes. + + auto output = outputs->make_iterator(); + auto spend = spends->make_iterator(); + + std::vector metas{}; + + resp.transactions.reserve(outputs->count()); + metas.reserve(resp.transactions.capacity()); + + db::transaction_link next_output{}; + db::transaction_link next_spend{}; + + if (!output.is_end()) + next_output = output.get_value(); + if (!spend.is_end()) + next_spend = spend.get_value(); + + while (!output.is_end() || !spend.is_end()) + { + if (!resp.transactions.empty()) + { + db::transaction_link const& last = resp.transactions.back().info.link; + + if ((!output.is_end() && next_output < last) || (!spend.is_end() && next_spend < last)) + { + throw std::logic_error{"DB has unexpected sort order"}; + } + } + + if (spend.is_end() || (!output.is_end() && next_output <= next_spend)) + { + std::uint64_t amount = 0; + if (resp.transactions.empty() || resp.transactions.back().info.link.tx_hash != next_output.tx_hash) + { + resp.transactions.push_back({*output}); + amount = resp.transactions.back().info.spend_meta.amount; + } + else + { + amount = output.get_value(); + resp.transactions.back().info.spend_meta.amount += amount; + } + + const db::output_id this_id = resp.transactions.back().info.spend_meta.id; + if (metas.empty() || metas.back().id < this_id) + metas.push_back(resp.transactions.back().info.spend_meta); + else + metas.insert(find_metadata(metas, this_id), resp.transactions.back().info.spend_meta); + + resp.total_received = rpc::safe_uint64(std::uint64_t(resp.total_received) + amount); + + ++output; + if (!output.is_end()) + next_output = output.get_value(); + } + else if (output.is_end() || (next_spend < next_output)) + { + const db::output_id source_id = spend.get_value(); + const auto meta = find_metadata(metas, source_id); + if (meta == metas.end() || meta->id != source_id) + { + throw std::logic_error{ + "Serious database error, no receive for spend" + }; + } + + if (resp.transactions.empty() || resp.transactions.back().info.link.tx_hash != next_spend.tx_hash) + { + resp.transactions.push_back({}); + resp.transactions.back().spends.push_back({*meta, *spend}); + resp.transactions.back().info.link.height = resp.transactions.back().spends.back().possible_spend.link.height; + resp.transactions.back().info.link.tx_hash = resp.transactions.back().spends.back().possible_spend.link.tx_hash; + resp.transactions.back().info.spend_meta.mixin_count = + resp.transactions.back().spends.back().possible_spend.mixin_count; + resp.transactions.back().info.timestamp = resp.transactions.back().spends.back().possible_spend.timestamp; + resp.transactions.back().info.unlock_time = resp.transactions.back().spends.back().possible_spend.unlock_time; + } + else + resp.transactions.back().spends.push_back({*meta, *spend}); + + resp.transactions.back().spent += meta->amount; + + ++spend; + if (!spend.is_end()) + next_spend = spend.get_value(); + } + } + + return resp; + } + }; + + struct get_random_outs + { + using request = rpc::get_random_outs_request; + using response = rpc::get_random_outs_response; + + static expect handle(request req, const db::storage&, rpc::client const& gclient) + { + using distribution_rpc = cryptonote::rpc::GetOutputDistribution; + using histogram_rpc = cryptonote::rpc::GetOutputHistogram; + using distribution_rpc = cryptonote::rpc::GetOutputDistribution; + + std::vector amounts = std::move(req.amounts.values); + + if (50 < req.count || 20 < amounts.size()) + return {lws::error::exceeded_rest_request_limit}; + + expect client = gclient.clone(); + if (!client) + return client.error(); + + const std::greater rsort{}; + std::sort(amounts.begin(), amounts.end(), rsort); + const std::size_t ringct_count = + amounts.end() - std::lower_bound(amounts.begin(), amounts.end(), 0, rsort); + + std::vector histograms{}; + if (ringct_count < amounts.size()) + { + // reuse allocated vector memory + amounts.resize(amounts.size() - ringct_count); + + histogram_rpc::Request histogram_req{}; + histogram_req.amounts = std::move(amounts); + histogram_req.min_count = 0; + histogram_req.max_count = 0; + histogram_req.unlocked = true; + histogram_req.recent_cutoff = 0; + + epee::byte_slice msg = rpc::client::make_message("get_output_histogram", histogram_req); + MONERO_CHECK(client->send(std::move(msg), std::chrono::seconds{10})); + + auto histogram_resp = client->receive(std::chrono::minutes{3}); + if (!histogram_resp) + return histogram_resp.error(); + if (histogram_resp->histogram.size() != histogram_req.amounts.size()) + return {lws::error::bad_daemon_response}; + + histograms = std::move(histogram_resp->histogram); + + amounts = std::move(histogram_req.amounts); + amounts.insert(amounts.end(), ringct_count, 0); + } + + std::vector distributions{}; + if (ringct_count) + { + distribution_rpc::Request distribution_req{}; + if (ringct_count == amounts.size()) + distribution_req.amounts = std::move(amounts); + + distribution_req.amounts.resize(1); + distribution_req.from_height = 0; + distribution_req.to_height = 0; + distribution_req.cumulative = true; + + epee::byte_slice msg = + rpc::client::make_message("get_output_distribution", distribution_req); + MONERO_CHECK(client->send(std::move(msg), std::chrono::seconds{10})); + + auto distribution_resp = + client->receive(std::chrono::minutes{3}); + if (!distribution_resp) + return distribution_resp.error(); + + if (distribution_resp->distributions.size() != 1) + return {lws::error::bad_daemon_response}; + if (distribution_resp->distributions[0].amount != 0) + return {lws::error::bad_daemon_response}; + + distributions = std::move(distribution_resp->distributions[0].data.distribution); + + if (amounts.empty()) + { + amounts = std::move(distribution_req.amounts); + amounts.insert(amounts.end(), ringct_count - 1, 0); + } + } + + class zmq_fetch_keys + { + /* `std::function` needs a copyable functor. The functor was made + const and copied in the function instead of using a reference to + make the callback in `std::function` thread-safe. This shouldn't + be a problem now, but this is just-in-case of a future refactor. */ + rpc::client gclient; + public: + zmq_fetch_keys(rpc::client src) noexcept + : gclient(std::move(src)) + {} + + zmq_fetch_keys(zmq_fetch_keys&&) = default; + zmq_fetch_keys(zmq_fetch_keys const& rhs) + : gclient(MONERO_UNWRAP(rhs.gclient.clone())) + {} + + expect> operator()(std::vector ids) const + { + using get_keys_rpc = cryptonote::rpc::GetOutputKeys; + + get_keys_rpc::Request keys_req{}; + keys_req.outputs = std::move(ids); + + expect client = gclient.clone(); + if (!client) + return client.error(); + + epee::byte_slice msg = rpc::client::make_message("get_output_keys", keys_req); + MONERO_CHECK(client->send(std::move(msg), std::chrono::seconds{10})); + + auto keys_resp = client->receive(std::chrono::seconds{10}); + if (!keys_resp) + return keys_resp.error(); + + return {std::move(keys_resp->keys)}; + } + }; + + lws::gamma_picker pick_rct{std::move(distributions)}; + auto rings = pick_random_outputs( + req.count, + epee::to_span(amounts), + pick_rct, + epee::to_mut_span(histograms), + zmq_fetch_keys{std::move(*client)} + ); + if (!rings) + return rings.error(); + + return response{std::move(*rings)}; + } + }; + + struct get_unspent_outs + { + using request = rpc::get_unspent_outs_request; + using response = rpc::get_unspent_outs_response; + + static expect handle(request req, db::storage disk, rpc::client const& gclient) + { + using rpc_command = cryptonote::rpc::GetFeeEstimate; + + auto user = open_account(req.creds, std::move(disk)); + if (!user) + return user.error(); + + expect client = gclient.clone(); + if (!client) + return client.error(); + + { + rpc_command::Request req{}; + req.num_grace_blocks = 10; + epee::byte_slice msg = rpc::client::make_message("get_dynamic_fee_estimate", req); + MONERO_CHECK(client->send(std::move(msg), std::chrono::seconds{10})); + } + + if ((req.use_dust && req.use_dust) || !req.dust_threshold) + req.dust_threshold = rpc::safe_uint64(0); + + if (!req.mixin) + req.mixin = 0; + + auto outputs = user->second.get_outputs(user->first.id); + if (!outputs) + return outputs.error(); + + std::uint64_t received = 0; + std::vector>> unspent; + + unspent.reserve(outputs->count()); + for (db::output const& out : outputs->make_range()) + { + if (out.spend_meta.amount < std::uint64_t(*req.dust_threshold) || out.spend_meta.mixin_count < *req.mixin) + continue; + + received += out.spend_meta.amount; + unspent.push_back({out, {}}); + + auto images = user->second.get_images(out.spend_meta.id); + if (!images) + return images.error(); + + unspent.back().second.reserve(images->count()); + auto range = images->make_range(); + std::copy(range.begin(), range.end(), std::back_inserter(unspent.back().second)); + } + + if (received < std::uint64_t(req.amount)) + return {lws::error::account_not_found}; + + const auto resp = client->receive(std::chrono::seconds{20}); + if (!resp) + return resp.error(); + + if (resp->size_scale == 0 || 1024 < resp->size_scale || resp->fee_mask == 0) + return {lws::error::bad_daemon_response}; + + const std::uint64_t per_kb_fee = + resp->estimated_base_fee * (1024 / resp->size_scale); + const std::uint64_t per_kb_fee_masked = + ((per_kb_fee + (resp->fee_mask - 1)) / resp->fee_mask) * resp->fee_mask; + + return response{per_kb_fee_masked, resp->fee_mask, rpc::safe_uint64(received), std::move(unspent), std::move(req.creds.key)}; + } + }; + + struct import_request + { + using request = rpc::account_credentials; + using response = rpc::import_response; + + static expect handle(request req, db::storage disk, rpc::client const&) + { + bool new_request = false; + bool fulfilled = false; + { + auto user = open_account(req, disk.clone()); + if (!user) + return user.error(); + + if (user->first.start_height == db::block_id(0)) + fulfilled = true; + else + { + const expect info = + user->second.get_request(db::request::import_scan, req.address); + + if (!info) + { + if (info != lmdb::error(MDB_NOTFOUND)) + return info.error(); + + new_request = true; + } + } + } // close reader + + if (new_request) + MONERO_CHECK(disk.import_request(req.address, db::block_id(0))); + + const char* status = new_request ? + "Accepted, waiting for approval" : (fulfilled ? "Approved" : "Waiting for Approval"); + return response{rpc::safe_uint64(0), status, new_request, fulfilled}; + } + }; + + struct login + { + using request = rpc::login_request; + using response = rpc::login_response; + + static expect handle(request req, db::storage disk, rpc::client const&) + { + if (!key_check(req.creds)) + return {lws::error::bad_view_key}; + + { + auto reader = disk.start_read(); + if (!reader) + return reader.error(); + + const auto account = reader->get_account(req.creds.address); + reader->finish_read(); + + if (account) + { + if (is_hidden(account->first)) + return {lws::error::account_not_found}; + + // Do not count a request for account creation as login + return response{false, bool(account->second.flags & db::account_generated_locally)}; + } + else if (!req.create_account || account != lws::error::account_not_found) + return account.error(); + } + + const auto flags = req.generated_locally ? db::account_generated_locally : db::default_account; + MONERO_CHECK(disk.creation_request(req.creds.address, req.creds.key, flags)); + return response{true, req.generated_locally}; + } + }; + + struct submit_raw_tx + { + using request = rpc::submit_raw_tx_request; + using response = rpc::submit_raw_tx_response; + + static expect handle(request req, const db::storage& disk, const rpc::client& gclient) + { + using transaction_rpc = cryptonote::rpc::SendRawTxHex; + + expect client = gclient.clone(); + if (!client) + return client.error(); + + transaction_rpc::Request daemon_req{}; + daemon_req.relay = true; + daemon_req.tx_as_hex = std::move(req.tx); + + epee::byte_slice message = rpc::client::make_message("send_raw_tx_hex", daemon_req); + MONERO_CHECK(client->send(std::move(message), std::chrono::seconds{10})); + + const auto daemon_resp = client->receive(std::chrono::seconds{20}); + if (!daemon_resp) + return daemon_resp.error(); + if (!daemon_resp->relayed) + return {lws::error::tx_relay_failed}; + + return response{"OK"}; + } + }; + + template + expect call(std::string&& root, db::storage disk, const rpc::client& gclient) + { + using request = typename E::request; + using response = typename E::response; + + expect req = wire::json::from_bytes(std::move(root)); + if (!req) + return req.error(); + + expect resp = E::handle(std::move(*req), std::move(disk), gclient); + if (!resp) + return resp.error(); + return wire::json::to_bytes(*resp); + } + + struct endpoint + { + char const* const name; + expect (*const run)(std::string&&, db::storage, rpc::client const&); + const unsigned max_size; + }; + + constexpr const endpoint endpoints[] = + { + {"/get_address_info", call, 2 * 1024}, + {"/get_address_txs", call, 2 * 1024}, + {"/get_random_outs", call, 2 * 1024}, + {"/get_txt_records", nullptr, 0 }, + {"/get_unspent_outs", call, 2 * 1024}, + {"/import_wallet_request", call, 2 * 1024}, + {"/login", call, 2 * 1024}, + {"/submit_raw_tx", call, 50 * 1024} + }; + + struct by_name_ + { + bool operator()(endpoint const& left, endpoint const& right) const noexcept + { + if (left.name && right.name) + return std::strcmp(left.name, right.name) < 0; + return false; + } + bool operator()(const boost::string_ref left, endpoint const& right) const noexcept + { + if (right.name) + return left < right.name; + return false; + } + bool operator()(endpoint const& left, const boost::string_ref right) const noexcept + { + if (left.name) + return left.name < right; + return false; + } + }; + constexpr const by_name_ by_name{}; + } // anonymous + + struct rest_server::internal final : public lws::http_server_impl_base + { + db::storage disk; + rpc::client client; + + explicit internal(boost::asio::io_service& io_service, lws::db::storage disk, rpc::client client) + : lws::http_server_impl_base(io_service) + , disk(std::move(disk)) + , client(std::move(client)) + { + assert(std::is_sorted(std::begin(endpoints), std::end(endpoints), by_name)); + } + + virtual bool + handle_http_request(const http::http_request_info& query, http::http_response_info& response, context& ctx) + override final + { + const auto handler = std::lower_bound( + std::begin(endpoints), std::end(endpoints), query.m_URI, by_name + ); + if (handler == std::end(endpoints) || handler->name != query.m_URI) + { + response.m_response_code = 404; + response.m_response_comment = "Not Found"; + return true; + } + + if (handler->run == nullptr) + { + response.m_response_code = 501; + response.m_response_comment = "Not Implemented"; + return true; + } + + if (handler->max_size < query.m_body.size()) + { + MINFO("Client exceeded maximum body size (" << handler->max_size << " bytes)"); + response.m_response_code = 400; + response.m_response_comment = "Bad Request"; + return true; + } + + if (query.m_http_method != http::http_method_post) + { + response.m_response_code = 405; + response.m_response_comment = "Method Not Allowed"; + return true; + } + + // \TODO remove copy of json string here :/ + auto body = handler->run(std::string{query.m_body}, disk.clone(), client); + if (!body) + { + MINFO(body.error().message() << " from " << ctx.m_remote_address.str() << " on " << handler->name); + + if (body.error().category() == wire::error::rapidjson_category()) + { + response.m_response_code = 400; + response.m_response_comment = "Bad Request"; + } + else if (body == lws::error::account_not_found) + { + response.m_response_code = 403; + response.m_response_comment = "Forbidden"; + } + else if (body.matches(std::errc::timed_out) || body.matches(std::errc::no_lock_available)) + { + response.m_response_code = 503; + response.m_response_comment = "Service Unavailable"; + } + else + { + response.m_response_code = 500; + response.m_response_comment = "Internal Server Error"; + } + return true; + } + + response.m_response_code = 200; + response.m_response_comment = "OK"; + response.m_mime_tipe = "application/json"; + response.m_header_info.m_content_type = "application/json"; + response.m_body.assign(reinterpret_cast(body->data()), body->size()); // \TODO Remove copy here too! + response.m_additional_fields.push_back({"Access-Control-Allow-Credentials", "true"}); + return true; + } + }; + + rest_server::rest_server(epee::span addresses, db::storage disk, rpc::client client, configuration config) + : io_service_(), ports_() + { + ports_.emplace_back(io_service_, std::move(disk), std::move(client)); + + if (addresses.empty()) + MONERO_THROW(common_error::kInvalidArgument, "REST server requires 1 or more addresses"); + + const auto init_port = [] (internal& port, const std::string& address, configuration config) -> bool + { + epee::net_utils::http::url_content url{}; + if (!epee::net_utils::parse_url(address, url)) + MONERO_THROW(lws::error::configuration, "REST Server URL/address is invalid"); + + const bool https = url.schema == "https"; + if (!https && url.schema != "http") + MONERO_THROW(lws::error::configuration, "Unsupported scheme, only http or https supported"); + + if (std::numeric_limits::max() < url.port) + MONERO_THROW(lws::error::configuration, "Specified port for rest server is out of range"); + + if (!https) + { + boost::system::error_code error{}; + const boost::asio::ip::address ip_host = + ip_host.from_string(url.host, error); + if (error) + MONERO_THROW(lws::error::configuration, "Invalid IP address for REST server"); + if (!ip_host.is_loopback() && !config.allow_external) + MONERO_THROW(lws::error::configuration, "Binding to external interface with http - consider using https or secure tunnel (ssh, etc). Use --confirm-external-bind to override"); + } + + if (url.port == 0) + url.port = https ? 8443 : 8080; + + epee::net_utils::ssl_options_t ssl_options = https ? + epee::net_utils::ssl_support_t::e_ssl_support_enabled : + epee::net_utils::ssl_support_t::e_ssl_support_disabled; + ssl_options.verification = epee::net_utils::ssl_verification_t::none; // clients verified with view key + ssl_options.auth = std::move(config.auth); + + if (!port.init(std::to_string(url.port), std::move(url.host), std::move(config.access_controls), std::move(ssl_options))) + MONERO_THROW(lws::error::http_server, "REST server failed to initialize"); + return https; + }; + + bool any_ssl = false; + for (std::size_t index = 1; index < addresses.size(); ++index) + { + ports_.emplace_back(io_service_, ports_.front().disk.clone(), MONERO_UNWRAP(ports_.front().client.clone())); + any_ssl |= init_port(ports_.back(), addresses[index], config); + } + + const bool expect_ssl = !config.auth.private_key_path.empty(); + const std::size_t threads = config.threads; + any_ssl |= init_port(ports_.front(), addresses[0], std::move(config)); + if (!any_ssl && expect_ssl) + MONERO_THROW(lws::error::configuration, "Specified SSL key/cert without specifying https capable REST server"); + + if (!ports_.front().run(threads, false)) + MONERO_THROW(lws::error::http_server, "REST server failed to run"); + } + + rest_server::~rest_server() noexcept + {} +} // lws diff --git a/src/rest_server.h b/src/rest_server.h new file mode 100644 index 0000000..25532c2 --- /dev/null +++ b/src/rest_server.h @@ -0,0 +1,69 @@ +// Copyright (c) 2018-2019, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include +#include +#include +#include + +#include "db/storage.h" +#include "net/net_ssl.h" +#include "rpc/client.h" +#include "span.h" + +namespace lws +{ + class rest_server + { + struct internal; + + boost::asio::io_service io_service_; + std::list ports_; + + public: + struct configuration + { + epee::net_utils::ssl_authentication_t auth; + std::vector access_controls; + std::size_t threads; + bool allow_external; + }; + + explicit rest_server(epee::span addresses, db::storage disk, rpc::client client, configuration config); + + rest_server(rest_server&&) = delete; + rest_server(rest_server const&) = delete; + + ~rest_server() noexcept; + + rest_server& operator=(rest_server&&) = delete; + rest_server& operator=(rest_server const&) = delete; + }; +} diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt new file mode 100644 index 0000000..3646686 --- /dev/null +++ b/src/rpc/CMakeLists.txt @@ -0,0 +1,33 @@ +# Copyright (c) 2020, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(monero-lws-rpc_sources client.cpp daemon_zmq.cpp light_wallet.cpp rates.cpp) +set(monero-lws-rpc_headers client.h daemon_zmq.h fwd.h json.h light_wallet.h rates.h) + +add_library(monero-lws-rpc ${monero-lws-rpc_sources} ${monero-lws-rpc_headers}) +target_link_libraries(monero-lws-rpc monero::libraries monero-lws-wire-json) diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp new file mode 100644 index 0000000..6a4a26b --- /dev/null +++ b/src/rpc/client.cpp @@ -0,0 +1,325 @@ +// Copyright (c) 2018-2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "client.h" + +#include +#include +#include + +#include "common/error.h" // monero/contrib/epee/include +#include "error.h" +#include "net/http_client.h" // monero/contrib/epee/include/net +#include "net/zmq.h" // monero/src + +namespace lws +{ +namespace rpc +{ + namespace http = epee::net_utils::http; + + namespace + { + constexpr const char signal_endpoint[] = "inproc://signal"; + constexpr const char abort_scan_signal[] = "SCAN"; + constexpr const char abort_process_signal[] = "PROCESS"; + constexpr const int daemon_zmq_linger = 0; + + struct terminate + { + void operator()(void* ptr) const noexcept + { + if (ptr) + { + while (zmq_term(ptr)) + { + if (zmq_errno() != EINTR) + break; + } + } + } + }; + using zcontext = std::unique_ptr; + + expect do_wait(void* daemon, void* signal_sub, short events, std::chrono::milliseconds timeout) noexcept + { + if (timeout <= std::chrono::seconds{0}) + return {lws::error::daemon_timeout}; + + zmq_pollitem_t items[2] { + {daemon, 0, short(events | ZMQ_POLLERR), 0}, + {signal_sub, 0, short(ZMQ_POLLIN | ZMQ_POLLERR), 0} + }; + + for (;;) + { + const auto start = std::chrono::steady_clock::now(); + const int ready = zmq_poll(items, 2, timeout.count()); + const auto end = std::chrono::steady_clock::now(); + const auto spent = std::chrono::duration_cast(start - end); + timeout -= std::min(spent, timeout); + + if (ready == 0) + return {lws::error::daemon_timeout}; + if (0 < ready) + break; + const int err = zmq_errno(); + if (err != EINTR) + return net::zmq::make_error_code(err); + } + if (items[0].revents) + return success(); + + char buf[1]; + MONERO_ZMQ_CHECK(zmq_recv(signal_sub, buf, 1, 0)); + + switch (buf[0]) + { + case 'P': + return {lws::error::signal_abort_process}; + case 'S': + return {lws::error::signal_abort_scan}; + default: + break; + } + return {lws::error::signal_unknown}; + } + + template + expect do_signal(void* signal_pub, const char (&signal)[N]) noexcept + { + MONERO_ZMQ_CHECK(zmq_send(signal_pub, signal, sizeof(signal), 0)); + return success(); + } + + template + expect do_subscribe(void* signal_sub, const char (&signal)[N]) noexcept + { + MONERO_ZMQ_CHECK(zmq_setsockopt(signal_sub, ZMQ_SUBSCRIBE, signal, sizeof(signal))); + return success(); + } + } // anonymous + + namespace detail + { + struct context + { + explicit context(zcontext comm, socket signal_pub, std::string daemon_addr, std::chrono::minutes interval) + : comm(std::move(comm)) + , signal_pub(std::move(signal_pub)) + , daemon_addr(std::move(daemon_addr)) + , rates_conn() + , cache_time() + , cache_interval(interval) + , cached{} + , sync_rates() + { + if (std::chrono::minutes{0} < cache_interval) + rates_conn.set_server(crypto_compare.host, boost::none, epee::net_utils::ssl_support_t::e_ssl_support_enabled); + } + + zcontext comm; + socket signal_pub; + std::string daemon_addr; + http::http_simple_client rates_conn; + std::chrono::steady_clock::time_point cache_time; + const std::chrono::minutes cache_interval; + rates cached; + boost::mutex sync_rates; + }; + } // detail + + expect client::get_message(std::chrono::seconds timeout) + { + MONERO_PRECOND(ctx != nullptr); + assert(daemon != nullptr); + assert(signal_sub != nullptr); + + expect msg{common_error::kInvalidArgument}; + while (!(msg = net::zmq::receive(daemon.get(), ZMQ_DONTWAIT))) + { + if (msg != net::zmq::make_error_code(EAGAIN)) + break; + + MONERO_CHECK(do_wait(daemon.get(), signal_sub.get(), ZMQ_POLLIN, timeout)); + timeout = std::chrono::seconds{0}; + } + // std::string move constructor is noexcept + return msg; + } + + expect client::make(std::shared_ptr ctx) noexcept + { + MONERO_PRECOND(ctx != nullptr); + + const int linger = daemon_zmq_linger; + client out{std::move(ctx)}; + + out.daemon.reset(zmq_socket(out.ctx->comm.get(), ZMQ_REQ)); + if (out.daemon.get() == nullptr) + return net::zmq::get_error_code(); + MONERO_ZMQ_CHECK(zmq_connect(out.daemon.get(), out.ctx->daemon_addr.c_str())); + MONERO_ZMQ_CHECK(zmq_setsockopt(out.daemon.get(), ZMQ_LINGER, &linger, sizeof(linger))); + + out.signal_sub.reset(zmq_socket(out.ctx->comm.get(), ZMQ_SUB)); + if (out.signal_sub.get() == nullptr) + return net::zmq::get_error_code(); + MONERO_ZMQ_CHECK(zmq_connect(out.signal_sub.get(), signal_endpoint)); + + MONERO_CHECK(do_subscribe(out.signal_sub.get(), abort_process_signal)); + return {std::move(out)}; + } + + client::~client() noexcept + {} + + expect client::watch_scan_signals() noexcept + { + MONERO_PRECOND(ctx != nullptr); + assert(signal_sub != nullptr); + return do_subscribe(signal_sub.get(), abort_scan_signal); + } + + expect client::wait(std::chrono::seconds timeout) noexcept + { + MONERO_PRECOND(ctx != nullptr); + assert(daemon != nullptr); + assert(signal_sub != nullptr); + return do_wait(daemon.get(), signal_sub.get(), 0, timeout); + } + + expect client::send(epee::byte_slice message, std::chrono::seconds timeout) noexcept + { + MONERO_PRECOND(ctx != nullptr); + assert(daemon != nullptr); + assert(signal_sub != nullptr); + + expect sent; + while (!(sent = net::zmq::send(message.clone(), daemon.get(), ZMQ_DONTWAIT))) + { + if (sent != net::zmq::make_error_code(EAGAIN)) + return sent.error(); + + MONERO_CHECK(do_wait(daemon.get(), signal_sub.get(), ZMQ_POLLOUT, timeout)); + timeout = std::chrono::seconds{0}; + } + return success(); + } + + expect client::get_rates() const + { + MONERO_PRECOND(ctx != nullptr); + if (ctx->cache_interval <= std::chrono::minutes{0}) + return {lws::error::exchange_rates_disabled}; + + const auto now = std::chrono::steady_clock::now(); + const boost::unique_lock lock{ctx->sync_rates}; + if (now - ctx->cache_time >= ctx->cache_interval + std::chrono::seconds{30}) + return {lws::error::exchange_rates_old}; + return ctx->cached; + } + + context context::make(std::string daemon_addr, std::chrono::minutes rates_interval) + { + zcontext comm{zmq_init(1)}; + if (comm == nullptr) + MONERO_THROW(net::zmq::get_error_code(), "zmq_init"); + + detail::socket pub{zmq_socket(comm.get(), ZMQ_PUB)}; + if (pub == nullptr) + MONERO_THROW(net::zmq::get_error_code(), "zmq_socket"); + if (zmq_bind(pub.get(), signal_endpoint) < 0) + MONERO_THROW(net::zmq::get_error_code(), "zmq_bind"); + + return context{ + std::make_shared( + std::move(comm), std::move(pub), std::move(daemon_addr), rates_interval + ) + }; + } + + context::~context() noexcept + { + if (ctx) + raise_abort_process(); + } + + std::string const& context::daemon_address() const + { + if (ctx == nullptr) + MONERO_THROW(common_error::kInvalidArgument, "Invalid lws::rpc::context"); + return ctx->daemon_addr; + } + + expect context::raise_abort_scan() noexcept + { + MONERO_PRECOND(ctx != nullptr); + assert(ctx->signal_pub != nullptr); + return do_signal(ctx->signal_pub.get(), abort_scan_signal); + } + + expect context::raise_abort_process() noexcept + { + MONERO_PRECOND(ctx != nullptr); + assert(ctx->signal_pub != nullptr); + return do_signal(ctx->signal_pub.get(), abort_process_signal); + } + + expect> context::retrieve_rates() + { + MONERO_PRECOND(ctx != nullptr); + + if (ctx->cache_interval <= std::chrono::minutes{0}) + return boost::make_optional(false, ctx->cached); + + const auto now = std::chrono::steady_clock::now(); + if (now - ctx->cache_time < ctx->cache_interval) + return boost::make_optional(false, ctx->cached); + + expect fresh{lws::error::exchange_rates_fetch}; + + const http::http_response_info* info = nullptr; + const bool retrieved = + ctx->rates_conn.invoke_get(crypto_compare.path, std::chrono::seconds{20}, std::string{}, std::addressof(info)) && + info != nullptr && + info->m_response_code == 200; + + // \TODO Remove copy below + if (retrieved) + fresh = crypto_compare(std::string{info->m_body}); + + const boost::unique_lock lock{ctx->sync_rates}; + ctx->cache_time = now; + if (fresh) + { + ctx->cached = *fresh; + return boost::make_optional(*fresh); + } + return fresh.error(); + } +} // rpc +} // lws diff --git a/src/rpc/client.h b/src/rpc/client.h new file mode 100644 index 0000000..92178ae --- /dev/null +++ b/src/rpc/client.h @@ -0,0 +1,219 @@ +// Copyright (c) 2018-2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "byte_slice.h" // monero/contrib/epee/include +#include "common/expect.h" // monero/src +#include "rates.h" +#include "rpc/message.h" // monero/src + +namespace lws +{ +namespace rpc +{ + namespace detail + { + struct close + { + void operator()(void* ptr) const noexcept + { + if (ptr) + zmq_close(ptr); + } + }; + using socket = std::unique_ptr; + + struct context; + } + + //! Abstraction for ZMQ RPC client. Only `get_rates()` thread-safe; use `clone()`. + class client + { + std::shared_ptr ctx; + detail::socket daemon; + detail::socket signal_sub; + + explicit client(std::shared_ptr ctx) + : ctx(std::move(ctx)), daemon(), signal_sub() + {} + + public: + //! A client with no connection (all send/receive functions fail). + explicit client() noexcept + : ctx(), daemon(), signal_sub() + {} + + static expect make(std::shared_ptr ctx) noexcept; + + client(client&&) = default; + client(client const&) = delete; + + ~client() noexcept; + + client& operator=(client&&) = default; + client& operator=(client const&) = delete; + + /*! + \note `watch_scan_signals()` status is not cloned. + \note The copy is not cheap - it creates a new ZMQ socket. + \return A client connected to same daemon as `this`. + */ + expect clone() const noexcept + { + return make(ctx); + } + + //! \return True if `this` is valid (i.e. not default or moved from). + explicit operator bool() const noexcept + { + return ctx != nullptr; + } + + //! `wait`, `send`, and `receive` will watch for `raise_abort_scan()`. + expect watch_scan_signals() noexcept; + + //! Block until `timeout` or until `context::stop()` is invoked. + expect wait(std::chrono::seconds timeout) noexcept; + + //! \return A JSON message for RPC request `M`. + template + static epee::byte_slice make_message(char const* const name, const M& message) + { + return cryptonote::rpc::FullMessage::getRequest(name, message, 0); + } + + /*! + Queue `message` for sending to daemon. If the queue is full, wait a + maximum of `timeout` seconds or until `context::raise_abort_scan` or + `context::raise_abort_process()` is called. + */ + expect send(epee::byte_slice message, std::chrono::seconds timeout) noexcept; + + //! \return Next available RPC message response from server + expect get_message(std::chrono::seconds timeout); + + //! \return RPC response `M`, waiting a max of `timeout` seconds. + template + expect receive(std::chrono::seconds timeout) + { + expect message = get_message(timeout); + if (!message) + return message.error(); + + cryptonote::rpc::FullMessage fm{std::move(*message)}; + M out{}; + out.fromJson(fm.getMessage()); + return out; + } + + /*! + \note This is the one function that IS thread-safe. Multiple threads can + call this function with the same `this` argument. + + \return Recent exchange rates. + */ + expect get_rates() const; + }; + + //! Owns ZMQ context, and ZMQ PUB socket for signalling child `client`s. + class context + { + std::shared_ptr ctx; + + explicit context(std::shared_ptr ctx) + : ctx(std::move(ctx)) + {} + + public: + /*! Use `daemon_addr` for call child client objects. + + \throw std::bad_alloc if internal `shared_ptr` allocation failed. + \throw std::system_error if any ZMQ errors occur. + + \note All errors are exceptions; no recovery can occur. + + \param daemon_addr Location of ZMQ enabled `monerod` RPC. + \param rates_interval Frequency to retrieve exchange rates. Set value to + `<= 0` to disable exchange rate retrieval. + */ + static context make(std::string daemon_addr, std::chrono::minutes rates_interval); + + context(context&&) = default; + context(context const&) = delete; + + //! Calls `raise_abort_process()`. Clients can safely destruct later. + ~context() noexcept; + + context& operator=(context&&) = default; + context& operator=(context const&) = delete; + + // Do not create clone method, only one of these should exist right now. + + //! \return The full address of the monerod ZMQ daemon. + std::string const& daemon_address() const; + + //! \return Client connection. Thread-safe. + expect connect() const noexcept + { + return client::make(ctx); + } + + /*! + All block `client::send`, `client::receive`, and `client::wait` calls + originating from `this` object AND whose `watch_scan_signal` method was + invoked, will immediately return with `lws::error::kSignlAbortScan`. This + is NOT signal-safe NOR signal-safe NOR thread-safe. + */ + expect raise_abort_scan() noexcept; + + /*! + All blocked `client::send`, `client::receive`, and `client::wait` calls + originating from `this` object will immediately return with + `lws::error::kSignalAbortProcess`. This call is NOT signal-safe NOR + thread-safe. + */ + expect raise_abort_process() noexcept; + + /*! + Retrieve exchange rates, if enabled and past cache interval. Not + thread-safe (this can be invoked from one thread only, but this is + thread-safe with `client::get_rates()`). All clients will see new rates + immediately. + + \return Rates iff they were updated. + */ + expect> retrieve_rates(); + }; +} // rpc +} // lws diff --git a/src/rpc/daemon_zmq.cpp b/src/rpc/daemon_zmq.cpp new file mode 100644 index 0000000..787c450 --- /dev/null +++ b/src/rpc/daemon_zmq.cpp @@ -0,0 +1,169 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "daemon_zmq.h" + +#include "crypto/crypto.h" // monero/src +#include "rpc/message_data_structs.h" // monero/src +#include "wire/crypto.h" +#include "wire/json.h" +#include "wire/vector.h" + +namespace +{ + constexpr const std::size_t default_blocks_fetched = 1000; + constexpr const std::size_t default_transaction_count = 100; + constexpr const std::size_t default_inputs = 2; + constexpr const std::size_t default_outputs = 4; + constexpr const std::size_t default_txextra_size = 2048; +} + +namespace rct +{ + static void read_bytes(wire::json_reader& source, ctkey& self) + { + self.dest = {}; + read_bytes(source, self.mask); + } + + static void read_bytes(wire::json_reader& source, ecdhTuple& self) + { + wire::object(source, WIRE_FIELD(mask), WIRE_FIELD(amount)); + } + + static void read_bytes(wire::json_reader& source, rctSig& self) + { + self.outPk.reserve(default_inputs); + wire::object(source, + WIRE_FIELD(type), + wire::field("encrypted", std::ref(self.ecdhInfo)), + wire::field("commitments", std::ref(self.outPk)), + wire::field("fee", std::ref(self.txnFee)) + ); + } +} // rct + +namespace cryptonote +{ + static void read_bytes(wire::json_reader& source, txout_to_script& self) + { + wire::object(source, WIRE_FIELD(keys), WIRE_FIELD(script)); + } + static void read_bytes(wire::json_reader& source, txout_to_scripthash& self) + { + wire::object(source, WIRE_FIELD(hash)); + } + static void read_bytes(wire::json_reader& source, txout_to_key& self) + { + wire::object(source, WIRE_FIELD(key)); + } + static void read_bytes(wire::json_reader& source, tx_out& self) + { + wire::object(source, + WIRE_FIELD(amount), + wire::variant_field("transaction output variant", std::ref(self.target), + wire::option{"to_key"}, + wire::option{"to_script"}, + wire::option{"to_scripthash"} + ) + ); + } + + static void read_bytes(wire::json_reader& source, txin_gen& self) + { + wire::object(source, WIRE_FIELD(height)); + } + static void read_bytes(wire::json_reader& source, txin_to_script& self) + { + wire::object(source, WIRE_FIELD(prev), WIRE_FIELD(prevout), WIRE_FIELD(sigset)); + } + static void read_bytes(wire::json_reader& source, txin_to_scripthash& self) + { + wire::object(source, WIRE_FIELD(prev), WIRE_FIELD(prevout), WIRE_FIELD(script), WIRE_FIELD(sigset)); + } + static void read_bytes(wire::json_reader& source, txin_to_key& self) + { + wire::object(source, WIRE_FIELD(amount), WIRE_FIELD(key_offsets), wire::field("key_image", std::ref(self.k_image))); + } + static void read_bytes(wire::json_reader& source, txin_v& self) + { + wire::object(source, + wire::variant_field("transaction input variant", std::ref(self), + wire::option{"to_key"}, + wire::option{"gen"}, + wire::option{"to_script"}, + wire::option{"to_scripthash"} + ) + ); + } + + static void read_bytes(wire::json_reader& source, transaction& self) + { + self.vin.reserve(default_inputs); + self.vout.reserve(default_outputs); + self.extra.reserve(default_txextra_size); + wire::object(source, + WIRE_FIELD(version), + WIRE_FIELD(unlock_time), + wire::field("inputs", std::ref(self.vin)), + wire::field("outputs", std::ref(self.vout)), + WIRE_FIELD(extra), + wire::field("ringct", std::ref(self.rct_signatures)) + ); + } + + static void read_bytes(wire::json_reader& source, block& self) + { + self.tx_hashes.reserve(default_transaction_count); + wire::object(source, + WIRE_FIELD(major_version), + WIRE_FIELD(minor_version), + WIRE_FIELD(timestamp), + WIRE_FIELD(miner_tx), + WIRE_FIELD(tx_hashes), + WIRE_FIELD(prev_id), + WIRE_FIELD(nonce) + ); + } + + namespace rpc + { + static void read_bytes(wire::json_reader& source, block_with_transactions& self) + { + self.transactions.reserve(default_transaction_count); + wire::object(source, WIRE_FIELD(block), WIRE_FIELD(transactions)); + } + } // rpc +} // cryptonote + +void lws::rpc::read_bytes(wire::json_reader& source, get_blocks_fast_response& self) +{ + self.blocks.reserve(default_blocks_fetched); + self.output_indices.reserve(default_blocks_fetched); + wire::object(source, WIRE_FIELD(blocks), WIRE_FIELD(output_indices), WIRE_FIELD(start_height), WIRE_FIELD(current_height)); +} + diff --git a/src/rpc/daemon_zmq.h b/src/rpc/daemon_zmq.h new file mode 100644 index 0000000..3f694e1 --- /dev/null +++ b/src/rpc/daemon_zmq.h @@ -0,0 +1,75 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include + +#include "common/pod-class.h" // monero/src +#include "wire/json/fwd.h" + +namespace crypto +{ + POD_CLASS hash; +} + +namespace cryptonote +{ + namespace rpc + { + struct block_with_transactions; + } +} + +namespace lws +{ +namespace rpc +{ + struct get_blocks_fast_request + { + get_blocks_fast_request() = delete; + std::vector block_ids; + std::uint64_t start_height; + bool prune; + }; + struct get_blocks_fast_response + { + get_blocks_fast_response() = delete; + std::vector blocks; + std::vector>> output_indices; + std::uint64_t start_height; + std::uint64_t current_height; + }; + struct get_blocks_fast + { + using request = get_blocks_fast_request; + using response = get_blocks_fast_response; + }; + void read_bytes(wire::json_reader&, get_blocks_fast_response&); +} // rpc +} // lws diff --git a/src/rpc/fwd.h b/src/rpc/fwd.h new file mode 100644 index 0000000..6c9c317 --- /dev/null +++ b/src/rpc/fwd.h @@ -0,0 +1,33 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +namespace lws +{ + struct rates; +} diff --git a/src/rpc/json.h b/src/rpc/json.h new file mode 100644 index 0000000..11c1480 --- /dev/null +++ b/src/rpc/json.h @@ -0,0 +1,96 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "wire/json.h" + +namespace lws +{ +namespace rpc +{ + struct json_request_base + { + static constexpr const char jsonrpc[] = "2.0"; + + //! `method` must be in static memory. + explicit json_request_base(const char* method) + : id(0), method(method) + {} + + unsigned id; + const char* method; //!< Must be in static memory + }; + const char json_request_base::jsonrpc[]; + + //! \tparam W implements the WRITE concept \tparam M implements the METHOD concept + template + struct json_request : json_request_base + { + template + explicit json_request(U&&... args) + : json_request_base(M::name()), + params{std::forward(args)...} + {} + + W params; + }; + + template + inline void write_bytes(wire::json_writer& dest, const json_request& self) + { + // pull fields from base class into the same object + wire::object(dest, WIRE_FIELD_COPY(id), WIRE_FIELD_COPY(jsonrpc), WIRE_FIELD_COPY(method), WIRE_FIELD(params)); + } + + + //! \tparam R implements the READ concept + template + struct json_response + { + json_response() = delete; + + unsigned id; + R result; + }; + + template + inline void read_bytes(wire::json_reader& source, json_response& self) + { + wire::object(source, WIRE_FIELD(id), WIRE_FIELD(result)); + } + + + /*! Implements the RPC concept (JSON-RPC 2.0). + \tparam M must implement the METHOD concept. */ + template + struct json + { + using wire_type = wire::json; + using request = json_request; + using response = json_response; + }; +} // rpc +} // lws diff --git a/src/rpc/light_wallet.cpp b/src/rpc/light_wallet.cpp new file mode 100644 index 0000000..9e45a22 --- /dev/null +++ b/src/rpc/light_wallet.cpp @@ -0,0 +1,338 @@ +// Copyright (c) 2018-2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "light_wallet.h" + +#include +#include +#include +#include +#include + +#include "db/string.h" +#include "error.h" +#include "misc_os_dependent.h" // monero/contrib/epee/include +#include "ringct/rctOps.h" // monero/src +#include "span.h" // monero/contrib/epee/include +#include "util/random_outputs.h" +#include "wire/crypto.h" +#include "wire/error.h" +#include "wire/json.h" +#include "wire/traits.h" +#include "wire/vector.h" + +namespace +{ + enum class iso_timestamp : std::uint64_t {}; + + struct rct_bytes + { + rct::key commitment; + rct::key mask; + rct::key amount; + }; + static_assert(sizeof(rct_bytes) == 32 * 3, "padding in rct struct"); + + struct expand_outputs + { + const std::pair>& data; + const crypto::secret_key& user_key; + }; +} // anonymous + +namespace wire +{ + template<> + struct is_blob + : std::true_type + {}; +} + +namespace +{ + void write_bytes(wire::json_writer& dest, const iso_timestamp self) + { + static_assert(std::is_integral::value, "unexpected time_t type"); + if (std::numeric_limits::max() < std::uint64_t(self)) + throw std::runtime_error{"Exceeded max time_t value"}; + + std::tm value; + if (!epee::misc_utils::get_gmt_time(std::time_t(self), value)) + throw std::runtime_error{"Failed to convert std::time_t to std::tm"}; + + char buf[28] = {0}; + if (sizeof(buf) - 1 != std::strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S.0-00:00", std::addressof(value))) + throw std::runtime_error{"strftime failed"}; + + dest.string({buf, sizeof(buf) - 1}); + } + + void write_bytes(wire::json_writer& dest, const expand_outputs self) + { + /*! \TODO Sending the public key for the output isn't necessary, as it can be + re-computed from the other parts. Same with the rct commitment and rct + amount. Consider dropping these from the API after client upgrades. Not + storing them in the DB saves 96-bytes per received out. */ + + rct_bytes rct{}; + rct_bytes const* optional_rct = nullptr; + if (unpack(self.data.first.extra).first & lws::db::ringct_output) + { + crypto::key_derivation derived; + if (!crypto::generate_key_derivation(self.data.first.spend_meta.tx_public, self.user_key, derived)) + MONERO_THROW(lws::error::crypto_failure, "generate_key_derivation failed"); + + crypto::secret_key scalar; + rct::ecdhTuple encrypted{self.data.first.ringct_mask, rct::d2h(self.data.first.spend_meta.amount)}; + + crypto::derivation_to_scalar(derived, self.data.first.spend_meta.index, scalar); + rct::ecdhEncode(encrypted, rct::sk2rct(scalar), false); + + rct.commitment = rct::commit(self.data.first.spend_meta.amount, self.data.first.ringct_mask); + rct.mask = encrypted.mask; + rct.amount = encrypted.amount; + + optional_rct = std::addressof(rct); + } + + wire::object(dest, + wire::field("amount", lws::rpc::safe_uint64(self.data.first.spend_meta.amount)), + wire::field("public_key", self.data.first.pub), + wire::field("index", self.data.first.spend_meta.index), + wire::field("global_index", self.data.first.spend_meta.id.low), + wire::field("tx_id", self.data.first.spend_meta.id.low), + wire::field("tx_hash", std::cref(self.data.first.link.tx_hash)), + wire::field("tx_prefix_hash", std::cref(self.data.first.tx_prefix_hash)), + wire::field("tx_pub_key", self.data.first.spend_meta.tx_public), + wire::field("timestamp", iso_timestamp(self.data.first.timestamp)), + wire::field("height", self.data.first.link.height), + wire::field("spend_key_images", std::cref(self.data.second)), + wire::optional_field("rct", optional_rct) + ); + } + + void convert_address(const boost::string_ref source, lws::db::account_address& dest) + { + expect bytes = lws::db::address_string(source); + if (!bytes) + WIRE_DLOG_THROW(wire::error::schema::fixed_binary, "invalid Monero address format - " << bytes.error()); + dest = std::move(*bytes); + } +} // anonymous + +namespace lws +{ + static void write_bytes(wire::json_writer& dest, random_output const& self) + { + const rct_bytes rct{self.keys.mask, rct::zero(), rct::zero()}; + wire::object(dest, + wire::field("global_index", rpc::safe_uint64(self.index)), + wire::field("public_key", std::cref(self.keys.key)), + wire::field("rct", std::cref(rct)) + ); + } + static void write_bytes(wire::json_writer& dest, random_ring const& self) + { + wire::object(dest, + wire::field("amount", rpc::safe_uint64(self.amount)), + wire::field("outputs", std::cref(self.ring)) + ); + }; + + void rpc::read_bytes(wire::json_reader& source, safe_uint64& self) + { + self = safe_uint64(wire::integer::convert_to(source.safe_unsigned_integer())); + } + void rpc::write_bytes(wire::json_writer& dest, const safe_uint64 self) + { + auto buf = wire::json_writer::to_string(std::uint64_t(self)); + dest.string(buf.data()); + } + void rpc::read_bytes(wire::json_reader& source, safe_uint64_array& self) + { + for (std::size_t count = source.start_array(); !source.is_array_end(count); --count) + self.values.emplace_back(wire::integer::convert_to(source.safe_unsigned_integer())); + source.end_array(); + } + + void rpc::read_bytes(wire::json_reader& source, account_credentials& self) + { + std::string address; + wire::object(source, + wire::field("address", std::ref(address)), + wire::field("view_key", std::ref(unwrap(unwrap(self.key)))) + ); + convert_address(address, self.address); + } + + void rpc::write_bytes(wire::json_writer& dest, const transaction_spend& self) + { + wire::object(dest, + wire::field("amount", safe_uint64(self.meta.amount)), + wire::field("key_image", std::cref(self.possible_spend.image)), + wire::field("tx_pub_key", std::cref(self.meta.tx_public)), + wire::field("out_index", self.meta.index), + wire::field("mixin", self.possible_spend.mixin_count) + ); + } + + void rpc::write_bytes(wire::json_writer& dest, const get_address_info_response& self) + { + wire::object(dest, + WIRE_FIELD_COPY(locked_funds), + WIRE_FIELD_COPY(total_received), + WIRE_FIELD_COPY(total_sent), + WIRE_FIELD_COPY(scanned_height), + WIRE_FIELD_COPY(scanned_block_height), + WIRE_FIELD_COPY(start_height), + WIRE_FIELD_COPY(transaction_height), + WIRE_FIELD_COPY(blockchain_height), + WIRE_FIELD(spent_outputs), + WIRE_OPTIONAL_FIELD(rates) + ); + } + + namespace rpc + { + static void write_bytes(wire::json_writer& dest, boost::range::index_value self) + { + epee::span const* payment_id = nullptr; + epee::span payment_id_bytes; + + const auto extra = db::unpack(self.value().info.extra); + if (extra.second) + { + payment_id = std::addressof(payment_id_bytes); + + if (extra.second == sizeof(self.value().info.payment_id.short_)) + payment_id_bytes = epee::as_byte_span(self.value().info.payment_id.short_); + else + payment_id_bytes = epee::as_byte_span(self.value().info.payment_id.long_); + } + + const bool is_coinbase = (extra.first & db::coinbase_output); + + wire::object(dest, + wire::field("id", std::uint64_t(self.index())), + wire::field("hash", std::cref(self.value().info.link.tx_hash)), + wire::field("timestamp", iso_timestamp(self.value().info.timestamp)), + wire::field("total_received", safe_uint64(self.value().info.spend_meta.amount)), + wire::field("total_sent", safe_uint64(self.value().spent)), + wire::field("unlock_time", self.value().info.unlock_time), + wire::field("height", self.value().info.link.height), + wire::optional_field("payment_id", payment_id), + wire::field("coinbase", is_coinbase), + wire::field("mempool", false), + wire::field("mixin", self.value().info.spend_meta.mixin_count), + wire::field("spent_outputs", std::cref(self.value().spends)) + ); + } + } // rpc + void rpc::write_bytes(wire::json_writer& dest, const get_address_txs_response& self) + { + wire::object(dest, + wire::field("total_received", safe_uint64(self.total_received)), + WIRE_FIELD_COPY(scanned_height), + WIRE_FIELD_COPY(scanned_block_height), + WIRE_FIELD_COPY(start_height), + WIRE_FIELD_COPY(transaction_height), + WIRE_FIELD_COPY(blockchain_height), + wire::field("transactions", wire::as_array(boost::adaptors::index(self.transactions))) + ); + } + + void rpc::read_bytes(wire::json_reader& source, get_random_outs_request& self) + { + wire::object(source, WIRE_FIELD(count), WIRE_FIELD(amounts)); + } + void rpc::write_bytes(wire::json_writer& dest, const get_random_outs_response& self) + { + wire::object(dest, WIRE_FIELD(amount_outs)); + } + + void rpc::read_bytes(wire::json_reader& source, get_unspent_outs_request& self) + { + std::string address; + wire::object(source, + wire::field("address", std::ref(address)), + wire::field("view_key", std::ref(unwrap(unwrap(self.creds.key)))), + WIRE_FIELD(amount), + WIRE_OPTIONAL_FIELD(mixin), + WIRE_OPTIONAL_FIELD(use_dust), + WIRE_OPTIONAL_FIELD(dust_threshold) + ); + convert_address(address, self.creds.address); + } + void rpc::write_bytes(wire::json_writer& dest, const get_unspent_outs_response& self) + { + const auto expand = [&self] (const std::pair>& src) + { + return expand_outputs{src, self.user_key}; + }; + wire::object(dest, + WIRE_FIELD_COPY(per_kb_fee), + WIRE_FIELD_COPY(fee_mask), + WIRE_FIELD_COPY(amount), + wire::field("outputs", wire::as_array(std::cref(self.outputs), expand)) + ); + } + + void rpc::write_bytes(wire::json_writer& dest, const import_response& self) + { + wire::object(dest, + WIRE_FIELD_COPY(import_fee), + WIRE_FIELD_COPY(status), + WIRE_FIELD_COPY(new_request), + WIRE_FIELD_COPY(request_fulfilled) + ); + } + + void rpc::read_bytes(wire::json_reader& source, login_request& self) + { + std::string address; + wire::object(source, + wire::field("address", std::ref(address)), + wire::field("view_key", std::ref(unwrap(unwrap(self.creds.key)))), + WIRE_FIELD(create_account), + WIRE_FIELD(generated_locally) + ); + convert_address(address, self.creds.address); + } + void rpc::write_bytes(wire::json_writer& dest, const login_response self) + { + wire::object(dest, WIRE_FIELD_COPY(new_address), WIRE_FIELD_COPY(generated_locally)); + } + + void rpc::read_bytes(wire::json_reader& source, submit_raw_tx_request& self) + { + wire::object(source, WIRE_FIELD(tx)); + } + void rpc::write_bytes(wire::json_writer& dest, const submit_raw_tx_response self) + { + wire::object(dest, WIRE_FIELD_COPY(status)); + } +} // lws diff --git a/src/rpc/light_wallet.h b/src/rpc/light_wallet.h new file mode 100644 index 0000000..cc9cb81 --- /dev/null +++ b/src/rpc/light_wallet.h @@ -0,0 +1,198 @@ +// Copyright (c) 2018-2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/expect.h" // monero/src +#include "crypto/crypto.h" // monero/src +#include "db/data.h" +#include "rpc/rates.h" +#include "util/fwd.h" +#include "wire/json/fwd.h" + +namespace lws +{ +namespace rpc +{ + //! Read/write uint64 value as JSON string. + enum class safe_uint64 : std::uint64_t {}; + void read_bytes(wire::json_reader&, safe_uint64&); + void write_bytes(wire::json_writer&, safe_uint64); + + //! Read an array of uint64 values as JSON strings. + struct safe_uint64_array + { + std::vector values; // so this can be passed to another function without copy + }; + void read_bytes(wire::json_reader&, safe_uint64_array&); + + + struct account_credentials + { + lws::db::account_address address; + crypto::secret_key key; + }; + void read_bytes(wire::json_reader&, account_credentials&); + + + struct transaction_spend + { + transaction_spend() = delete; + lws::db::output::spend_meta_ meta; + lws::db::spend possible_spend; + }; + void write_bytes(wire::json_writer&, const transaction_spend&); + + + struct get_address_info_response + { + get_address_info_response() = delete; + safe_uint64 locked_funds; + safe_uint64 total_received; + safe_uint64 total_sent; + std::uint64_t scanned_height; + std::uint64_t scanned_block_height; + std::uint64_t start_height; + std::uint64_t transaction_height; + std::uint64_t blockchain_height; + std::vector spent_outputs; + expect rates; + }; + void write_bytes(wire::json_writer&, const get_address_info_response&); + + + struct get_address_txs_response + { + get_address_txs_response() = delete; + struct transaction + { + transaction() = delete; + db::output info; + std::vector spends; + std::uint64_t spent; + }; + + safe_uint64 total_received; + std::uint64_t scanned_height; + std::uint64_t scanned_block_height; + std::uint64_t start_height; + std::uint64_t transaction_height; + std::uint64_t blockchain_height; + std::vector transactions; + }; + void write_bytes(wire::json_writer&, const get_address_txs_response&); + + + struct get_random_outs_request + { + get_random_outs_request() = delete; + std::uint64_t count; + safe_uint64_array amounts; + }; + void read_bytes(wire::json_reader&, get_random_outs_request&); + + struct get_random_outs_response + { + get_random_outs_response() = delete; + std::vector amount_outs; + }; + void write_bytes(wire::json_writer&, const get_random_outs_response&); + + + struct get_unspent_outs_request + { + get_unspent_outs_request() = delete; + safe_uint64 amount; + boost::optional dust_threshold; + boost::optional mixin; + boost::optional use_dust; + account_credentials creds; + }; + void read_bytes(wire::json_reader&, get_unspent_outs_request&); + + struct get_unspent_outs_response + { + get_unspent_outs_response() = delete; + std::uint64_t per_kb_fee; + std::uint64_t fee_mask; + safe_uint64 amount; + std::vector>> outputs; + crypto::secret_key user_key; + }; + void write_bytes(wire::json_writer&, const get_unspent_outs_response&); + + + struct import_response + { + import_response() = delete; + safe_uint64 import_fee; + const char* status; + bool new_request; + bool request_fulfilled; + }; + void write_bytes(wire::json_writer&, const import_response&); + + + struct login_request + { + login_request() = delete; + account_credentials creds; + bool create_account; + bool generated_locally; + }; + void read_bytes(wire::json_reader&, login_request&); + + struct login_response + { + login_response() = delete; + bool new_address; + bool generated_locally; + }; + void write_bytes(wire::json_writer&, login_response); + + + struct submit_raw_tx_request + { + submit_raw_tx_request() = delete; + std::string tx; + }; + void read_bytes(wire::json_reader&, submit_raw_tx_request&); + + struct submit_raw_tx_response + { + submit_raw_tx_response() = delete; + const char* status; + }; + void write_bytes(wire::json_writer&, submit_raw_tx_response); +} // rpc +} // lws diff --git a/src/rpc/rates.cpp b/src/rpc/rates.cpp new file mode 100644 index 0000000..399265d --- /dev/null +++ b/src/rpc/rates.cpp @@ -0,0 +1,78 @@ +// Copyright (c) 2018-2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "rates.h" + +#include "wire/json.h" + +namespace +{ + template + void map_rates(F& format, T& self) + { + wire::object(format, + WIRE_FIELD(AUD), + WIRE_FIELD(BRL), + WIRE_FIELD(BTC), + WIRE_FIELD(CAD), + WIRE_FIELD(CHF), + WIRE_FIELD(CNY), + WIRE_FIELD(EUR), + WIRE_FIELD(GBP), + WIRE_FIELD(HKD), + WIRE_FIELD(INR), + WIRE_FIELD(JPY), + WIRE_FIELD(KRW), + WIRE_FIELD(MXN), + WIRE_FIELD(NOK), + WIRE_FIELD(NZD), + WIRE_FIELD(SEK), + WIRE_FIELD(SGD), + WIRE_FIELD(TRY), + WIRE_FIELD(USD), + WIRE_FIELD(RUB), + WIRE_FIELD(ZAR) + ); + } +} + +namespace lws +{ + WIRE_JSON_DEFINE_OBJECT(rates, map_rates); + + namespace rpc + { + const char crypto_compare_::host[] = "https://min-api.cryptocompare.com:443"; + const char crypto_compare_::path[] = + "/data/price?fsym=XMR&tsyms=AUD,BRL,BTC,CAD,CHF,CNY,EUR,GBP," + "HKD,INR,JPY,KRW,MXN,NOK,NZD,SEK,SGD,TRY,USD,RUB,ZAR"; + + expect crypto_compare_::operator()(std::string&& body) const + { + return wire::json::from_bytes(std::move(body)); + } + } // rpc +} // lws diff --git a/src/rpc/rates.h b/src/rpc/rates.h new file mode 100644 index 0000000..d5ceaa2 --- /dev/null +++ b/src/rpc/rates.h @@ -0,0 +1,74 @@ +// Copyright (c) 2018-2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include + +#include "byte_slice.h" +#include "common/expect.h" +#include "wire/json/fwd.h" + +namespace lws +{ + struct rates + { + double AUD; + double BRL; + double BTC; + double CAD; + double CHF; + double CNY; + double EUR; + double GBP; + double HKD; + double INR; + double JPY; + double KRW; + double MXN; + double NOK; + double NZD; + double SEK; + double SGD; + double TRY; + double USD; + double RUB; + double ZAR; + }; + WIRE_JSON_DECLARE_OBJECT(rates); + + namespace rpc + { + struct crypto_compare_ + { + static const char host[]; + static const char path[]; + + expect operator()(std::string&& body) const; + }; + constexpr const crypto_compare_ crypto_compare{}; + } // rpc +} // lws diff --git a/src/scanner.cpp b/src/scanner.cpp new file mode 100644 index 0000000..fc3294c --- /dev/null +++ b/src/scanner.cpp @@ -0,0 +1,780 @@ +// Copyright (c) 2018-2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "scanner.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/error.h" // monero/src +#include "crypto/crypto.h" // monero/src +#include "crypto/wallet/crypto.h" // monero/src +#include "cryptonote_basic/cryptonote_basic.h" // monero/src +#include "cryptonote_basic/cryptonote_format_utils.h" // monero/src +#include "db/account.h" +#include "db/data.h" +#include "error.h" +#include "misc_log_ex.h" // monero/contrib/epee/include +#include "rpc/daemon_messages.h" // monero/src +#include "rpc/daemon_zmq.h" +#include "rpc/json.h" +#include "util/transactions.h" +#include "wire/json.h" + +#include "serialization/json_object.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "lws" + +namespace lws +{ + std::atomic scanner::running{true}; + + // Not in `rates.h` - defaulting to JSON output seems odd + std::ostream& operator<<(std::ostream& out, lws::rates const& src) + { + wire::json_stream_writer dest{out}; + lws::write_bytes(dest, src); + dest.finish(); + return out; + } + + namespace + { + constexpr const std::chrono::seconds account_poll_interval{10}; + constexpr const std::chrono::seconds block_poll_interval{20}; + constexpr const std::chrono::minutes block_rpc_timeout{2}; + constexpr const std::chrono::seconds send_timeout{30}; + constexpr const std::chrono::seconds sync_rpc_timeout{30}; + + struct thread_sync + { + boost::mutex sync; + boost::condition_variable user_poll; + std::atomic update; + }; + + struct thread_data + { + explicit thread_data(rpc::client client, db::storage disk, std::vector users) + : client(std::move(client)), disk(std::move(disk)), users(std::move(users)) + {} + + rpc::client client; + db::storage disk; + std::vector users; + }; + + // until we have a signal-handler safe notification system + void checked_wait(const std::chrono::nanoseconds wait) + { + static constexpr const std::chrono::milliseconds interval{500}; + + const auto start = std::chrono::steady_clock::now(); + while (scanner::is_running()) + { + const auto current = std::chrono::steady_clock::now() - start; + if (wait <= current) + break; + const auto sleep_time = std::min(wait - current, std::chrono::nanoseconds{interval}); + boost::this_thread::sleep_for(boost::chrono::nanoseconds{sleep_time.count()}); + } + } + + bool send(rpc::client& client, epee::byte_slice message) + { + const expect sent = client.send(std::move(message), send_timeout); + if (!sent) + { + if (sent.matches(std::errc::interrupted)) + return false; + MONERO_THROW(sent.error(), "Failed to send ZMQ RPC message"); + } + return true; + } + + struct by_height + { + bool operator()(account const& left, account const& right) const noexcept + { + return left.scan_height() < right.scan_height(); + } + }; + + void scan_transaction( + epee::span users, + const db::block_id height, + const std::uint64_t timestamp, + crypto::hash const& tx_hash, + cryptonote::transaction const& tx, + std::vector const& out_ids) + { + if (2 < tx.version) + throw std::runtime_error{"Unsupported tx version"}; + + cryptonote::tx_extra_pub_key key; + boost::optional prefix_hash; + boost::optional extra_nonce; + std::pair payment_id; + + { + std::vector extra; + cryptonote::parse_tx_extra(tx.extra, extra); + // allow partial parsing of tx extra (similar to wallet2.cpp) + + if (!cryptonote::find_tx_extra_field_by_type(extra, key)) + return; + + extra_nonce.emplace(); + if (cryptonote::find_tx_extra_field_by_type(extra, *extra_nonce)) + { + if (cryptonote::get_payment_id_from_tx_extra_nonce(extra_nonce->nonce, payment_id.second.long_)) + payment_id.first = sizeof(crypto::hash); + } + else + extra_nonce = boost::none; + } // destruct `extra` vector + + for (account& user : users) + { + if (height <= user.scan_height()) + continue; // to next user + + crypto::key_derivation derived; + if (!crypto::wallet::generate_key_derivation(key.pub_key, user.view_key(), derived)) + throw std::runtime_error{"Key derivation failed"}; + + db::extra ext{}; + std::uint32_t mixin = 0; + for (auto const& in : tx.vin) + { + cryptonote::txin_to_key const* const in_data = + boost::get(std::addressof(in)); + if (in_data) + { + mixin = boost::numeric_cast( + std::max(std::size_t(1), in_data->key_offsets.size()) - 1 + ); + + std::uint64_t goffset = 0; + for (std::uint64_t offset : in_data->key_offsets) + { + goffset += offset; + if (user.has_spendable(db::output_id{in_data->amount, goffset})) + { + user.add_spend( + db::spend{ + db::transaction_link{height, tx_hash}, + in_data->k_image, + db::output_id{in_data->amount, goffset}, + timestamp, + tx.unlock_time, + mixin, + {0, 0, 0}, // reserved + payment_id.first, + payment_id.second.long_ + } + ); + } + } + } + else if (boost::get(std::addressof(in))) + ext = db::extra(ext | db::coinbase_output); + } + + std::size_t index = -1; + for (auto const& out : tx.vout) + { + ++index; + + cryptonote::txout_to_key const* const out_data = + boost::get(std::addressof(out.target)); + if (!out_data) + continue; // to next output + + crypto::public_key derived_pub; + const bool received = + crypto::wallet::derive_subaddress_public_key(out_data->key, derived, index, derived_pub) && + derived_pub == user.spend_public(); + + if (!received) + continue; // to next output + + if (!prefix_hash) + { + prefix_hash.emplace(); + cryptonote::get_transaction_prefix_hash(tx, *prefix_hash); + } + + std::uint64_t amount = out.amount; + rct::key mask = rct::identity(); + if (!amount && !(ext & db::coinbase_output) && 1 < tx.version) + { + const bool bulletproof2 = (tx.rct_signatures.type == rct::RCTTypeBulletproof2); + const auto decrypted = lws::decode_amount( + tx.rct_signatures.outPk.at(index).mask, tx.rct_signatures.ecdhInfo.at(index), derived, index, bulletproof2 + ); + if (!decrypted) + { + MWARNING(user.address() << " failed to decrypt amount for tx " << tx_hash << ", skipping output"); + continue; // to next output + } + amount = decrypted->first; + mask = decrypted->second; + ext = db::extra(ext | db::ringct_output); + } + + if (extra_nonce) + { + if (!payment_id.first && cryptonote::get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce->nonce, payment_id.second.short_)) + { + payment_id.first = sizeof(crypto::hash8); + lws::decrypt_payment_id(payment_id.second.short_, derived); + } + } + + const bool added = user.add_out( + db::output{ + db::transaction_link{height, tx_hash}, + db::output::spend_meta_{ + db::output_id{out.amount, out_ids.at(index)}, + amount, + mixin, + boost::numeric_cast(index), + key.pub_key + }, + timestamp, + tx.unlock_time, + *prefix_hash, + out_data->key, + mask, + {0, 0, 0, 0, 0, 0, 0}, // reserved bytes + db::pack(ext, payment_id.first), + payment_id.second + } + ); + + if (!added) + MWARNING("Output not added, duplicate public key encountered"); + } // for all tx outs + } // for all users + } + + void update_rates(rpc::context& ctx) + { + const expect> new_rates = ctx.retrieve_rates(); + if (!new_rates) + MERROR("Failed to retrieve exchange rates: " << new_rates.error().message()); + else if (*new_rates) + MINFO("Updated exchange rates: " << *(*new_rates)); + } + + void scan_loop(thread_sync& self, std::shared_ptr data) noexcept + { + try + { + // boost::thread doesn't support move-only types + attributes + rpc::client client{std::move(data->client)}; + db::storage disk{std::move(data->disk)}; + std::vector users{std::move(data->users)}; + + assert(!users.empty()); + assert(std::is_sorted(users.begin(), users.end(), by_height{})); + + data.reset(); + + struct stop_ + { + thread_sync& self; + ~stop_() noexcept + { + self.update = true; + self.user_poll.notify_one(); + } + } stop{self}; + + // RPC server assumes that `start_height == 0` means use + // block ids. This technically skips genesis block. + cryptonote::rpc::GetBlocksFast::Request req{}; + req.start_height = std::uint64_t(users.begin()->scan_height()); + req.start_height = std::max(std::uint64_t(1), req.start_height); + req.prune = true; + + epee::byte_slice block_request = rpc::client::make_message("get_blocks_fast", req); + if (!send(client, block_request.clone())) + return; + + std::vector blockchain{}; + + while (!self.update && scanner::is_running()) + { + blockchain.clear(); + + auto resp = client.get_message(block_rpc_timeout); + if (!resp) + { + if (resp.matches(std::errc::interrupted)) + return; // a signal was sent over ZMQ + if (resp.matches(std::errc::timed_out)) + { + MWARNING("Block retrieval timeout, retrying"); + if (!send(client, block_request.clone())) + return; + continue; + } + MONERO_THROW(resp.error(), "Failed to retrieve blocks from daemon"); + } + + auto fetched = MONERO_UNWRAP(wire::json::from_bytes::response>(std::move(*resp))); + if (fetched.result.blocks.empty()) + throw std::runtime_error{"Daemon unexpectedly returned zero blocks"}; + + if (fetched.result.start_height != req.start_height) + { + MWARNING("Daemon sent wrong blocks, resetting state"); + return; + } + + // retrieve next blocks in background + req.start_height = fetched.result.start_height + fetched.result.blocks.size() - 1; + block_request = rpc::client::make_message("get_blocks_fast", req); + if (!send(client, block_request.clone())) + return; + + if (fetched.result.blocks.size() <= 1) + { + // ... how about some ZMQ push stuff? we can only dream ... + if (client.wait(block_poll_interval).matches(std::errc::interrupted)) + return; + continue; + } + + if (fetched.result.blocks.size() != fetched.result.output_indices.size()) + throw std::runtime_error{"Bad daemon response - need same number of blocks and indices"}; + + blockchain.push_back(cryptonote::get_block_hash(fetched.result.blocks.front().block)); + + auto blocks = epee::to_span(fetched.result.blocks); + auto indices = epee::to_span(fetched.result.output_indices); + + if (fetched.result.start_height != 1) + { + // skip overlap block + blocks.remove_prefix(1); + indices.remove_prefix(1); + } + else + fetched.result.start_height = 0; + + for (auto block_data : boost::combine(blocks, indices)) + { + ++(fetched.result.start_height); + + cryptonote::block const& block = boost::get<0>(block_data).block; + auto const& txes = boost::get<0>(block_data).transactions; + + if (block.tx_hashes.size() != txes.size()) + throw std::runtime_error{"Bad daemon response - need same number of txes and tx hashes"}; + + auto indices = epee::to_span(boost::get<1>(block_data)); + if (indices.empty()) + throw std::runtime_error{"Bad daemon response - missing /coinbase tx indices"}; + + crypto::hash miner_tx_hash; + if (!cryptonote::get_transaction_hash(block.miner_tx, miner_tx_hash)) + throw std::runtime_error{"Failed to calculate miner tx hash"}; + + scan_transaction( + epee::to_mut_span(users), + db::block_id(fetched.result.start_height), + block.timestamp, + miner_tx_hash, + block.miner_tx, + *(indices.begin()) + ); + + indices.remove_prefix(1); + if (txes.size() != indices.size()) + throw std::runtime_error{"Bad daemon respnse - need same number of txes and indices"}; + + for (auto tx_data : boost::combine(block.tx_hashes, txes, indices)) + { + scan_transaction( + epee::to_mut_span(users), + db::block_id(fetched.result.start_height), + block.timestamp, + boost::get<0>(tx_data), + boost::get<1>(tx_data), + boost::get<2>(tx_data) + ); + } + + blockchain.push_back(cryptonote::get_block_hash(block)); + } // for each block + + expect updated = disk.update( + users.front().scan_height(), epee::to_span(blockchain), epee::to_span(users) + ); + if (!updated) + { + if (updated == lws::error::blockchain_reorg) + { + epee::byte_stream dest{}; + { + rapidjson::Writer out{dest}; + cryptonote::json::toJsonValue(out, blocks[998]); + } + MINFO("Blockchain reorg detected, resetting state"); + return; + } + MONERO_THROW(updated.error(), "Failed to update accounts on disk"); + } + + MINFO("Processed " << blocks.size() << " block(s) against " << users.size() << " account(s)"); + if (*updated != users.size()) + { + MWARNING("Only updated " << *updated << " account(s) out of " << users.size() << ", resetting"); + return; + } + + for (account& user : users) + user.updated(db::block_id(fetched.result.start_height)); + } + } + catch (std::exception const& e) + { + scanner::stop(); + MERROR(e.what()); + } + catch (...) + { + scanner::stop(); + MERROR("Unknown exception"); + } + } + + /*! + Launches `thread_count` threads to run `scan_loop`, and then polls for + active account changes in background + */ + void check_loop(db::storage disk, rpc::context& ctx, std::size_t thread_count, std::vector users, std::vector active) + { + assert(0 < thread_count); + assert(0 < users.size()); + + thread_sync self{}; + std::vector threads{}; + + struct join_ + { + thread_sync& self; + std::vector& threads; + rpc::context& ctx; + + ~join_() noexcept + { + self.update = true; + ctx.raise_abort_scan(); + for (auto& thread : threads) + thread.join(); + } + } join{self, threads, ctx}; + + /* + The algorithm here is extremely basic. Users are divided evenly amongst + the configurable thread count, and grouped by scan height. If an old + account appears, some accounts (grouped on that thread) will be delayed + in processing waiting for that account to catch up. Its not the greatest, + but this "will have to do" for the first cut. + Its not expected that many people will be running + "enterprise level" of nodes where accounts are constantly added. + + Another "issue" is that each thread works independently instead of more + cooperatively for scanning. This requires a bit more synchronization, so + was left for later. Its likely worth doing to reduce the number of + transfers from the daemon, and the bottleneck on the writes into LMDB. + + If the active user list changes, all threads are stopped/joined, and + everything is re-started. + */ + + boost::thread::attributes attrs; + attrs.set_stack_size(THREAD_STACK_SIZE); + + threads.reserve(thread_count); + std::sort(users.begin(), users.end(), by_height{}); + + MINFO("Starting scan loops on " << std::min(thread_count, users.size()) << " thread(s) with " << users.size() << " account(s)"); + + while (!users.empty() && --thread_count) + { + const std::size_t per_thread = std::max(std::size_t(1), users.size() / (thread_count + 1)); + const std::size_t count = std::min(per_thread, users.size()); + std::vector thread_users{ + std::make_move_iterator(users.end() - count), std::make_move_iterator(users.end()) + }; + users.erase(users.end() - count, users.end()); + + rpc::client client = MONERO_UNWRAP(ctx.connect()); + client.watch_scan_signals(); + + auto data = std::make_shared( + std::move(client), disk.clone(), std::move(thread_users) + ); + threads.emplace_back(attrs, std::bind(&scan_loop, std::ref(self), std::move(data))); + } + + if (!users.empty()) + { + rpc::client client = MONERO_UNWRAP(ctx.connect()); + client.watch_scan_signals(); + + auto data = std::make_shared( + std::move(client), disk.clone(), std::move(users) + ); + threads.emplace_back(attrs, std::bind(&scan_loop, std::ref(self), std::move(data))); + } + + auto last_check = std::chrono::steady_clock::now(); + + lmdb::suspended_txn read_txn{}; + db::cursor::accounts accounts_cur{}; + boost::unique_lock lock{self.sync}; + + while (scanner::is_running()) + { + update_rates(ctx); + + for (;;) + { + //! \TODO use signalfd + ZMQ? Windows is the difficult case... + self.user_poll.wait_for(lock, boost::chrono::seconds{1}); + if (self.update || !scanner::is_running()) + return; + auto this_check = std::chrono::steady_clock::now(); + if (account_poll_interval <= (this_check - last_check)) + { + last_check = this_check; + break; + } + } + + auto reader = disk.start_read(std::move(read_txn)); + if (!reader) + { + if (reader.matches(std::errc::no_lock_available)) + { + MWARNING("Failed to open DB read handle, retrying later"); + continue; + } + MONERO_THROW(reader.error(), "Failed to open DB read handle"); + } + + auto current_users = MONERO_UNWRAP( + reader->get_accounts(db::account_status::active, std::move(accounts_cur)) + ); + if (current_users.count() != active.size()) + { + MINFO("Change in active user accounts detected, stopping scan threads..."); + return; + } + + for (auto user = current_users.make_iterator(); !user.is_end(); ++user) + { + const db::account_id user_id = user.get_value(); + if (!std::binary_search(active.begin(), active.end(), user_id)) + { + MINFO("Change in active user accounts detected, stopping scan threads..."); + return; + } + } + + read_txn = reader->finish_read(); + accounts_cur = current_users.give_cursor(); + } // while scanning + } + } // anonymous + + expect scanner::sync(db::storage disk, rpc::client client) + { + using get_hashes = cryptonote::rpc::GetHashesFast; + + MINFO("Starting blockchain sync with daemon"); + + get_hashes::Request req{}; + req.start_height = 0; + { + auto reader = disk.start_read(); + if (!reader) + return reader.error(); + + auto chain = reader->get_chain_sync(); + if (!chain) + return chain.error(); + + req.known_hashes = std::move(*chain); + } + + for (;;) + { + if (req.known_hashes.empty()) + return {lws::error::bad_blockchain}; + + expect sent{lws::error::daemon_timeout}; + + epee::byte_slice msg = rpc::client::make_message("get_hashes_fast", req); + auto start = std::chrono::steady_clock::now(); + + while (!(sent = client.send(std::move(msg), std::chrono::seconds{1}))) + { + if (!scanner::is_running()) + return {lws::error::signal_abort_process}; + + if (sync_rpc_timeout <= (std::chrono::steady_clock::now() - start)) + return {lws::error::daemon_timeout}; + + if (!sent.matches(std::errc::timed_out)) + return sent.error(); + } + + expect resp{lws::error::daemon_timeout}; + start = std::chrono::steady_clock::now(); + + while (!(resp = client.receive(std::chrono::seconds{1}))) + { + if (!scanner::is_running()) + return {lws::error::signal_abort_process}; + + if (sync_rpc_timeout <= (std::chrono::steady_clock::now() - start)) + return {lws::error::daemon_timeout}; + + if (!resp.matches(std::errc::timed_out)) + return resp.error(); + } + + // + // Exit loop if it appears we have synced to top of chain + // + if (resp->hashes.size() <= 1 || resp->hashes.back() == req.known_hashes.front()) + return {std::move(client)}; + + MONERO_CHECK(disk.sync_chain(db::block_id(resp->start_height), epee::to_span(resp->hashes))); + + req.known_hashes.erase(req.known_hashes.begin(), --(req.known_hashes.end())); + for (std::size_t num = 0; num < 10; ++num) + { + if (resp->hashes.empty()) + break; + + req.known_hashes.insert(--(req.known_hashes.end()), resp->hashes.back()); + } + } + + return {std::move(client)}; + } + + void scanner::run(db::storage disk, rpc::context ctx, std::size_t thread_count) + { + thread_count = std::max(std::size_t(1), thread_count); + + rpc::client client{}; + for (;;) + { + const auto last = std::chrono::steady_clock::now(); + update_rates(ctx); + + std::vector active; + std::vector users; + + { + MINFO("Retrieving current active account list"); + + auto reader = MONERO_UNWRAP(disk.start_read()); + auto accounts = MONERO_UNWRAP( + reader.get_accounts(db::account_status::active) + ); + + for (db::account user : accounts.make_range()) + { + std::vector receives{}; + std::vector pubs{}; + auto receive_list = MONERO_UNWRAP(reader.get_outputs(user.id)); + + const std::size_t elems = receive_list.count(); + receives.reserve(elems); + pubs.reserve(elems); + + for (auto output = receive_list.make_iterator(); !output.is_end(); ++output) + { + receives.emplace_back(output.get_value()); + pubs.emplace_back(output.get_value()); + } + + users.emplace_back(user, std::move(receives), std::move(pubs)); + active.insert( + std::lower_bound(active.begin(), active.end(), user.id), user.id + ); + } + + reader.finish_read(); + } // cleanup DB reader + + if (users.empty()) + { + MINFO("No active accounts"); + checked_wait(account_poll_interval - (std::chrono::steady_clock::now() - last)); + } + else + check_loop(disk.clone(), ctx, thread_count, std::move(users), std::move(active)); + + if (!scanner::is_running()) + return; + + if (!client) + client = MONERO_UNWRAP(ctx.connect()); + + expect synced = sync(disk.clone(), std::move(client)); + if (!synced) + { + if (!synced.matches(std::errc::timed_out)) + MONERO_THROW(synced.error(), "Unable to sync blockchain"); + + MWARNING("Failed to connect to daemon at " << ctx.daemon_address()); + } + else + client = std::move(*synced); + } // while scanning + } +} // lws diff --git a/src/scanner.h b/src/scanner.h new file mode 100644 index 0000000..c7bc7e5 --- /dev/null +++ b/src/scanner.h @@ -0,0 +1,59 @@ +// Copyright (c) 2018-2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include +#include +#include +#include + +#include "db/storage.h" +#include "rpc/client.h" + +namespace lws +{ + //! Scans all active `db::account`s. Detects if another process changes active list. + class scanner + { + static std::atomic running; + + scanner() = delete; + + public: + //! Use `client` to sync blockchain data, and \return client if successful. + static expect sync(db::storage disk, rpc::client client); + + //! Poll daemon until `stop()` is called, using `thread_count` threads. + static void run(db::storage disk, rpc::context ctx, std::size_t thread_count); + + //! \return True if `stop()` has never been called. + static bool is_running() noexcept { return running; } + + //! Stops all scanner instances globally. + static void stop() noexcept { running = false; } + }; +} // lws diff --git a/src/server_main.cpp b/src/server_main.cpp new file mode 100644 index 0000000..0caaf3a --- /dev/null +++ b/src/server_main.cpp @@ -0,0 +1,241 @@ +// Copyright (c) 2018-2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/command_line.h" // monero/src/ +#include "common/expect.h" // monero/src/ +#include "common/util.h" // monero/src/ +#include "config.h" +#include "cryptonote_config.h" // monero/src/ +#include "db/storage.h" +#include "options.h" +#include "rest_server.h" +#include "scanner.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "lws" + +namespace +{ + struct options : lws::options + { + const command_line::arg_descriptor daemon_rpc; + const command_line::arg_descriptor> rest_servers; + const command_line::arg_descriptor rest_ssl_key; + const command_line::arg_descriptor rest_ssl_cert; + const command_line::arg_descriptor rest_threads; + const command_line::arg_descriptor scan_threads; + const command_line::arg_descriptor> access_controls; + const command_line::arg_descriptor external_bind; + const command_line::arg_descriptor create_queue_max; + const command_line::arg_descriptor rates_interval; + const command_line::arg_descriptor log_level; + + static std::string get_default_zmq() + { + static constexpr const char base[] = "tcp://127.0.0.1:"; + switch (lws::config::network) + { + case cryptonote::TESTNET: + return base + std::to_string(config::testnet::ZMQ_RPC_DEFAULT_PORT); + case cryptonote::STAGENET: + return base + std::to_string(config::stagenet::ZMQ_RPC_DEFAULT_PORT); + case cryptonote::MAINNET: + default: + break; + } + return base + std::to_string(config::ZMQ_RPC_DEFAULT_PORT); + } + + options() + : lws::options() + , daemon_rpc{"daemon", "://
: of a monerod ZMQ RPC", get_default_zmq()} + , rest_servers{"rest-server", "[(https|http)://
:] for incoming connections, multiple declarations allowed"} + , rest_ssl_key{"rest-ssl-key", " to PEM formatted SSL key for https REST server", ""} + , rest_ssl_cert{"rest-ssl-certificate", " to PEM formatted SSL certificate (chains supported) for https REST server", ""} + , rest_threads{"rest-threads", "Number of threads to process REST connections", 1} + , scan_threads{"scan-threads", "Maximum number of threads for account scanning", boost::thread::hardware_concurrency()} + , access_controls{"access-control-origin", "Specify a whitelisted HTTP control origin domain"} + , external_bind{"confirm-external-bind", "Allow listening for external connections", false} + , create_queue_max{"create-queue-max", "Set pending create account requests maximum", 10000} + , rates_interval{"exchange-rate-interval", "Retrieve exchange rates in minute intervals from cryptocompare.com if greater than 0", 0} + , log_level{"log-level", "Log level [0-4]", 1} + {} + + void prepare(boost::program_options::options_description& description) const + { + static constexpr const char rest_default[] = "https://0.0.0.0:8443"; + + lws::options::prepare(description); + command_line::add_arg(description, daemon_rpc); + description.add_options()(rest_servers.name, boost::program_options::value>()->default_value({rest_default}, rest_default), rest_servers.description); + command_line::add_arg(description, rest_ssl_key); + command_line::add_arg(description, rest_ssl_cert); + command_line::add_arg(description, rest_threads); + command_line::add_arg(description, scan_threads); + command_line::add_arg(description, access_controls); + command_line::add_arg(description, external_bind); + command_line::add_arg(description, create_queue_max); + command_line::add_arg(description, rates_interval); + command_line::add_arg(description, log_level); + } + }; + + struct program + { + std::string db_path; + std::vector rest_servers; + lws::rest_server::configuration rest_config; + std::string daemon_rpc; + std::chrono::minutes rates_interval; + std::size_t scan_threads; + unsigned create_queue_max; + }; + + void print_help(std::ostream& out) + { + boost::program_options::options_description description{"Options"}; + options{}.prepare(description); + + out << "Usage: [options]" << std::endl; + out << description; + } + + boost::optional get_program(int argc, char** argv) + { + namespace po = boost::program_options; + + const options opts{}; + po::variables_map args{}; + { + po::options_description description{"Options"}; + opts.prepare(description); + + po::store( + po::command_line_parser(argc, argv).options(description).run(), args + ); + po::notify(args); + } + + if (command_line::get_arg(args, command_line::arg_help)) + { + print_help(std::cout); + return boost::none; + } + + opts.set_network(args); // do this first, sets global variable :/ + mlog_set_log_level(command_line::get_arg(args, opts.log_level)); + + program prog{ + command_line::get_arg(args, opts.db_path), + command_line::get_arg(args, opts.rest_servers), + lws::rest_server::configuration{ + {command_line::get_arg(args, opts.rest_ssl_key), command_line::get_arg(args, opts.rest_ssl_cert)}, + command_line::get_arg(args, opts.access_controls), + command_line::get_arg(args, opts.rest_threads), + command_line::get_arg(args, opts.external_bind) + }, + command_line::get_arg(args, opts.daemon_rpc), + std::chrono::minutes{command_line::get_arg(args, opts.rates_interval)}, + command_line::get_arg(args, opts.scan_threads), + command_line::get_arg(args, opts.create_queue_max), + }; + + prog.rest_config.threads = std::max(std::size_t(1), prog.rest_config.threads); + prog.scan_threads = std::max(std::size_t(1), prog.scan_threads); + + if (command_line::is_arg_defaulted(args, opts.daemon_rpc)) + prog.daemon_rpc = options::get_default_zmq(); + + return prog; + } + + void run(program prog) + { + std::signal(SIGINT, [] (int) { lws::scanner::stop(); }); + + boost::filesystem::create_directories(prog.db_path); + auto disk = lws::db::storage::open(prog.db_path.c_str(), prog.create_queue_max); + auto ctx = lws::rpc::context::make(std::move(prog.daemon_rpc), prog.rates_interval); + + MINFO("Using monerod ZMQ RPC at " << ctx.daemon_address()); + auto client = lws::scanner::sync(disk.clone(), ctx.connect().value()).value(); + + lws::rest_server server{epee::to_span(prog.rest_servers), disk.clone(), std::move(client), std::move(prog.rest_config)}; + for (const std::string& address : prog.rest_servers) + MINFO("Listening for REST clients at " << address); + + // blocks until SIGINT + lws::scanner::run(std::move(disk), std::move(ctx), prog.scan_threads); + } +} // anonymous + +int main(int argc, char** argv) +{ + tools::on_startup(); // if it throws, don't use MERROR just print default msg + + try + { + boost::optional prog; + + try + { + prog = get_program(argc, argv); + } + catch (std::exception const& e) + { + std::cerr << e.what() << std::endl << std::endl; + print_help(std::cerr); + return EXIT_FAILURE; + } + + if (prog) + run(std::move(*prog)); + } + catch (std::exception const& e) + { + MERROR(e.what()); + return EXIT_FAILURE; + } + catch (...) + { + MERROR("Unknown exception"); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt new file mode 100644 index 0000000..66816cf --- /dev/null +++ b/src/util/CMakeLists.txt @@ -0,0 +1,33 @@ +# Copyright (c) 2020, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(monero-lws-util_sources gamma_picker.cpp random_outputs.cpp transactions.cpp) +set(monero-lws-util_headers fwd.h gamma_picker.h http_server.h random_outputs.h transactions.h) + +add_library(monero-lws-util ${monero-lws-util_sources} ${monero-lws-util_headers}) +target_link_libraries(monero-lws-util monero::libraries) diff --git a/src/util/fwd.h b/src/util/fwd.h new file mode 100644 index 0000000..7a05724 --- /dev/null +++ b/src/util/fwd.h @@ -0,0 +1,35 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +namespace lws +{ + class gamma_picker; + struct random_output; + struct random_ring; +} diff --git a/src/util/gamma_picker.cpp b/src/util/gamma_picker.cpp new file mode 100644 index 0000000..29273cd --- /dev/null +++ b/src/util/gamma_picker.cpp @@ -0,0 +1,112 @@ +// Copyright (c) 2019-2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "gamma_picker.h" + +#include +#include + +#include "crypto/crypto.h" +#include "cryptonote_config.h" + +namespace lws +{ + namespace + { + constexpr const double gamma_shape = 19.28; + constexpr const double gamma_scale = 1 / double(1.61); + constexpr const std::size_t blocks_in_a_year = (86400 * 365) / DIFFICULTY_TARGET_V2; + } + + gamma_picker::gamma_picker(std::vector rct_offsets) + : gamma_picker(std::move(rct_offsets), gamma_shape, gamma_scale) + {} + + gamma_picker::gamma_picker(std::vector offsets_in, double shape, double scale) + : rct_offsets(std::move(offsets_in)), + gamma(shape, scale), + outputs_per_second(0) + { + if (!rct_offsets.empty()) + { + const std::size_t blocks_to_consider = std::min(rct_offsets.size(), blocks_in_a_year); + const std::uint64_t initial = blocks_to_consider < rct_offsets.size() ? + rct_offsets[rct_offsets.size() - blocks_to_consider - 1] : 0; + const std::size_t outputs_to_consider = rct_offsets.back() - initial; + + static_assert(0 < DIFFICULTY_TARGET_V2, "block target time cannot be zero"); + // this assumes constant target over the whole rct range + outputs_per_second = outputs_to_consider / double(DIFFICULTY_TARGET_V2 * blocks_to_consider); + } + } + + bool gamma_picker::is_valid() const noexcept + { + return CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE < rct_offsets.size(); + } + + std::uint64_t gamma_picker::spendable_upper_bound() const noexcept + { + if (!is_valid()) + return 0; + return *(rct_offsets.end() - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE - 1); + } + + std::uint64_t gamma_picker::operator()() + { + if (!is_valid()) + throw std::logic_error{"Cannot select random output - blockchain height too small"}; + + static_assert(std::is_empty(), "random_device is no longer cheap to construct"); + static constexpr const crypto::random_device engine{}; + const auto end = offsets().end() - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE; + + for (unsigned tries = 0; tries < 100; ++tries) + { + std::uint64_t output_index = std::exp(gamma(engine)) * outputs_per_second; + if (offsets().back() <= output_index) + continue; // gamma selected older than blockchain height (rare) + + output_index = offsets().back() - 1 - output_index; + const auto selection = std::lower_bound(offsets().begin(), end, output_index); + if (selection == end) + continue; // gamma selected within locked/non-spendable range (rare) + + const std::uint64_t first_rct = offsets().begin() == selection ? 0 : *(selection - 1); + const std::uint64_t n_rct = *selection - first_rct; + if (n_rct != 0) + return first_rct + crypto::rand_idx(n_rct); + // block had zero outputs (miner didn't collect XMR?) + } + throw std::runtime_error{"Unable to select random output in spendable range using gamma distribution after 1,024 attempts"}; + } + + std::vector gamma_picker::take_offsets() + { + return std::vector{std::move(rct_offsets)}; + } +} // lws diff --git a/src/util/gamma_picker.h b/src/util/gamma_picker.h new file mode 100644 index 0000000..851d504 --- /dev/null +++ b/src/util/gamma_picker.h @@ -0,0 +1,88 @@ +// Copyright (c) 2019-2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include + +namespace lws +{ + //! Select outputs using a gamma distribution with Sarang's output-lineup method + class gamma_picker + { + std::vector rct_offsets; + std::gamma_distribution gamma; + double outputs_per_second; + + gamma_picker(const gamma_picker&) = default; // force explicit usage of `clone()` to copy. + public: + //! \post `!is_valid()` since the chain of offsets is empty. + gamma_picker() + : gamma_picker(std::vector{}) + {} + + //! Use default (recommended) gamma parameters with `rct_offsets`. + explicit gamma_picker(std::vector rct_offsets); + explicit gamma_picker(std::vector rct_offsets, double shape, double scale); + + //! \post Source of move `!is_valid()`. + gamma_picker(gamma_picker&&) = default; + + //! \post Source of move `!is_valid()`. + gamma_picker& operator=(gamma_picker&&) = default; + + //! \return A copy of `this`. + gamma_picker clone() const { return gamma_picker{*this}; } + + //! \return `is_valid()`. + explicit operator bool() const noexcept { return is_valid(); } + + //! \return True if `operator()()` can pick an output using `offsets()`. + bool is_valid() const noexcept; + + //! \return An upper-bound on the number of unlocked/spendable outputs based on block age. + std::uint64_t spendable_upper_bound() const noexcept; + + /*! + Select a random output index for use in a ring. Outputs in the unspendable + range (too new) and older than the chain (too old) are filtered out by + retrying the gamma distribution. + + \throw std::logic_error if `!is_valid()` - considered unrecoverable. + \throw std::runtiime_error if no output within spendable range was selected + after 100 attempts. + \return Selected output using gamma distribution. + */ + std::uint64_t operator()(); + + //! \return Current ringct distribution used for `operator()()` output selection. + const std::vector& offsets() const noexcept { return rct_offsets; } + + //! \return Ownership of `offsets()` by move. \post `!is_valid()` + std::vector take_offsets(); + }; +} // lws diff --git a/src/util/http_server.h b/src/util/http_server.h new file mode 100644 index 0000000..d19bb57 --- /dev/null +++ b/src/util/http_server.h @@ -0,0 +1,124 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include + +#include "misc_log_ex.h" +#include "net/abstract_tcp_server2.h" // monero/contrib/epee/include +#include "net/http_protocol_handler.h" // monero/contrib/epee/include +#include "net/http_server_handlers_map2.h" // monero/contrib/epee/include + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "net.http" + + +namespace lws +{ + template + class http_server_impl_base: public epee::net_utils::http::i_http_server_handler + { + + public: + http_server_impl_base() + : m_net_server(epee::net_utils::e_connection_type_RPC) + {} + + explicit http_server_impl_base(boost::asio::io_service& external_io_service) + : m_net_server(external_io_service, epee::net_utils::e_connection_type_RPC) + {} + + bool init(const std::string& bind_port, const std::string& bind_ip, + std::vector access_control_origins, epee::net_utils::ssl_options_t ssl_options) + { + + //set self as callback handler + m_net_server.get_config_object().m_phandler = static_cast(this); + + //here set folder for hosting reqests + m_net_server.get_config_object().m_folder = ""; + + //set access control allow origins if configured + std::sort(access_control_origins.begin(), access_control_origins.end()); + m_net_server.get_config_object().m_access_control_origins = std::move(access_control_origins); + + + MGINFO("Binding on " << bind_ip << " (IPv4):" << bind_port); + bool res = m_net_server.init_server(bind_port, bind_ip, bind_port, std::string{}, false, true, std::move(ssl_options)); + if(!res) + { + LOG_ERROR("Failed to bind server"); + return false; + } + return true; + } + + bool run(size_t threads_count, bool wait = true) + { + //go to loop + MINFO("Run net_service loop( " << threads_count << " threads)..."); + if(!m_net_server.run_server(threads_count, wait)) + { + LOG_ERROR("Failed to run net tcp server!"); + } + + if(wait) + MINFO("net_service loop stopped."); + return true; + } + + bool deinit() + { + return m_net_server.deinit_server(); + } + + bool timed_wait_server_stop(uint64_t ms) + { + return m_net_server.timed_wait_server_stop(ms); + } + + bool send_stop_signal() + { + m_net_server.send_stop_signal(); + return true; + } + + int get_binded_port() + { + return m_net_server.get_binded_port(); + } + + long get_connections_count() const + { + return m_net_server.get_connections_count(); + } + + protected: + epee::net_utils::boosted_tcp_server > m_net_server; + }; +} // lws diff --git a/src/util/random_outputs.cpp b/src/util/random_outputs.cpp new file mode 100644 index 0000000..7e89934 --- /dev/null +++ b/src/util/random_outputs.cpp @@ -0,0 +1,309 @@ +// Copyright (c) 2018-2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "random_outputs.h" + +#include +#include +#include +#include +#include +#include + +#include "cryptonote_config.h" // monero/src +#include "error.h" +#include "util/gamma_picker.h" + +namespace lws +{ + namespace + { + struct by_amount + { + template + bool operator()(T const& left, T const& right) const noexcept + { + return left.amount < right.amount; + } + + template + bool operator()(std::uint64_t left, T const& right) const noexcept + { + return left < right.amount; + } + + template + bool operator()(T const& left, std::uint64_t right) const noexcept + { + return left.amount < right; + } + }; + + struct by_index + { + template + bool operator()(T const& left, U const& right) const noexcept + { + return left.index < right.index; + } + }; + + struct same_index + { + bool operator()(lws::output_ref const& left, lws::output_ref const& right) const noexcept + { + return left.index == right.index; + } + }; + + expect pick_all(epee::span out, const std::uint64_t amount) noexcept + { + static_assert( + std::numeric_limits::max() <= std::numeric_limits::max(), + "size_t is really large" + ); + + std::size_t index = 0; + for (auto& entry : out) + { + entry.amount = amount; + entry.index = index++; + } + return success(); + } + + expect triangular_pick(epee::span out, lws::histogram const& hist) + { + MONERO_PRECOND(hist.unlocked_count <= hist.total_count); + + if (hist.unlocked_count < out.size()) + return {lws::error::not_enough_mixin}; + + if (hist.unlocked_count == out.size()) + return pick_all(out, hist.amount); + + /* This does not match the wallet2 selection code - recents are not + considered. There should be no new recents for this selection + algorithm because it is only used for non-ringct outputs. */ + + static constexpr const std::uint64_t max = std::uint64_t(1) << 53; + for (auto& entry : out) + { + entry.amount = hist.amount; + + /* \TODO Update REST API to send real outputs so selection + algorithm can use fork information (like wallet2). */ + + do + { + const std::uint64_t r = crypto::rand() % max; + const double frac = std::sqrt(double(r) / double(max)); + entry.index = std::uint64_t(frac * double(hist.total_count)); + } while (hist.unlocked_count < entry.index); + } + + return success(); + } + + expect gamma_pick(epee::span out, gamma_picker& pick_rct) + { + if (!pick_rct) + return {lws::error::not_enough_mixin}; + + const std::uint64_t spendable = pick_rct.spendable_upper_bound(); + if (spendable < out.size()) + return {lws::error::not_enough_mixin}; + if (spendable == out.size()) + return pick_all(out, 0); + + for (auto& entry : out) + { + entry.amount = 0; + entry.index = pick_rct(); + + /* \TODO Update REST API to send real outputs so selection + algorithm can use fork information (like wallet2). */ + } + return success(); + } + } + + expect> pick_random_outputs( + const std::uint32_t mixin, + const epee::span amounts, + gamma_picker& pick_rct, + epee::span histograms, + const std::function fetch + ) { + if (mixin == 0 || amounts.empty()) + return std::vector{amounts.size()}; + + const std::size_t sizet_max = std::numeric_limits::max(); + MONERO_PRECOND(bool(fetch)); + MONERO_PRECOND(mixin <= (sizet_max / amounts.size())); + + std::vector proposed{}; + std::vector rings{}; + rings.resize(amounts.size()); + + for (auto ring : boost::combine(amounts, rings)) + boost::get<1>(ring).amount = boost::get<0>(ring); + + std::sort(histograms.begin(), histograms.end(), by_amount{}); + for (unsigned tries = 0; tries < 64; ++tries) + { + proposed.clear(); + proposed.reserve(rings.size() * mixin); + + // select indexes foreach ring below mixin count + for (auto ring = rings.begin(); ring != rings.end(); /* handled below */) + { + const std::size_t count = proposed.size(); + if (ring->ring.size() < mixin) + { + const std::size_t diff = mixin - ring->ring.size(); + proposed.resize(proposed.size() + diff); + { + const epee::span latest{proposed.data() + count, diff}; + + expect picked{}; + const std::uint64_t amount = ring->amount; + if (amount == 0) + picked = gamma_pick(latest, pick_rct); + else + { + const auto match = + std::lower_bound(histograms.begin(), histograms.end(), amount, by_amount{}); + MONERO_PRECOND(match != histograms.end() && match->amount == amount); + picked = triangular_pick(latest, *match); + } + + if (!picked) + { + if (picked == lws::error::not_enough_mixin) + { + proposed.resize(proposed.size() - diff); + ring = rings.erase(ring); + continue; + } + return picked.error(); + } + + // drop dupes in latest selection + std::sort(latest.begin(), latest.end(), by_index{}); + const auto last = std::unique(latest.begin(), latest.end(), same_index{}); + proposed.resize(last - proposed.data()); + } + + ring->ring.reserve(mixin); + epee::span current = epee::to_mut_span(ring->ring); + std::sort(current.begin(), current.end(), by_index{}); + + // See if new list has duplicates with existing ring + for (auto ref = proposed.begin() + count; ref < proposed.end(); /* see branches */ ) + { + // must update after push_back call + current = {ring->ring.data(), current.size()}; + + const auto match = + std::lower_bound(current.begin(), current.end(), *ref, by_index{}); + if (match == current.end() || match->index != ref->index) + { + ring->ring.push_back(random_output{{}, ref->index}); + ring->ring.back().keys.unlocked = false; // for tracking below + ++ref; + } + else // dupe + ref = proposed.erase(ref); + } + } + + ++ring; + } + + // all amounts lack enough mixin + if (rings.empty()) + return rings; + + /* \TODO For maximum privacy, the real outputs need to be fetched + below. This requires an update of the REST API. */ + + // fetch all new keys in one shot + const std::size_t expected = proposed.size(); + auto result = fetch(std::move(proposed)); + if (!result) + return result.error(); + + if (expected != result->size()) + return {lws::error::bad_daemon_response}; + + bool done = true; + std::size_t offset = 0; + for (auto& ring : rings) + { + // this should never fail, else the logic in here is bad + assert(ring.ring.size() <= ring.ring.size()); + + // if we dropped a selection due to dupe, must try again + done = (done && mixin <= ring.ring.size()); + + for (auto entry = ring.ring.begin(); entry < ring.ring.end(); /* see branches */) + { + // check only new keys + if (entry->keys.unlocked) + ++entry; + else + { + if (result->size() <= offset) + return {lws::error::bad_daemon_response}; + + output_keys const& keys = result->at(offset); + ++offset; + + if (keys.unlocked) + { + entry->keys = keys; + ++entry; + } + else + { + done = false; + entry = ring.ring.erase(entry); + } + } + } + } + assert(offset == result->size()); + + if (done) + return {std::move(rings)}; + } + + return {lws::error::not_enough_mixin}; + } +} + diff --git a/src/util/random_outputs.h b/src/util/random_outputs.h new file mode 100644 index 0000000..b5bd8e9 --- /dev/null +++ b/src/util/random_outputs.h @@ -0,0 +1,92 @@ +// Copyright (c) 2018-2019, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include +#include + +#include "common/expect.h" // monero/src +#include "rpc/message_data_structs.h" // monero/src +#include "span.h" // monero/src +#include "util/fwd.h" + +namespace lws +{ + using histogram = cryptonote::rpc::output_amount_count; + using output_ref = cryptonote::rpc::output_amount_and_index; + using output_keys = cryptonote::rpc::output_key_mask_unlocked; + + struct random_output + { + output_keys keys; + std::uint64_t index; + }; + + struct random_ring + { + std::vector ring; + std::uint64_t amount; + }; + + using key_fetcher = expect>(std::vector); + + /*! + Selects random outputs for use in a ring signature. `amounts` of `0` + use a gamma distribution algorithm and all other amounts use a + triangular distribution. + + \param mixin The number of dummy outputs per ring. + \param amounts The amounts that need dummy outputs to be selected. + \param pick_rct Ring-ct distribution from the daemon + \param histograms A histogram from the daemon foreach non-zero value + in `amounts`. + \param fetch A function that can retrieve the keys for the randomly + selected outputs. + + \note `histograms` is modified - the list is sorted by amount. + + \note This currenty leaks the real outputs to `fetch`, because the + real output is not provided alongside the dummy outputs. This is a + limitation of the current openmonero/mymonero API. When this is + resolved, this function can possibly be moved outside of the `lws` + namespace for use by simple wallet. + + \return Randomly selected outputs in rings of size `mixin`, one for + each element in `amounts`. Amounts with less than `mixin` available + are not returned. All outputs are unlocked. + */ + expect> pick_random_outputs( + std::uint32_t mixin, + epee::span amounts, + gamma_picker& pick_rct, + epee::span histograms, + std::function fetch + ); +} + diff --git a/src/util/transactions.cpp b/src/util/transactions.cpp new file mode 100644 index 0000000..197563c --- /dev/null +++ b/src/util/transactions.cpp @@ -0,0 +1,61 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "transactions.h" + +#include "cryptonote_config.h" +#include "crypto/crypto.h" +#include "crypto/hash.h" +#include "ringct/rctOps.h" + +void lws::decrypt_payment_id(crypto::hash8& out, const crypto::key_derivation& key) +{ + crypto::hash hash; + char data[33]; /* A hash, and an extra byte */ + + memcpy(data, &key, 32); + data[32] = config::HASH_KEY_ENCRYPTED_PAYMENT_ID; + cn_fast_hash(data, 33, hash); + + for (size_t b = 0; b < 8; ++b) + out.data[b] ^= hash.data[b]; +} + +boost::optional> lws::decode_amount(const rct::key& commitment, const rct::ecdhTuple& info, const crypto::key_derivation& sk, std::size_t index, const bool bulletproof2) +{ + crypto::secret_key scalar{}; + crypto::derivation_to_scalar(sk, index, scalar); + + rct::ecdhTuple copy{info}; + rct::ecdhDecode(copy, rct::sk2rct(scalar), bulletproof2); + + rct::key Ctmp; + rct::addKeys2(Ctmp, copy.mask, copy.amount, rct::H); + if (rct::equalKeys(commitment, Ctmp)) + return {{rct::h2d(copy.amount), copy.mask}}; + return boost::none; +} diff --git a/src/util/transactions.h b/src/util/transactions.h new file mode 100644 index 0000000..cb9b519 --- /dev/null +++ b/src/util/transactions.h @@ -0,0 +1,45 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include + +#include "common/pod-class.h" +#include "ringct/rctTypes.h" + +namespace crypto +{ + POD_CLASS hash8; + POD_CLASS key_derivation; +} + +namespace lws +{ + void decrypt_payment_id(crypto::hash8& out, const crypto::key_derivation& key); + boost::optional> decode_amount(const rct::key& commitment, const rct::ecdhTuple& info, const crypto::key_derivation& sk, std::size_t index, const bool bulletproof2); +} diff --git a/src/wire.h b/src/wire.h new file mode 100644 index 0000000..f24d3ca --- /dev/null +++ b/src/wire.h @@ -0,0 +1,77 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include +#include +#include + +#include "common/expect.h" // monero/src +#include "wire/error.h" +#include "wire/read.h" +#include "wire/write.h" + +#define WIRE_DEFINE_ENUM(type_, map) \ + static_assert(std::is_enum::value, "get_string will fail"); \ + static_assert(!std::is_signed::value, "write_bytes will fail"); \ + const char* get_string(const type_ source) noexcept \ + { \ + using native_type = std::underlying_type::type; \ + const native_type value = native_type(source); \ + if (value < std::end(map) - std::begin(map)) \ + return map[value]; \ + return "invalid enum value"; \ + } \ + expect type_ ## _from_string(const boost::string_ref source) noexcept \ + { \ + if (const auto elem = std::find(std::begin(map), std::end(map), source)) \ + { \ + if (elem != std::end(map)) \ + return type_(elem - std::begin(map)); \ + } \ + return {::wire::error::schema::enumeration}; \ + } \ + void read_bytes(::wire::reader& source, type_& dest) \ + { \ + dest = type_(source.enumeration(map)); \ + } \ + void write_bytes(::wire::writer& dest, const type_ source) \ + { \ + dest.enumeration(std::size_t(source), map); \ + } + +#define WIRE_DEFINE_OBJECT(type, map) \ + void read_bytes(::wire::reader& source, type& dest) \ + { \ + map(source, dest); \ + } \ + void write_bytes(::wire::writer& dest, const type& source) \ + { \ + map(dest, source); \ + } diff --git a/src/wire/CMakeLists.txt b/src/wire/CMakeLists.txt new file mode 100644 index 0000000..4a09e65 --- /dev/null +++ b/src/wire/CMakeLists.txt @@ -0,0 +1,36 @@ +# Copyright (c) 2020, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(monero-lws-wire_sources error.cpp read.cpp write.cpp) +set(monero-lws-wire_headers crypto.h error.h field.h filters.h fwd.h json.h read.h traits.h vector.h write.h) + +add_library(monero-lws-wire ${monero-lws-wire_sources} ${monero-lws-wire_headers}) +target_include_directories(monero-lws-wire PUBLIC "${LMDB_INCLUDE}") +target_link_libraries(monero-lws-wire PRIVATE monero::libraries) + +add_subdirectory(json) diff --git a/src/wire/crypto.h b/src/wire/crypto.h new file mode 100644 index 0000000..2ad8f51 --- /dev/null +++ b/src/wire/crypto.h @@ -0,0 +1,72 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include + +#include "crypto/crypto.h" // monero/src +#include "ringct/rctTypes.h" // monero/src +#include "wire/traits.h" + +namespace wire +{ + template<> + struct is_blob + : std::true_type + {}; + + template<> + struct is_blob + : std::true_type + {}; + + template<> + struct is_blob + : std::true_type + {}; + + template<> + struct is_blob + : std::true_type + {}; + + template<> + struct is_blob + : std::true_type + {}; + + template<> + struct is_blob + : std::true_type + {}; + + template<> + struct is_blob + : std::true_type + {}; +} diff --git a/src/wire/error.cpp b/src/wire/error.cpp new file mode 100644 index 0000000..790d220 --- /dev/null +++ b/src/wire/error.cpp @@ -0,0 +1,93 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "wire/error.h" + +namespace wire +{ + namespace error + { + const char* get_string(const schema value) noexcept + { + switch (value) + { + default: + break; + + case schema::none: + return "No schema errors"; + case schema::array: + return "Schema expected array"; + case schema::binary: + return "Schema expected binary value of variable size"; + case schema::boolean: + return "Schema expected boolean value"; + case schema::enumeration: + return "Schema expected a specific of enumeration value(s)"; + case schema::fixed_binary: + return "Schema expected binary of fixed size"; + case schema::integer: + return "Schema expected integer value"; + case schema::invalid_key: + return "Schema does not allow object field key"; + case schema::larger_integer: + return "Schema expected a larger integer value"; + case schema::maximum_depth: + return "Schema hit maximum array+object depth tracking"; + case schema::missing_key: + return "Schema missing required field key"; + case schema::number: + return "Schema expected number (integer or float) value"; + case schema::object: + return "Schema expected object"; + case schema::smaller_integer: + return "Schema expected a smaller integer value"; + case schema::string: + return "Schema expected string"; + } + return "Unknown schema error"; + } + + const std::error_category& schema_category() noexcept + { + struct category final : std::error_category + { + virtual const char* name() const noexcept override final + { + return "wire::error::schema_category()"; + } + + virtual std::string message(int value) const override final + { + return get_string(schema(value)); + } + }; + static const category instance{}; + return instance; + } + } +} diff --git a/src/wire/error.h b/src/wire/error.h new file mode 100644 index 0000000..c67a978 --- /dev/null +++ b/src/wire/error.h @@ -0,0 +1,135 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include +#include + +#include "misc_log_ex.h" // monero/contrib/epee/include + +//! Print default `code` message followed by optional message to debug log then throw `code`. +#define WIRE_DLOG_THROW_(code, ...) \ + do \ + { \ + MDEBUG( get_string(code) __VA_ARGS__ ); \ + throw ::wire::exception_t{code}; \ + } \ + while (0) + +//! Print default `code` message followed by `msg` to debug log then throw `code`. +#define WIRE_DLOG_THROW(code, msg) \ + WIRE_DLOG_THROW_(code, << ": " << msg) + +namespace wire +{ + namespace error + { + enum class schema : int + { + none = 0, //!< Must be zero for `expect<..>` + array, //!< Expected an array value + binary, //!< Expected a binary value of variable length + boolean, //!< Expected a boolean value + enumeration, //!< Expected a value from a specific set + fixed_binary, //!< Expected a binary value of fixed length + integer, //!< Expected an integer value + invalid_key, //!< Key for object is invalid + larger_integer, //!< Expected a larger integer value + maximum_depth, //!< Hit maximum number of object+array tracking + missing_key, //!< Missing required key for object + number, //!< Expected a number (integer or float) value + object, //!< Expected object value + smaller_integer, //!< Expected a smaller integer value + string, //!< Expected string value + }; + + //! \return Error message string. + const char* get_string(schema value) noexcept; + + //! \return Category for `schema_error`. + const std::error_category& schema_category() noexcept; + + //! \return Error code with `value` and `schema_category()`. + inline std::error_code make_error_code(const schema value) noexcept + { + return std::error_code{int(value), schema_category()}; + } + } // error + + //! `std::exception` doesn't require dynamic memory like `std::runtime_error` + struct exception : std::exception + { + exception() noexcept + : std::exception() + {} + + exception(const exception&) = default; + exception& operator=(const exception&) = default; + virtual ~exception() noexcept + {} + + virtual std::error_code code() const noexcept = 0; + }; + + template + class exception_t final : public wire::exception + { + static_assert(std::is_enum(), "only enumerated types allowed"); + T value; + + public: + exception_t(T value) noexcept + : value(value) + {} + + exception_t(const exception_t&) = default; + ~exception_t() = default; + exception_t& operator=(const exception_t&) = default; + + const char* what() const noexcept override final + { + static_assert(noexcept(noexcept(get_string(value))), "get_string function must be noexcept"); + return get_string(value); + } + + std::error_code code() const noexcept override final + { + static_assert(noexcept(noexcept(make_error_code(value))), "make_error_code funcion must be noexcept"); + return make_error_code(value); + } + }; +} + +namespace std +{ + template<> + struct is_error_code_enum + : true_type + {}; +} diff --git a/src/wire/field.h b/src/wire/field.h new file mode 100644 index 0000000..8b2f435 --- /dev/null +++ b/src/wire/field.h @@ -0,0 +1,280 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include + +#include "wire/filters.h" +#include "wire/traits.h" + +//! A required field has the same key name and C/C++ name +#define WIRE_FIELD(name) \ + ::wire::field( #name , std::ref( self . name )) + +//! A required field has the same key name and C/C++ name AND is cheap to copy (faster output). +#define WIRE_FIELD_COPY(name) \ + ::wire::field( #name , self . name ) + +//! The optional field has the same key name and C/C++ name +#define WIRE_OPTIONAL_FIELD(name) \ + ::wire::optional_field( #name , std::ref( self . name )) + +namespace wire +{ + template + struct unwrap_reference + { + using type = T; + }; + + template + struct unwrap_reference> + { + using type = T; + }; + + + //! Links `name` to a `value` for object serialization. + template + struct field_ + { + using value_type = typename unwrap_reference::type; + static constexpr bool is_required() noexcept { return Required; } + static constexpr std::size_t count() noexcept { return 1; } + + const char* name; + T value; + + //! \return `value` with `std::reference_wrapper` removed. + constexpr const value_type& get_value() const noexcept + { + return value; + } + + //! \return `value` with `std::reference_wrapper` removed. + value_type& get_value() noexcept + { + return value; + } + }; + + //! Links `name` to `value`. Use `std::ref` if de-serializing. + template + constexpr inline field_ field(const char* name, T value) + { + return {name, std::move(value)}; + } + + //! Links `name` to `value`. Use `std::ref` if de-serializing. + template + constexpr inline field_ optional_field(const char* name, T value) + { + return {name, std::move(value)}; + } + + + //! Links `name` to a type `T` for variant serialization. + template + struct option + { + const char* name; + }; + + //! \return Name associated with type `T` for variant `field`. + template + constexpr const char* get_option_name(const U& field) noexcept + { + return static_cast< const option& >(field).name; + } + + //! Links each type in a variant to a string key. + template + struct variant_field_ : option... + { + using value_type = typename unwrap_reference::type; + static constexpr bool is_required() noexcept { return Required; } + static constexpr std::size_t count() noexcept { return sizeof...(U); } + + constexpr variant_field_(const char* name, T value, option... opts) + : option(std::move(opts))..., name(name), value(std::move(value)) + {} + + const char* name; + T value; + + constexpr const value_type& get_value() const noexcept + { + return value; + } + + value_type& get_value() noexcept + { + return value; + } + + template + struct wrap + { + using result_type = void; + + variant_field_ self; + V visitor; + + template + void operator()(const X& value) const + { + visitor(get_option_name(self), value); + } + }; + + template + void visit(V visitor) const + { + apply_visitor(wrap{*this, std::move(visitor)}, get_value()); + } + }; + + //! Links variant `value` to a unique name per type in `opts`. Use `std::ref` for `value` if de-serializing. + template + constexpr inline variant_field_ variant_field(const char* name, T value, option... opts) + { + return {name, std::move(value), std::move(opts)...}; + } + + + //! Indicates a field value should be written as an array + template + struct as_array_ + { + using value_type = typename unwrap_reference::type; + + T value; + F filter; //!< Each element in `value` given to this callable before `write_bytes`. + + //! \return `value` with `std::reference_wrapper` removed. + constexpr const value_type& get_value() const noexcept + { + return value; + } + + //! \return `value` with `std::reference_wrapper` removed. + value_type& get_value() noexcept + { + return value; + } + }; + + //! Callable that can filter `as_object` values or be used immediately. + template + struct as_array_filter + { + Default default_filter; + + template + constexpr as_array_ operator()(T value) const + { + return {std::move(value), default_filter}; + } + + template + constexpr as_array_ operator()(T value, F filter) const + { + return {std::move(value), std::move(filter)}; + } + }; + //! Usage: `wire::field("foo", wire::as_array(self.foo, to_string{})`. Consider `std::ref`. + constexpr as_array_filter as_array{}; + + + //! Indicates a field value should be written as an object + template + struct as_object_ + { + using map_type = typename unwrap_reference::type; + + T map; + F key_filter; //!< Each key (`.first`) in `map` given to this callable before writing field key. + G value_filter; //!< Each value (`.second`) in `map` given to this callable before `write_bytes`. + + //! \return `map` with `std::reference_wrapper` removed. + constexpr const map_type& get_map() const noexcept + { + return map; + } + + //! \return `map` with `std::reference_wrapper` removed. + map_type& get_map() noexcept + { + return map; + } + }; + + //! Usage: `wire::field("foo", wire::as_object(self.foo, to_string{}, wire::as_array))`. Consider `std::ref`. + template + inline constexpr as_object_ as_object(T map, F key_filter = F{}, G value_filter = G{}) + { + return {std::move(map), std::move(key_filter), std::move(value_filter)}; + } + + + template + inline constexpr bool available(const field_&) noexcept + { + return true; + } + template + inline bool available(const field_& elem) + { + return bool(elem.get_value()); + } + template + inline constexpr bool available(const variant_field_&) noexcept + { + return true; + } + template + inline constexpr bool available(const variant_field_& elem) + { + return elem != nullptr; + } + + + // example usage : `wire::sum(std::size_t(wire::available(fields))...)` + + inline constexpr int sum() noexcept + { + return 0; + } + template + inline constexpr T sum(const T head, const U... tail) noexcept + { + return head + sum(tail...); + } +} + diff --git a/src/wire/filters.h b/src/wire/filters.h new file mode 100644 index 0000000..6b46709 --- /dev/null +++ b/src/wire/filters.h @@ -0,0 +1,73 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include + +#include "lmdb/util.h" + +// These functions are to be used with `wire::as_object(...)` key filtering + +namespace wire +{ + //! Callable that returns the value unchanged; default filter for `as_array` and `as_object`. + struct identity_ + { + template + const T& operator()(const T& value) const noexcept + { + return value; + } + }; + constexpr const identity_ identity{}; + + //! Callable that forwards enum to get_string. + struct enum_as_string_ + { + template + auto operator()(const T value) const noexcept(noexcept(get_string(value))) -> decltype(get_string(value)) + { + return get_string(value); + } + }; + constexpr const enum_as_string_ enum_as_string{}; + + //! Callable that converts C++11 enum class or integer to integer value. + struct as_integer_ + { + template + lmdb::native_type operator()(const T value) const noexcept + { + using native = lmdb::native_type; + static_assert(!std::is_signed::value, "integer cannot be signed"); + return native(value); + } + }; + constexpr const as_integer_ as_integer{}; +} diff --git a/src/wire/fwd.h b/src/wire/fwd.h new file mode 100644 index 0000000..01f4f29 --- /dev/null +++ b/src/wire/fwd.h @@ -0,0 +1,68 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include + +#include "common/expect.h" // monero/src + +//! Declare an enum to be serialized as an integer +#define WIRE_AS_INTEGER(type_) \ + static_assert(std::is_enum(), "AS_INTEGER only enum types"); \ + template \ + inline void read_bytes(R& source, type_& dest) \ + { \ + std::underlying_type::type temp{}; \ + read_bytes(source, temp); \ + dest = type_(temp); \ + } \ + template \ + inline void write_bytes(W& dest, const type_ source) \ + { \ + write_bytes(dest, std::underlying_type::type(source)); \ + } + +//! Declare an enum to be serialized as a string (json) or integer (msgpack) +#define WIRE_DECLARE_ENUM(type) \ + const char* get_string(type) noexcept; \ + expect type ## _from_string(const boost::string_ref) noexcept; \ + void read_bytes(::wire::reader&, type&); \ + void write_bytes(::wire::writer&, type) + +//! Declare a class/struct serialization for all available formats +#define WIRE_DECLARE_OBJECT(type) \ + void read_bytes(::wire::reader&, type&); \ + void write_bytes(::wire::writer&, const type&) + +namespace wire +{ + class reader; + struct writer; +} + diff --git a/src/wire/json.h b/src/wire/json.h new file mode 100644 index 0000000..b3e3351 --- /dev/null +++ b/src/wire/json.h @@ -0,0 +1,54 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "wire/json/base.h" +#include "wire/json/error.h" +#include "wire/json/read.h" +#include "wire/json/write.h" + +#define WIRE_JSON_DEFINE_ENUM(type, map) \ + void read_bytes(::wire::json_reader& source, type& dest) \ + { \ + dest = type(source.enumeration(map)); \ + } \ + void write_bytes(::wire::json_writer& dest, const type source) \ + { \ + dest.enumeration(std::size_t(source), map); \ + } + +#define WIRE_JSON_DEFINE_OBJECT(type, map) \ + void read_bytes(::wire::json_reader& source, type& dest) \ + { \ + map(source, dest); \ + } \ + void write_bytes(::wire::json_writer& dest, const type& source) \ + { \ + map(dest, source); \ + } + diff --git a/src/wire/json/CMakeLists.txt b/src/wire/json/CMakeLists.txt new file mode 100644 index 0000000..aca1d81 --- /dev/null +++ b/src/wire/json/CMakeLists.txt @@ -0,0 +1,33 @@ +# Copyright (c) 2020, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(monero-lws_wire-json_sources error.cpp read.cpp write.cpp) +set(monero-lws_wire-json_headers base.h error.h fwd.h read.h write.h) + +add_library(monero-lws-wire-json ${monero-lws_wire-json_sources} ${monero-lws-wire-json_headers}) +target_link_libraries(monero-lws-wire-json monero::libraries monero-lws-wire) diff --git a/src/wire/json/base.h b/src/wire/json/base.h new file mode 100644 index 0000000..5027ffb --- /dev/null +++ b/src/wire/json/base.h @@ -0,0 +1,50 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include + +#include "byte_slice.h" +#include "common/expect.h" +#include "wire/json/fwd.h" + +namespace wire +{ + struct json + { + using input_type = json_reader; + using output_type = json_writer; + + template + static expect from_bytes(std::string&& source); + + template + static epee::byte_slice to_bytes(const T& source); + }; +} + diff --git a/src/wire/json/error.cpp b/src/wire/json/error.cpp new file mode 100644 index 0000000..a745c6d --- /dev/null +++ b/src/wire/json/error.cpp @@ -0,0 +1,108 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "error.h" + +namespace wire +{ +namespace error +{ + const char* get_string(const rapidjson_e value) noexcept + { + switch (rapidjson::ParseErrorCode(value)) + { + default: + break; + + case rapidjson::kParseErrorNone: + return "No JSON parsing errors"; + + // from rapidjson + case rapidjson::kParseErrorDocumentEmpty: + return "JSON parser expected non-empty document"; + case rapidjson::kParseErrorDocumentRootNotSingular: + return "JSON parser expected one value at root level"; + + case rapidjson::kParseErrorValueInvalid: + return "JSON parser found invalid value"; + + case rapidjson::kParseErrorObjectMissName: + return "JSON parser expected name for object field"; + case rapidjson::kParseErrorObjectMissColon: + return "JSON parser expected ':' between name and value"; + case rapidjson::kParseErrorObjectMissCommaOrCurlyBracket: + return "JSON parser expected ',' or '}'"; + + case rapidjson::kParseErrorArrayMissCommaOrSquareBracket: + return "JSON parser expected ',' or ']'"; + + case rapidjson::kParseErrorStringUnicodeEscapeInvalidHex: + return "JSON parser found invalid unicode escape"; + case rapidjson::kParseErrorStringUnicodeSurrogateInvalid: + return "JSON parser found invalid unicode surrogate value"; + case rapidjson::kParseErrorStringEscapeInvalid: + return "JSON parser found invalid escape sequence in string value"; + case rapidjson::kParseErrorStringMissQuotationMark: + return "JSON parser expected '\"'"; + case rapidjson::kParseErrorStringInvalidEncoding: + return "JSON parser found invalid encoding"; + + case rapidjson::kParseErrorNumberTooBig: + return "JSON parser found number value larger than double float precision"; + case rapidjson::kParseErrorNumberMissFraction: + return "JSON parser found number missing fractional component"; + case rapidjson::kParseErrorNumberMissExponent: + return "JSON parser found number missing exponent"; + + case rapidjson::kParseErrorTermination: + return "JSON parser was stopped"; + case rapidjson::kParseErrorUnspecificSyntaxError: + return "JSON parser found syntax error"; + } + + return "Unknown JSON parser error"; + } + + const std::error_category& rapidjson_category() noexcept + { + struct category final : std::error_category + { + virtual const char* name() const noexcept override final + { + return "wire::error::rapidjson_category()"; + } + + virtual std::string message(int value) const override final + { + return get_string(rapidjson_e(value)); + } + }; + static const category instance{}; + return instance; + } +} // error +} // wire diff --git a/src/wire/json/error.h b/src/wire/json/error.h new file mode 100644 index 0000000..728d1c8 --- /dev/null +++ b/src/wire/json/error.h @@ -0,0 +1,53 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include + +namespace wire +{ +namespace error +{ + //! Type wrapper to "grab" rapidjson errors + enum class rapidjson_e : int {}; + + //! \return Static string describing error `value`. + const char* get_string(rapidjson_e value) noexcept; + + //! \return Category for rapidjson generated errors. + const std::error_category& rapidjson_category() noexcept; + + //! \return Error code with `value` and `rapidjson_category()`. + inline std::error_code make_error_code(rapidjson_e value) noexcept + { + return std::error_code{int(value), rapidjson_category()}; + } +} +} + diff --git a/src/wire/json/fwd.h b/src/wire/json/fwd.h new file mode 100644 index 0000000..b0f8032 --- /dev/null +++ b/src/wire/json/fwd.h @@ -0,0 +1,45 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#define WIRE_JSON_DECLARE_ENUM(type) \ + const char* get_string(type) noexcept; \ + void read_bytes(::wire::json_reader&, type&); \ + void write_bytes(:wire::json_writer&, type) + +#define WIRE_JSON_DECLARE_OBJECT(type) \ + void read_bytes(::wire::json_reader&, type&); \ + void write_bytes(::wire::json_writer&, const type&) + +namespace wire +{ + struct json; + class json_reader; + class json_writer; +} + diff --git a/src/wire/json/read.cpp b/src/wire/json/read.cpp new file mode 100644 index 0000000..497c278 --- /dev/null +++ b/src/wire/json/read.cpp @@ -0,0 +1,413 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "read.h" + +#include +#include +#include +#include + +#include "common/expect.h" // monero/src +#include "hex.h" // monero/contrib/epee/include +#include "wire/error.h" +#include "wire/json/error.h" + +namespace +{ + //! Maximum number of bytes to display "near" JSON error. + constexpr const std::size_t snippet_size = 30; + + struct json_default_reject : rapidjson::BaseReaderHandler, json_default_reject> + { + bool Default() const noexcept { return false; } + }; + + //! \throw std::system_error by converting `code` into a std::error_code + [[noreturn]] void throw_json_error(const epee::span source, const rapidjson::Reader& reader, const wire::error::schema expected) + { + const std::size_t offset = std::min(source.size(), reader.GetErrorOffset()); + const std::size_t start = offset;//std::max(snippet_size / 2, offset) - (snippet_size / 2); + const std::size_t end = start + std::min(snippet_size, source.size() - start); + + const boost::string_ref text{source.data() + start, end - start}; + const rapidjson::ParseErrorCode parse_error = reader.GetParseErrorCode(); + switch (parse_error) + { + default: + WIRE_DLOG_THROW(wire::error::rapidjson_e(parse_error), "near \"" << text << '"'); + case rapidjson::kParseErrorNone: + case rapidjson::kParseErrorTermination: // the handler returned false + break; + } + WIRE_DLOG_THROW(expected, "near '" << text << '\''); + } +} + +namespace wire +{ + struct json_reader::rapidjson_sax + { + struct string_contents + { + const char* ptr; + std::size_t length; + }; + + union + { + bool boolean; + std::intmax_t integer; + std::uintmax_t unsigned_integer; + double number; + string_contents string; + } value; + + error::schema expected_; + bool negative; + + explicit rapidjson_sax(error::schema expected) noexcept + : expected_(expected), negative(false) + {} + + bool Null() const noexcept + { + return expected_ == error::schema::none; + } + + bool Bool(bool i) noexcept + { + value.boolean = i; + return expected_ == error::schema::boolean || expected_ == error::schema::none; + } + + bool Int(int i) noexcept + { + return Int64(i); + } + bool Uint(unsigned i) noexcept + { + return Uint64(i); + } + bool Int64(std::int64_t i) noexcept + { + negative = true; + switch(expected_) + { + default: + return false; + case error::schema::integer: + value.integer = i; + break; + case error::schema::number: + value.number = i; + break; + case error::schema::none: + break; + } + return true; + } + bool Uint64(std::uint64_t i) noexcept + { + switch (expected_) + { + default: + return false; + case error::schema::integer: + value.unsigned_integer = i; + break; + case error::schema::number: + value.number = i; + break; + case error::schema::none: + break; + } + return true; + } + + bool Double(double i) noexcept + { + value.number = i; + return expected_ == error::schema::number || expected_ == error::schema::none; + } + + bool RawNumber(const char*, std::size_t, bool) const noexcept + { + return false; + } + + bool String(const char* str, std::size_t length, bool) noexcept + { + value.string = {str, length}; + return expected_ == error::schema::string || expected_ == error::schema::none; + } + bool Key(const char* str, std::size_t length, bool) + { + return String(str, length, true); + } + + bool StartArray() const noexcept { return expected_ == error::schema::none; } + bool EndArray(std::size_t) const noexcept { return expected_ == error::schema::none; } + bool StartObject() const noexcept { return expected_ == error::schema::none; } + bool EndObject(std::size_t) const noexcept { return expected_ == error::schema::none; } + }; + + void json_reader::read_next_value(rapidjson_sax& handler) + { + rapidjson::InsituStringStream stream{current_.data()}; + if (!reader_.Parse(stream, handler)) + throw_json_error(current_, reader_, handler.expected_); + current_.remove_prefix(stream.Tell()); + } + + char json_reader::get_next_token() + { + rapidjson::InsituStringStream stream{current_.data()}; + rapidjson::SkipWhitespace(stream); + current_.remove_prefix(stream.Tell()); + return stream.Peek(); + } + + boost::string_ref json_reader::get_next_string() + { + if (get_next_token() != '"') + WIRE_DLOG_THROW_(error::schema::string); + current_.remove_prefix(1); + + void const* const end = std::memchr(current_.data(), '"', current_.size()); + if (!end) + WIRE_DLOG_THROW_(error::rapidjson_e(rapidjson::kParseErrorStringMissQuotationMark)); + + char const* const begin = current_.data(); + const std::size_t length = current_.remove_prefix(static_cast(end) - current_.data() + 1); + return {begin, length - 1}; + } + + void json_reader::skip_value() + { + rapidjson_sax accept_all{error::schema::none}; + read_next_value(accept_all); + } + + json_reader::json_reader(std::string&& source) + : reader(), + source_(std::move(source)), + current_(std::addressof(source_[0]), source_.size()), + reader_() + {} + + void json_reader::check_complete() const + { + if (depth()) + WIRE_DLOG_THROW(error::rapidjson_e(rapidjson::kParseErrorUnspecificSyntaxError), "Unexpected end"); + } + + bool json_reader::boolean() + { + rapidjson_sax json_bool{error::schema::boolean}; + read_next_value(json_bool); + return json_bool.value.boolean; + } + + std::intmax_t json_reader::integer() + { + rapidjson_sax json_int{error::schema::integer}; + read_next_value(json_int); + if (json_int.negative) + return json_int.value.integer; + return integer::convert_to(json_int.value.unsigned_integer); + } + + std::uintmax_t json_reader::unsigned_integer() + { + rapidjson_sax json_uint{error::schema::integer}; + read_next_value(json_uint); + if (!json_uint.negative) + return json_uint.value.unsigned_integer; + return integer::convert_to(json_uint.value.integer); + } + /* + const std::vector& json_reader::unsigned_integer_array() + { + read_next_unsigned_array( + }*/ + + std::uintmax_t json_reader::safe_unsigned_integer() + { + if (get_next_token() != '"') + WIRE_DLOG_THROW_(error::schema::string); + current_.remove_prefix(1); + + const std::uintmax_t out = unsigned_integer(); + + if (get_next_token() != '"') + WIRE_DLOG_THROW_(error::rapidjson_e(rapidjson::kParseErrorStringMissQuotationMark)); + current_.remove_prefix(1); + + return out; + } + + double json_reader::real() + { + rapidjson_sax json_number{error::schema::number}; + read_next_value(json_number); + return json_number.value.number; + } + + std::string json_reader::string() + { + rapidjson_sax json_string{error::schema::string}; + read_next_value(json_string); + return std::string{json_string.value.string.ptr, json_string.value.string.length}; + } + + std::vector json_reader::binary() + { + const boost::string_ref value = get_next_string(); + + std::vector out; + out.resize(value.size() / 2); + + if (!epee::from_hex::to_buffer(epee::to_mut_span(out), value)) + WIRE_DLOG_THROW_(error::schema::binary); + + return out; + } + + void json_reader::binary(epee::span dest) + { + const boost::string_ref value = get_next_string(); + if (!epee::from_hex::to_buffer(dest, value)) + WIRE_DLOG_THROW(error::schema::fixed_binary, "of size" << dest.size() * 2 << " but got " << value.size()); + } + + std::size_t json_reader::enumeration(epee::span enums) + { + rapidjson_sax json_enum{error::schema::string}; + read_next_value(json_enum); + + const boost::string_ref value{json_enum.value.string.ptr, json_enum.value.string.length}; + for (std::size_t i = 0; i < enums.size(); ++i) + { + if (value == enums[i]) + return i; + } + + WIRE_DLOG_THROW(error::schema::enumeration, value << " is not a valid enum"); + return enums.size(); + } + + std::size_t json_reader::start_array() + { + if (get_next_token() != '[') + WIRE_DLOG_THROW_(error::schema::array); + current_.remove_prefix(1); + increment_depth(); + return 0; + } + + bool json_reader::is_array_end(const std::size_t count) + { + const char next = get_next_token(); + if (next == 0) + WIRE_DLOG_THROW_(error::rapidjson_e(rapidjson::kParseErrorArrayMissCommaOrSquareBracket)); + if (next == ']') + { + current_.remove_prefix(1); + return true; + } + + if (count) + { + if (next != ',') + WIRE_DLOG_THROW_(error::rapidjson_e(rapidjson::kParseErrorArrayMissCommaOrSquareBracket)); + current_.remove_prefix(1); + } + return false; + } + + std::size_t json_reader::start_object() + { + if (get_next_token() != '{') + WIRE_DLOG_THROW_(error::schema::object); + current_.remove_prefix(1); + increment_depth(); + return 0; + } + + bool json_reader::key(const epee::span map, std::size_t& state, std::size_t& index) + { + rapidjson_sax json_key{error::schema::string}; + const auto process_key = [map] (const rapidjson_sax::string_contents value) + { + const boost::string_ref key{value.ptr, value.length}; + for (std::size_t i = 0; i < map.size(); ++i) + { + if (map[i].name == key) + return i; + } + return map.size(); + }; + + index = map.size(); + for (;;) + { + // check for object or text end + const char next = get_next_token(); + if (next == 0) + WIRE_DLOG_THROW_(error::rapidjson_e(rapidjson::kParseErrorObjectMissCommaOrCurlyBracket)); + if (next == '}') + { + current_.remove_prefix(1); + return false; + } + + // parse next field token + if (state) + { + if (next != ',') + WIRE_DLOG_THROW_(error::rapidjson_e(rapidjson::kParseErrorObjectMissCommaOrCurlyBracket)); + current_.remove_prefix(1); + } + ++state; + + // parse key + read_next_value(json_key); + index = process_key(json_key.value.string); + if (get_next_token() != ':') + WIRE_DLOG_THROW_(error::rapidjson_e(rapidjson::kParseErrorObjectMissColon)); + current_.remove_prefix(1); + + // parse value + if (index != map.size()) + break; + skip_value(); + } + return true; + } +} + diff --git a/src/wire/json/read.h b/src/wire/json/read.h new file mode 100644 index 0000000..a9bfd7e --- /dev/null +++ b/src/wire/json/read.h @@ -0,0 +1,134 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "wire/field.h" +#include "wire/json/base.h" +#include "wire/read.h" +#include "wire/traits.h" + +namespace wire +{ + //! Reads JSON tokens one-at-a-time for DOMless parsing + class json_reader : public reader + { + struct rapidjson_sax; + + std::string source_; + epee::span current_; + rapidjson::Reader reader_; + + void read_next_value(rapidjson_sax& handler); + char get_next_token(); + boost::string_ref get_next_string(); + + //! Skips next value. \throw wire::exception if invalid JSON syntax. + void skip_value(); + + public: + explicit json_reader(std::string&& source); + + //! \throw wire::exception if JSON parsing is incomplete. + void check_complete() const override final; + + //! \throw wire::exception if next token not a boolean. + bool boolean() override final; + + //! \throw wire::expception if next token not an integer. + std::intmax_t integer() override final; + + //! \throw wire::exception if next token not an unsigned integer. + std::uintmax_t unsigned_integer() override final; + + //! \throw wire::exception if next token is not an integer encoded as string + std::uintmax_t safe_unsigned_integer(); + + //! \throw wire::exception if next token not a valid real number + double real() override final; + + //! \throw wire::exception if next token not a string + std::string string() override final; + + //! \throw wire::exception if next token cannot be read as hex + std::vector binary() override final; + + //! \throw wire::exception if next token cannot be read as hex into `dest`. + void binary(epee::span dest) override final; + + //! \throw wire::exception if invalid next token invalid enum. \return Index in `enums`. + std::size_t enumeration(epee::span enums) override final; + + + //! \throw wire::exception if next token not `[`. + std::size_t start_array() override final; + + //! Skips whitespace to next token. \return True if next token is eof or ']'. + bool is_array_end(std::size_t count) override final; + + + //! \throw wire::exception if next token not `{`. + std::size_t start_object() override final; + + /*! \throw wire::exception if next token not key or `}`. + \param[out] index of key match within `map`. + \return True if another value to read. */ + bool key(epee::span map, std::size_t&, std::size_t& index) override final; + }; + + + // Don't call `read` directly in this namespace, do it from `wire_read`. + + template + expect json::from_bytes(std::string&& bytes) + { + json_reader source{std::move(bytes)}; + return wire_read::to(source); + } + + // specialization prevents type "downgrading" to base type in cpp files + + template + inline void array(json_reader& source, T& dest) + { + wire_read::array(source, dest); + } + + template + inline void object(json_reader& source, T... fields) + { + wire_read::object(source, wire_read::tracker{std::move(fields)}...); + } +} // wire diff --git a/src/wire/json/write.cpp b/src/wire/json/write.cpp new file mode 100644 index 0000000..bef2164 --- /dev/null +++ b/src/wire/json/write.cpp @@ -0,0 +1,167 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "write.h" + +#include + +#include "hex.h" // monero/contrib/epee/include + +namespace +{ + constexpr const unsigned flush_threshold = 100; +} + +namespace wire +{ + void json_writer::do_flush(epee::span) + {} + + void json_writer::check_flush() + { + if (needs_flush_ && (bytes_.increase_size() < flush_threshold || bytes_.increase_size() - flush_threshold < bytes_.size())) + flush(); + } + + void json_writer::check_complete() + { + if (!formatter_.IsComplete()) + throw std::logic_error{"json_writer::take_json() failed with incomplete JSON tree"}; + } + epee::byte_slice json_writer::take_json() + { + check_complete(); + epee::byte_slice out{std::move(bytes_)}; + formatter_.Reset(bytes_); + return out; + } + + json_writer::~json_writer() noexcept + {} + + std::array json_writer::to_string(const std::uintmax_t value) noexcept + { + static_assert(std::numeric_limits::max() <= std::numeric_limits::max(), "bad uint conversion"); + std::array buf{{}}; + rapidjson::internal::u64toa(std::uint64_t(value), buf.data()); + return buf; + } + + void json_writer::integer(const int source) + { + formatter_.Int(source); + check_flush(); + } + void json_writer::integer(const std::intmax_t source) + { + static_assert(std::numeric_limits::min() <= std::numeric_limits::min(), "too small"); + static_assert(std::numeric_limits::max() <= std::numeric_limits::max(), "too large"); + formatter_.Int64(source); + check_flush(); + } + void json_writer::unsigned_integer(const unsigned source) + { + formatter_.Uint(source); + check_flush(); + } + void json_writer::unsigned_integer(const std::uintmax_t source) + { + static_assert(std::numeric_limits::max() <= std::numeric_limits::max(), "too large"); + formatter_.Uint64(source); + check_flush(); + } + void json_writer::real(const double source) + { + formatter_.Double(source); + check_flush(); + } + + void json_writer::string(const boost::string_ref source) + { + formatter_.String(source.data(), source.size()); + check_flush(); + } + void json_writer::binary(epee::span source) + {/* TODO update monero project + std::array buffer; + if (source.size() <= buffer.size() / 2) + { + if (!epee::to_hex::buffer({buffer.data(), source.size() * 2}, source)) + throw std::logic_error{"Invalid buffer size for binary->hex conversion"}; + string({buffer.data(), source.size() * 2}); + } + else + {*/ + const auto hex = epee::to_hex::string(source); + string(hex); + //} + } + + void json_writer::enumeration(const std::size_t index, const epee::span enums) + { + if (enums.size() < index) + throw std::logic_error{"Invalid enum/string value"}; + string({enums[index], std::strlen(enums[index])}); + } + + void json_writer::start_array(std::size_t) + { + formatter_.StartArray(); + } + void json_writer::end_array() + { + formatter_.EndArray(); + } + + void json_writer::start_object(std::size_t) + { + formatter_.StartObject(); + } + void json_writer::key(const boost::string_ref str) + { + formatter_.Key(str.data(), str.size()); + check_flush(); + } + void json_writer::key(const std::uintmax_t id) + { + auto str = json_writer::to_string(id); + key(str.data()); + } + void json_writer::key(unsigned, const boost::string_ref str) + { + key(str); + } + void json_writer::end_object() + { + formatter_.EndObject(); + } + + void json_stream_writer::do_flush(epee::span bytes) + { + dest.write(reinterpret_cast(bytes.data()), bytes.size()); + } +} diff --git a/src/wire/json/write.h b/src/wire/json/write.h new file mode 100644 index 0000000..bea13f7 --- /dev/null +++ b/src/wire/json/write.h @@ -0,0 +1,184 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include +#include +#include +#include + +#include "byte_stream.h" // monero/contrib/epee/include +#include "span.h" // monero/contrib/epee/include +#include "wire/field.h" +#include "wire/filters.h" +#include "wire/json/base.h" +#include "wire/traits.h" +#include "wire/write.h" + +namespace wire +{ + constexpr const std::size_t uint_to_string_size = + std::numeric_limits::digits10 + 2; + + //! Writes JSON tokens one-at-a-time for DOMless output. + class json_writer : public writer + { + epee::byte_stream bytes_; + rapidjson::Writer formatter_; + bool needs_flush_; + + //! \return True if buffer needs to be cleared + virtual void do_flush(epee::span); + + //! Flush written bytes to `do_flush(...)` if configured + void check_flush(); + + protected: + json_writer(bool needs_flush) + : writer(), bytes_(), formatter_(bytes_), needs_flush_(needs_flush) + {} + + //! \throw std::logic_error if incomplete JSON tree + void check_complete(); + + //! \throw std::logic_error if incomplete JSON tree. \return JSON bytes + epee::byte_slice take_json(); + + //! Flush bytes in local buffer to `do_flush(...)` + void flush() + { + do_flush({bytes_.data(), bytes_.size()}); + bytes_ = epee::byte_stream{}; // TODO create .clear() method in monero project + } + + public: + json_writer(const json_writer&) = delete; + virtual ~json_writer() noexcept; + json_writer& operator=(const json_writer&) = delete; + + //! \return Null-terminated buffer containing uint as decimal ascii + static std::array to_string(std::uintmax_t) noexcept; + + void integer(int) override final; + void integer(std::intmax_t) override final; + + void unsigned_integer(unsigned) override final; + void unsigned_integer(std::uintmax_t) override final; + + void real(double) override final; + + void string(boost::string_ref) override final; + void binary(epee::span source) override final; + + void enumeration(std::size_t index, epee::span enums) override final; + + void start_array(std::size_t) override final; + void end_array() override final; + + void start_object(std::size_t) override final; + void key(std::uintmax_t) override final; + void key(boost::string_ref) override final; + void key(unsigned, boost::string_ref) override final; + void end_object() override final; + }; + + //! Buffers entire JSON message in memory + struct json_slice_writer final : json_writer + { + explicit json_slice_writer() + : json_writer(false) + {} + + //! \throw std::logic_error if incomplete JSON tree \return JSON bytes + epee::byte_slice take_bytes() + { + return json_writer::take_json(); + } + }; + + //! Periodically flushes JSON data to `std::ostream` + class json_stream_writer final : public json_writer + { + std::ostream& dest; + + virtual void do_flush(epee::span) override final; + public: + explicit json_stream_writer(std::ostream& dest) + : json_writer(true), dest(dest) + {} + + //! Flush remaining bytes to stream \throw std::logic_error if incomplete JSON tree + void finish() + { + check_complete(); + flush(); + } + }; + + template + epee::byte_slice json::to_bytes(const T& source) + { + return wire_write::to_bytes(source); + } + + template + inline void array(json_writer& dest, const T& source, F filter = F{}) + { + // works with "lazily" computed ranges + wire_write::array(dest, source, 0, std::move(filter)); + } + template + inline void write_bytes(json_writer& dest, as_array_ source) + { + wire::array(dest, source.get_value(), std::move(source.filter)); + } + template + inline enable_if::value> write_bytes(json_writer& dest, const T& source) + { + wire::array(dest, source); + } + + template + inline void dynamic_object(json_writer& dest, const T& source, F key_filter = F{}, G value_filter = G{}) + { + // works with "lazily" computed ranges + wire_write::dynamic_object(dest, source, 0, std::move(key_filter), std::move(value_filter)); + } + template + inline void write_bytes(json_writer& dest, as_object_ source) + { + wire::dynamic_object(dest, source.get_map(), std::move(source.key_filter), std::move(source.value_filter)); + } + + template + inline void object(json_writer& dest, T... fields) + { + wire_write::object(dest, std::move(fields)...); + } +} diff --git a/src/wire/read.cpp b/src/wire/read.cpp new file mode 100644 index 0000000..85d2977 --- /dev/null +++ b/src/wire/read.cpp @@ -0,0 +1,61 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "wire/read.h" + +#include + +void wire::reader::increment_depth() +{ + if (++depth_ == max_read_depth()) + WIRE_DLOG_THROW_(error::schema::maximum_depth); +} + +[[noreturn]] void wire::integer::throw_exception(std::intmax_t source, std::intmax_t min) +{ + WIRE_DLOG_THROW(error::schema::larger_integer, source << " given when " << min << " is minimum permitted"); +} +[[noreturn]] void wire::integer::throw_exception(std::uintmax_t source, std::uintmax_t max) +{ + WIRE_DLOG_THROW(error::schema::smaller_integer, source << " given when " << max << "is maximum permitted"); +} + +[[noreturn]] void wire_read::throw_exception(const wire::error::schema code, const char* display, epee::span names) +{ + const char* name = nullptr; + for (const char* elem : names) + { + if (elem != nullptr) + { + name = elem; + break; + } + } + WIRE_DLOG_THROW(code, display << (name ? name : "")); +} + + diff --git a/src/wire/read.h b/src/wire/read.h new file mode 100644 index 0000000..bc5ce1b --- /dev/null +++ b/src/wire/read.h @@ -0,0 +1,471 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "common/expect.h" // monero/src +#include "span.h" // monero/contrib/epee/include +#include "wire/error.h" +#include "wire/field.h" +#include "wire/traits.h" + +namespace wire +{ + //! Interface for converting "wire" (byte) formats to C/C++ objects without a DOM. + class reader + { + std::size_t depth_; //!< Tracks number of recursive objects and arrays + + protected: + //! \throw wire::exception if max depth is reached + void increment_depth(); + void decrement_depth() noexcept { --depth_; } + + reader(const reader&) = default; + reader(reader&&) = default; + reader& operator=(const reader&) = default; + reader& operator=(reader&&) = default; + + public: + struct key_map + { + const char* name; + unsigned id; // binary() = 0; + + //! \throw wire::exception if next value cannot be read as binary into `dest`. + virtual void binary(epee::span dest) = 0; + + //! \throw wire::exception if next value invalid enum. \return Index in `enums`. + virtual std::size_t enumeration(epee::span enums) = 0; + + /*! \throw wire::exception if next value not array + \return Number of values to read before calling `is_array_end()`. */ + virtual std::size_t start_array() = 0; + + //! \return True if there is another element to read. + virtual bool is_array_end(std::size_t count) = 0; + + //! \throw wire::exception if array end delimiter not present. + void end_array() noexcept { decrement_depth(); } + + + //! \throw wire::exception if not object begin. \return State to be given to `key(...)` function. + virtual std::size_t start_object() = 0; + + /*! Read a key of an object field and match against a known list of keys. + Skips or throws exceptions on unknown fields depending on implementation + settings. + + \param map of known keys (strings and integer) that are valid. + \param[in,out] state returned by `start_object()` or `key(...)` whichever + was last. + \param[out] index of match found in `map`. + + \throw wire::exception if next value not a key. + \throw wire::exception if next key not found in `map` and skipping + fields disabled. + + \return True if this function found a field in `map` to process. + */ + virtual bool key(epee::span map, std::size_t& state, std::size_t& index) = 0; + + void end_object() noexcept { decrement_depth(); } + }; + + inline void read_bytes(reader& source, bool& dest) + { + dest = source.boolean(); + } + + inline void read_bytes(reader& source, double& dest) + { + dest = source.real(); + } + + inline void read_bytes(reader& source, std::string& dest) + { + dest = source.string(); + } + + template + inline void read_bytes(R& source, std::vector& dest) + { + dest = source.binary(); + } + + template + inline enable_if::value> read_bytes(reader& source, T& dest) + { + source.binary(epee::as_mut_byte_span(dest)); + } + + namespace integer + { + [[noreturn]] void throw_exception(std::intmax_t source, std::intmax_t min); + [[noreturn]] void throw_exception(std::uintmax_t source, std::uintmax_t max); + + template + inline Target convert_to(const U source) + { + using common = typename std::common_type::type; + static constexpr const Target target_min = std::numeric_limits::min(); + static constexpr const Target target_max = std::numeric_limits::max(); + + /* After optimizations, this is: + * 1 check for unsigned -> unsigned (uint, uint) + * 2 checks for signed -> signed (int, int) + * 2 checks for signed -> unsigned-- ( + * 1 check for unsigned -> signed (uint, uint) + + Put `WIRE_DLOG_THROW` in cpp to reduce code/ASM duplication. Do not + remove first check, signed values can be implicitly converted to + unsigned in some checks. */ + if (!std::numeric_limits::is_signed && source < 0) + throw_exception(std::intmax_t(source), std::intmax_t(0)); + else if (common(source) < common(target_min)) + throw_exception(std::intmax_t(source), std::intmax_t(target_min)); + else if (common(target_max) < common(source)) + throw_exception(std::uintmax_t(source), std::uintmax_t(target_max)); + + return Target(source); + } + } + + inline void read_bytes(reader& source, char& dest) + { + dest = integer::convert_to(source.integer()); + } + inline void read_bytes(reader& source, short& dest) + { + dest = integer::convert_to(source.integer()); + } + inline void read_bytes(reader& source, int& dest) + { + dest = integer::convert_to(source.integer()); + } + inline void read_bytes(reader& source, long& dest) + { + dest = integer::convert_to(source.integer()); + } + inline void read_bytes(reader& source, long long& dest) + { + dest = integer::convert_to(source.integer()); + } + + inline void read_bytes(reader& source, unsigned char& dest) + { + dest = integer::convert_to(source.unsigned_integer()); + } + inline void read_bytes(reader& source, unsigned short& dest) + { + dest = integer::convert_to(source.unsigned_integer()); + } + inline void read_bytes(reader& source, unsigned& dest) + { + dest = integer::convert_to(source.unsigned_integer()); + } + inline void read_bytes(reader& source, unsigned long& dest) + { + dest = integer::convert_to(source.unsigned_integer()); + } + inline void read_bytes(reader& source, unsigned long long& dest) + { + dest = integer::convert_to(source.unsigned_integer()); + } +} // wire + +namespace wire_read +{ + /*! Don't add a function called `read_bytes` to this namespace, it will prevent + ADL lookup. ADL lookup delays the function searching until the template + is used instead of when its defined. This allows the unqualified calls to + `read_bytes` in this namespace to "find" user functions that are declared + after these functions (the technique behind `boost::serialization`). */ + + [[noreturn]] void throw_exception(wire::error::schema code, const char* display, epee::span name_list); + + //! \return `T` converted from `source` or error. + template + inline expect to(R& source) + { + try + { + T dest{}; + read_bytes(source, dest); + source.check_complete(); + return dest; + } + catch (const wire::exception& e) + { + return e.code(); + } + } + + template + inline void array(R& source, T& dest) + { + using value_type = typename T::value_type; + static_assert(!std::is_same::value, "read array of chars as binary"); + static_assert(!std::is_same::value, "read array of unsigned chars as binary"); + + std::size_t count = source.start_array(); + + dest.clear(); + dest.reserve(count); + + bool more = count; + while (more || !source.is_array_end(count)) + { + dest.emplace_back(); + read_bytes(source, dest.back()); + --count; + more &= bool(count); + } + + return source.end_array(); + } + + // `unpack_variant_field` identifies which of the variant types was selected. starts with index-0 + + template + inline void unpack_variant_field(std::size_t, R&, const T&) + {} + + template + inline void unpack_variant_field(const std::size_t index, R& source, T& variant, const wire::option& head, const wire::option&... tail) + { + if (index) + unpack_variant_field(index - 1, source, variant, tail...); + else + { + U dest{}; + read_bytes(source, dest); + variant = std::move(dest); + } + } + + // `unpack_field` expands `variant_field_`s or reads `field_`s directly + + template + inline void unpack_field(const std::size_t index, R& source, wire::variant_field_& dest) + { + unpack_variant_field(index, source, dest.get_value(), static_cast< const wire::option& >(dest)...); + } + + template + inline void unpack_field(std::size_t, R& source, wire::field_& dest) + { + read_bytes(source, dest.get_value()); + } + + template + inline void unpack_field(std::size_t, R& source, wire::field_& dest) + { + dest.get_value().emplace(); + read_bytes(source, *dest.get_value()); + } + + // `expand_field_map` writes a single `field_` name or all option names in a `variant_field_` to a table + + template + inline void expand_field_map(std::size_t, wire::reader::key_map (&)[N]) + {} + + template + inline void expand_field_map(std::size_t index, wire::reader::key_map (&map)[N], const T& head, const U&... tail) + { + map[index].name = head.name; + map[index].id = 0; + expand_field_map(index + 1, map, tail...); + } + + template + inline void expand_field_map(std::size_t index, wire::reader::key_map (&map)[N], const wire::variant_field_& field) + { + expand_field_map(index, map, static_cast< const wire::option & >(field)...); + } + + //! Tracks read status of every object field instance. + template + class tracker + { + T field_; + std::size_t our_index_; + bool read_; + + public: + static constexpr bool is_required() noexcept { return T::is_required(); } + static constexpr std::size_t count() noexcept { return T::count(); } + + explicit tracker(T field) + : field_(std::move(field)), our_index_(0), read_(false) + {} + + //! \return Field name if required and not read, otherwise `nullptr`. + const char* name_if_missing() const noexcept + { + return (is_required() && !read_) ? field_.name : nullptr; + } + + + //! Set all entries in `map` related to this field (expand variant types!). + template + std::size_t set_mapping(std::size_t index, wire::reader::key_map (&map)[N]) + { + our_index_ = index; + expand_field_map(index, map, field_); // expands possible inner options + return index + count(); + } + + //! Try to read next value if `index` matches `this`. \return 0 if no match, 1 if optional field read, and 2 if required field read + template + std::size_t try_read(R& source, const std::size_t index) + { + if (index < our_index_ || our_index_ + count() <= index) + return 0; + if (read_) + throw_exception(wire::error::schema::invalid_key, "duplicate", {std::addressof(field_.name), 1}); + + unpack_field(index - our_index_, source, field_); + read_ = true; + return 1 + is_required(); + } + }; + + // `expand_tracker_map` writes all `tracker` types to a table + + template + inline constexpr std::size_t expand_tracker_map(std::size_t index, const wire::reader::key_map (&)[N]) + { + return index; + } + + template + inline void expand_tracker_map(std::size_t index, wire::reader::key_map (&map)[N], tracker& head, tracker&... tail) + { + expand_tracker_map(head.set_mapping(index, map), map, tail...); + } + + template + inline void object(R& source, tracker... fields) + { + static constexpr const std::size_t total_subfields = wire::sum(fields.count()...); + static_assert(total_subfields < 100, "algorithm uses too much stack space and linear searching"); + + std::size_t state = source.start_object(); + std::size_t required = wire::sum(std::size_t(fields.is_required())...); + + wire::reader::key_map map[total_subfields] = {}; + expand_tracker_map(0, map, fields...); + + std::size_t next = 0; + while (source.key(map, state, next)) + { + switch (wire::sum(fields.try_read(source, next)...)) + { + default: + case 0: + throw_exception(wire::error::schema::invalid_key, "bad map setup", nullptr); + break; + case 2: + --required; /* fallthrough */ + case 1: + break; + } + } + + if (required) + { + const char* missing[] = {fields.name_if_missing()...}; + throw_exception(wire::error::schema::missing_key, "", missing); + } + + source.end_object(); + } +} // wire_read + +namespace wire +{ + template + inline void array(reader& source, T& dest) + { + wire_read::array(source, dest); + } + template + inline enable_if::value> read_bytes(R& source, T& dest) + { + wire_read::array(source, dest); + } + + template + inline void object(reader& source, T... fields) + { + wire_read::object(source, wire_read::tracker{std::move(fields)}...); + } +} diff --git a/src/wire/traits.h b/src/wire/traits.h new file mode 100644 index 0000000..29b0542 --- /dev/null +++ b/src/wire/traits.h @@ -0,0 +1,46 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include + +namespace wire +{ + template + using enable_if = typename std::enable_if::type; + + template + struct is_array : std::false_type + {}; + + template + struct is_blob : std::false_type + {}; +} + diff --git a/src/wire/vector.h b/src/wire/vector.h new file mode 100644 index 0000000..fde8743 --- /dev/null +++ b/src/wire/vector.h @@ -0,0 +1,41 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include + +#include "wire/traits.h" + +namespace wire +{ + template + struct is_array> + : std::true_type + {}; +} diff --git a/src/wire/write.cpp b/src/wire/write.cpp new file mode 100644 index 0000000..96361ea --- /dev/null +++ b/src/wire/write.cpp @@ -0,0 +1,31 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "write.h" + +wire::writer::~writer() noexcept +{} diff --git a/src/wire/write.h b/src/wire/write.h new file mode 100644 index 0000000..a8b2c06 --- /dev/null +++ b/src/wire/write.h @@ -0,0 +1,224 @@ +// Copyright (c) 2020, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include +#include +#include + +#include "byte_slice.h" // monero/contrib/epee/include +#include "span.h" // monero/contrib/epee/include +#include "wire/field.h" +#include "wire/filters.h" +#include "wire/traits.h" + +namespace wire +{ + //! Interface for converting C/C++ objects to "wire" (byte) formats. + struct writer + { + writer() = default; + + virtual ~writer() noexcept; + + virtual void integer(int) = 0; + virtual void integer(std::intmax_t) = 0; + + virtual void unsigned_integer(unsigned) = 0; + virtual void unsigned_integer(std::uintmax_t) = 0; + + virtual void real(double) = 0; + + virtual void string(boost::string_ref) = 0; + virtual void binary(epee::span bytes) = 0; + + virtual void enumeration(std::size_t index, epee::span enums) = 0; + + virtual void start_array(std::size_t) = 0; + virtual void end_array() = 0; + + virtual void start_object(std::size_t) = 0; + virtual void key(std::uintmax_t) = 0; + virtual void key(boost::string_ref) = 0; + virtual void key(unsigned, boost::string_ref) = 0; //!< Implementation should output fastest key + virtual void end_object() = 0; + + protected: + writer(const writer&) = default; + writer(writer&&) = default; + writer& operator=(const writer&) = default; + writer& operator=(writer&&) = default; + }; + + // leave in header, compiler can de-virtualize when final type is given + + inline void write_bytes(writer& dest, const int source) + { + dest.integer(source); + } + inline void write_bytes(writer& dest, const std::intmax_t source) + { + dest.integer(source); + } + + inline void write_bytes(writer& dest, const unsigned source) + { + dest.unsigned_integer(source); + } + inline void write_bytes(writer& dest, const std::uintmax_t source) + { + dest.unsigned_integer(source); + } + + inline void write_bytes(writer& dest, const double source) + { + dest.real(source); + } + + inline void write_bytes(writer& dest, const boost::string_ref source) + { + dest.string(source); + } + + template + inline enable_if::value> write_bytes(writer& dest, const T& source) + { + dest.binary(epee::as_byte_span(source)); + } + + inline void write_bytes(writer& dest, const epee::span source) + { + dest.binary(source); + } +} + +namespace wire_write +{ + /*! Don't add a function called `write_bytes` to this namespace, it will + prevent ADL lookup. ADL lookup delays the function searching until the + template is used instead of when its defined. This allows the unqualified + calls to `write_bytes` in this namespace to "find" user functions that are + declared after these functions. */ + + template + inline epee::byte_slice to_bytes(const T& value) + { + W dest{}; + write_bytes(dest, value); + return dest.take_bytes(); + } + + template + inline void array(W& dest, const T& source, const std::size_t count, F filter = F{}) + { + using value_type = typename T::value_type; + static_assert(!std::is_same::value, "write array of chars as binary"); + static_assert(!std::is_same::value, "write array of unsigned chars as binary"); + + dest.start_array(count); + for (const auto& elem : source) + write_bytes(dest, filter(elem)); + dest.end_array(); + } + + template + inline bool field(W& dest, const wire::field_ elem) + { + dest.key(0, elem.name); + write_bytes(dest, elem.get_value()); + return true; + } + + template + inline bool field(W& dest, const wire::field_ elem) + { + if (bool(elem.get_value())) + { + dest.key(0, elem.name); + write_bytes(dest, *elem.get_value()); + } + return true; + } + + template + inline void object(W& dest, T... fields) + { + dest.start_object(wire::sum(std::size_t(wire::available(fields))...)); + const bool dummy[] = {field(dest, std::move(fields))...}; + dest.end_object(); + } + + template + inline void dynamic_object(W& dest, const T& values, const std::size_t count, F key_filter, G value_filter) + { + dest.start_object(count); + for (const auto& elem : values) + { + dest.key(key_filter(elem.first)); + write_bytes(dest, value_filter(elem.second)); + } + dest.end_object(); + } +} // wire_write + +namespace wire +{ + template + inline void array(writer& dest, const T& source, F filter = F{}) + { + wire_write::array(dest, source, source.size(), std::move(filter)); + } + template + inline void write_bytes(writer& dest, as_array_ source) + { + wire::array(dest, source.get_value(), std::move(source.filter)); + } + template + inline enable_if::value> write_bytes(writer& dest, const T& source) + { + wire::array(dest, source); + } + + template + inline void dynamic_object(writer& dest, const T& source, F key_filter = F{}, G value_filter = G{}) + { + wire_write::dynamic_object(dest, source, source.size(), std::move(key_filter), std::move(value_filter)); + } + template + inline void write_bytes(writer& dest, as_object_ source) + { + wire::dynamic_object(dest, source.get_map(), std::move(source.key_filter), std::move(source.value_filter)); + } + + template + inline void object(writer& dest, T... fields) + { + wire_write::object(dest, std::move(fields)...); + } +}