diff --git a/src/fcmp_pp/CMakeLists.txt b/src/fcmp_pp/CMakeLists.txt index 56989b087..aba6ceafd 100644 --- a/src/fcmp_pp/CMakeLists.txt +++ b/src/fcmp_pp/CMakeLists.txt @@ -52,6 +52,8 @@ endif() target_link_libraries(fcmp_pp PUBLIC cncrypto + common + epee PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/fcmp_pp_rust/libfcmp_pp_rust.a ${EXTRA_LIBRARIES} diff --git a/src/fcmp_pp/curve_trees.cpp b/src/fcmp_pp/curve_trees.cpp index 49650cd12..3c2885194 100644 --- a/src/fcmp_pp/curve_trees.cpp +++ b/src/fcmp_pp/curve_trees.cpp @@ -28,11 +28,15 @@ #include "curve_trees.h" +#include "common/threadpool.h" #include "crypto/crypto.h" #include "fcmp_pp_crypto.h" #include "fcmp_pp_types.h" #include "profile_tools.h" +#include +#include + namespace { // Struct composed of ec elems needed to get a full-fledged leaf tuple @@ -154,5 +158,199 @@ static PreLeafTuple output_to_pre_leaf_tuple(const OutputPair &output_pair) } //---------------------------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------------------------- +// CurveTrees private member functions +//---------------------------------------------------------------------------------------------------------------------- +template +void CurveTrees::outputs_to_leaves(std::vector &&new_outputs, + std::vector &flattened_leaves_out, + std::vector &valid_outputs_out) const +{ + flattened_leaves_out.clear(); + valid_outputs_out.clear(); + + TIME_MEASURE_START(set_valid_leaves); + + // Keep track of valid outputs to make sure we only use leaves from valid outputs. Can't use std::vector + // because std::vector concurrent access is not thread safe. + enum Boolean : uint8_t { + False = 0, + True = 1, + }; + std::vector valid_outputs(new_outputs.size(), False); + + tools::threadpool& tpool = tools::threadpool::getInstanceForCompute(); + tools::threadpool::waiter waiter(tpool); + const std::size_t n_threads = std::max(1, tpool.get_max_concurrency()); + + TIME_MEASURE_START(convert_valid_leaves); + // Step 1. Multithreaded convert valid outputs into Edwards derivatives needed to get Wei coordinates + std::vector pre_leaves; + pre_leaves.resize(new_outputs.size()); + const std::size_t LEAF_CONVERT_BATCH_SIZE = std::max(1, (new_outputs.size() / n_threads)); + for (std::size_t i = 0; i < new_outputs.size(); i += LEAF_CONVERT_BATCH_SIZE) + { + const std::size_t end = std::min(i + LEAF_CONVERT_BATCH_SIZE, new_outputs.size()); + tpool.submit(&waiter, + [ + &new_outputs, + &valid_outputs, + &pre_leaves, + i, + end + ]() + { + for (std::size_t j = i; j < end; ++j) + { + CHECK_AND_ASSERT_THROW_MES(!valid_outputs.at(j), "unexpected already valid output"); + + const auto &output_pair = new_outputs.at(j).output_pair; + try + { + pre_leaves.at(j) = output_to_pre_leaf_tuple(output_pair); + } + catch(...) + { + /* Invalid outputs can't be added to the tree */ + LOG_PRINT_L2("Output " << new_outputs.at(j).unified_id << " is invalid (out pubkey " + << output_pubkey_cref(output_pair) + << " , commitment " << commitment_cref(output_pair) << ")"); + continue; + } + + valid_outputs.at(j) = True; + } + }, + true + ); + } + + CHECK_AND_ASSERT_THROW_MES(waiter.wait(), "failed to convert outputs to ed derivatives"); + TIME_MEASURE_FINISH(convert_valid_leaves); + + TIME_MEASURE_START(collect_derivatives); + // Step 2. Collect valid Edwards y derivatives + const std::size_t n_valid_outputs = std::count(valid_outputs.begin(), valid_outputs.end(), True); + const std::size_t n_valid_leaf_points = n_valid_outputs * LEAF_TUPLE_POINTS; + + // Collecting [(1+y),(1-y),((1-y)*x)] for batch inversion + std::unique_ptr one_plus_y_vec = std::make_unique(n_valid_leaf_points); + std::unique_ptr fe_batch = std::make_unique(n_valid_leaf_points * 2); + std::unique_ptr batch_inv_res = std::make_unique(n_valid_leaf_points * 2); + + std::size_t valid_i = 0, batch_i = 0; + for (std::size_t i = 0; i < valid_outputs.size(); ++i) + { + if (!valid_outputs[i]) + continue; + + CHECK_AND_ASSERT_THROW_MES(n_valid_leaf_points > valid_i, "unexpected valid_i"); + + auto &pl = pre_leaves.at(i); + + auto &O_derivatives = pl.O_derivatives; + auto &I_derivatives = pl.I_derivatives; + auto &C_derivatives = pl.C_derivatives; + + static_assert(LEAF_TUPLE_POINTS == 3, "unexpected n leaf tuple points"); + + // TODO: avoid copying underlying (tried using pointer to pointers, but wasn't clean) + memcpy(&one_plus_y_vec[valid_i++], &O_derivatives.one_plus_y, sizeof(fe)); + memcpy(&one_plus_y_vec[valid_i++], &I_derivatives.one_plus_y, sizeof(fe)); + memcpy(&one_plus_y_vec[valid_i++], &C_derivatives.one_plus_y, sizeof(fe)); + + memcpy(&fe_batch[batch_i++], &O_derivatives.one_minus_y, sizeof(fe)); + memcpy(&fe_batch[batch_i++], &O_derivatives.one_minus_y_mul_x, sizeof(fe)); + + memcpy(&fe_batch[batch_i++], &I_derivatives.one_minus_y, sizeof(fe)); + memcpy(&fe_batch[batch_i++], &I_derivatives.one_minus_y_mul_x, sizeof(fe)); + + memcpy(&fe_batch[batch_i++], &C_derivatives.one_minus_y, sizeof(fe)); + memcpy(&fe_batch[batch_i++], &C_derivatives.one_minus_y_mul_x, sizeof(fe)); + } + + CHECK_AND_ASSERT_THROW_MES(n_valid_leaf_points == valid_i, "unexpected end valid_i"); + CHECK_AND_ASSERT_THROW_MES((n_valid_leaf_points * 2) == batch_i, "unexpected end batch_i"); + TIME_MEASURE_FINISH(collect_derivatives); + + TIME_MEASURE_START(batch_invert); + // Step 3. Get batch inverse of all valid (1-y)'s and ((1-y)*x)'s + // - Batch inversion is significantly faster than inverting 1 at a time + CHECK_AND_ASSERT_THROW_MES(fe_batch_invert(batch_inv_res.get(), fe_batch.get(), n_valid_leaf_points * 2) == 0, + "failed to batch invert ed derivatives"); + TIME_MEASURE_FINISH(batch_invert); + + TIME_MEASURE_START(get_selene_scalars); + // Step 4. Multithreaded get Wei coordinates and convert to Selene scalars + const std::size_t n_valid_leaf_elems = n_valid_outputs * LEAF_TUPLE_SIZE; + flattened_leaves_out.resize(n_valid_leaf_elems); + CHECK_AND_ASSERT_THROW_MES(flattened_leaves_out.size() == (2 * n_valid_leaf_points), + "unexpected size of flattened leaves"); + + const std::size_t DERIVATION_BATCH_SIZE = std::max(1, (n_valid_leaf_points / n_threads)); + for (std::size_t i = 0; i < n_valid_leaf_points; i += DERIVATION_BATCH_SIZE) + { + const std::size_t end = std::min(n_valid_leaf_points, i + DERIVATION_BATCH_SIZE); + tpool.submit(&waiter, + [ + &batch_inv_res, + &one_plus_y_vec, + &flattened_leaves_out, + i, + end + ]() + { + std::size_t point_idx = i * 2; + for (std::size_t j = i; j < end; ++j) + { + crypto::ec_coord wei_x; + crypto::ec_coord wei_y; + fe_ed_derivatives_to_wei_x_y( + to_bytes(wei_x), + to_bytes(wei_y), + batch_inv_res[point_idx]/*inv_one_minus_y*/, + one_plus_y_vec[j], + batch_inv_res[point_idx+1]/*inv_one_minus_y_mul_x*/ + ); + + flattened_leaves_out[point_idx++] = tower_cycle::selene_scalar_from_bytes(wei_x); + flattened_leaves_out[point_idx++] = tower_cycle::selene_scalar_from_bytes(wei_y); + } + }, + true + ); + } + + CHECK_AND_ASSERT_THROW_MES(waiter.wait(), "failed to convert outputs to wei coords"); + TIME_MEASURE_FINISH(get_selene_scalars); + + // Step 5. Set valid tuples to be stored in the db + valid_outputs_out.reserve(n_valid_outputs); + for (std::size_t i = 0; i < valid_outputs.size(); ++i) + { + if (!valid_outputs[i]) + continue; + + // We can derive leaf tuples from output pairs, so we store just the unified output in the db to save 32 bytes + valid_outputs_out.emplace_back(std::move(new_outputs.at(i))); + } + CHECK_AND_ASSERT_THROW_MES(valid_outputs_out.size() == n_valid_outputs, "unexpected size of valid_outputs_out"); + + TIME_MEASURE_FINISH(set_valid_leaves); + + m_convert_valid_leaves_ms += convert_valid_leaves; + m_collect_derivatives_ms += collect_derivatives; + m_batch_invert_ms += batch_invert; + m_get_selene_scalars_ms += get_selene_scalars; + + m_set_valid_leaves_ms += set_valid_leaves; + + LOG_PRINT_L2("Total time spent setting leaves: " << m_set_valid_leaves_ms / 1000 + << " , converting valid leaves: " << m_convert_valid_leaves_ms / 1000 + << " , collecting derivatives: " << m_collect_derivatives_ms / 1000 + << " , batch invert: " << m_batch_invert_ms / 1000 + << " , get selene scalars: " << m_get_selene_scalars_ms / 1000); +} +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- } //namespace curve_trees } //namespace fcmp_pp diff --git a/src/fcmp_pp/curve_trees.h b/src/fcmp_pp/curve_trees.h index ceb62f354..0861407da 100644 --- a/src/fcmp_pp/curve_trees.h +++ b/src/fcmp_pp/curve_trees.h @@ -30,8 +30,11 @@ #include "crypto/crypto.h" #include "fcmp_pp_types.h" +#include "tower_cycle.h" #include "misc_log_ex.h" +#include +#include namespace fcmp_pp { @@ -42,5 +45,77 @@ namespace curve_trees OutputTuple output_to_tuple(const OutputPair &output_pair); //---------------------------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------------------------- +// This class is useful to help update the curve trees merkle tree without needing to keep the entire tree in memory +// - It requires instantiation with the C1 and C2 curve classes and widths, hardening the tree structure +// - It ties the C1 curve in the tree to the leaf layer (the leaf layer is composed of C1 scalars) +template +class CurveTrees +{ +public: + CurveTrees(std::unique_ptr &&c1, + std::unique_ptr &&c2, + const std::size_t c1_width, + const std::size_t c2_width): + m_c1{std::move(c1)}, + m_c2{std::move(c2)}, + m_c1_width{c1_width}, + m_c2_width{c2_width}, + m_leaf_layer_chunk_width{LEAF_TUPLE_SIZE * c1_width} + { + assert(c1_width > 0); + assert(c2_width > 0); + }; + +//member structs +public: + // Tuple that composes a single leaf in the tree + struct LeafTuple final + { + // Output ed25519 point wei x and y coordinates + typename C1::Scalar O_x; + typename C1::Scalar O_y; + // Key image generator wei x and y coordinates + typename C1::Scalar I_x; + typename C1::Scalar I_y; + // Commitment wei x and y coordinates + typename C1::Scalar C_x; + typename C1::Scalar C_y; + }; + + static const std::size_t LEAF_TUPLE_POINTS = 3; + static constexpr std::size_t LEAF_TUPLE_SIZE = LEAF_TUPLE_POINTS * 2; + + static_assert(sizeof(LeafTuple) == (sizeof(typename C1::Scalar) * LEAF_TUPLE_SIZE), "unexpected LeafTuple size"); + +//member functions +private: + // Multithreaded helper function to convert valid outputs to leaf tuples ready for insertion to the tree & db + void outputs_to_leaves(std::vector &&new_outputs, + std::vector &flattened_leaves_out, + std::vector &valid_outputs_out) const; + +//private state +private: + mutable uint64_t m_set_valid_leaves_ms{0}; + mutable uint64_t m_get_selene_scalars_ms{0}; + mutable uint64_t m_batch_invert_ms{0}; + mutable uint64_t m_collect_derivatives_ms{0}; + mutable uint64_t m_convert_valid_leaves_ms{0}; + +//public member variables +public: + // The curve interfaces + const std::unique_ptr m_c1; + const std::unique_ptr m_c2; + + // The leaf layer has a distinct chunk width than the other layers + const std::size_t m_leaf_layer_chunk_width; + + // The chunk widths of the layers in the tree tied to each curve + const std::size_t m_c1_width; + const std::size_t m_c2_width; +}; +//---------------------------------------------------------------------------------------------------------------------- +//---------------------------------------------------------------------------------------------------------------------- } //namespace curve_trees } //namespace fcmp_pp diff --git a/src/fcmp_pp/fcmp_pp_types.h b/src/fcmp_pp/fcmp_pp_types.h index 80a783931..8dcc1321d 100644 --- a/src/fcmp_pp/fcmp_pp_types.h +++ b/src/fcmp_pp/fcmp_pp_types.h @@ -99,6 +99,25 @@ const crypto::ec_point &commitment_cref(const OutputPair &output_pair); bool output_checked_for_torsion(const OutputPair &output_pair); bool use_biased_hash_to_point(const OutputPair &output_pair); + +// Wrapper for outputs with context to insert the output into the FCMP++ curve tree +struct UnifiedOutput final +{ + // Output's unique id in the chain. + // When a batch of outputs is added to the tree, the batch + // is internally sorted by unique ids, which correspond with + // the order the outputs entered the chain. Batch-added outputs + // may not be globally ordered because e.g. timelocked + // outputs were added to 'late batches'. Batch refers to + // outputs that become unlocked/spendable at the same time. + uint64_t unified_id{0}; + OutputPair output_pair; + + bool operator==(const UnifiedOutput &other) const + { + return unified_id == other.unified_id && output_pair == other.output_pair; + } +}; //---------------------------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------------------------- }//namespace fcmp_pp