mirror of
https://github.com/wownero/wownero.git
synced 2026-10-17 07:31:35 +02:00
FCMP++: CurveTrees class + outputs_to_leaves impl
- The CurveTrees class is a useful object for updating the curve trees merkle tree. It is instantiated with the widths of each layer in the tree, as well as the curve objects themselves. - The outputs_to_leaves function takes in outputs that enter the chain, and converts them into valid leaves prepared to be inserted to the curve trees merkle tree. We do not add invalid outputs to the tree, so outputs_to_leaves checks outputs for validity and only includes the valid outputs in the final response.
This commit is contained in:
j-berman
@@ -52,6 +52,8 @@ endif()
|
||||
target_link_libraries(fcmp_pp
|
||||
PUBLIC
|
||||
cncrypto
|
||||
common
|
||||
epee
|
||||
PRIVATE
|
||||
${CMAKE_CURRENT_BINARY_DIR}/fcmp_pp_rust/libfcmp_pp_rust.a
|
||||
${EXTRA_LIBRARIES}
|
||||
|
||||
@@ -28,11 +28,15 @@
|
||||
|
||||
#include "curve_trees.h"
|
||||
|
||||
#include "common/threadpool.h"
|
||||
#include "crypto/crypto.h"
|
||||
#include "fcmp_pp_crypto.h"
|
||||
#include "fcmp_pp_types.h"
|
||||
#include "profile_tools.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstring>
|
||||
|
||||
namespace
|
||||
{
|
||||
// Struct composed of ec elems needed to get a full-fledged leaf tuple
|
||||
@@ -154,5 +158,199 @@ static PreLeafTuple output_to_pre_leaf_tuple(const OutputPair &output_pair)
|
||||
}
|
||||
//----------------------------------------------------------------------------------------------------------------------
|
||||
//----------------------------------------------------------------------------------------------------------------------
|
||||
// CurveTrees private member functions
|
||||
//----------------------------------------------------------------------------------------------------------------------
|
||||
template<typename C1, typename C2>
|
||||
void CurveTrees<C1, C2>::outputs_to_leaves(std::vector<UnifiedOutput> &&new_outputs,
|
||||
std::vector<typename C1::Scalar> &flattened_leaves_out,
|
||||
std::vector<UnifiedOutput> &valid_outputs_out) const
|
||||
{
|
||||
flattened_leaves_out.clear();
|
||||
valid_outputs_out.clear();
|
||||
|
||||
TIME_MEASURE_START(set_valid_leaves);
|
||||
|
||||
// Keep track of valid outputs to make sure we only use leaves from valid outputs. Can't use std::vector<bool>
|
||||
// because std::vector<bool> concurrent access is not thread safe.
|
||||
enum Boolean : uint8_t {
|
||||
False = 0,
|
||||
True = 1,
|
||||
};
|
||||
std::vector<Boolean> valid_outputs(new_outputs.size(), False);
|
||||
|
||||
tools::threadpool& tpool = tools::threadpool::getInstanceForCompute();
|
||||
tools::threadpool::waiter waiter(tpool);
|
||||
const std::size_t n_threads = std::max<std::size_t>(1, tpool.get_max_concurrency());
|
||||
|
||||
TIME_MEASURE_START(convert_valid_leaves);
|
||||
// Step 1. Multithreaded convert valid outputs into Edwards derivatives needed to get Wei coordinates
|
||||
std::vector<PreLeafTuple> pre_leaves;
|
||||
pre_leaves.resize(new_outputs.size());
|
||||
const std::size_t LEAF_CONVERT_BATCH_SIZE = std::max<std::size_t>(1, (new_outputs.size() / n_threads));
|
||||
for (std::size_t i = 0; i < new_outputs.size(); i += LEAF_CONVERT_BATCH_SIZE)
|
||||
{
|
||||
const std::size_t end = std::min(i + LEAF_CONVERT_BATCH_SIZE, new_outputs.size());
|
||||
tpool.submit(&waiter,
|
||||
[
|
||||
&new_outputs,
|
||||
&valid_outputs,
|
||||
&pre_leaves,
|
||||
i,
|
||||
end
|
||||
]()
|
||||
{
|
||||
for (std::size_t j = i; j < end; ++j)
|
||||
{
|
||||
CHECK_AND_ASSERT_THROW_MES(!valid_outputs.at(j), "unexpected already valid output");
|
||||
|
||||
const auto &output_pair = new_outputs.at(j).output_pair;
|
||||
try
|
||||
{
|
||||
pre_leaves.at(j) = output_to_pre_leaf_tuple(output_pair);
|
||||
}
|
||||
catch(...)
|
||||
{
|
||||
/* Invalid outputs can't be added to the tree */
|
||||
LOG_PRINT_L2("Output " << new_outputs.at(j).unified_id << " is invalid (out pubkey "
|
||||
<< output_pubkey_cref(output_pair)
|
||||
<< " , commitment " << commitment_cref(output_pair) << ")");
|
||||
continue;
|
||||
}
|
||||
|
||||
valid_outputs.at(j) = True;
|
||||
}
|
||||
},
|
||||
true
|
||||
);
|
||||
}
|
||||
|
||||
CHECK_AND_ASSERT_THROW_MES(waiter.wait(), "failed to convert outputs to ed derivatives");
|
||||
TIME_MEASURE_FINISH(convert_valid_leaves);
|
||||
|
||||
TIME_MEASURE_START(collect_derivatives);
|
||||
// Step 2. Collect valid Edwards y derivatives
|
||||
const std::size_t n_valid_outputs = std::count(valid_outputs.begin(), valid_outputs.end(), True);
|
||||
const std::size_t n_valid_leaf_points = n_valid_outputs * LEAF_TUPLE_POINTS;
|
||||
|
||||
// Collecting [(1+y),(1-y),((1-y)*x)] for batch inversion
|
||||
std::unique_ptr<fe[]> one_plus_y_vec = std::make_unique<fe[]>(n_valid_leaf_points);
|
||||
std::unique_ptr<fe[]> fe_batch = std::make_unique<fe[]>(n_valid_leaf_points * 2);
|
||||
std::unique_ptr<fe[]> batch_inv_res = std::make_unique<fe[]>(n_valid_leaf_points * 2);
|
||||
|
||||
std::size_t valid_i = 0, batch_i = 0;
|
||||
for (std::size_t i = 0; i < valid_outputs.size(); ++i)
|
||||
{
|
||||
if (!valid_outputs[i])
|
||||
continue;
|
||||
|
||||
CHECK_AND_ASSERT_THROW_MES(n_valid_leaf_points > valid_i, "unexpected valid_i");
|
||||
|
||||
auto &pl = pre_leaves.at(i);
|
||||
|
||||
auto &O_derivatives = pl.O_derivatives;
|
||||
auto &I_derivatives = pl.I_derivatives;
|
||||
auto &C_derivatives = pl.C_derivatives;
|
||||
|
||||
static_assert(LEAF_TUPLE_POINTS == 3, "unexpected n leaf tuple points");
|
||||
|
||||
// TODO: avoid copying underlying (tried using pointer to pointers, but wasn't clean)
|
||||
memcpy(&one_plus_y_vec[valid_i++], &O_derivatives.one_plus_y, sizeof(fe));
|
||||
memcpy(&one_plus_y_vec[valid_i++], &I_derivatives.one_plus_y, sizeof(fe));
|
||||
memcpy(&one_plus_y_vec[valid_i++], &C_derivatives.one_plus_y, sizeof(fe));
|
||||
|
||||
memcpy(&fe_batch[batch_i++], &O_derivatives.one_minus_y, sizeof(fe));
|
||||
memcpy(&fe_batch[batch_i++], &O_derivatives.one_minus_y_mul_x, sizeof(fe));
|
||||
|
||||
memcpy(&fe_batch[batch_i++], &I_derivatives.one_minus_y, sizeof(fe));
|
||||
memcpy(&fe_batch[batch_i++], &I_derivatives.one_minus_y_mul_x, sizeof(fe));
|
||||
|
||||
memcpy(&fe_batch[batch_i++], &C_derivatives.one_minus_y, sizeof(fe));
|
||||
memcpy(&fe_batch[batch_i++], &C_derivatives.one_minus_y_mul_x, sizeof(fe));
|
||||
}
|
||||
|
||||
CHECK_AND_ASSERT_THROW_MES(n_valid_leaf_points == valid_i, "unexpected end valid_i");
|
||||
CHECK_AND_ASSERT_THROW_MES((n_valid_leaf_points * 2) == batch_i, "unexpected end batch_i");
|
||||
TIME_MEASURE_FINISH(collect_derivatives);
|
||||
|
||||
TIME_MEASURE_START(batch_invert);
|
||||
// Step 3. Get batch inverse of all valid (1-y)'s and ((1-y)*x)'s
|
||||
// - Batch inversion is significantly faster than inverting 1 at a time
|
||||
CHECK_AND_ASSERT_THROW_MES(fe_batch_invert(batch_inv_res.get(), fe_batch.get(), n_valid_leaf_points * 2) == 0,
|
||||
"failed to batch invert ed derivatives");
|
||||
TIME_MEASURE_FINISH(batch_invert);
|
||||
|
||||
TIME_MEASURE_START(get_selene_scalars);
|
||||
// Step 4. Multithreaded get Wei coordinates and convert to Selene scalars
|
||||
const std::size_t n_valid_leaf_elems = n_valid_outputs * LEAF_TUPLE_SIZE;
|
||||
flattened_leaves_out.resize(n_valid_leaf_elems);
|
||||
CHECK_AND_ASSERT_THROW_MES(flattened_leaves_out.size() == (2 * n_valid_leaf_points),
|
||||
"unexpected size of flattened leaves");
|
||||
|
||||
const std::size_t DERIVATION_BATCH_SIZE = std::max<std::size_t>(1, (n_valid_leaf_points / n_threads));
|
||||
for (std::size_t i = 0; i < n_valid_leaf_points; i += DERIVATION_BATCH_SIZE)
|
||||
{
|
||||
const std::size_t end = std::min(n_valid_leaf_points, i + DERIVATION_BATCH_SIZE);
|
||||
tpool.submit(&waiter,
|
||||
[
|
||||
&batch_inv_res,
|
||||
&one_plus_y_vec,
|
||||
&flattened_leaves_out,
|
||||
i,
|
||||
end
|
||||
]()
|
||||
{
|
||||
std::size_t point_idx = i * 2;
|
||||
for (std::size_t j = i; j < end; ++j)
|
||||
{
|
||||
crypto::ec_coord wei_x;
|
||||
crypto::ec_coord wei_y;
|
||||
fe_ed_derivatives_to_wei_x_y(
|
||||
to_bytes(wei_x),
|
||||
to_bytes(wei_y),
|
||||
batch_inv_res[point_idx]/*inv_one_minus_y*/,
|
||||
one_plus_y_vec[j],
|
||||
batch_inv_res[point_idx+1]/*inv_one_minus_y_mul_x*/
|
||||
);
|
||||
|
||||
flattened_leaves_out[point_idx++] = tower_cycle::selene_scalar_from_bytes(wei_x);
|
||||
flattened_leaves_out[point_idx++] = tower_cycle::selene_scalar_from_bytes(wei_y);
|
||||
}
|
||||
},
|
||||
true
|
||||
);
|
||||
}
|
||||
|
||||
CHECK_AND_ASSERT_THROW_MES(waiter.wait(), "failed to convert outputs to wei coords");
|
||||
TIME_MEASURE_FINISH(get_selene_scalars);
|
||||
|
||||
// Step 5. Set valid tuples to be stored in the db
|
||||
valid_outputs_out.reserve(n_valid_outputs);
|
||||
for (std::size_t i = 0; i < valid_outputs.size(); ++i)
|
||||
{
|
||||
if (!valid_outputs[i])
|
||||
continue;
|
||||
|
||||
// We can derive leaf tuples from output pairs, so we store just the unified output in the db to save 32 bytes
|
||||
valid_outputs_out.emplace_back(std::move(new_outputs.at(i)));
|
||||
}
|
||||
CHECK_AND_ASSERT_THROW_MES(valid_outputs_out.size() == n_valid_outputs, "unexpected size of valid_outputs_out");
|
||||
|
||||
TIME_MEASURE_FINISH(set_valid_leaves);
|
||||
|
||||
m_convert_valid_leaves_ms += convert_valid_leaves;
|
||||
m_collect_derivatives_ms += collect_derivatives;
|
||||
m_batch_invert_ms += batch_invert;
|
||||
m_get_selene_scalars_ms += get_selene_scalars;
|
||||
|
||||
m_set_valid_leaves_ms += set_valid_leaves;
|
||||
|
||||
LOG_PRINT_L2("Total time spent setting leaves: " << m_set_valid_leaves_ms / 1000
|
||||
<< " , converting valid leaves: " << m_convert_valid_leaves_ms / 1000
|
||||
<< " , collecting derivatives: " << m_collect_derivatives_ms / 1000
|
||||
<< " , batch invert: " << m_batch_invert_ms / 1000
|
||||
<< " , get selene scalars: " << m_get_selene_scalars_ms / 1000);
|
||||
}
|
||||
//----------------------------------------------------------------------------------------------------------------------
|
||||
//----------------------------------------------------------------------------------------------------------------------
|
||||
} //namespace curve_trees
|
||||
} //namespace fcmp_pp
|
||||
|
||||
@@ -30,8 +30,11 @@
|
||||
|
||||
#include "crypto/crypto.h"
|
||||
#include "fcmp_pp_types.h"
|
||||
#include "tower_cycle.h"
|
||||
#include "misc_log_ex.h"
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
namespace fcmp_pp
|
||||
{
|
||||
@@ -42,5 +45,77 @@ namespace curve_trees
|
||||
OutputTuple output_to_tuple(const OutputPair &output_pair);
|
||||
//----------------------------------------------------------------------------------------------------------------------
|
||||
//----------------------------------------------------------------------------------------------------------------------
|
||||
// This class is useful to help update the curve trees merkle tree without needing to keep the entire tree in memory
|
||||
// - It requires instantiation with the C1 and C2 curve classes and widths, hardening the tree structure
|
||||
// - It ties the C1 curve in the tree to the leaf layer (the leaf layer is composed of C1 scalars)
|
||||
template<typename C1, typename C2>
|
||||
class CurveTrees
|
||||
{
|
||||
public:
|
||||
CurveTrees(std::unique_ptr<C1> &&c1,
|
||||
std::unique_ptr<C2> &&c2,
|
||||
const std::size_t c1_width,
|
||||
const std::size_t c2_width):
|
||||
m_c1{std::move(c1)},
|
||||
m_c2{std::move(c2)},
|
||||
m_c1_width{c1_width},
|
||||
m_c2_width{c2_width},
|
||||
m_leaf_layer_chunk_width{LEAF_TUPLE_SIZE * c1_width}
|
||||
{
|
||||
assert(c1_width > 0);
|
||||
assert(c2_width > 0);
|
||||
};
|
||||
|
||||
//member structs
|
||||
public:
|
||||
// Tuple that composes a single leaf in the tree
|
||||
struct LeafTuple final
|
||||
{
|
||||
// Output ed25519 point wei x and y coordinates
|
||||
typename C1::Scalar O_x;
|
||||
typename C1::Scalar O_y;
|
||||
// Key image generator wei x and y coordinates
|
||||
typename C1::Scalar I_x;
|
||||
typename C1::Scalar I_y;
|
||||
// Commitment wei x and y coordinates
|
||||
typename C1::Scalar C_x;
|
||||
typename C1::Scalar C_y;
|
||||
};
|
||||
|
||||
static const std::size_t LEAF_TUPLE_POINTS = 3;
|
||||
static constexpr std::size_t LEAF_TUPLE_SIZE = LEAF_TUPLE_POINTS * 2;
|
||||
|
||||
static_assert(sizeof(LeafTuple) == (sizeof(typename C1::Scalar) * LEAF_TUPLE_SIZE), "unexpected LeafTuple size");
|
||||
|
||||
//member functions
|
||||
private:
|
||||
// Multithreaded helper function to convert valid outputs to leaf tuples ready for insertion to the tree & db
|
||||
void outputs_to_leaves(std::vector<UnifiedOutput> &&new_outputs,
|
||||
std::vector<typename C1::Scalar> &flattened_leaves_out,
|
||||
std::vector<UnifiedOutput> &valid_outputs_out) const;
|
||||
|
||||
//private state
|
||||
private:
|
||||
mutable uint64_t m_set_valid_leaves_ms{0};
|
||||
mutable uint64_t m_get_selene_scalars_ms{0};
|
||||
mutable uint64_t m_batch_invert_ms{0};
|
||||
mutable uint64_t m_collect_derivatives_ms{0};
|
||||
mutable uint64_t m_convert_valid_leaves_ms{0};
|
||||
|
||||
//public member variables
|
||||
public:
|
||||
// The curve interfaces
|
||||
const std::unique_ptr<C1> m_c1;
|
||||
const std::unique_ptr<C2> m_c2;
|
||||
|
||||
// The leaf layer has a distinct chunk width than the other layers
|
||||
const std::size_t m_leaf_layer_chunk_width;
|
||||
|
||||
// The chunk widths of the layers in the tree tied to each curve
|
||||
const std::size_t m_c1_width;
|
||||
const std::size_t m_c2_width;
|
||||
};
|
||||
//----------------------------------------------------------------------------------------------------------------------
|
||||
//----------------------------------------------------------------------------------------------------------------------
|
||||
} //namespace curve_trees
|
||||
} //namespace fcmp_pp
|
||||
|
||||
@@ -99,6 +99,25 @@ const crypto::ec_point &commitment_cref(const OutputPair &output_pair);
|
||||
|
||||
bool output_checked_for_torsion(const OutputPair &output_pair);
|
||||
bool use_biased_hash_to_point(const OutputPair &output_pair);
|
||||
|
||||
// Wrapper for outputs with context to insert the output into the FCMP++ curve tree
|
||||
struct UnifiedOutput final
|
||||
{
|
||||
// Output's unique id in the chain.
|
||||
// When a batch of outputs is added to the tree, the batch
|
||||
// is internally sorted by unique ids, which correspond with
|
||||
// the order the outputs entered the chain. Batch-added outputs
|
||||
// may not be globally ordered because e.g. timelocked
|
||||
// outputs were added to 'late batches'. Batch refers to
|
||||
// outputs that become unlocked/spendable at the same time.
|
||||
uint64_t unified_id{0};
|
||||
OutputPair output_pair;
|
||||
|
||||
bool operator==(const UnifiedOutput &other) const
|
||||
{
|
||||
return unified_id == other.unified_id && output_pair == other.output_pair;
|
||||
}
|
||||
};
|
||||
//----------------------------------------------------------------------------------------------------------------------
|
||||
//----------------------------------------------------------------------------------------------------------------------
|
||||
}//namespace fcmp_pp
|
||||
|
||||
Reference in New Issue
Block a user