Memory usage reduction

This commit is contained in:
Javier Smooth
2015-08-14 07:37:54 -07:00
parent 443652e48c
commit 86d8cb9b7b
2 changed files with 131 additions and 39 deletions
+38 -23
View File
@@ -957,16 +957,18 @@ size_t blockchain_storage::get_alternative_blocks_count()
return m_alternative_chains.size();
}
//------------------------------------------------------------------
bool blockchain_storage::add_out_to_get_random_outs(std::vector<std::pair<crypto::hash, size_t> >& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i)
bool blockchain_storage::add_out_to_get_random_outs(std::vector<std::tuple<uint32_t,uint32_t,uint32_t>>& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i)
{
CRITICAL_REGION_LOCAL(m_blockchain_lock);
transactions_container::iterator tx_it = m_transactions.find(amount_outs[i].first);
CHECK_AND_ASSERT_MES(tx_it != m_transactions.end(), false, "internal error: transaction with id " << amount_outs[i].first << ENDL <<
auto txid = get_txid_by_out_index(amount_outs[i]);
transactions_container::iterator tx_it = m_transactions.find(txid);
CHECK_AND_ASSERT_MES(tx_it != m_transactions.end(), false, "internal error: transaction with id " << txid<< ENDL <<
", used in mounts global index for amount=" << amount << ": i=" << i << "not found in transactions index");
CHECK_AND_ASSERT_MES(tx_it->second.tx.vout.size() > amount_outs[i].second, false, "internal error: in global outs index, transaction out index="
<< amount_outs[i].second << " more than transaction outputs = " << tx_it->second.tx.vout.size() << ", for tx id = " << amount_outs[i].first);
size_t out_idx = std::get<2>(amount_outs[i]);
CHECK_AND_ASSERT_MES(tx_it->second.tx.vout.size() > out_idx, false, "internal error: in global outs index, transaction out index="
<< out_idx<< " more than transaction outputs = " << tx_it->second.tx.vout.size() << ", for tx id = " << txid);
transaction& tx = tx_it->second.tx;
CHECK_AND_ASSERT_MES(tx.vout[amount_outs[i].second].target.type() == typeid(txout_to_key), false, "unknown tx out type");
CHECK_AND_ASSERT_MES(tx.vout[out_idx].target.type() == typeid(txout_to_key), false, "unknown tx out type");
//check if transaction is unlocked
if(!is_tx_spendtime_unlocked(tx.unlock_time))
@@ -974,11 +976,18 @@ bool blockchain_storage::add_out_to_get_random_outs(std::vector<std::pair<crypto
COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry& oen = *result_outs.outs.insert(result_outs.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry());
oen.global_amount_index = i;
oen.out_key = boost::get<txout_to_key>(tx.vout[amount_outs[i].second].target).key;
oen.out_key = boost::get<txout_to_key>(tx.vout[out_idx].target).key;
return true;
}
//------------------------------------------------------------------
size_t blockchain_storage::find_end_of_allowed_index(const std::vector<std::pair<crypto::hash, size_t> >& amount_outs)
crypto::hash blockchain_storage::get_txid_by_out_index(const std::tuple<uint32_t,uint32_t,uint32_t>& out_entry) const
{
auto& bl = m_blocks[std::get<0>(out_entry)].bl;
auto index_in_block = std::get<1>(out_entry);
return index_in_block==0?get_transaction_hash(bl.miner_tx):bl.tx_hashes[index_in_block-1];
}
//------------------------------------------------------------------
size_t blockchain_storage::find_end_of_allowed_index(const std::vector<std::tuple<uint32_t,uint32_t,uint32_t>>& amount_outs)
{
CRITICAL_REGION_LOCAL(m_blockchain_lock);
if(!amount_outs.size())
@@ -987,8 +996,9 @@ size_t blockchain_storage::find_end_of_allowed_index(const std::vector<std::pair
do
{
--i;
transactions_container::iterator it = m_transactions.find(amount_outs[i].first);
CHECK_AND_ASSERT_MES(it != m_transactions.end(), 0, "internal error: failed to find transaction from outputs index with tx_id=" << amount_outs[i].first);
auto txid = get_txid_by_out_index(amount_outs[i]);
transactions_container::iterator it = m_transactions.find(txid);
CHECK_AND_ASSERT_MES(it != m_transactions.end(), 0, "internal error: failed to find transaction from outputs index with tx_id=" << txid);
if(it->second.m_keeper_block_height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW <= get_current_blockchain_height() )
return i+1;
} while (i != 0);
@@ -1008,7 +1018,7 @@ bool blockchain_storage::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDO
LOG_ERROR("COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS: not outs for amount " << amount << ", wallet should use some real outs when it lookup for some mix, so, at least one out for this amount should exist");
continue;//actually this is strange situation, wallet should use some real outs when it lookup for some mix, so, at least one out for this amount should exist
}
std::vector<std::pair<crypto::hash, size_t> >& amount_outs = it->second;
std::vector<std::tuple<uint32_t,uint32_t,uint32_t>>& amount_outs = it->second;
//it is not good idea to use top fresh outs, because it increases possibility of transaction canceling on split
//lets find upper bound of not fresh outs
size_t up_index_limit = find_end_of_allowed_index(amount_outs);
@@ -1132,6 +1142,7 @@ void blockchain_storage::print_blockchain_index()
//------------------------------------------------------------------
void blockchain_storage::print_blockchain_outs(const std::string& file)
{
#if 0
std::stringstream ss;
CRITICAL_REGION_LOCAL(m_blockchain_lock);
BOOST_FOREACH(const outputs_container::value_type& v, m_outputs)
@@ -1151,6 +1162,7 @@ void blockchain_storage::print_blockchain_outs(const std::string& file)
{
LOG_PRINT_L0("Failed to write current outputs index to file: " << file);
}
#endif
}
//------------------------------------------------------------------
bool blockchain_storage::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp)
@@ -1229,14 +1241,14 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, block_verif
return handle_block_to_main_chain(bl, id, bvc);
}
//------------------------------------------------------------------
bool blockchain_storage::push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector<uint64_t>& global_indexes)
bool blockchain_storage::push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector<uint64_t>& global_indexes, uint64_t bl_height, size_t index_in_block)
{
CRITICAL_REGION_LOCAL(m_blockchain_lock);
size_t i = 0;
BOOST_FOREACH(const auto& ot, tx.vout)
{
outputs_container::mapped_type& amount_index = m_outputs[ot.amount];
amount_index.push_back(std::pair<crypto::hash, size_t>(tx_id, i));
amount_index.push_back(std::make_tuple((uint32_t)bl_height,(uint32_t)index_in_block,(uint32_t)i));
global_indexes.push_back(amount_index.size()-1);
++i;
}
@@ -1258,11 +1270,13 @@ bool blockchain_storage::get_outs(uint64_t amount, std::list<crypto::public_key>
BOOST_FOREACH(const auto& out_entry, it->second)
{
auto tx_it = m_transactions.find(out_entry.first);
auto txid = get_txid_by_out_index(out_entry);
auto tx_it = m_transactions.find(txid);
auto out_index_in_tx = std::get<2>(out_entry);
CHECK_AND_ASSERT_MES(tx_it != m_transactions.end(), false, "transactions outs global index consistency broken: wrong tx id in index");
CHECK_AND_ASSERT_MES(tx_it->second.tx.vout.size() > out_entry.second, false, "transactions outs global index consistency broken: index in tx_outx more then size");
CHECK_AND_ASSERT_MES(tx_it->second.tx.vout[out_entry.second].target.type() == typeid(txout_to_key), false, "transactions outs global index consistency broken: index in tx_outx more then size");
pkeys.push_back(boost::get<txout_to_key>(tx_it->second.tx.vout[out_entry.second].target).key);
CHECK_AND_ASSERT_MES(tx_it->second.tx.vout.size() > out_index_in_tx, false, "transactions outs global index consistency broken: index in tx_outx more then size");
CHECK_AND_ASSERT_MES(tx_it->second.tx.vout[out_index_in_tx].target.type() == typeid(txout_to_key), false, "transactions outs global index consistency broken: index in tx_outx more then size");
pkeys.push_back(boost::get<txout_to_key>(tx_it->second.tx.vout[out_index_in_tx].target).key);
}
return true;
@@ -1277,15 +1291,16 @@ bool blockchain_storage::pop_transaction_from_global_index(const transaction& tx
auto it = m_outputs.find(ot.amount);
CHECK_AND_ASSERT_MES(it != m_outputs.end(), false, "transactions outs global index consistency broken");
CHECK_AND_ASSERT_MES(it->second.size(), false, "transactions outs global index: empty index for amount: " << ot.amount);
CHECK_AND_ASSERT_MES(it->second.back().first == tx_id , false, "transactions outs global index consistency broken: tx id missmatch");
CHECK_AND_ASSERT_MES(it->second.back().second == i, false, "transactions outs global index consistency broken: in transaction index missmatch");
const auto out_tx_id = get_txid_by_out_index(it->second.back());
CHECK_AND_ASSERT_MES(out_tx_id == tx_id , false, "transactions outs global index consistency broken: tx id missmatch");
CHECK_AND_ASSERT_MES(std::get<2>(it->second.back())== i, false, "transactions outs global index consistency broken: in transaction index missmatch");
it->second.pop_back();
--i;
}
return true;
}
//------------------------------------------------------------------
bool blockchain_storage::add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height)
bool blockchain_storage::add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height, size_t index_in_block)
{
CRITICAL_REGION_LOCAL(m_blockchain_lock);
struct add_transaction_input_visitor: public boost::static_visitor<bool>
@@ -1331,7 +1346,7 @@ bool blockchain_storage::add_transaction_from_block(const transaction& tx, const
LOG_PRINT_L0("tx with id: " << tx_id << " in block id: " << bl_id << " already in blockchain");
return false;
}
bool r = push_transaction_to_global_outs_index(tx, tx_id, i_r.first->second.m_global_output_indexes);
bool r = push_transaction_to_global_outs_index(tx, tx_id, i_r.first->second.m_global_output_indexes,bl_height,index_in_block);
CHECK_AND_ASSERT_MES(r, false, "failed to return push_transaction_to_global_outs_index tx id " << tx_id);
LOG_PRINT_L2("Added transaction to blockchain history:" << ENDL
<< "tx_id: " << tx_id << ENDL
@@ -1593,7 +1608,7 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt
size_t coinbase_blob_size = get_object_blobsize(bl.miner_tx);
size_t cumulative_block_size = coinbase_blob_size;
//process transactions
if(!add_transaction_from_block(bl.miner_tx, get_transaction_hash(bl.miner_tx), id, get_current_blockchain_height()))
if(!add_transaction_from_block(bl.miner_tx, get_transaction_hash(bl.miner_tx), id, get_current_blockchain_height(),0))
{
LOG_PRINT_L0("Block with id: " << id << " failed to add transaction to blockchain storage");
bvc.m_verifivation_failed = true;
@@ -1627,7 +1642,7 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt
return false;
}
if(!add_transaction_from_block(tx, tx_id, id, get_current_blockchain_height()))
if(!add_transaction_from_block(tx, tx_id, id, get_current_blockchain_height(),tx_processed_count+1))
{
LOG_PRINT_L0("Block with id: " << id << " failed to add transaction to blockchain storage");
cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc);
+93 -16
View File
@@ -53,6 +53,20 @@
#include "crypto/hash.h"
#include "checkpoints.h"
namespace boost
{
namespace serialization
{
template<class Archive>
void serialize(Archive & ar, std::tuple<uint32_t,uint32_t,uint32_t>& v, const unsigned int version)
{
ar & std::get<0>(v);
ar & std::get<1>(v);
ar & std::get<2>(v);
}
}
}
namespace cryptonote
{
@@ -187,7 +201,7 @@ namespace cryptonote
typedef std::unordered_set<crypto::key_image> key_images_container;
typedef std::vector<block_extended_info> blocks_container;
typedef std::unordered_map<crypto::hash, block_extended_info> blocks_ext_by_hash;
typedef std::map<uint64_t, std::vector<std::pair<crypto::hash, size_t>>> outputs_container; //crypto::hash - tx hash, size_t - index of out in transaction
typedef std::map<uint64_t, std::vector<std::tuple<uint32_t,uint32_t,uint32_t>>> outputs_container; // block, tx in block, output in tx
tx_memory_pool& m_tx_pool;
epee::critical_section m_blockchain_lock; // TODO: add here reader/writer lock
@@ -199,7 +213,6 @@ namespace cryptonote
key_images_container m_spent_keys;
size_t m_current_block_cumul_sz_limit;
// all alternative chains
blocks_ext_by_hash m_alternative_chains; // crypto::hash -> block_extended_info
@@ -207,7 +220,6 @@ namespace cryptonote
blocks_ext_by_hash m_invalid_blocks; // crypto::hash -> block_extended_info
outputs_container m_outputs;
std::string m_config_folder;
checkpoints m_checkpoints;
std::atomic<bool> m_is_in_checkpoint_zone;
@@ -227,15 +239,16 @@ namespace cryptonote
bool validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, size_t height);
bool validate_transaction(const block& b, uint64_t height, const transaction& tx);
bool rollback_blockchain_switching(std::list<block>& original_chain, size_t rollback_height);
bool add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height);
bool push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector<uint64_t>& global_indexes);
bool add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height, size_t index_in_block);
bool push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector<uint64_t>& global_indexes, uint64_t bl_height, size_t index_in_block);
bool pop_transaction_from_global_index(const transaction& tx, const crypto::hash& tx_id);
bool get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count);
bool add_out_to_get_random_outs(std::vector<std::pair<crypto::hash, size_t> >& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i);
bool add_out_to_get_random_outs(std::vector<std::tuple<uint32_t,uint32_t,uint32_t>>& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i);
bool is_tx_spendtime_unlocked(uint64_t unlock_time);
bool add_block_as_invalid(const block& bl, const crypto::hash& h);
bool add_block_as_invalid(const block_extended_info& bei, const crypto::hash& h);
size_t find_end_of_allowed_index(const std::vector<std::pair<crypto::hash, size_t> >& amount_outs);
crypto::hash get_txid_by_out_index(const std::tuple<uint32_t,uint32_t,uint32_t>& out_entry) const;
size_t find_end_of_allowed_index(const std::vector<std::tuple<uint32_t,uint32_t,uint32_t>>& amount_outs);
bool check_block_timestamp_main(const block& b);
bool check_block_timestamp(std::vector<uint64_t> timestamps, const block& b);
uint64_t get_adjusted_time();
@@ -248,7 +261,7 @@ namespace cryptonote
/* */
/************************************************************************/
#define CURRENT_BLOCKCHAIN_STORAGE_ARCHIVE_VER 12
#define CURRENT_BLOCKCHAIN_STORAGE_ARCHIVE_VER 13
template<class archive_t>
void blockchain_storage::serialize(archive_t & ar, const unsigned int version)
@@ -261,7 +274,47 @@ namespace cryptonote
ar & m_transactions;
ar & m_spent_keys;
ar & m_alternative_chains;
ar & m_outputs;
if (version < 13)
{
LOG_PRINT_L0("Upgrading blockchain format...");
typedef std::map<uint64_t, std::vector<std::pair<crypto::hash, size_t>>> old_outputs_container; //crypto::hash - tx hash, size_t - index of out in transaction
old_outputs_container old_outs;
ar & old_outs;
LOG_PRINT_L0("Out outs size " << old_outs.size());
for (const auto& entry : old_outs)
{
if (entry.second.size() == 0) {
LOG_PRINT_L0("empty out size: " << entry.first);
m_outputs[entry.first];
}
for (const auto& p : entry.second)
{
const auto& txid = p.first;
const auto& tce = m_transactions[txid];
const auto& bei = m_blocks[tce.m_keeper_block_height];
uint32_t i=0;
if (txid != get_transaction_hash(bei.bl.miner_tx))
{
++i;
for (const auto& t : bei.bl.tx_hashes)
{
if (txid == t)
break;
++i;
}
if (i > bei.bl.tx_hashes.size())
{
LOG_ERROR("Blockchain storage data corruption detected. unable to mach txid to block");
throw std::runtime_error("Blockchain data corruption");
}
}
m_outputs[entry.first].push_back(std::make_tuple((uint32_t)tce.m_keeper_block_height,i,(uint32_t)p.second));
}
}
LOG_PRINT_L0("Converted outs size " << m_outputs.size());
}
else
ar & m_outputs;
ar & m_invalid_blocks;
ar & m_current_block_cumul_sz_limit;
/*serialization bug workaround*/
@@ -295,13 +348,36 @@ namespace cryptonote
}
size_t count =0;
for (const auto& vec : m_outputs)
count += vec.second.size();
size_t total_blob_size =0;
for (const auto& bei : m_blocks) {
total_blob_size += block_to_blob(bei.bl).size();
for (const auto& txid : bei.bl.tx_hashes)
total_blob_size += tx_to_blob(m_transactions[txid].tx).size();
}
size_t total_blob_size_headers =0;
for (const auto& bei : m_blocks) {
total_blob_size_headers += block_to_blob(bei.bl).size();
for (const auto& txid : bei.bl.tx_hashes) {
transaction_prefix p = m_transactions[txid].tx;
total_blob_size_headers += t_serializable_object_to_blob(p).size();
}
}
LOG_PRINT_L2("Blockchain storage:" << ENDL <<
"m_blocks: " << m_blocks.size() << ENDL <<
"total_blob_size: " << total_blob_size << ENDL <<
"total_blob_size_headers: " << total_blob_size_headers << ENDL <<
"m_blocks_index: " << m_blocks_index.size() << ENDL <<
"m_transactions: " << m_transactions.size() << ENDL <<
"m_spent_keys: " << m_spent_keys.size() << ENDL <<
"m_alternative_chains: " << m_alternative_chains.size() << ENDL <<
"m_outputs: " << m_outputs.size() << ENDL <<
"m_outputs_total_entries: " << count << ENDL <<
"m_invalid_blocks: " << m_invalid_blocks.size() << ENDL <<
"m_current_block_cumul_sz_limit: " << m_current_block_cumul_sz_limit);
}
@@ -317,8 +393,7 @@ namespace cryptonote
std::vector<uint64_t> absolute_offsets = relative_output_offsets_to_absolute(tx_in_to_key.key_offsets);
std::vector<std::pair<crypto::hash, size_t> >& amount_outs_vec = it->second;
std::vector<std::tuple<uint32_t,uint32_t,uint32_t>>& amount_outs_vec = it->second;
size_t count = 0;
BOOST_FOREACH(uint64_t i, absolute_offsets)
{
@@ -327,11 +402,13 @@ namespace cryptonote
LOG_PRINT_L0("Wrong index in transaction inputs: " << i << ", expected maximum " << amount_outs_vec.size() - 1);
return false;
}
transactions_container::iterator tx_it = m_transactions.find(amount_outs_vec[i].first);
CHECK_AND_ASSERT_MES(tx_it != m_transactions.end(), false, "Wrong transaction id in output indexes: " << epee::string_tools::pod_to_hex(amount_outs_vec[i].first));
CHECK_AND_ASSERT_MES(amount_outs_vec[i].second < tx_it->second.tx.vout.size(), false,
"Wrong index in transaction outputs: " << amount_outs_vec[i].second << ", expected less then " << tx_it->second.tx.vout.size());
if(!vis.handle_output(tx_it->second.tx, tx_it->second.tx.vout[amount_outs_vec[i].second]))
auto tx_id = get_txid_by_out_index(amount_outs_vec[i]);
transactions_container::iterator tx_it = m_transactions.find(tx_id);
CHECK_AND_ASSERT_MES(tx_it != m_transactions.end(), false, "Wrong transaction id in output indexes: " << tx_id);
size_t out_idx = std::get<2>(amount_outs_vec[i]);
CHECK_AND_ASSERT_MES(out_idx < tx_it->second.tx.vout.size(), false,
"Wrong index in transaction outputs: " << out_idx << ", expected less then " << tx_it->second.tx.vout.size());
if(!vis.handle_output(tx_it->second.tx, tx_it->second.tx.vout[out_idx]))
{
LOG_PRINT_L0("Failed to handle_output for output no = " << count << ", with absolute offset " << i);
return false;