diff --git a/src/cryptonote_core/blockchain_storage.cpp b/src/cryptonote_core/blockchain_storage.cpp index 61903c2..db0caa6 100644 --- a/src/cryptonote_core/blockchain_storage.cpp +++ b/src/cryptonote_core/blockchain_storage.cpp @@ -957,16 +957,18 @@ size_t blockchain_storage::get_alternative_blocks_count() return m_alternative_chains.size(); } //------------------------------------------------------------------ -bool blockchain_storage::add_out_to_get_random_outs(std::vector >& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) +bool blockchain_storage::add_out_to_get_random_outs(std::vector>& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) { CRITICAL_REGION_LOCAL(m_blockchain_lock); - transactions_container::iterator tx_it = m_transactions.find(amount_outs[i].first); - CHECK_AND_ASSERT_MES(tx_it != m_transactions.end(), false, "internal error: transaction with id " << amount_outs[i].first << ENDL << + auto txid = get_txid_by_out_index(amount_outs[i]); + transactions_container::iterator tx_it = m_transactions.find(txid); + CHECK_AND_ASSERT_MES(tx_it != m_transactions.end(), false, "internal error: transaction with id " << txid<< ENDL << ", used in mounts global index for amount=" << amount << ": i=" << i << "not found in transactions index"); - CHECK_AND_ASSERT_MES(tx_it->second.tx.vout.size() > amount_outs[i].second, false, "internal error: in global outs index, transaction out index=" - << amount_outs[i].second << " more than transaction outputs = " << tx_it->second.tx.vout.size() << ", for tx id = " << amount_outs[i].first); + size_t out_idx = std::get<2>(amount_outs[i]); + CHECK_AND_ASSERT_MES(tx_it->second.tx.vout.size() > out_idx, false, "internal error: in global outs index, transaction out index=" + << out_idx<< " more than transaction outputs = " << tx_it->second.tx.vout.size() << ", for tx id = " << txid); transaction& tx = tx_it->second.tx; - CHECK_AND_ASSERT_MES(tx.vout[amount_outs[i].second].target.type() == typeid(txout_to_key), false, "unknown tx out type"); + CHECK_AND_ASSERT_MES(tx.vout[out_idx].target.type() == typeid(txout_to_key), false, "unknown tx out type"); //check if transaction is unlocked if(!is_tx_spendtime_unlocked(tx.unlock_time)) @@ -974,11 +976,18 @@ bool blockchain_storage::add_out_to_get_random_outs(std::vector(tx.vout[amount_outs[i].second].target).key; + oen.out_key = boost::get(tx.vout[out_idx].target).key; return true; } //------------------------------------------------------------------ -size_t blockchain_storage::find_end_of_allowed_index(const std::vector >& amount_outs) +crypto::hash blockchain_storage::get_txid_by_out_index(const std::tuple& out_entry) const +{ + auto& bl = m_blocks[std::get<0>(out_entry)].bl; + auto index_in_block = std::get<1>(out_entry); + return index_in_block==0?get_transaction_hash(bl.miner_tx):bl.tx_hashes[index_in_block-1]; +} +//------------------------------------------------------------------ +size_t blockchain_storage::find_end_of_allowed_index(const std::vector>& amount_outs) { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(!amount_outs.size()) @@ -987,8 +996,9 @@ size_t blockchain_storage::find_end_of_allowed_index(const std::vector >& amount_outs = it->second; + std::vector>& amount_outs = it->second; //it is not good idea to use top fresh outs, because it increases possibility of transaction canceling on split //lets find upper bound of not fresh outs size_t up_index_limit = find_end_of_allowed_index(amount_outs); @@ -1132,6 +1142,7 @@ void blockchain_storage::print_blockchain_index() //------------------------------------------------------------------ void blockchain_storage::print_blockchain_outs(const std::string& file) { +#if 0 std::stringstream ss; CRITICAL_REGION_LOCAL(m_blockchain_lock); BOOST_FOREACH(const outputs_container::value_type& v, m_outputs) @@ -1151,6 +1162,7 @@ void blockchain_storage::print_blockchain_outs(const std::string& file) { LOG_PRINT_L0("Failed to write current outputs index to file: " << file); } +#endif } //------------------------------------------------------------------ bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) @@ -1229,14 +1241,14 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, block_verif return handle_block_to_main_chain(bl, id, bvc); } //------------------------------------------------------------------ -bool blockchain_storage::push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector& global_indexes) +bool blockchain_storage::push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector& global_indexes, uint64_t bl_height, size_t index_in_block) { CRITICAL_REGION_LOCAL(m_blockchain_lock); size_t i = 0; BOOST_FOREACH(const auto& ot, tx.vout) { outputs_container::mapped_type& amount_index = m_outputs[ot.amount]; - amount_index.push_back(std::pair(tx_id, i)); + amount_index.push_back(std::make_tuple((uint32_t)bl_height,(uint32_t)index_in_block,(uint32_t)i)); global_indexes.push_back(amount_index.size()-1); ++i; } @@ -1258,11 +1270,13 @@ bool blockchain_storage::get_outs(uint64_t amount, std::list BOOST_FOREACH(const auto& out_entry, it->second) { - auto tx_it = m_transactions.find(out_entry.first); + auto txid = get_txid_by_out_index(out_entry); + auto tx_it = m_transactions.find(txid); + auto out_index_in_tx = std::get<2>(out_entry); CHECK_AND_ASSERT_MES(tx_it != m_transactions.end(), false, "transactions outs global index consistency broken: wrong tx id in index"); - CHECK_AND_ASSERT_MES(tx_it->second.tx.vout.size() > out_entry.second, false, "transactions outs global index consistency broken: index in tx_outx more then size"); - CHECK_AND_ASSERT_MES(tx_it->second.tx.vout[out_entry.second].target.type() == typeid(txout_to_key), false, "transactions outs global index consistency broken: index in tx_outx more then size"); - pkeys.push_back(boost::get(tx_it->second.tx.vout[out_entry.second].target).key); + CHECK_AND_ASSERT_MES(tx_it->second.tx.vout.size() > out_index_in_tx, false, "transactions outs global index consistency broken: index in tx_outx more then size"); + CHECK_AND_ASSERT_MES(tx_it->second.tx.vout[out_index_in_tx].target.type() == typeid(txout_to_key), false, "transactions outs global index consistency broken: index in tx_outx more then size"); + pkeys.push_back(boost::get(tx_it->second.tx.vout[out_index_in_tx].target).key); } return true; @@ -1277,15 +1291,16 @@ bool blockchain_storage::pop_transaction_from_global_index(const transaction& tx auto it = m_outputs.find(ot.amount); CHECK_AND_ASSERT_MES(it != m_outputs.end(), false, "transactions outs global index consistency broken"); CHECK_AND_ASSERT_MES(it->second.size(), false, "transactions outs global index: empty index for amount: " << ot.amount); - CHECK_AND_ASSERT_MES(it->second.back().first == tx_id , false, "transactions outs global index consistency broken: tx id missmatch"); - CHECK_AND_ASSERT_MES(it->second.back().second == i, false, "transactions outs global index consistency broken: in transaction index missmatch"); + const auto out_tx_id = get_txid_by_out_index(it->second.back()); + CHECK_AND_ASSERT_MES(out_tx_id == tx_id , false, "transactions outs global index consistency broken: tx id missmatch"); + CHECK_AND_ASSERT_MES(std::get<2>(it->second.back())== i, false, "transactions outs global index consistency broken: in transaction index missmatch"); it->second.pop_back(); --i; } return true; } //------------------------------------------------------------------ -bool blockchain_storage::add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height) +bool blockchain_storage::add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height, size_t index_in_block) { CRITICAL_REGION_LOCAL(m_blockchain_lock); struct add_transaction_input_visitor: public boost::static_visitor @@ -1331,7 +1346,7 @@ bool blockchain_storage::add_transaction_from_block(const transaction& tx, const LOG_PRINT_L0("tx with id: " << tx_id << " in block id: " << bl_id << " already in blockchain"); return false; } - bool r = push_transaction_to_global_outs_index(tx, tx_id, i_r.first->second.m_global_output_indexes); + bool r = push_transaction_to_global_outs_index(tx, tx_id, i_r.first->second.m_global_output_indexes,bl_height,index_in_block); CHECK_AND_ASSERT_MES(r, false, "failed to return push_transaction_to_global_outs_index tx id " << tx_id); LOG_PRINT_L2("Added transaction to blockchain history:" << ENDL << "tx_id: " << tx_id << ENDL @@ -1593,7 +1608,7 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt size_t coinbase_blob_size = get_object_blobsize(bl.miner_tx); size_t cumulative_block_size = coinbase_blob_size; //process transactions - if(!add_transaction_from_block(bl.miner_tx, get_transaction_hash(bl.miner_tx), id, get_current_blockchain_height())) + if(!add_transaction_from_block(bl.miner_tx, get_transaction_hash(bl.miner_tx), id, get_current_blockchain_height(),0)) { LOG_PRINT_L0("Block with id: " << id << " failed to add transaction to blockchain storage"); bvc.m_verifivation_failed = true; @@ -1627,7 +1642,7 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt return false; } - if(!add_transaction_from_block(tx, tx_id, id, get_current_blockchain_height())) + if(!add_transaction_from_block(tx, tx_id, id, get_current_blockchain_height(),tx_processed_count+1)) { LOG_PRINT_L0("Block with id: " << id << " failed to add transaction to blockchain storage"); cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); diff --git a/src/cryptonote_core/blockchain_storage.h b/src/cryptonote_core/blockchain_storage.h index 091dcf9..cba5b89 100644 --- a/src/cryptonote_core/blockchain_storage.h +++ b/src/cryptonote_core/blockchain_storage.h @@ -53,6 +53,20 @@ #include "crypto/hash.h" #include "checkpoints.h" +namespace boost +{ + namespace serialization + { + template + void serialize(Archive & ar, std::tuple& v, const unsigned int version) + { + ar & std::get<0>(v); + ar & std::get<1>(v); + ar & std::get<2>(v); + } + } +} + namespace cryptonote { @@ -187,7 +201,7 @@ namespace cryptonote typedef std::unordered_set key_images_container; typedef std::vector blocks_container; typedef std::unordered_map blocks_ext_by_hash; - typedef std::map>> outputs_container; //crypto::hash - tx hash, size_t - index of out in transaction + typedef std::map>> outputs_container; // block, tx in block, output in tx tx_memory_pool& m_tx_pool; epee::critical_section m_blockchain_lock; // TODO: add here reader/writer lock @@ -199,7 +213,6 @@ namespace cryptonote key_images_container m_spent_keys; size_t m_current_block_cumul_sz_limit; - // all alternative chains blocks_ext_by_hash m_alternative_chains; // crypto::hash -> block_extended_info @@ -207,7 +220,6 @@ namespace cryptonote blocks_ext_by_hash m_invalid_blocks; // crypto::hash -> block_extended_info outputs_container m_outputs; - std::string m_config_folder; checkpoints m_checkpoints; std::atomic m_is_in_checkpoint_zone; @@ -227,15 +239,16 @@ namespace cryptonote bool validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, size_t height); bool validate_transaction(const block& b, uint64_t height, const transaction& tx); bool rollback_blockchain_switching(std::list& original_chain, size_t rollback_height); - bool add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height); - bool push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector& global_indexes); + bool add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height, size_t index_in_block); + bool push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector& global_indexes, uint64_t bl_height, size_t index_in_block); bool pop_transaction_from_global_index(const transaction& tx, const crypto::hash& tx_id); bool get_last_n_blocks_sizes(std::vector& sz, size_t count); - bool add_out_to_get_random_outs(std::vector >& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i); + bool add_out_to_get_random_outs(std::vector>& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i); bool is_tx_spendtime_unlocked(uint64_t unlock_time); bool add_block_as_invalid(const block& bl, const crypto::hash& h); bool add_block_as_invalid(const block_extended_info& bei, const crypto::hash& h); - size_t find_end_of_allowed_index(const std::vector >& amount_outs); + crypto::hash get_txid_by_out_index(const std::tuple& out_entry) const; + size_t find_end_of_allowed_index(const std::vector>& amount_outs); bool check_block_timestamp_main(const block& b); bool check_block_timestamp(std::vector timestamps, const block& b); uint64_t get_adjusted_time(); @@ -248,7 +261,7 @@ namespace cryptonote /* */ /************************************************************************/ - #define CURRENT_BLOCKCHAIN_STORAGE_ARCHIVE_VER 12 + #define CURRENT_BLOCKCHAIN_STORAGE_ARCHIVE_VER 13 template void blockchain_storage::serialize(archive_t & ar, const unsigned int version) @@ -261,7 +274,47 @@ namespace cryptonote ar & m_transactions; ar & m_spent_keys; ar & m_alternative_chains; - ar & m_outputs; + if (version < 13) + { + LOG_PRINT_L0("Upgrading blockchain format..."); + typedef std::map>> old_outputs_container; //crypto::hash - tx hash, size_t - index of out in transaction + old_outputs_container old_outs; + ar & old_outs; + LOG_PRINT_L0("Out outs size " << old_outs.size()); + for (const auto& entry : old_outs) + { + if (entry.second.size() == 0) { + LOG_PRINT_L0("empty out size: " << entry.first); + m_outputs[entry.first]; + } + for (const auto& p : entry.second) + { + const auto& txid = p.first; + const auto& tce = m_transactions[txid]; + const auto& bei = m_blocks[tce.m_keeper_block_height]; + uint32_t i=0; + if (txid != get_transaction_hash(bei.bl.miner_tx)) + { + ++i; + for (const auto& t : bei.bl.tx_hashes) + { + if (txid == t) + break; + ++i; + } + if (i > bei.bl.tx_hashes.size()) + { + LOG_ERROR("Blockchain storage data corruption detected. unable to mach txid to block"); + throw std::runtime_error("Blockchain data corruption"); + } + } + m_outputs[entry.first].push_back(std::make_tuple((uint32_t)tce.m_keeper_block_height,i,(uint32_t)p.second)); + } + } + LOG_PRINT_L0("Converted outs size " << m_outputs.size()); + } + else + ar & m_outputs; ar & m_invalid_blocks; ar & m_current_block_cumul_sz_limit; /*serialization bug workaround*/ @@ -295,13 +348,36 @@ namespace cryptonote } + size_t count =0; + for (const auto& vec : m_outputs) + count += vec.second.size(); + + size_t total_blob_size =0; + for (const auto& bei : m_blocks) { + total_blob_size += block_to_blob(bei.bl).size(); + for (const auto& txid : bei.bl.tx_hashes) + total_blob_size += tx_to_blob(m_transactions[txid].tx).size(); + } + + size_t total_blob_size_headers =0; + for (const auto& bei : m_blocks) { + total_blob_size_headers += block_to_blob(bei.bl).size(); + for (const auto& txid : bei.bl.tx_hashes) { + transaction_prefix p = m_transactions[txid].tx; + total_blob_size_headers += t_serializable_object_to_blob(p).size(); + } + } + LOG_PRINT_L2("Blockchain storage:" << ENDL << "m_blocks: " << m_blocks.size() << ENDL << + "total_blob_size: " << total_blob_size << ENDL << + "total_blob_size_headers: " << total_blob_size_headers << ENDL << "m_blocks_index: " << m_blocks_index.size() << ENDL << "m_transactions: " << m_transactions.size() << ENDL << "m_spent_keys: " << m_spent_keys.size() << ENDL << "m_alternative_chains: " << m_alternative_chains.size() << ENDL << "m_outputs: " << m_outputs.size() << ENDL << + "m_outputs_total_entries: " << count << ENDL << "m_invalid_blocks: " << m_invalid_blocks.size() << ENDL << "m_current_block_cumul_sz_limit: " << m_current_block_cumul_sz_limit); } @@ -317,8 +393,7 @@ namespace cryptonote std::vector absolute_offsets = relative_output_offsets_to_absolute(tx_in_to_key.key_offsets); - - std::vector >& amount_outs_vec = it->second; + std::vector>& amount_outs_vec = it->second; size_t count = 0; BOOST_FOREACH(uint64_t i, absolute_offsets) { @@ -327,11 +402,13 @@ namespace cryptonote LOG_PRINT_L0("Wrong index in transaction inputs: " << i << ", expected maximum " << amount_outs_vec.size() - 1); return false; } - transactions_container::iterator tx_it = m_transactions.find(amount_outs_vec[i].first); - CHECK_AND_ASSERT_MES(tx_it != m_transactions.end(), false, "Wrong transaction id in output indexes: " << epee::string_tools::pod_to_hex(amount_outs_vec[i].first)); - CHECK_AND_ASSERT_MES(amount_outs_vec[i].second < tx_it->second.tx.vout.size(), false, - "Wrong index in transaction outputs: " << amount_outs_vec[i].second << ", expected less then " << tx_it->second.tx.vout.size()); - if(!vis.handle_output(tx_it->second.tx, tx_it->second.tx.vout[amount_outs_vec[i].second])) + auto tx_id = get_txid_by_out_index(amount_outs_vec[i]); + transactions_container::iterator tx_it = m_transactions.find(tx_id); + CHECK_AND_ASSERT_MES(tx_it != m_transactions.end(), false, "Wrong transaction id in output indexes: " << tx_id); + size_t out_idx = std::get<2>(amount_outs_vec[i]); + CHECK_AND_ASSERT_MES(out_idx < tx_it->second.tx.vout.size(), false, + "Wrong index in transaction outputs: " << out_idx << ", expected less then " << tx_it->second.tx.vout.size()); + if(!vis.handle_output(tx_it->second.tx, tx_it->second.tx.vout[out_idx])) { LOG_PRINT_L0("Failed to handle_output for output no = " << count << ", with absolute offset " << i); return false;