monero_blockchain_export: >2.5x blocksdat export speed

Uses efficient DB traversal to slash disk read time, which is a large chunk of the run-time.

src/blockchain_db/blockchain_db.h

@@ -1709,6 +1709,32 @@ public:
    */
   virtual bool for_blocks_range(const uint64_t& h1, const uint64_t& h2, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const = 0;
 
+  /**
+   * @brief runs a function over all block info in height range in order
+   *
+   * The subclass should run the passed function for each block in the
+   * specified range, passing (height, block ID, block weight) as its parameters.
+   *
+   * If any call to the function returns false, the subclass should return
+   * false. Otherwise, the subclass returns true.
+   *
+   * The subclass should throw DB_ERROR if any of the expected values are
+   * not found.
+   *
+   * @param start_height start height, inclusive
+   * @param stop_height stop height, inclusive
+   * @param f the function to run
+   *
+   * @return false if the function returns false for any block, otherwise true
+   */
+  virtual bool for_all_block_info(uint64_t start_height
+    , uint64_t stop_height
+    , const std::function<bool(/*height*/ uint64_t
+      , /*block ID*/ const crypto::hash&
+      , /*block weight*/ uint64_t
+      )> &f
+  ) const = 0;
+
   /**
    * @brief runs a function over all transactions stored
    *

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -3694,6 +3694,49 @@ bool BlockchainLMDB::for_blocks_range(const uint64_t& h1, const uint64_t& h2, st
   return fret;
 }
 
+bool BlockchainLMDB::for_all_block_info(const uint64_t start_height
+    , const uint64_t stop_height
+    , const std::function<bool(/*height*/ uint64_t
+      , /*block ID*/ const crypto::hash&
+      , /*block weight*/ uint64_t
+      )> &f
+  ) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
+  if (stop_height < start_height)
+    throw0(DB_ERROR("Invalid stop height, lower than start height"));
+
+  TXN_PREFIX_RDONLY();
+  RCURSOR(block_info);
+
+  MDB_val_set(k, zerokey);
+  MDB_val_set(v, start_height);
+  bool fret = true;
+  MDB_cursor_op op = MDB_GET_BOTH;
+
+  for (uint64_t height = start_height; height <= stop_height; ++height)
+  {
+    int ret = mdb_cursor_get(m_cur_block_info, &k, &v, op);
+    op = MDB_NEXT;
+    if (ret)
+      throw0(DB_ERROR(lmdb_error("Failed to get block info at given height", ret).c_str()));
+    assert(v.mv_size == sizeof(mdb_block_info));
+    assert(reinterpret_cast<uintptr_t>(v.mv_data) % alignof(mdb_block_info) == 0);
+    const mdb_block_info *bi = (const mdb_block_info *)v.mv_data;
+    assert(bi->bi_height == height);
+    if (!f(height, bi->bi_hash, bi->bi_weight)) {
+      fret = false;
+      break;
+    }
+  }
+
+  TXN_POSTFIX_RDONLY();
+
+  return fret;
+}
+
 bool BlockchainLMDB::for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)> f, bool pruned) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);

src/blockchain_db/lmdb/db_lmdb.h

@@ -305,6 +305,13 @@ public:
 
   bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const override;
   bool for_blocks_range(const uint64_t& h1, const uint64_t& h2, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const override;
+  bool for_all_block_info(uint64_t start_height
+    , uint64_t stop_height
+    , const std::function<bool(/*height*/ uint64_t
+      , /*block ID*/ const crypto::hash&
+      , /*block weight*/ uint64_t
+      )> &f
+  ) const override;
   bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>, bool pruned) const override;
   bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const override;
   bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const override;

src/blockchain_db/testdb.h

@@ -121,6 +121,7 @@ public:
 
   virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const override { return true; }
   virtual bool for_blocks_range(const uint64_t&, const uint64_t&, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const override { return true; }
+  virtual bool for_all_block_info(uint64_t, uint64_t, const std::function<bool(uint64_t, const crypto::hash&, uint64_t)> &f) const override { return true; }
   virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>, bool pruned) const override { return true; }
   virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const override { return true; }
   virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const override { return true; }

src/blockchain_utilities/blocksdat_file.cpp

@@ -156,11 +156,12 @@ bool BlocksdatFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
     MFATAL("failed to open raw file for write");
     return false;
   }
-  for (m_cur_height = block_start; m_cur_height <= block_stop; ++m_cur_height)
+  m_blockchain_storage->get_db().for_all_block_info(block_start,
-  {
+    block_stop,
+    [&, this](const uint64_t height, const crypto::hash &hash, const uint64_t weight) -> bool {
+
     // this method's height refers to 0-based height (genesis block = height 0)
-    crypto::hash hash = m_blockchain_storage->get_block_id_by_height(m_cur_height);
+    m_cur_height = height;
-    uint64_t weight = m_blockchain_storage->get_db().get_block_weight(m_cur_height);
     write_block(hash, weight);
     if (m_cur_height % NUM_BLOCKS_PER_CHUNK == 0) {
       num_blocks_written += NUM_BLOCKS_PER_CHUNK;
@@ -169,10 +170,12 @@ bool BlocksdatFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
       std::cout << refresh_string;
       std::cout << "block " << m_cur_height << "/" << block_stop << std::flush;
     }
-  }
+
+    return true;
+  });
   // print message for last block, which may not have been printed yet due to progress_interval
   std::cout << refresh_string;
-  std::cout << "block " << m_cur_height-1 << "/" << block_stop << ENDL;
+  std::cout << "block " << m_cur_height << "/" << block_stop << ENDL;
 
   MINFO("Number of blocks exported: " << num_blocks_written);