Merge pull request #11068

91d851a blockchain_db: resume interrupted pruning (Samy)

ACKs: selsta, jpk68

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -2074,8 +2074,29 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed)
     throw0(DB_ERROR(lmdb_error("Failed to retrieve or create pruning seed: ", result).c_str()));
   }
 
+  MDB_val_str(pruning_progress_key, "pruning_progress");
+  crypto::hash pruning_progress{};
+  bool resume_pruning = false;
+  if (mode != prune_mode_check)
+  {
+    result = mdb_get(txn, m_properties, &pruning_progress_key, &v);
+    if (result == 0)
+    {
+      if (v.mv_size != sizeof(pruning_progress))
+        throw0(DB_ERROR("Failed to retrieve pruning progress: unexpected value size"));
+      memcpy(&pruning_progress, v.mv_data, sizeof(pruning_progress));
+      resume_pruning = true;
+    }
+    else if (result != MDB_NOTFOUND)
+    {
+      throw0(DB_ERROR(lmdb_error("Failed to retrieve pruning progress: ", result).c_str()));
+    }
+  }
+
   if (mode == prune_mode_check)
     MINFO("Checking blockchain pruning...");
+  else if (resume_pruning)
+    MINFO("Resuming interrupted blockchain pruning...");
   else
     MINFO("Pruning blockchain...");
 
@@ -2091,7 +2112,7 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed)
     throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable_tip: ", result).c_str()));
   const uint64_t blockchain_height = height();
 
-  if (prune_tip_table)
+  if (prune_tip_table || resume_pruning)
   {
     MDB_cursor_op op = MDB_FIRST;
     while (1)
@@ -2152,13 +2173,25 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed)
       }
     }
   }
-  else
+  // A resumed operation still needs the historical pass after pruning newly matured tip records.
+  if (!prune_tip_table || resume_pruning)
   {
     MDB_cursor *c_tx_indices;
     result = mdb_cursor_open(txn, m_tx_indices, &c_tx_indices);
     if (result)
       throw0(DB_ERROR(lmdb_error("Failed to open a cursor for tx_indices: ", result).c_str()));
     MDB_cursor_op op = MDB_FIRST;
+    if (resume_pruning)
+    {
+      MDB_val_set(progress, pruning_progress);
+      result = mdb_cursor_get(c_tx_indices, (MDB_val*)&zerokval, &progress, MDB_GET_BOTH);
+      if (result == 0)
+        op = MDB_NEXT;
+      else if (result == MDB_NOTFOUND)
+        MWARNING("Pruning progress transaction not found, restarting from the beginning");
+      else
+        throw0(DB_ERROR(lmdb_error("Failed to restore pruning progress: ", result).c_str()));
+    }
     while (1)
     {
       int ret = mdb_cursor_get(c_tx_indices, &k, &v, op);
@@ -2239,6 +2272,11 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed)
       if (mode != prune_mode_check && commit_counter >= 4096)
       {
         MDEBUG("Committing txn at checkpoint...");
+        // Commit the cursor position with the deletions so a restart cannot skip uncommitted work.
+        MDB_val_set(progress, ti.key);
+        result = mdb_put(txn, m_properties, &pruning_progress_key, &progress, 0);
+        if (result)
+          throw0(DB_ERROR(lmdb_error("Failed to save pruning progress: ", result).c_str()));
         txn.commit();
         result = mdb_txn_begin(m_env, NULL, 0, txn);
         if (result)
@@ -2265,6 +2303,13 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed)
       }
     }
     mdb_cursor_close(c_tx_indices);
+
+    if (mode != prune_mode_check)
+    {
+      result = mdb_del(txn, m_properties, &pruning_progress_key, NULL);
+      if (result && result != MDB_NOTFOUND)
+        throw0(DB_ERROR(lmdb_error("Failed to clear pruning progress: ", result).c_str()));
+    }
   }
 
   if ((result = mdb_stat(txn, m_txs_prunable, &db_stats)))

tests/unit_tests/blockchain_db.cpp

@@ -38,6 +38,7 @@
 #include "string_tools.h"
 #include "blockchain_db/blockchain_db.h"
 #include "blockchain_db/lmdb/db_lmdb.h"
+#include "common/pruning.h"
 #include "cryptonote_basic/cryptonote_format_utils.h"
 
 using namespace cryptonote;
@@ -149,6 +150,180 @@ std::string h2b(const std::string& s)
   return result;
 }
 
+void check_lmdb(int result, const char* message)
+{
+  if (result)
+    throw std::runtime_error(std::string(message) + ": " + mdb_strerror(result));
+}
+
+class test_lmdb_env
+{
+public:
+  explicit test_lmdb_env(const std::string& path)
+  {
+    check_lmdb(mdb_env_create(&m_env), "Failed to create LMDB environment");
+    try
+    {
+      check_lmdb(mdb_env_set_maxdbs(m_env, 32), "Failed to set LMDB max databases");
+      check_lmdb(mdb_env_open(m_env, path.c_str(), 0, 0644), "Failed to open LMDB environment");
+    }
+    catch (...)
+    {
+      mdb_env_close(m_env);
+      m_env = nullptr;
+      throw;
+    }
+  }
+
+  ~test_lmdb_env()
+  {
+    mdb_env_close(m_env);
+  }
+
+  MDB_env* get() const { return m_env; }
+
+private:
+  MDB_env* m_env = nullptr;
+};
+
+crypto::hash pruning_test_hash(uint32_t value)
+{
+  crypto::hash hash{};
+  memcpy(hash.data + sizeof(hash.data) - sizeof(value), &value, sizeof(value));
+  return hash;
+}
+
+MDB_dbi open_test_table(MDB_txn* txn, const char* name)
+{
+  MDB_dbi table;
+  check_lmdb(mdb_dbi_open(txn, name, 0, &table), "Failed to open LMDB table");
+  return table;
+}
+
+void create_interrupted_pruning_data(const std::string& path, size_t records, size_t missing_pruned_record)
+{
+  test_lmdb_env env(path);
+  mdb_txn_safe txn;
+  check_lmdb(mdb_txn_begin(env.get(), nullptr, 0, txn), "Failed to begin LMDB transaction");
+  const MDB_dbi blocks = open_test_table(txn, "blocks");
+  const MDB_dbi txs_pruned = open_test_table(txn, "txs_pruned");
+  const MDB_dbi txs_prunable = open_test_table(txn, "txs_prunable");
+  const MDB_dbi tx_indices = open_test_table(txn, "tx_indices");
+
+  check_lmdb(mdb_set_compare(txn, blocks, BlockchainLMDB::compare_uint64), "Failed to set blocks comparator");
+  check_lmdb(mdb_set_compare(txn, txs_pruned, BlockchainLMDB::compare_uint64),
+      "Failed to set txs_pruned comparator");
+  check_lmdb(mdb_set_compare(txn, txs_prunable, BlockchainLMDB::compare_uint64),
+      "Failed to set txs_prunable comparator");
+  check_lmdb(mdb_set_dupsort(txn, tx_indices, BlockchainLMDB::compare_hash32),
+      "Failed to set tx_indices comparator");
+
+  const char block_blob = 0;
+  MDB_val block_value{sizeof(block_blob), const_cast<char*>(&block_blob)};
+  for (uint64_t height = 0; height <= CRYPTONOTE_PRUNING_TIP_BLOCKS; ++height)
+  {
+    MDB_val block_key{sizeof(height), &height};
+    check_lmdb(mdb_put(txn, blocks, &block_key, &block_value, MDB_APPEND), "Failed to add test block");
+  }
+
+  const char pruned_blob = 2;
+  const char prunable_blob = 1;
+  const uint64_t zero = 0;
+  MDB_val tx_indices_key{sizeof(zero), const_cast<uint64_t*>(&zero)};
+  for (uint64_t tx_id = 0; tx_id < records; ++tx_id)
+  {
+    MDB_val tx_key{sizeof(tx_id), &tx_id};
+    MDB_val prunable_value{sizeof(prunable_blob), const_cast<char*>(&prunable_blob)};
+    check_lmdb(mdb_put(txn, txs_prunable, &tx_key, &prunable_value, MDB_APPEND),
+        "Failed to add prunable test transaction");
+
+    if (tx_id != missing_pruned_record)
+    {
+      MDB_val pruned_value{sizeof(pruned_blob), const_cast<char*>(&pruned_blob)};
+      check_lmdb(mdb_put(txn, txs_pruned, &tx_key, &pruned_value, MDB_APPEND),
+          "Failed to add pruned test transaction");
+    }
+
+    txindex tx_index{};
+    tx_index.key = pruning_test_hash(tx_id + 1);
+    tx_index.data.tx_id = tx_id;
+    tx_index.data.block_id = 0;
+    MDB_val tx_index_value{sizeof(tx_index), &tx_index};
+    check_lmdb(mdb_put(txn, tx_indices, &tx_indices_key, &tx_index_value, MDB_APPENDDUP),
+        "Failed to add test transaction index");
+  }
+
+  txn.commit();
+}
+
+void prepare_pruning_resume(const std::string& path, uint64_t missing_pruned_record,
+    uint64_t matured_tip_record, uint64_t skipped_record)
+{
+  test_lmdb_env env(path);
+  mdb_txn_safe txn;
+  check_lmdb(mdb_txn_begin(env.get(), nullptr, 0, txn), "Failed to begin LMDB transaction");
+  const MDB_dbi txs_pruned = open_test_table(txn, "txs_pruned");
+  const MDB_dbi txs_prunable = open_test_table(txn, "txs_prunable");
+  const MDB_dbi txs_prunable_tip = open_test_table(txn, "txs_prunable_tip");
+  check_lmdb(mdb_set_compare(txn, txs_pruned, BlockchainLMDB::compare_uint64),
+      "Failed to set txs_pruned comparator");
+  check_lmdb(mdb_set_compare(txn, txs_prunable, BlockchainLMDB::compare_uint64),
+      "Failed to set txs_prunable comparator");
+  check_lmdb(mdb_set_dupsort(txn, txs_prunable_tip, BlockchainLMDB::compare_uint64),
+      "Failed to set txs_prunable_tip comparator");
+
+  const char pruned_blob = 2;
+  MDB_val missing_key{sizeof(missing_pruned_record), &missing_pruned_record};
+  MDB_val pruned_value{sizeof(pruned_blob), const_cast<char*>(&pruned_blob)};
+  check_lmdb(mdb_put(txn, txs_pruned, &missing_key, &pruned_value, 0),
+      "Failed to repair pruned test transaction");
+
+  const char prunable_blob = 1;
+  MDB_val tip_key{sizeof(matured_tip_record), &matured_tip_record};
+  MDB_val prunable_value{sizeof(prunable_blob), const_cast<char*>(&prunable_blob)};
+  check_lmdb(mdb_put(txn, txs_prunable, &tip_key, &prunable_value, 0), "Failed to add matured tip transaction");
+
+  MDB_val skipped_key{sizeof(skipped_record), &skipped_record};
+  check_lmdb(mdb_put(txn, txs_prunable, &skipped_key, &prunable_value, 0), "Failed to add skipped transaction");
+
+  const uint64_t block_height = 0;
+  MDB_val tip_value{sizeof(block_height), const_cast<uint64_t*>(&block_height)};
+  check_lmdb(mdb_put(txn, txs_prunable_tip, &tip_key, &tip_value, 0), "Failed to add matured tip record");
+  txn.commit();
+}
+
+size_t test_table_entries(const std::string& path, const char* name)
+{
+  test_lmdb_env env(path);
+  mdb_txn_safe txn;
+  check_lmdb(mdb_txn_begin(env.get(), nullptr, MDB_RDONLY, txn), "Failed to begin LMDB transaction");
+  const MDB_dbi table = open_test_table(txn, name);
+  MDB_stat stat;
+  check_lmdb(mdb_stat(txn, table, &stat), "Failed to read LMDB table stats");
+  return stat.ms_entries;
+}
+
+bool get_pruning_progress(const std::string& path, crypto::hash& progress)
+{
+  test_lmdb_env env(path);
+  mdb_txn_safe txn;
+  check_lmdb(mdb_txn_begin(env.get(), nullptr, MDB_RDONLY, txn), "Failed to begin LMDB transaction");
+  const MDB_dbi properties = open_test_table(txn, "properties");
+  check_lmdb(mdb_set_compare(txn, properties, BlockchainLMDB::compare_string),
+      "Failed to set properties comparator");
+  const char key_data[] = "pruning_progress";
+  MDB_val key{sizeof(key_data), const_cast<char*>(key_data)};
+  MDB_val value;
+  const int result = mdb_get(txn, properties, &key, &value);
+  if (result == MDB_NOTFOUND)
+    return false;
+  check_lmdb(result, "Failed to read pruning progress");
+  if (value.mv_size != sizeof(progress))
+    throw std::runtime_error("Unexpected pruning progress size");
+  memcpy(&progress, value.mv_data, sizeof(progress));
+  return true;
+}
+
 template <typename T>
 class BlockchainDBTest : public testing::Test
 {
@@ -341,4 +516,54 @@ TYPED_TEST(BlockchainDBTest, RetrieveBlockData)
   ASSERT_HASH_EQ(get_block_hash(this->m_blocks[1].first), hashes[1]);
 }
 
+TYPED_TEST(BlockchainDBTest, ResumeInterruptedPruning)
+{
+  static constexpr size_t checkpoint_records = 4096;
+  static constexpr size_t records = checkpoint_records + 1;
+  static constexpr size_t missing_pruned_record = checkpoint_records;
+  static constexpr size_t matured_tip_record = 0;
+  static constexpr size_t skipped_record = 1;
+
+  const auto test_resume = [&](bool update)
+  {
+    SCOPED_TRACE(update ? "update" : "manual");
+    const boost::filesystem::path temp_path =
+        boost::filesystem::temp_directory_path() / boost::filesystem::unique_path();
+    const std::string dir_path = temp_path.string();
+    this->set_prefix(dir_path);
+
+    ASSERT_NO_THROW(this->m_db->open(dir_path));
+    this->get_filenames();
+    ASSERT_NO_THROW(this->m_db->close());
+    ASSERT_NO_THROW(create_interrupted_pruning_data(dir_path, records, missing_pruned_record));
+
+    const uint32_t pruning_seed = tools::make_pruning_seed(2, CRYPTONOTE_PRUNING_LOG_STRIPES);
+    ASSERT_NO_THROW(this->m_db->open(dir_path));
+    ASSERT_THROW(this->m_db->prune_blockchain(pruning_seed), DB_ERROR);
+    ASSERT_NO_THROW(this->m_db->close());
+
+    crypto::hash progress{};
+    ASSERT_EQ(1, test_table_entries(dir_path, "txs_prunable"));
+    ASSERT_TRUE(get_pruning_progress(dir_path, progress));
+    ASSERT_HASH_EQ(pruning_test_hash(checkpoint_records), progress);
+
+    ASSERT_NO_THROW(prepare_pruning_resume(dir_path, missing_pruned_record, matured_tip_record, skipped_record));
+    ASSERT_EQ(3, test_table_entries(dir_path, "txs_prunable"));
+    ASSERT_EQ(1, test_table_entries(dir_path, "txs_prunable_tip"));
+    ASSERT_NO_THROW(this->m_db->open(dir_path));
+    ASSERT_TRUE(update ? this->m_db->update_pruning() : this->m_db->prune_blockchain(pruning_seed));
+    ASSERT_NO_THROW(this->m_db->close());
+
+    ASSERT_EQ(1, test_table_entries(dir_path, "txs_prunable"));
+    ASSERT_EQ(0, test_table_entries(dir_path, "txs_prunable_tip"));
+    ASSERT_FALSE(get_pruning_progress(dir_path, progress));
+
+    boost::filesystem::remove_all(dir_path);
+    this->m_filenames.clear();
+  };
+
+  ASSERT_NO_FATAL_FAILURE(test_resume(true));
+  ASSERT_NO_FATAL_FAILURE(test_resume(false));
+}
+
 }  // anonymous namespace