blockchain_db/rpc: faster is_key_image_spent

Does the following to speedup the `/is_key_image_spent` RPC endpoint:
  - Reads all on-chain key images in one LMDB read transacion
  - Uses `cryptonote_core::are_key_images_spent_in_pool()` instead of `cryptonote_core::get_pool_transactions_and_spent_keys_info()` for pool querying. This only does a LMDB read per key image if in the pool.
  - Filters known on-chain spent key images before querying for key images spent in pool

This RPC endpoint was causing major daemon slowdowns in the Carrot/FCMP++ Alpha Stressnet when using the `rescan_spent` command, especially for large wallets.
The effect was much worse if the mempool was full.

src/blockchain_db/blockchain_db.cpp

@@ -407,6 +407,17 @@ transaction BlockchainDB::get_pruned_tx(const crypto::hash& h) const
   return tx;
 }
 
+std::vector<bool> BlockchainDB::has_key_images(const epee::span<const crypto::key_image> img) const
+{
+  std::vector<bool> spent(img.size(), true);
+  for (std::size_t i = 0; i < img.size(); ++i)
+  {
+    const crypto::key_image &ki = img[i];
+    spent[i] = this->has_key_image(ki);
+  }
+  return spent;
+}
+
 void BlockchainDB::reset_stats()
 {
   num_calls = 0;

src/blockchain_db/blockchain_db.h

@@ -1510,6 +1510,15 @@ public:
    */
   virtual bool has_key_image(const crypto::key_image& img) const = 0;
 
+  /**
+   * @brief check if key images are stored as spent
+   *
+   * @param img the key images to check for
+   *
+   * @return true at element `i` if the `img[i]` is present, otherwise false
+   */
+  virtual std::vector<bool> has_key_images(const epee::span<const crypto::key_image> img) const;
+
   /**
    * @brief add a txpool transaction
    *

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -3612,6 +3612,27 @@ bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const
   return ret;
 }
 
+std::vector<bool> BlockchainLMDB::has_key_images(const epee::span<const crypto::key_image> img) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+
+  std::vector<bool> ret(img.size(), true);
+
+  TXN_PREFIX_RDONLY();
+  RCURSOR(spent_keys);
+
+  for (std::size_t i = 0; i < img.size(); ++i)
+  {
+    crypto::key_image ki = img[i];
+    MDB_val k = {sizeof(ki), reinterpret_cast<void*>(&ki)};
+    ret[i] = (mdb_cursor_get(m_cur_spent_keys, const_cast<MDB_val *>(&zerokval), &k, MDB_GET_BOTH) == 0);
+  }
+
+  TXN_POSTFIX_RDONLY();
+  return ret;
+}
+
 bool BlockchainLMDB::for_all_key_images(std::function<bool(const crypto::key_image&)> f) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);

src/blockchain_db/lmdb/db_lmdb.h

@@ -282,6 +282,8 @@ public:
 
   bool has_key_image(const crypto::key_image& img) const override;
 
+  std::vector<bool> has_key_images(const epee::span<const crypto::key_image> img) const override;
+
   void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata_ref &blob, const txpool_tx_meta_t& meta) override;
   void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t& meta) override;
   uint64_t get_txpool_tx_count(relay_category category = relay_category::broadcasted) const override;

src/cryptonote_core/blockchain.cpp

@@ -3206,6 +3206,17 @@ bool Blockchain::have_tx_keyimges_as_spent(const transaction &tx) const
   }
   return false;
 }
+//------------------------------------------------------------------
+std::vector<bool> Blockchain::have_tx_keyimges_as_spent(const epee::span<const crypto::key_image> key_imgs) const
+{
+  LOG_PRINT_L3("Blockchain::" << __func__);
+  // WARNING: this function does not take m_blockchain_lock, and thus should only call read only
+  // m_db functions which do not depend on one another (ie, no getheight + gethash(height-1), as
+  // well as not accessing class members, even read only (ie, m_invalid_blocks). The caller must
+  // lock if it is otherwise needed.
+  return m_db->has_key_images(key_imgs);
+}
+//------------------------------------------------------------------
 bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys)
 {
   PERF_TIMER(expand_transaction_2);

src/cryptonote_core/blockchain.h

@@ -286,6 +286,15 @@ namespace cryptonote
      */
     bool have_tx_keyimges_as_spent(const transaction &tx) const;
 
+    /**
+     * @brief check if key images are already spent on the blockchain
+     *
+     * @param key_imgs the key images to search for
+     *
+     * @return true at element `i` iff `key_imgs[i]` is already spent in the blockchain, else false
+     */
+    std::vector<bool> have_tx_keyimges_as_spent(const epee::span<const crypto::key_image> key_imgs) const;
+
     /**
      * @brief check if a key image is already spent on the blockchain
      *

src/cryptonote_core/cryptonote_core.cpp

@@ -933,11 +933,7 @@ namespace cryptonote
   //-----------------------------------------------------------------------------------------------
   bool core::are_key_images_spent(const std::vector<crypto::key_image>& key_im, std::vector<bool> &spent) const
   {
-    spent.clear();
+    spent = m_blockchain_storage.have_tx_keyimges_as_spent(epee::to_span(key_im));
-    for(auto& ki: key_im)
-    {
-      spent.push_back(m_blockchain_storage.have_tx_keyimg_as_spent(ki));
-    }
     return true;
   }
   //-----------------------------------------------------------------------------------------------

src/cryptonote_core/tx_pool.h

@@ -37,7 +37,6 @@
 #include <unordered_map>
 #include <unordered_set>
 #include <queue>
-#include <boost/serialization/version.hpp>
 #include <boost/utility.hpp>
 #include <boost/bimap.hpp>
 #include <boost/bimap/set_of.hpp>
@@ -450,9 +449,6 @@ namespace cryptonote
      */
     void reduce_txpool_weight(size_t weight);
 
-#define CURRENT_MEMPOOL_ARCHIVE_VER    11
-#define CURRENT_MEMPOOL_TX_DETAILS_ARCHIVE_VER    13
-
     /**
      * @brief information about a single transaction
      */
@@ -724,38 +720,3 @@ private:
     friend struct BlockchainAndPool;
   };
 }
-
-namespace boost
-{
-  namespace serialization
-  {
-    template<class archive_t>
-    void serialize(archive_t & ar, cryptonote::tx_memory_pool::tx_details& td, const unsigned int version)
-    {
-      ar & td.blob_size;
-      ar & td.fee;
-      ar & td.tx;
-      ar & td.max_used_block_height;
-      ar & td.max_used_block_id;
-      ar & td.last_failed_height;
-      ar & td.last_failed_id;
-      ar & td.receive_time;
-      ar & td.last_relayed_time;
-      ar & td.relayed;
-      if (version < 11)
-        return;
-      ar & td.kept_by_block;
-      if (version < 12)
-        return;
-      ar & td.do_not_relay;
-      if (version < 13)
-        return;
-      ar & td.weight;
-    }
-  }
-}
-BOOST_CLASS_VERSION(cryptonote::tx_memory_pool, CURRENT_MEMPOOL_ARCHIVE_VER)
-BOOST_CLASS_VERSION(cryptonote::tx_memory_pool::tx_details, CURRENT_MEMPOOL_TX_DETAILS_ARCHIVE_VER)
-
-
-

src/rpc/core_rpc_server.cpp

@@ -1269,6 +1269,7 @@ namespace cryptonote
 
     CHECK_PAYMENT_MIN1(req, res, req.key_images.size() * COST_PER_KEY_IMAGE, false);
 
+    // parse key images from request
     std::vector<crypto::key_image> key_images;
     for(const auto& ki_hex_str: req.key_images)
     {
@@ -1286,44 +1287,53 @@ namespace cryptonote
       crypto::key_image &ki = key_images.emplace_back();
       memcpy(&ki, b.data(), sizeof(crypto::key_image));
     }
+
+    // check key images in blockchain
     std::vector<bool> spent_status;
     bool r = m_core.are_key_images_spent(key_images, spent_status);
-    if(!r)
+    if (!r || spent_status.size() != key_images.size())
     {
       res.status = "Failed";
       return true;
     }
     res.spent_status.clear();
+    res.spent_status.reserve(spent_status.size());
     for (size_t n = 0; n < spent_status.size(); ++n)
       res.spent_status.push_back(spent_status[n] ? COMMAND_RPC_IS_KEY_IMAGE_SPENT::SPENT_IN_BLOCKCHAIN : COMMAND_RPC_IS_KEY_IMAGE_SPENT::UNSPENT);
 
-    // check the pool too
+    // filter out known spent key images
-    std::vector<cryptonote::tx_info> txs;
+    std::vector<crypto::key_image> filtered_key_images;
-    std::vector<cryptonote::spent_key_image_info> ki;
+    std::vector<std::size_t> filtered_key_image_idxs;
-    r = m_core.get_pool_transactions_and_spent_keys_info(txs, ki, !request_has_rpc_origin || !restricted);
+    filtered_key_images.reserve(key_images.size());
-    if(!r)
+    filtered_key_image_idxs.reserve(key_images.size());
+    for (std::size_t i = 0; i < key_images.size(); ++i)
+    {
+      if (!spent_status.at(i))
+      {
+        filtered_key_images.push_back(key_images.at(i));
+        filtered_key_image_idxs.push_back(i);
+      }
+    }
+    if (filtered_key_images.size() != filtered_key_image_idxs.size())
     {
       res.status = "Failed";
       return true;
     }
-    for (std::vector<cryptonote::spent_key_image_info>::const_iterator i = ki.begin(); i != ki.end(); ++i)
+
+    // check the pool too
+    spent_status.clear();
+    r = m_core.are_key_images_spent_in_pool(filtered_key_images, spent_status);
+    if (!r || spent_status.size() != filtered_key_images.size())
     {
-      crypto::hash hash;
+      res.status = "Failed";
-      crypto::key_image spent_key_image;
+      return true;
-      if (parse_hash256(i->id_hash, hash))
+    }
+    for (std::size_t i = 0; i < spent_status.size(); ++i)
+    {
+      if (spent_status.at(i))
       {
-        memcpy(&spent_key_image, &hash, sizeof(hash)); // a bit dodgy, should be other parse functions somewhere
+        const std::size_t res_idx = filtered_key_image_idxs.at(i);
-        for (size_t n = 0; n < res.spent_status.size(); ++n)
+        res.spent_status.at(res_idx) = COMMAND_RPC_IS_KEY_IMAGE_SPENT::SPENT_IN_POOL;
-        {
-          if (res.spent_status[n] == COMMAND_RPC_IS_KEY_IMAGE_SPENT::UNSPENT)
-          {
-            if (key_images[n] == spent_key_image)
-            {
-              res.spent_status[n] = COMMAND_RPC_IS_KEY_IMAGE_SPENT::SPENT_IN_POOL;
-              break;
-            }
-          }
-        }
       }
     }