rpc: speedup block&tx RPC calls

In RPC calls `get_blocks.bin` and `get_transactions`:
  1. Skip deserializing then re-serializing tx blobs
  2. Provide full unpruned transaction from mempool in `get_transactions` when `!req.prune`
  3. Change `on_get_transactions()` response sort time from O(N^2) to O(N)
  4. Generally improve memory usage and lookups in `on_get_transactions()`

Reviewed-by: selsta <selsta@sent.at>
Reviewed-by: j-berman <justinberman@protonmail.com>
Reviewed-by: jpk68 <jpk68@tutanota.com>

src/cryptonote_basic/cryptonote_format_utils.cpp

@@ -283,6 +283,17 @@ namespace cryptonote
     get_transaction_prefix_hash(tx, tx_prefix_hash);
     return true;
   }
+  //------------------------------------------------------------------
+  size_t get_tx_version(const blobdata_ref tx_blob)
+  {
+    size_t version;
+    const char* begin = reinterpret_cast<const char*>(tx_blob.data());
+    const char* end = begin + tx_blob.size();
+    int read = tools::read_varint(begin, end, version);
+    if (read <= 0)
+      throw std::runtime_error("Internal error getting transaction version");
+    return version;
+  }
   //---------------------------------------------------------------
   bool is_v1_tx(const blobdata_ref& tx_blob)
   {
@@ -300,6 +311,126 @@ namespace cryptonote
     return is_v1_tx(blobdata_ref{tx_blob.data(), tx_blob.size()});
   }
   //---------------------------------------------------------------
+  bool get_transaction_unprunable_summary(const blobdata_ref tx_blob, unprunable_summary_t &summary_out)
+  {
+    //! @TODO: update for FCMP++:
+    //!    * Allow rct::RctTypeFcmpPlusPlus
+    //!    * Set output length for txout_to_carrot_v1
+
+    const unsigned char *p = reinterpret_cast<const unsigned char*>(tx_blob.data());
+    const unsigned char *end = reinterpret_cast<const unsigned char*>(tx_blob.data() + tx_blob.size());
+
+    #define READ_VARINT(v) if (tools::read_varint(p, end, v) <= 0) return false
+    #define READ_BYTE(v) if (end <= p ) { return false; } else { v = *p; ++p; }
+    #define SKIP(n) if (end - p < static_cast<std::ptrdiff_t>(n)) { return false; } else { p += (n); }
+
+    // read and validate tx version
+    READ_VARINT(summary_out.version);
+    if (summary_out.version < 1 || summary_out.version > 2)
+      return false;
+
+    // skip unlock_time
+    std::uint64_t dummy;
+    READ_VARINT(dummy);
+
+    // read number of inputs
+    READ_VARINT(summary_out.n_inputs);
+
+    // skip n_inputs inputs, checking for coinbase inputs
+    summary_out.is_coinbase = false;
+    for (std::size_t i = 0; i < summary_out.n_inputs; ++i)
+    {
+      std::size_t input_tag = 0;
+      READ_BYTE(input_tag);
+      switch (input_tag)
+      {
+      case 0xff: //txin_gen
+        summary_out.is_coinbase = true;
+        READ_VARINT(dummy); //height
+        break;
+      case 0x02: //txin_to_key
+        READ_VARINT(dummy); //amount
+        READ_VARINT(input_tag); //key_offsets.size()
+        for (;input_tag-->0;)
+          READ_VARINT(dummy);
+        SKIP(32); //k_image
+        break;
+      default:
+        return false;
+      }
+    }
+
+    // read number of outputs
+    READ_VARINT(summary_out.n_outputs);
+
+    // skip n_outputs outputs
+    for (std::size_t i = 0; i < summary_out.n_outputs; ++i)
+    {
+      READ_VARINT(dummy); //amount
+      std::size_t output_tag = 0;
+      READ_BYTE(output_tag); //target variant tag
+      std::ptrdiff_t output_length = 0;
+      switch (output_tag)
+      {
+      case 0x02: //txout_to_key
+        output_length = 32;
+        break;
+      case 0x03: //txout_to_tagged_key
+        output_length = 32 + 1;
+        break;
+      default:
+        return false;
+      }
+      SKIP(output_length);
+    }
+
+    // read extra length and skip that
+    READ_VARINT(summary_out.extra_len);
+    if (summary_out.extra_len > CRYPTONOTE_MAX_TX_SIZE)
+      return false;
+    SKIP(summary_out.extra_len);
+
+    // now `p` is at end of tx prefix...
+    summary_out.prefix_size = reinterpret_cast<const char*>(p) - tx_blob.data();
+
+    if (2 == summary_out.version)
+    {
+      // read RingCT type
+      std::uint8_t rct_type = std::numeric_limits<std::uint8_t>::max();
+      READ_BYTE(rct_type);
+      if (rct_type > rct::RCTTypeBulletproofPlus)
+        return false;
+
+      // skip unprunable RingCT fields
+      if (rct_type != rct::RCTTypeNull)
+      {
+        // skip txnFee
+        READ_VARINT(dummy);
+
+        // if RCTTypeSimple, skip pseudoOutputs
+        if (rct_type == rct::RCTTypeSimple)
+        {
+          SKIP(32 * summary_out.n_inputs);
+        }
+
+        // skip ECDH info and amount commitments
+        const bool short_amount = rct_type >= rct::RCTTypeBulletproof2;
+        const std::ptrdiff_t ecdh_tuple_len = short_amount ? 8 : 64;
+        const std::ptrdiff_t output_stuff_len = summary_out.n_outputs * (ecdh_tuple_len + 32);
+        SKIP(output_stuff_len);
+      }
+    }
+
+    #undef READ_VARINT
+    #undef READ_BYTE
+    #undef SKIP
+
+    // now `p` is at end of unprunable part of v2 tx...
+    summary_out.unprunable_size = reinterpret_cast<const char*>(p) - tx_blob.data();
+
+    return true;
+  }
+  //---------------------------------------------------------------
   bool generate_key_image_helper(const account_keys& ack, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, const crypto::public_key& out_key, const crypto::public_key& tx_public_key, const std::vector<crypto::public_key>& additional_tx_public_keys, size_t real_output_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev)
   {
     crypto::key_derivation recv_derivation{};
@@ -535,6 +666,19 @@ namespace cryptonote
     return tx.blob_size;
   }
   //---------------------------------------------------------------
+  bool prune_transaction_blob(cryptonote::blobdata &tx_blob)
+  {
+    // Deserialize unprunable part to retrieve size
+    unprunable_summary_t desc;
+    if (!get_transaction_unprunable_summary(tx_blob, desc))
+      return false;
+
+    CHECK_AND_ASSERT_MES(desc.unprunable_size <= tx_blob.size(), false, "Unprunable size is larger than tx blob");
+    tx_blob.resize(desc.unprunable_size);
+
+    return true;
+  }
+  //---------------------------------------------------------------
   bool get_tx_fee(const transaction& tx, uint64_t & fee)
   {
     if (tx.version > 1)
@@ -1293,6 +1437,18 @@ namespace cryptonote
     return get_transaction_hash(t, res, NULL);
   }
   //---------------------------------------------------------------
+  bool calculate_transaction_prunable_hash(const std::size_t tx_version, const blobdata_ref prunable_tx_blob, crypto::hash &res)
+  {
+    switch (tx_version)
+    {
+    case 2:
+      cryptonote::get_blob_hash(prunable_tx_blob, res);
+      return true;
+    default:
+      return false;
+    }
+  }
+  //---------------------------------------------------------------
   bool calculate_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata_ref *blob, crypto::hash& res)
   {
     if (t.version == 1)
@@ -1301,7 +1457,8 @@ namespace cryptonote
     if (blob && unprunable_size)
     {
       CHECK_AND_ASSERT_MES(unprunable_size <= blob->size(), false, "Inconsistent transaction unprunable and blob sizes");
-      cryptonote::get_blob_hash(blobdata_ref(blob->data() + unprunable_size, blob->size() - unprunable_size), res);
+      return calculate_transaction_prunable_hash(t.version,
+        blobdata_ref(blob->data() + unprunable_size, blob->size() - unprunable_size), res);
     }
     else
     {

src/cryptonote_basic/cryptonote_format_utils.h

@@ -58,8 +58,26 @@ namespace cryptonote
   bool parse_and_validate_tx_from_blob(const blobdata_ref& tx_blob, transaction& tx, crypto::hash& tx_hash, const bool max_size_check = false);
   bool parse_and_validate_tx_from_blob(const blobdata_ref& tx_blob, transaction& tx, const bool max_size_check = false);
   bool parse_and_validate_tx_base_from_blob(const blobdata_ref& tx_blob, transaction& tx, const bool max_size_check = false);
+  /**
+   * @brief extract transaction version from transaction blob
+   * @param tx_blob transction blob
+   * @return transaction version
+   * @throw std::runtime_error if the version could not be deserialized
+   */
+  size_t get_tx_version(const blobdata_ref tx_blob);
   bool is_v1_tx(const blobdata_ref& tx_blob);
   bool is_v1_tx(const blobdata& tx_blob);
+  struct unprunable_summary_t
+  {
+    std::size_t version;
+    std::size_t n_inputs;
+    std::size_t n_outputs;
+    std::size_t extra_len;
+    bool is_coinbase;
+    std::size_t prefix_size;
+    std::size_t unprunable_size;
+  };
+  bool get_transaction_unprunable_summary(const blobdata_ref tx_blob, unprunable_summary_t &summary_out);
 
   template<typename T>
   bool find_tx_extra_field_by_type(const std::vector<tx_extra_field>& tx_extra_fields, T& field, size_t index = 0)
@@ -116,6 +134,14 @@ namespace cryptonote
   bool get_transaction_hash(const transaction& t, crypto::hash& res);
   bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t& blob_size);
   bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size);
+  /**
+   * @brief calculate transaction prunable hash from the prunable blob and version
+   * @param tx_version transaction version
+   * @param prunable_tx_blob blob of prunable part of transaction
+   * @param[out] res transaction prunable hash
+   * @return true on success, false otherwise
+   */
+  bool calculate_transaction_prunable_hash(const std::size_t tx_version, const blobdata_ref prunable_tx_blob, crypto::hash &res);
   bool calculate_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata_ref *blob, crypto::hash& res);
   crypto::hash get_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata_ref *blob = NULL);
   bool calculate_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size);
@@ -139,6 +165,12 @@ namespace cryptonote
   uint64_t get_transaction_weight(const transaction &tx, size_t blob_size);
   uint64_t get_pruned_transaction_weight(const transaction &tx);
   uint64_t get_transaction_blob_size(const transaction& tx);
+  /**
+   * @brief prune a transaction blob in-place to just its unprunable part
+   * @param[inout] tx_blob transaction blob
+   * @return true on success, false otherwise
+   */
+  bool prune_transaction_blob(cryptonote::blobdata &tx_blob);
 
   bool check_money_overflow(const transaction& tx);
   bool check_outs_overflow(const transaction& tx);

src/cryptonote_core/blockchain.cpp

@@ -2074,13 +2074,13 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id
     bei.block_cumulative_weight = cryptonote::get_transaction_weight(b.miner_tx);
     for (const crypto::hash &txid: b.tx_hashes)
     {
-      cryptonote::tx_memory_pool::tx_details td;
       cryptonote::blobdata blob;
       if (m_tx_pool.have_tx(txid, relay_category::legacy))
       {
-        if (m_tx_pool.get_transaction_info(txid, td, true/*include_sensitive_data*/))
+        cryptonote::txpool_tx_meta_t tx_meta;
+        if (this->get_txpool_tx_meta(txid, tx_meta))
         {
-          bei.block_cumulative_weight += td.weight;
+          bei.block_cumulative_weight += tx_meta.weight;
         }
         else
         {

src/cryptonote_core/tx_pool.cpp

@@ -594,7 +594,7 @@ namespace cryptonote
     return true;
   }
   //---------------------------------------------------------------------------------
-  bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data, bool include_blob) const
+  bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data) const
   {
     PERF_TIMER(get_transaction_info);
     CRITICAL_REGION_LOCAL(m_transactions_lock);
@@ -614,22 +614,11 @@ namespace cryptonote
         // We don't want sensitive data && the tx is sensitive, so no need to return it
         return false;
       }
-      cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid, relay_category::all);
-      auto ci = m_parsed_tx_cache.find(txid);
-      if (ci != m_parsed_tx_cache.end())
-      {
-        td.tx = ci->second;
-      }
-      else if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(txblob, td.tx) : parse_and_validate_tx_from_blob(txblob, td.tx)))
-      {
-        MERROR("Failed to parse tx from txpool");
-        return false;
-      }
-      else
-      {
-        td.tx.set_hash(txid);
-      }
-      td.blob_size = txblob.size();
+
+      // Fetch tx blob
+      td.tx_blob = m_blockchain.get_txpool_tx_blob(txid, relay_category::all);
+
+      // Fill in other details from meta entry
       td.weight = meta.weight;
       td.fee = meta.fee;
       td.max_used_block_id = meta.max_used_block_id;
@@ -642,8 +631,6 @@ namespace cryptonote
       td.relayed = meta.relayed;
       td.do_not_relay = meta.do_not_relay;
       td.double_spend_seen = meta.double_spend_seen;
-      if (include_blob)
-        td.tx_blob = std::move(txblob);
     }
     catch (const std::exception &e)
     {
@@ -670,7 +657,7 @@ namespace cryptonote
       {
         const crypto::hash &it{txids[i]};
         tx_details details;
-        const bool success = get_transaction_info(it, details, include_sensitive, true /*include_blob*/);
+        const bool success = get_transaction_info(it, details, include_sensitive);
         if (!success)
           continue;
 

src/cryptonote_core/tx_pool.h

@@ -432,9 +432,7 @@ namespace cryptonote
      */
     struct tx_details
     {
-      transaction tx;  //!< the transaction
       cryptonote::blobdata tx_blob; //!< the transaction's binary blob
-      size_t blob_size;  //!< the transaction's size
       size_t weight;  //!< the transaction's weight
       uint64_t fee;  //!< the transaction's fee amount
       crypto::hash max_used_block_id;  //!< the hash of the highest block referenced by an input
@@ -472,7 +470,7 @@ namespace cryptonote
     /**
      * @brief get information about a single transaction
      */
-    bool get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data, bool include_blob = false) const;
+    bool get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data) const;
 
     /**
      * @brief get information about multiple transactions

src/rpc/core_rpc_server.cpp

@@ -99,7 +99,7 @@ namespace
   {
     store_128(difficulty, sdiff, swdiff, stop64);
   }
-}
+} //anonymous namespace
 
 namespace cryptonote
 {
@@ -433,16 +433,6 @@ namespace cryptonote
     return true;
   }
   //------------------------------------------------------------------------------------------------------------------------------
-  class pruned_transaction {
-    transaction& tx;
-  public:
-    pruned_transaction(transaction& tx) : tx(tx) {}
-    BEGIN_SERIALIZE_OBJECT()
-      bool r = tx.serialize_base(ar);
-      if (!r) return false;
-    END_SERIALIZE()
-  };
-  //------------------------------------------------------------------------------------------------------------------------------
   bool core_rpc_server::on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res, const connection_context *ctx)
   {
     RPC_TRACKER(get_blocks);
@@ -505,23 +495,19 @@ namespace cryptonote
       }
 
       res.added_pool_txs.clear();
-      for (const auto &added_pool_tx: added_pool_txs)
+      res.added_pool_txs.reserve(added_pool_txs.size());
+      for (auto &added_pool_tx: added_pool_txs)
       {
-        COMMAND_RPC_GET_BLOCKS_FAST::pool_tx_info info;
+        COMMAND_RPC_GET_BLOCKS_FAST::pool_tx_info &info = res.added_pool_txs.emplace_back();
         info.tx_hash = added_pool_tx.first;
-        std::stringstream oss;
-        binary_archive<true> ar(oss);
-        bool r = req.prune
-          ? const_cast<cryptonote::transaction&>(added_pool_tx.second.tx).serialize_base(ar)
-          : ::serialization::serialize(ar, const_cast<cryptonote::transaction&>(added_pool_tx.second.tx));
-        if (!r)
+        tx_memory_pool::tx_details &tx_details = added_pool_tx.second;
+        info.tx_blob = std::move(tx_details.tx_blob);
+        if (req.prune && !prune_transaction_blob(info.tx_blob))
         {
-          res.status = "Failed to serialize transaction";
+          res.status = "Failed to prune pool transaction";
           return true;
         }
-        info.tx_blob = oss.str();
-        info.double_spend_seen = added_pool_tx.second.double_spend_seen;
-        res.added_pool_txs.push_back(std::move(info));
+        info.double_spend_seen = tx_details.double_spend_seen;
       }
 
       res.pool_info_extent = incremental
@@ -697,7 +683,6 @@ namespace cryptonote
     RPC_TRACKER(get_transactions);
 
     const bool restricted = m_restricted && ctx;
-    const bool request_has_rpc_origin = ctx != NULL;
 
     if (restricted && req.txs_hashes.size() > RESTRICTED_TRANSACTIONS_COUNT)
     {
@@ -731,67 +716,92 @@ namespace cryptonote
     }
     LOG_PRINT_L2("Found " << txs.size() << "/" << vh.size() << " transactions on the blockchain");
 
-    // try the pool for any missing txes
-    size_t found_in_pool = 0;
-    std::unordered_set<crypto::hash> pool_tx_hashes;
+    /**
+     * @brief maps TXID -> mempol tx details
+     * @note: `per_tx_pool_tx_details` *will* be missing blobs
+     */
     std::unordered_map<crypto::hash, tx_memory_pool::tx_details> per_tx_pool_tx_details;
+
+    // try the pool for any missing txes
     if (!missed_txs.empty())
     {
       std::vector<std::pair<crypto::hash, tx_memory_pool::tx_details>> pool_txs;
-      bool r = m_core.get_pool_transactions_info(missed_txs, pool_txs, !request_has_rpc_origin || !restricted);
+      bool r = m_core.get_pool_transactions_info(missed_txs, pool_txs, !restricted);
       if(r)
       {
-        // sort to match original request
+        // merge blockchain and pool txs into original request order
         std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>> sorted_txs;
-        const std::unordered_set<crypto::hash> missed_set(missed_txs.begin(), missed_txs.end());
-        const std::unordered_map<crypto::hash, tx_memory_pool::tx_details> pool_tx_map(pool_txs.begin(), pool_txs.end());
-        std::unordered_map<crypto::hash, tx_memory_pool::tx_details>::const_iterator i;
+        sorted_txs.reserve(txs.size() + pool_txs.size());
         unsigned txs_processed = 0;
-        missed_txs.clear();
+        unsigned pool_txs_processed = 0;
+        unsigned missed_txs_counted = 0;
         for (const crypto::hash &h: vh)
         {
-          if (missed_set.find(h) == missed_set.end())
+          // The below logic assumes that both blockchain and mempool responses are returned in the
+          // same order that they requested, minus txs which were missed. Also, `missed_txs` at this
+          // point is a superset of `pool_txs`
+          if (txs_processed < txs.size() && std::get<0>(txs[txs_processed]) == h)
           {
-            if (txs.size() == txs_processed)
-            {
-              res.status = "Failed: internal error - txs is empty";
-              return true;
-            }
-            // core returns the ones it finds in the right order
-            if (std::get<0>(txs[txs_processed]) != h)
-            {
-              res.status = "Failed: tx hash mismatch";
-              return true;
-            }
             sorted_txs.push_back(std::move(txs[txs_processed]));
             ++txs_processed;
           }
-          else if ((i = pool_tx_map.find(h)) != pool_tx_map.end())
+          else if (pool_txs_processed < pool_txs.size() && pool_txs[pool_txs_processed].first == h)
           {
-            const tx_memory_pool::tx_details &td = i->second;
-            std::stringstream ss;
-            binary_archive<true> ba(ss);
-            bool r = const_cast<cryptonote::transaction&>(td.tx).serialize_base(ba);
-            if (!r)
+            tx_memory_pool::tx_details &td = pool_txs[pool_txs_processed].second;
+
+            // split pruned/prunable tx blobs
+            blobdata pruned_tx_blob = td.tx_blob;
+            blobdata &prunable_tx_blob = td.tx_blob;
+            if (!prune_transaction_blob(pruned_tx_blob))
             {
-              res.status = "Failed to serialize transaction base";
+              res.status = "Failed to prune transaction blob";
               return true;
             }
-            const cryptonote::blobdata pruned = ss.str();
-            const crypto::hash prunable_hash = td.tx.version == 1 ? crypto::null_hash : get_transaction_prunable_hash(td.tx);
-            sorted_txs.push_back(std::make_tuple(h, pruned, prunable_hash, std::string(td.tx_blob, pruned.size())));
-            pool_tx_hashes.insert(h);
-            per_tx_pool_tx_details.insert(std::make_pair(h, td));
-            ++found_in_pool;
+            assert(pruned_tx_blob.size() <= prunable_tx_blob.size());
+            prunable_tx_blob.erase(prunable_tx_blob.cbegin(), prunable_tx_blob.cbegin() + pruned_tx_blob.size());
+
+            // calculate unprunable hash if applicable
+            crypto::hash prunable_hash = crypto::null_hash;
+            {
+              const std::size_t tx_version = get_tx_version(pruned_tx_blob);
+              if (1 != tx_version && !calculate_transaction_prunable_hash(tx_version, prunable_tx_blob, prunable_hash))
+              {
+                res.status = "Failed to calculate transaction prunable hash";
+                return true;
+              }
+            }
+
+            sorted_txs.emplace_back(h, std::move(pruned_tx_blob), prunable_hash, std::move(prunable_tx_blob));
+            per_tx_pool_tx_details.emplace(h, std::move(td));
+            ++pool_txs_processed;
+            ++missed_txs_counted;
+          }
+          else if (missed_txs_counted < missed_txs.size() && missed_txs[missed_txs_counted] == h)
+          {
+            ++missed_txs_counted;
           }
           else
           {
-            missed_txs.push_back(h);
+            res.status = "Business logic error in on_get_transactions() while trying to merge sorted tx list";
+            return true;
           }
         }
-        txs = sorted_txs;
+        txs = std::move(sorted_txs);
+        LOG_PRINT_L2("Found " << pool_txs_processed << "/" << vh.size() << " transactions in the pool");
+
+        // now erase missed_txs which are present in the mempool response
+        size_t erase_head = 0;
+        for (size_t erase_tail = 0; erase_tail < missed_txs.size(); ++erase_tail)
+        {
+          const bool erase = per_tx_pool_tx_details.count(missed_txs[erase_tail]);
+          if (!erase)
+          {
+            missed_txs[erase_head] = missed_txs[erase_tail];
+            ++erase_head;
+          }
+        }
+        missed_txs.resize(erase_head);
       }
-      LOG_PRINT_L2("Found " << found_in_pool << "/" << vh.size() << " transactions in the pool");
     }
 
     CHECK_AND_ASSERT_MES(txs.size() + missed_txs.size() == vh.size(), false, "mismatched number of txs");
@@ -800,6 +810,10 @@ namespace cryptonote
     auto vhi = vh.cbegin();
     auto missedi = missed_txs.cbegin();
 
+    res.txs.reserve(txs.size());
+    res.txs_as_hex.reserve(txs.size());
+    if (req.decode_as_json)
+      res.txs_as_json.reserve(txs.size());
     for(auto& tx: txs)
     {
       res.txs.push_back(COMMAND_RPC_GET_TRANSACTIONS::entry());
@@ -821,8 +835,8 @@ namespace cryptonote
       bool pruned = std::get<3>(tx).empty();
       if (pruned)
       {
-        cryptonote::transaction t;
-        if (cryptonote::parse_and_validate_tx_base_from_blob(std::get<1>(tx), t) && is_coinbase(t))
+        unprunable_summary_t tx_desc;
+        if (get_transaction_unprunable_summary(std::get<1>(tx), tx_desc) && tx_desc.is_coinbase)
           pruned = false;
       }
 
@@ -842,8 +856,8 @@ namespace cryptonote
             tx_data = std::get<1>(tx);
             if (cryptonote::parse_and_validate_tx_base_from_blob(tx_data, t))
             {
-              pruned_transaction pruned_tx{t};
-              e.as_json = obj_to_json_str(pruned_tx);
+              assert(t.pruned);
+              e.as_json = obj_to_json_str(t);
             }
             else
             {
@@ -886,25 +900,15 @@ namespace cryptonote
           }
         }
       }
-      e.in_pool = pool_tx_hashes.find(tx_hash) != pool_tx_hashes.end();
+      const auto pool_it = per_tx_pool_tx_details.find(tx_hash);
+      e.in_pool = pool_it != per_tx_pool_tx_details.cend();
       if (e.in_pool)
       {
         e.block_height = e.block_timestamp = std::numeric_limits<uint64_t>::max();
         e.confirmations = 0;
-        auto it = per_tx_pool_tx_details.find(tx_hash);
-        if (it != per_tx_pool_tx_details.end())
-        {
-          e.double_spend_seen = it->second.double_spend_seen;
-          e.relayed = it->second.relayed;
-          e.received_timestamp = it->second.receive_time;
-        }
-        else
-        {
-          MERROR("Failed to determine pool info for " << tx_hash);
-          e.double_spend_seen = false;
-          e.relayed = false;
-          e.received_timestamp = 0;
-        }
+        e.double_spend_seen = pool_it->second.double_spend_seen;
+        e.relayed = pool_it->second.relayed;
+        e.received_timestamp = pool_it->second.receive_time;
       }
       else
       {
@@ -922,7 +926,7 @@ namespace cryptonote
         res.txs_as_json.push_back(e.as_json);
 
       // output indices too if not in pool
-      if (pool_tx_hashes.find(tx_hash) == pool_tx_hashes.end())
+      if (!e.in_pool)
       {
         bool r = m_core.get_tx_outputs_gindexs(tx_hash, e.output_indices);
         if (!r)
@@ -933,6 +937,7 @@ namespace cryptonote
       }
     }
 
+    res.missed_tx.reserve(missed_txs.size());
     for(const auto& miss_tx: missed_txs)
     {
       res.missed_tx.push_back(string_tools::pod_to_hex(miss_tx));