Fix ringct+coinbase output tracking, and update rescan rollbacks (#178)

src/db/storage.cpp

@@ -2100,7 +2100,7 @@ namespace db
   {
     //! \return Success, even if `address` was not found (designed for
     expect<void>
-    change_height(MDB_cursor& accounts_cur, MDB_cursor& accounts_ba_cur, MDB_cursor& accounts_bh_cur, block_id height, account_address const& address)
+    change_height(MDB_cursor& accounts_cur, MDB_cursor& accounts_ba_cur, MDB_cursor& accounts_bh_cur, MDB_cursor& outputs_cur, MDB_cursor& spends_cur, MDB_cursor& images_cur, block_id height, account_address const& address)
     {
       MDB_val key = lmdb::to_val(by_address_version);
       MDB_val value = lmdb::to_val(address);
@@ -2146,6 +2146,9 @@ namespace db
         mdb_cursor_put(&accounts_bh_cur, &key, &value, MDB_NODUPDATA)
       );
 
+      MONERO_CHECK(rollback_outputs(user->id, height, outputs_cur));
+      MONERO_CHECK(rollback_spends(user->id, height, spends_cur, images_cur));
+
       return success();
     }
   }
@@ -2180,15 +2183,21 @@ namespace db
       cursor::accounts accounts_cur;
       cursor::accounts_by_address accounts_ba_cur;
       cursor::accounts_by_height accounts_bh_cur;
+      cursor::outputs outputs_cur;
+      cursor::spends spends_cur;
+      cursor::images images_cur;
 
       MONERO_CHECK(check_cursor(txn, this->db->tables.accounts, accounts_cur));
       MONERO_CHECK(check_cursor(txn, this->db->tables.accounts_ba, accounts_ba_cur));
       MONERO_CHECK(check_cursor(txn, this->db->tables.accounts_bh, accounts_bh_cur));
+      MONERO_CHECK(check_cursor(txn, this->db->tables.outputs, outputs_cur));
+      MONERO_CHECK(check_cursor(txn, this->db->tables.spends, spends_cur));
+      MONERO_CHECK(check_cursor(txn, this->db->tables.images, images_cur));
 
       for (account_address const& address : addresses)
       {
         const expect<void> changed = change_height(
-          *accounts_cur, *accounts_ba_cur, *accounts_bh_cur, height, address
+          *accounts_cur, *accounts_ba_cur, *accounts_bh_cur, *outputs_cur, *spends_cur, *images_cur, height, address
         );
         if (changed)
           updated.push_back(address);
@@ -2421,11 +2430,17 @@ namespace db
       cursor::accounts accounts_ba_cur;
       cursor::accounts accounts_bh_cur;
       cursor::requests requests_cur;
+      cursor::outputs outputs_cur;
+      cursor::spends spends_cur;
+      cursor::images images_cur;
 
       MONERO_CHECK(check_cursor(txn, tables.accounts, accounts_cur));
       MONERO_CHECK(check_cursor(txn, tables.accounts_ba, accounts_ba_cur));
       MONERO_CHECK(check_cursor(txn, tables.accounts_bh, accounts_bh_cur));
       MONERO_CHECK(check_cursor(txn, tables.requests, requests_cur));
+      MONERO_CHECK(check_cursor(txn, tables.outputs, outputs_cur));
+      MONERO_CHECK(check_cursor(txn, tables.spends, spends_cur));
+      MONERO_CHECK(check_cursor(txn, tables.images, images_cur));
 
       const request req = request::import_scan;
       for (account_address const& address : addresses)
@@ -2445,7 +2460,7 @@ namespace db
           return new_height.error();
 
         const expect<void> changed = change_height(
-          *accounts_cur, *accounts_ba_cur, *accounts_bh_cur, *new_height, address
+          *accounts_cur, *accounts_ba_cur, *accounts_bh_cur, *outputs_cur, *spends_cur, *images_cur, *new_height, address
         );
         if (changed)
           updated.push_back(address);

src/rpc/light_wallet.cpp

@@ -107,7 +107,10 @@ namespace
 
     rct_bytes rct{};
     rct_bytes const* optional_rct = nullptr;
-    if (unpack(self.data.first.extra).first & lws::db::ringct_output)
+    const auto flags = unpack(self.data.first.extra).first;
+    if (flags & lws::db::ringct_output)
+    {
+      if (!(flags & lws::db::coinbase_output))
       {
         crypto::key_derivation derived;
         if (!crypto::generate_key_derivation(self.data.first.spend_meta.tx_public, self.user_key, derived))
@@ -122,6 +125,9 @@ namespace
         rct.commitment = rct::commit(self.data.first.spend_meta.amount, self.data.first.ringct_mask);
         rct.mask = encrypted.mask;
         rct.amount = encrypted.amount;
+      }
+      else
+        rct.mask = rct::identity();
 
       optional_rct = std::addressof(rct);
     }

src/scanner.cpp

@@ -507,6 +507,8 @@ namespace lws
             mask = decrypted->second;
             ext = db::extra(ext | db::ringct_output);
           }
+          else if (1 < tx.version)
+            ext = db::extra(ext | db::ringct_output);
 
           if (extra_nonce)
           {

tests/unit/rest.test.cpp

@@ -79,10 +79,12 @@ namespace
     };
   }
 
-  rct_bytes get_rct_bytes(const crypto::secret_key& user_key, const crypto::public_key& tx_public, const rct::key& ringct_mask, const std::uint64_t amount, const std::uint32_t index)
+  rct_bytes get_rct_bytes(const crypto::secret_key& user_key, const crypto::public_key& tx_public, const rct::key& ringct_mask, const std::uint64_t amount, const std::uint32_t index, bool coinbase)
   {
     rct_bytes out{};
 
+    if (!coinbase)
+    {
       crypto::key_derivation derived;
       if (!crypto::generate_key_derivation(tx_public, user_key, derived))
         MONERO_THROW(lws::error::crypto_failure, "generate_key_derivation failed");
@@ -96,6 +98,10 @@ namespace
       out.commitment = rct::commit(amount, ringct_mask);
       out.mask = encrypted.mask;
       out.amount = encrypted.amount;
+    }
+    else
+      out.mask = rct::identity();
+
     return out;
   }
 }
@@ -289,7 +295,7 @@ LWS_CASE("rest_server")
       boost::thread server_thread(&lws_test::rpc_thread, context.zmq_context(), std::cref(messages));
       const join on_scope_exit{server_thread};
 
-      const auto ringct_expanded = get_rct_bytes(view, tx_public, ringct, 40000, 2);
+      const auto ringct_expanded = get_rct_bytes(view, tx_public, ringct, 40000, 2, true);
       message = "{\"address\":\"" + address + "\",\"view_key\":\"" + viewkey + "\",\"amount\":\"0\"}";
       response = invoke(client, "/get_unspent_outs", message);
       EXPECT(response ==
@@ -446,7 +452,7 @@ LWS_CASE("rest_server")
       boost::thread server_thread(&lws_test::rpc_thread, context.zmq_context(), std::cref(messages));
       const join on_scope_exit{server_thread};
 
-      const auto ringct_expanded = get_rct_bytes(view, tx_public, ringct, 40000, 2);
+      const auto ringct_expanded = get_rct_bytes(view, tx_public, ringct, 40000, 2, true);
       message = "{\"address\":\"" + address + "\",\"view_key\":\"" + viewkey + "\",\"amount\":\"0\"}";
       response = invoke(client, "/get_unspent_outs", message);
       EXPECT(response ==

tests/unit/scanner.test.cpp

@@ -560,7 +560,7 @@ LWS_CASE("lws::scanner::sync and lws::scanner::run")
               tx.spend_publics.at(0),
               rct::commit(35184372088830, rct::identity()),
               {},
-              lws::db::pack(lws::db::extra::coinbase_output, 0),
+              lws::db::pack(lws::db::extra(lws::db::extra::coinbase_output | lws::db::extra::ringct_output), 0),
               {},
               0, // fee
               lws::db::address_index{}