Add support for remote scanning via custom TCP (#118)

src/db/account.h

@@ -40,7 +40,7 @@
 #include "wire/msgpack/fwd.h"
 
 namespace lws
-{
+{ 
   //! Tracks a subset of DB account info for scanning/updating.
   class account
   {
@@ -127,4 +127,18 @@ namespace lws
     //! Track a possible `spend`.
     void add_spend(db::spend const& spend);
   };
+
+  struct by_height
+  {
+    bool operator()(account const& left, account const& right) const noexcept
+    {
+      return left.scan_height() < right.scan_height();
+    }
+
+    bool operator()(db::account const& left, db::account const& right) const noexcept
+    {
+      return left.scan_height < right.scan_height;
+    }
+  };
+
 } // lws

src/db/data.cpp

@@ -457,20 +457,37 @@ namespace db
     map_webhook_value(dest, source, payment_id);
   }
 
-  void write_bytes(wire::writer& dest, const webhook_tx_confirmation& self)
+  namespace
+  {
+    template<typename F, typename T, typename U>
+    void map_webhook_confirmation(F& format, T& self, U& payment_id)
+    {
+       wire::object(format,
+        wire::field<0>("event", std::ref(self.key.type)),
+        wire::field<1>("payment_id", std::ref(payment_id)),
+        wire::field<2>("token", std::ref(self.value.second.token)),
+        wire::field<3>("confirmations", std::ref(self.value.second.confirmations)),
+        wire::field<4>("event_id", std::ref(self.value.first.event_id)),
+        WIRE_FIELD_ID(5, tx_info)
+      );
+    }
+  }
+
+  void read_bytes(wire::reader& source, webhook_tx_confirmation& dest)
+  {
+    crypto::hash8 payment_id{};
+    map_webhook_confirmation(source, dest, payment_id);
+
+    static_assert(sizeof(payment_id) == sizeof(dest.value.first.payment_id), "bad memcpy");
+    std::memcpy(std::addressof(dest.value.first.payment_id), std::addressof(payment_id), sizeof(payment_id));
+  }
+  void write_bytes(wire::writer& dest, const webhook_tx_confirmation& source)
   {
     crypto::hash8 payment_id;
-    static_assert(sizeof(payment_id) == sizeof(self.value.first.payment_id), "bad memcpy");
-    std::memcpy(std::addressof(payment_id), std::addressof(self.value.first.payment_id), sizeof(payment_id));
-    // to be sent to remote url
-    wire::object(dest,
-      wire::field<0>("event", std::cref(self.key.type)),
-      wire::field<1>("payment_id", std::cref(payment_id)),
-      wire::field<2>("token", std::cref(self.value.second.token)),
-      wire::field<3>("confirmations", std::cref(self.value.second.confirmations)),
-      wire::field<4>("event_id", std::cref(self.value.first.event_id)),
-      WIRE_FIELD_ID(5, tx_info)
-    );
+    static_assert(sizeof(payment_id) == sizeof(source.value.first.payment_id), "bad memcpy");
+    std::memcpy(std::addressof(payment_id), std::addressof(source.value.first.payment_id), sizeof(payment_id));
+    
+    map_webhook_confirmation(dest, source, payment_id);
   }
 
   static void write_bytes(wire::writer& dest, const output::spend_meta_& self)

src/db/data.h

@@ -291,8 +291,7 @@ namespace db
     sizeof(output) == 8 + 32 + (8 * 3) + (4 * 2) + 32 + (8 * 2) + (32 * 3) + 7 + 1 + 32 + 8 + 2 * 4,
     "padding in output"
   );
-  void read_bytes(wire::reader&, output&);
-  void write_bytes(wire::writer&, const output&);
+  WIRE_DECLARE_OBJECT(output);
 
   //! Information about a possible spend of a received `output`.
   struct spend
@@ -384,7 +383,7 @@ namespace db
     webhook_value value;
     output tx_info;
   };
-  void write_bytes(wire::writer&, const webhook_tx_confirmation&);
+  WIRE_DECLARE_OBJECT(webhook_tx_confirmation);
 
   //! Returned by DB when a webhook event "tripped"
   struct webhook_tx_spend

src/db/storage.cpp

@@ -935,6 +935,29 @@ namespace db
     return accounts.get_value<account>(value);
   }
 
+  expect<lws::account> storage_reader::get_full_account(const account& user)
+  {
+    std::vector<std::pair<db::output_id, db::address_index>> receives{};
+    std::vector<crypto::public_key> pubs{};
+    auto receive_list = get_outputs(user.id);
+    if (!receive_list)
+      return receive_list.error();
+
+    const std::size_t elems = receive_list->count();
+    receives.reserve(elems);
+    pubs.reserve(elems);
+
+    for (auto output = receive_list->make_iterator(); !output.is_end(); ++output)
+    {
+      auto id = output.get_value<MONERO_FIELD(db::output, spend_meta.id)>();
+      auto subaddr = output.get_value<MONERO_FIELD(db::output, recipient)>();
+      receives.emplace_back(std::move(id), std::move(subaddr));
+      pubs.emplace_back(output.get_value<MONERO_FIELD(db::output, pub)>());
+    }
+
+    return lws::account{user, std::move(receives), std::move(pubs)};
+  }
+
   expect<std::pair<account_status, account>>
   storage_reader::get_account(account_address const& address) noexcept
   {
@@ -2811,8 +2834,13 @@ namespace db
             accounts_by_address.get_value<MONERO_FIELD(account_by_address, lookup)>(temp_value).value().status;
           MONERO_LMDB_CHECK(mdb_cursor_get(accounts_cur.get(), &key, &value, MDB_GET_BOTH));
         }
+
+        /* The check below is `<` instead of `!=` because of remote scanning -
+          a "check-in" can occur before the user accounts are replaced.
+          Duplicate writes should be supported as this (duplicate writes)
+          happened historically due to a different bug involving scan heights.*/
         expect<account> existing = accounts.get_value<account>(value);
-        if (!existing || existing->scan_height != user->scan_height())
+        if (!existing || existing->scan_height < user->scan_height())
           continue; // to next account
 
         // Don't re-store data if already scanned

src/db/storage.h

@@ -40,6 +40,7 @@
 #include "lmdb/transaction.h"
 #include "lmdb/key_stream.h"
 #include "lmdb/value_stream.h"
+#include "wire/msgpack/fwd.h"
 
 namespace cryptonote { class checkpoints; }
 namespace lws
@@ -132,6 +133,9 @@ namespace db
     //! \return Info for account `id` iff it has `status`.
     expect<account> get_account(const account_status status, const account_id id) noexcept;
 
+    //! \return Account with outputs and spends
+    expect<lws::account> get_full_account(const account&);
+
     //! \return Info related to `address`.
     expect<std::pair<account_status, account>>
       get_account(account_address const& address) noexcept;