Add (working draft) subaddress support (#83)

src/db/account.cpp

@@ -72,11 +72,11 @@ namespace lws
     crypto::secret_key view_key;
   };
 
-  account::account(std::shared_ptr<const internal> immutable, db::block_id height, std::vector<db::output_id> spendable, std::vector<crypto::public_key> pubs) noexcept
+  account::account(std::shared_ptr<const internal> immutable, db::block_id height, std::vector<std::pair<db::output_id, db::address_index>> spendable, std::vector<crypto::public_key> pubs) noexcept
     : immutable_(std::move(immutable))
     , spendable_(std::move(spendable))
     , pubs_(std::move(pubs))
-      , spends_()
+    , spends_()
     , outputs_()
     , height_(height)
   {}
@@ -87,7 +87,7 @@ namespace lws
       MONERO_THROW(::common_error::kInvalidArgument, "using moved from account");
   }
 
-  account::account(db::account const& source, std::vector<db::output_id> spendable, std::vector<crypto::public_key> pubs)
+  account::account(db::account const& source, std::vector<std::pair<db::output_id, db::address_index>> spendable, std::vector<crypto::public_key> pubs)
     : account(std::make_shared<internal>(source), source.scan_height, std::move(spendable), std::move(pubs))
   {
     std::sort(spendable_.begin(), spendable_.end());
@@ -151,9 +151,15 @@ namespace lws
     return immutable_->view_key;
   }
 
-  bool account::has_spendable(db::output_id const& id) const noexcept
+  boost::optional<db::address_index> account::get_spendable(db::output_id const& id) const noexcept
   {
-    return std::binary_search(spendable_.begin(), spendable_.end(), id);
+    const auto searchable = 
+      std::make_pair(id, db::address_index{db::major_index::primary, db::minor_index::primary});
+    const auto account =
+      std::lower_bound(spendable_.begin(), spendable_.end(), searchable);
+    if (account == spendable_.end() || account->first != id)
+      return boost::none;
+    return account->second;
   }
 
   bool account::add_out(db::output const& out)
@@ -163,9 +169,10 @@ namespace lws
       return false;
 
     pubs_.insert(existing_pub, out.pub);
+    auto spendable_value = std::make_pair(out.spend_meta.id, out.recipient);
     spendable_.insert(
-      std::lower_bound(spendable_.begin(), spendable_.end(), out.spend_meta.id),
-      out.spend_meta.id
+      std::lower_bound(spendable_.begin(), spendable_.end(), spendable_value),
+      spendable_value
     );
     outputs_.push_back(out);
     return true;

src/db/account.h

@@ -26,6 +26,7 @@
 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #pragma once
 
+#include <boost/optional/optional.hpp>
 #include <cstdint>
 #include <memory>
 #include <string>
@@ -33,6 +34,7 @@
 
 #include "crypto/crypto.h"
 #include "fwd.h"
+#include "db/data.h"
 #include "db/fwd.h"
 
 namespace lws
@@ -43,19 +45,19 @@ namespace lws
     struct internal;
 
     std::shared_ptr<const internal> immutable_;
-    std::vector<db::output_id> spendable_;
+    std::vector<std::pair<db::output_id, db::address_index>> spendable_;
     std::vector<crypto::public_key> pubs_;
     std::vector<db::spend> spends_;
     std::vector<db::output> outputs_;
     db::block_id height_;
 
-    explicit account(std::shared_ptr<const internal> immutable, db::block_id height, std::vector<db::output_id> spendable, std::vector<crypto::public_key> pubs) noexcept;
+    explicit account(std::shared_ptr<const internal> immutable, db::block_id height, std::vector<std::pair<db::output_id, db::address_index>> spendable, std::vector<crypto::public_key> pubs) noexcept;
     void null_check() const;
 
   public:
 
     //! Construct an account from `source` and current `spendable` outputs.
-    explicit account(db::account const& source, std::vector<db::output_id> spendable, std::vector<crypto::public_key> pubs);
+    explicit account(db::account const& source, std::vector<std::pair<db::output_id, db::address_index>> spendable, std::vector<crypto::public_key> pubs);
 
     /*!
       \return False if this is a "moved-from" account (i.e. the internal memory
@@ -96,8 +98,8 @@ namespace lws
     //! \return Current scan height of `this`.
     db::block_id scan_height() const noexcept { return height_; }
 
-    //! \return True iff `id` is spendable by `this`.
-    bool has_spendable(db::output_id const& id) const noexcept;
+    //! \return Subaddress index iff `id` is spendable by `this`.
+    boost::optional<db::address_index> get_spendable(db::output_id const& id) const noexcept;
 
     //! \return Outputs matched during the latest scan.
     std::vector<db::output> const& outputs() const noexcept { return outputs_; }

src/db/data.cpp

@@ -29,12 +29,18 @@
 #include <cstring>
 #include <memory>
 
+#include "cryptonote_config.h" // monero/src
 #include "db/string.h"
+#include "int-util.h"          // monero/contribe/epee/include
+#include "ringct/rctOps.h"     // monero/src
+#include "ringct/rctTypes.h"   // monero/src
 #include "wire.h"
+#include "wire/adapted/array.h"
 #include "wire/crypto.h"
 #include "wire/json/write.h"
 #include "wire/msgpack.h"
 #include "wire/uuid.h"
+#include "wire/vector.h"
 #include "wire/wrapper/defaulted.h"
 
 namespace lws
@@ -69,6 +75,102 @@ namespace db
   }
   WIRE_DEFINE_OBJECT(account_address, map_account_address);
 
+  namespace
+  {
+    template<typename F, typename T>
+    void map_subaddress_dict(F& format, T& self)
+    {
+      wire::object(format,
+        wire::field<0>("key", std::ref(self.first)),
+        wire::field<1>("value", std::ref(self.second))
+      );
+    }
+  }
+
+  bool check_subaddress_dict(const subaddress_dict& self)
+  {
+    bool is_first = true;
+    minor_index last = minor_index::primary;
+    for (const auto& elem : self.second)
+    {
+      if (elem[1] < elem[0])
+      {
+        MERROR("Invalid subaddress_range (last before first");
+        return false;
+      }
+      if (std::uint32_t(elem[0]) <= std::uint64_t(last) + 1 && !is_first)
+      {
+        MERROR("Invalid subaddress_range (overlapping with previous)");
+        return false;
+      }
+      is_first = false;
+      last = elem[1];
+    }
+    return true;
+  }
+  void read_bytes(wire::reader& source, subaddress_dict& dest)
+  {
+    map_subaddress_dict(source, dest);
+    if (!check_subaddress_dict(dest))
+      WIRE_DLOG_THROW_(wire::error::schema::array);
+  }
+  void write_bytes(wire::writer& dest, const subaddress_dict& source)
+  { 
+    if (!check_subaddress_dict(source))
+      WIRE_DLOG_THROW_(wire::error::schema::array);
+    map_subaddress_dict(dest, source);
+  }
+
+  namespace
+  {
+    template<typename F, typename T>
+    void map_address_index(F& format, T& self)
+    {
+      wire::object(format, WIRE_FIELD_ID(0, maj_i), WIRE_FIELD_ID(1, min_i));
+    }
+
+    crypto::secret_key get_subaddress_secret_key(const crypto::secret_key &a, const std::uint32_t major, const std::uint32_t minor)
+    {
+      char data[sizeof(config::HASH_KEY_SUBADDRESS) + sizeof(crypto::secret_key) + 2 * sizeof(uint32_t)];
+      memcpy(data, config::HASH_KEY_SUBADDRESS, sizeof(config::HASH_KEY_SUBADDRESS));
+      memcpy(data + sizeof(config::HASH_KEY_SUBADDRESS), &a, sizeof(crypto::secret_key));
+      std::uint32_t idx = SWAP32LE(major);
+      memcpy(data + sizeof(config::HASH_KEY_SUBADDRESS) + sizeof(crypto::secret_key), &idx, sizeof(uint32_t));
+      idx = SWAP32LE(minor);
+      memcpy(data + sizeof(config::HASH_KEY_SUBADDRESS) + sizeof(crypto::secret_key) + sizeof(uint32_t), &idx, sizeof(uint32_t));
+      crypto::secret_key m;
+      crypto::hash_to_scalar(data, sizeof(data), m);
+      return m;
+    }
+  }
+  WIRE_DEFINE_OBJECT(address_index, map_address_index);
+
+  crypto::public_key address_index::get_spend_public(account_address const& base, crypto::secret_key const& view) const
+  {
+    if (is_zero())
+      return base.spend_public;
+
+    // m = Hs(a || index_major || index_minor)
+    crypto::secret_key m = get_subaddress_secret_key(view, std::uint32_t(maj_i), std::uint32_t(min_i));
+
+    // M = m*G
+    crypto::public_key M;
+    crypto::secret_key_to_public_key(m, M);
+
+    // D = B + M
+    return rct::rct2pk(rct::addKeys(rct::pk2rct(base.spend_public), rct::pk2rct(M)));
+  }
+
+  namespace
+  {
+    template<typename F, typename T>
+    void map_subaddress_map(F& format, T& self)
+    {
+      wire::object(format, WIRE_FIELD_ID(0, subaddress), WIRE_FIELD_ID(1, index));
+    }
+  }
+  WIRE_DEFINE_OBJECT(subaddress_map, map_subaddress_map);
+
   void write_bytes(wire::writer& dest, const account& self, const bool show_key)
   {
     view_key const* const key =
@@ -144,7 +246,8 @@ namespace db
       wire::field<10>("unlock_time", self.unlock_time),
       wire::field<11>("mixin_count", self.spend_meta.mixin_count),
       wire::field<12>("coinbase", coinbase),
-      wire::field<13>("fee", self.fee)
+      wire::field<13>("fee", self.fee),
+      wire::field<14>("recipient", self.recipient)
     );
   }
 
@@ -161,7 +264,8 @@ namespace db
         WIRE_FIELD(timestamp),
         WIRE_FIELD(unlock_time),
         WIRE_FIELD(mixin_count),
-        wire::optional_field("payment_id", std::ref(payment_id))
+        wire::optional_field("payment_id", std::ref(payment_id)),
+        WIRE_FIELD(sender)
       );
     }
   }

src/db/data.h

@@ -26,13 +26,16 @@
 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #pragma once
 
+#include <array>
 #include <boost/uuid/uuid.hpp>
 #include <cassert>
 #include <cstdint>
 #include <iosfwd>
 #include <limits>
 #include <string>
+#include <type_traits>
 #include <utility>
+#include <vector>
 
 #include "crypto/crypto.h"
 #include "lmdb/util.h"
@@ -122,6 +125,49 @@ namespace db
   static_assert(sizeof(account_address) == 64, "padding in account_address");
   WIRE_DECLARE_OBJECT(account_address);
 
+  //! Major index of a subaddress
+  enum class major_index : std::uint32_t { primary = 0 };
+  WIRE_AS_INTEGER(major_index);
+
+  //! Minor index of a subaddress
+  enum class minor_index : std::uint32_t { primary = 0 };
+  WIRE_AS_INTEGER(minor_index);
+
+  //! Range within a major index
+  using index_range = std::array<minor_index, 2>;
+
+  //! Ranges within a major index
+  using index_ranges = std::vector<index_range>;
+
+  //! Compatible with msgpack_table
+  using subaddress_dict = std::pair<major_index, index_ranges>;
+  bool check_subaddress_dict(const subaddress_dict&);
+  WIRE_DECLARE_OBJECT(subaddress_dict);
+
+  //! A specific (sub)address index
+  struct address_index
+  {
+    major_index maj_i;
+    minor_index min_i;
+
+    crypto::public_key get_spend_public(account_address const& base, crypto::secret_key const& view) const;
+    constexpr bool is_zero() const noexcept
+    {
+      return maj_i == major_index::primary && min_i == minor_index::primary;
+    } 
+  };
+  static_assert(sizeof(address_index) == 4 * 2, "padding in address_index");
+  WIRE_DECLARE_OBJECT(address_index);
+
+  //! Maps a subaddress pubkey to its index values
+  struct subaddress_map
+  {
+    crypto::public_key subaddress; //!< Must be first for LMDB optimzations
+    address_index index;
+  };
+  static_assert(sizeof(subaddress_map) == 32 + 4 * 2, "padding in subaddress_map");
+  WIRE_DECLARE_OBJECT(subaddress_map);
+
   struct account
   {
     account_id id;          //!< Must be first for LMDB optimizations
@@ -205,9 +251,10 @@ namespace db
       crypto::hash long_;    //!< Long version of payment id (always decrypted)
     } payment_id;
     std::uint64_t fee;       //!< Total fee for transaction
+    address_index recipient;
   };
   static_assert(
-    sizeof(output) == 8 + 32 + (8 * 3) + (4 * 2) + 32 + (8 * 2) + (32 * 3) + 7 + 1 + 32 + 8,
+    sizeof(output) == 8 + 32 + (8 * 3) + (4 * 2) + 32 + (8 * 2) + (32 * 3) + 7 + 1 + 32 + 8 + 2 * 4,
     "padding in output"
   );
   void write_bytes(wire::writer&, const output&);
@@ -225,8 +272,9 @@ namespace db
     char reserved[3];
     std::uint8_t length;      //!< Length of `payment_id` field (0..32).
     crypto::hash payment_id;  //!< Unencrypted only, can't decrypt spend
+    address_index sender;
   };
-  static_assert(sizeof(spend) == 8 + 32 * 2 + 8 * 4 + 4 + 3 + 1 + 32, "padding in spend");
+  static_assert(sizeof(spend) == 8 + 32 * 2 + 8 * 4 + 4 + 3 + 1 + 32 + 2 * 4, "padding in spend");
   WIRE_DECLARE_OBJECT(spend);
 
   //! Key image and info needed to retrieve primary `spend` data.
@@ -325,6 +373,18 @@ namespace db
   };
   void write_bytes(wire::writer&, const webhook_new_account&);
 
+  inline constexpr bool operator==(address_index const& left, address_index const& right) noexcept
+  {
+    return left.maj_i == right.maj_i && left.min_i == right.min_i;
+  }
+
+  inline constexpr bool operator<(address_index const& left, address_index const& right) noexcept
+  {
+    return left.maj_i == right.maj_i ?
+      left.min_i < right.min_i : left.maj_i < right.maj_i;
+  }
+
+
   bool operator==(transaction_link const& left, transaction_link const& right) noexcept;
   bool operator<(transaction_link const& left, transaction_link const& right) noexcept;
   bool operator<=(transaction_link const& left, transaction_link const& right) noexcept;

src/db/fwd.h

@@ -39,10 +39,14 @@ namespace db
   enum class block_id : std::uint64_t;
   enum extra : std::uint8_t;
   enum class extra_and_length : std::uint8_t;
+  enum class major_index : std::uint32_t;
+  enum class minor_index : std::uint32_t;
   enum class request : std::uint8_t;
+  enum class webhook_type : std::uint8_t; 
 
   struct account;
   struct account_address;
+  struct address_index;
   struct block_info;
   struct key_image;
   struct output;
@@ -50,7 +54,15 @@ namespace db
   struct request_info;
   struct spend;
   class storage;
+  struct subaddress_map;
   struct transaction_link;
   struct view_key;
+  struct webhook_data;
+  struct webhook_dupsort;
+  struct webhook_event;
+  struct webhook_key;
+  struct webhook_new_account;
+  struct webhook_output;
+  struct webhook_tx_confirmation;
 } // db
 } // lws

src/db/storage.cpp

@@ -57,6 +57,7 @@
 #include "lmdb/value_stream.h"
 #include "net/net_parse_helpers.h" // monero/contrib/epee/include
 #include "span.h"
+#include "wire/adapted/array.h"
 #include "wire/filters.h"
 #include "wire/json.h"
 #include "wire/vector.h"
@@ -102,8 +103,63 @@ namespace db
       sizeof(output) == 8 + 32 + (8 * 3) + (4 * 2) + 32 + (8 * 2) + (32 * 3) + 7 + 1 + 32,
       "padding in output"
     );
+
+    //! Original db value, with no subaddress
+    struct spend
+    {
+      transaction_link link;    //!< Orders and links `spend` to `output`.
+      crypto::key_image image;  //!< Unique ID for the spend
+      // `link` and `image` must in this order for LMDB optimizations
+      output_id source;         //!< The output being spent
+      std::uint64_t timestamp;  //!< Timestamp of spend
+      std::uint64_t unlock_time;//!< Unlock time of spend
+      std::uint32_t mixin_count;//!< Ring-size of TX output
+      char reserved[3];
+      std::uint8_t length;      //!< Length of `payment_id` field (0..32).
+      crypto::hash payment_id;  //!< Unencrypted only, can't decrypt spend
+    };
+    static_assert(sizeof(spend) == 8 + 32 * 2 + 8 * 4 + 4 + 3 + 1 + 32, "padding in spend");
   }
 
+  namespace v1
+  {
+    //! Second DB value, with no subaddress
+    struct output
+    {
+      transaction_link link;        //! Orders and links `output` to `spend`s.
+
+      //! Data that a linked `spend` needs in some REST endpoints.
+      struct spend_meta_
+      {
+        output_id id;             //!< Unique id for output within monero
+        // `link` and `id` must be in this order for LMDB optimizations
+        std::uint64_t amount;
+        std::uint32_t mixin_count;//!< Ring-size of TX
+        std::uint32_t index;      //!< Offset within a tx
+        crypto::public_key tx_public;
+      } spend_meta;
+
+      std::uint64_t timestamp;
+      std::uint64_t unlock_time; //!< Not always a timestamp; mirrors chain value.
+      crypto::hash tx_prefix_hash;
+      crypto::public_key pub;    //!< One-time spendable public key.
+      rct::key ringct_mask;      //!< Unencrypted CT mask
+      char reserved[7];
+      extra_and_length extra;    //!< Extra info + length of payment id
+      union payment_id_
+      {
+        crypto::hash8 short_;  //!< Decrypted short payment id
+        crypto::hash long_;    //!< Long version of payment id (always decrypted)
+      } payment_id;
+      std::uint64_t fee;       //!< Total fee for transaction
+    };
+    static_assert(
+      sizeof(output) == 8 + 32 + (8 * 3) + (4 * 2) + 32 + (8 * 2) + (32 * 3) + 7 + 1 + 32 + 8,
+      "padding in output"
+    );
+  }
+
+
   namespace
   {
     //! Used for finding `account` instances by other indexes.
@@ -243,11 +299,17 @@ namespace db
     constexpr const lmdb::basic_table<account_id, v0::output> outputs_v0{
       "outputs_by_account_id,block_id,tx_hash,output_id", MDB_DUPSORT, &output_compare
     };
+    constexpr const lmdb::basic_table<account_id, v1::output> outputs_v1{
+      "outputs_v1_by_account_id,block_id,tx_hash,output_id", MDB_DUPSORT, &output_compare
+    };
     constexpr const lmdb::basic_table<account_id, output> outputs{
-      "outputs_v1_by_account_id,block_id,tx_hash,output_id", (MDB_CREATE | MDB_DUPSORT), &output_compare
+      "outputs_v2_by_account_id,block_id,tx_hash,output_id", (MDB_CREATE | MDB_DUPSORT), &output_compare
+    };
+    constexpr const lmdb::basic_table<account_id, v0::spend> spends_v0{
+      "spends_by_account_id,block_id,tx_hash,image", MDB_DUPSORT, &spend_compare
     };
     constexpr const lmdb::basic_table<account_id, spend> spends{
-      "spends_by_account_id,block_id,tx_hash,image", (MDB_CREATE | MDB_DUPSORT), &spend_compare
+      "spends_v1_by_account_id,block_id,tx_hash,image", (MDB_CREATE | MDB_DUPSORT), &spend_compare
     };
     constexpr const lmdb::basic_table<output_id, db::key_image> images{
       "key_images_by_output_id,image", (MDB_CREATE | MDB_DUPSORT), MONERO_COMPARE(db::key_image, value)
@@ -261,6 +323,12 @@ namespace db
     constexpr const lmdb::basic_table<account_id, webhook_event> events_by_account_id{
       "webhook_events_by_account_id,type,block_id,tx_hash,output_id,payment_id,event_id", (MDB_CREATE | MDB_DUPSORT), &lmdb::less<webhook_event>
     };
+    constexpr const lmdb::msgpack_table<account_id, major_index, index_ranges> subaddress_ranges{
+      "subaddress_ranges_by_account_id,major_index", (MDB_CREATE | MDB_DUPSORT), &lmdb::less<db::major_index>
+    };
+    constexpr const lmdb::basic_table<account_id, subaddress_map> subaddress_indexes{
+      "subaddress_indexes_by_account_id,public_key", (MDB_CREATE | MDB_DUPSORT), MONERO_COMPARE(subaddress_map, subaddress)
+    };
 
     template<typename D>
     expect<void> check_cursor(MDB_txn& txn, MDB_dbi tbl, std::unique_ptr<MDB_cursor, D>& cur) noexcept
@@ -553,6 +621,8 @@ namespace db
       MDB_dbi requests;
       MDB_dbi webhooks;
       MDB_dbi events;
+      MDB_dbi subaddress_ranges;
+      MDB_dbi subaddress_indexes;
     } tables;
 
     const unsigned create_queue_max;
@@ -573,6 +643,8 @@ namespace db
       tables.requests    = requests.open(*txn).value();
       tables.webhooks    = webhooks.open(*txn).value();
       tables.events      = events_by_account_id.open(*txn).value();
+      tables.subaddress_ranges  = subaddress_ranges.open(*txn).value();
+      tables.subaddress_indexes = subaddress_indexes.open(*txn).value(); 
 
       const auto v0_outputs = outputs_v0.open(*txn);
       if (v0_outputs)
@@ -580,6 +652,18 @@ namespace db
       else if (v0_outputs != lmdb::error(MDB_NOTFOUND))
         MONERO_THROW(v0_outputs.error(), "Error opening old outputs table");
 
+      const auto v1_outputs = outputs_v1.open(*txn);
+      if (v1_outputs)
+        MONERO_UNWRAP(convert_table<v1::output, output>(*txn, *v1_outputs, tables.outputs));
+      else if (v1_outputs != lmdb::error(MDB_NOTFOUND))
+        MONERO_THROW(v1_outputs.error(), "Error opening old outputs table");
+
+      const auto v0_spends = spends_v0.open(*txn);
+      if (v0_spends)
+        MONERO_UNWRAP(convert_table<v0::spend, spend>(*txn, *v0_spends, tables.spends));
+      else if (v0_spends != lmdb::error(MDB_NOTFOUND))
+        MONERO_THROW(v0_spends.error(), "Error opening old spends table");
+
       check_blockchain(*txn, tables.blocks);
 
       MONERO_UNWRAP(this->commit(std::move(txn)));
@@ -755,6 +839,52 @@ namespace db
     return requests.get_value<request_info>(value);
   }
 
+  expect<std::vector<subaddress_dict>>
+  storage_reader::get_subaddresses(account_id id, cursor::subaddress_ranges cur) noexcept
+  {
+    MONERO_PRECOND(txn != nullptr);
+    assert(db != nullptr);
+
+    MONERO_CHECK(check_cursor(*txn, db->tables.subaddress_ranges, cur));
+
+    MDB_val key = lmdb::to_val(id);
+    MDB_val value{};
+    std::vector<subaddress_dict> ranges{};
+    int err = mdb_cursor_get(cur.get(), &key, &value, MDB_SET_KEY);
+    if (!err)
+    {
+      std::size_t count = 0;
+      if (mdb_cursor_count(cur.get(), &count) == 0)
+        ranges.reserve(count);
+    }
+    for (;;)
+    {
+      if (err)
+      {
+        if (err == MDB_NOTFOUND)
+          break;
+        return {lmdb::error(err)};
+      }
+      ranges.push_back(MONERO_UNWRAP(subaddress_ranges.get_value(value)));
+      err = mdb_cursor_get(cur.get(), &key, &value, MDB_NEXT_DUP);
+    }
+    return {std::move(ranges)};
+  }
+
+  expect<address_index>
+  storage_reader::find_subaddress(account_id id, crypto::public_key const& address, cursor::subaddress_indexes& cur) noexcept
+  {
+    MONERO_PRECOND(txn != nullptr);
+    assert(db != nullptr);
+
+    MONERO_CHECK(check_cursor(*txn, db->tables.subaddress_indexes, cur));
+    MDB_val key = lmdb::to_val(id);
+    MDB_val value = lmdb::to_val(address);
+
+    MONERO_LMDB_CHECK(mdb_cursor_get(cur.get(), &key, &value, MDB_GET_BOTH));
+    return subaddress_indexes.get_value<MONERO_FIELD(subaddress_map, index)>(value);
+  }
+
   expect<std::vector<webhook_value>>
   storage_reader::find_webhook(webhook_key const& key, crypto::hash8 const& payment_id, cursor::webhooks cur)
   {
@@ -889,6 +1019,14 @@ namespace db
     );
   }
 
+  static void write_bytes(wire::json_writer& dest, const std::pair<lws::db::account_id, std::vector<std::pair<lws::db::major_index, std::vector<std::array<lws::db::minor_index, 2>>>>>& self)
+  {
+    wire::object(dest,
+      wire::field("id", std::cref(self.first)),
+      wire::field("subaddress_indexes", std::cref(self.second))
+    );
+  }
+
   expect<void> storage_reader::json_debug(std::ostream& out, bool show_keys)
   {
     using boost::adaptors::reverse;
@@ -909,6 +1047,8 @@ namespace db
     cursor::requests requests_cur;
     cursor::webhooks webhooks_cur;
     cursor::webhooks events_cur;
+    cursor::subaddress_ranges ranges_cur;
+    cursor::subaddress_indexes indexes_cur;
 
     MONERO_CHECK(check_cursor(*txn, db->tables.blocks, curs.blocks_cur));
     MONERO_CHECK(check_cursor(*txn, db->tables.accounts, accounts_cur));
@@ -920,6 +1060,8 @@ namespace db
     MONERO_CHECK(check_cursor(*txn, db->tables.requests, requests_cur));
     MONERO_CHECK(check_cursor(*txn, db->tables.webhooks, webhooks_cur));
     MONERO_CHECK(check_cursor(*txn, db->tables.events, events_cur));
+    MONERO_CHECK(check_cursor(*txn, db->tables.subaddress_ranges, ranges_cur));
+    MONERO_CHECK(check_cursor(*txn, db->tables.subaddress_indexes, indexes_cur));
 
     auto blocks_partial =
       get_blocks<boost::container::static_vector<block_info, 12>>(*curs.blocks_cur, 0);
@@ -958,6 +1100,14 @@ namespace db
     if (!requests_stream)
       return requests_stream.error();
 
+    const auto ranges_data = subaddress_ranges.get_all(*ranges_cur);
+    if (!ranges_data)
+        return ranges_data.error();
+
+    auto indexes_stream = subaddress_indexes.get_key_stream(std::move(indexes_cur));
+    if (!indexes_stream)
+      return indexes_stream.error();
+
     // This list should be smaller ... ?
     const auto webhooks_data = webhooks.get_all(*webhooks_cur);
     if (!webhooks_data)
@@ -978,6 +1128,8 @@ namespace db
       wire::field(spends.name, wire::as_object(spends_stream->make_range(), wire::as_integer, wire::as_array)),
       wire::field(images.name, wire::as_object(images_stream->make_range(), output_id_key{}, wire::as_array)),
       wire::field(requests.name, wire::as_object(requests_stream->make_range(), wire::enum_as_string, toggle_keys_filter)),
+      wire::field(subaddress_ranges.name, std::cref(*ranges_data)),
+      wire::field(subaddress_indexes.name, wire::as_object(indexes_stream->make_range(), wire::as_integer, wire::as_array)),
       wire::field(webhooks.name, std::cref(*webhooks_data)),
       wire::field(events_by_account_id.name, wire::as_object(events_stream->make_range(), wire::as_integer, wire::as_array))
     );
@@ -2215,6 +2367,173 @@ namespace db
     });
   }
 
+  expect<std::vector<subaddress_dict>>
+  storage::upsert_subaddresses(const account_id id, const account_address& address, const crypto::secret_key& view_key, std::vector<subaddress_dict> subaddrs, const std::uint32_t max_subaddr)
+  {
+    MONERO_PRECOND(db != nullptr);
+    std::sort(subaddrs.begin(), subaddrs.end());
+
+    return db->try_write([this, id, &address, &view_key, &subaddrs, max_subaddr] (MDB_txn& txn) -> expect<std::vector<subaddress_dict>>
+    {
+      std::size_t subaddr_count = 0;
+      std::vector<subaddress_dict> out{};
+      index_ranges new_dict{};
+      const auto add_out = [&out] (major_index major, index_range minor)
+      {
+        if (out.empty() || out.back().first != major)
+          out.emplace_back(major, index_ranges{minor});
+        else
+          out.back().second.push_back(minor);
+      };
+
+      const auto check_max_range = [&subaddr_count, max_subaddr] (const index_range& range) -> bool
+      {
+        const auto more = std::uint32_t(range[1]) - std::uint32_t(range[0]);
+        if (max_subaddr - subaddr_count <= more)
+          return false;
+        subaddr_count += more + 1;
+        return true;
+      };
+      const auto check_max_ranges = [&check_max_range] (const index_ranges& ranges) -> bool
+      {
+        for (const auto& range : ranges)
+        {
+          if (!check_max_range(range))
+            return false;
+        }
+        return true;
+      };
+
+      cursor::subaddress_ranges   ranges_cur;
+      cursor::subaddress_indexes  indexes_cur;
+
+      MONERO_CHECK(check_cursor(txn, this->db->tables.subaddress_ranges, ranges_cur));
+      MONERO_CHECK(check_cursor(txn, this->db->tables.subaddress_indexes, indexes_cur));
+
+      MDB_val key = lmdb::to_val(id);
+      MDB_val value{};
+      int err = mdb_cursor_get(indexes_cur.get(), &key, &value, MDB_SET);
+      if (err)
+      {
+        if (err != MDB_NOTFOUND)
+          return {lmdb::error(err)};
+      }
+      else
+      {
+        MONERO_LMDB_CHECK(mdb_cursor_count(indexes_cur.get(), &subaddr_count));
+        if (max_subaddr < subaddr_count)
+          return {error::max_subaddresses};
+      }
+
+      for (auto& major_entry : subaddrs)
+      {
+        new_dict.clear();
+        if (!check_subaddress_dict(major_entry))
+        {
+          MERROR("Invalid subaddress_dict given to storage::upsert_subaddrs");
+          return {wire::error::schema::array};
+        }
+
+        value = lmdb::to_val(major_entry.first);
+        err = mdb_cursor_get(ranges_cur.get(), &key, &value, MDB_GET_BOTH);
+        if (err)
+        {
+          if (err != MDB_NOTFOUND)
+            return {lmdb::error(err)};
+          if (!check_max_ranges(major_entry.second))
+            return {error::max_subaddresses};
+          out.push_back(major_entry);
+          new_dict = std::move(major_entry.second);
+        }
+        else // merge new minor index ranges with old
+        {
+          auto old_dict = subaddress_ranges.get_value(value);
+          if (!old_dict)
+            return old_dict.error();
+
+          auto& old_range = old_dict->second;
+          const auto& new_range = major_entry.second;
+
+          auto old_loc = old_range.begin();
+          auto new_loc = new_range.begin();
+          for ( ; old_loc != old_range.end() && new_loc != new_range.end(); )
+          {
+            if (std::uint64_t(new_loc->at(1)) + 1 < std::uint32_t(old_loc->at(0)))
+            { // new has no overlap with existing
+              if (!check_max_range(*new_loc))
+                return {error::max_subaddresses};
+
+              new_dict.push_back(*new_loc);
+              add_out(major_entry.first, *new_loc);
+              ++new_loc;
+            }
+            else if (std::uint64_t(old_loc->at(1)) + 1 < std::uint32_t(new_loc->at(0)))
+            { // existing has no overlap with new
+              new_dict.push_back(*old_loc);
+              ++old_loc;
+            }
+            else if (old_loc->at(0) <= new_loc->at(0) && new_loc->at(1) <= old_loc->at(1))
+            { // new is completely within existing
+              ++new_loc;
+            }
+            else // new overlap at beginning, end, or both
+            {
+              if (new_loc->at(0) < old_loc->at(0))
+              { // overlap at beginning
+                const index_range new_range{new_loc->at(0), minor_index(std::uint32_t(old_loc->at(0)) - 1)};
+                if (!check_max_range(new_range))
+                  return {error::max_subaddresses};
+                add_out(major_entry.first, new_range);
+                old_loc->at(0) = new_loc->at(0);
+              }
+              if (old_loc->at(1) < new_loc->at(1))
+              { // overlap at end
+                const index_range new_range{minor_index(std::uint32_t(old_loc->at(1)) + 1), new_loc->at(1)};
+                if (!check_max_range(new_range))
+                  return {error::max_subaddresses};
+                add_out(major_entry.first, new_range);
+                old_loc->at(1) = new_loc->at(1);
+              }
+              ++new_loc;
+            }
+          }
+
+          std::copy(old_loc, old_range.end(), std::back_inserter(new_dict));
+          for ( ; new_loc != new_range.end(); ++new_loc)
+          {
+            if (!check_max_range(*new_loc))
+              return {error::max_subaddresses};
+            new_dict.push_back(*new_loc);
+            add_out(major_entry.first, *new_loc);
+          }
+        }
+
+        for (const auto& new_indexes : new_dict)
+        {
+          for (std::uint64_t minor : boost::counting_range(std::uint64_t(new_indexes[0]), std::uint64_t(new_indexes[1]) + 1))
+          {
+            subaddress_map new_value{};
+            new_value.index = address_index{major_entry.first, minor_index(minor)};
+            new_value.subaddress = new_value.index.get_spend_public(address, view_key);
+
+            value = lmdb::to_val(new_value);
+
+            MONERO_LMDB_CHECK(mdb_cursor_put(indexes_cur.get(), &key, &value, 0));
+          }
+        }
+
+        const expect<epee::byte_slice> value_bytes =
+          subaddress_ranges.make_value(major_entry.first, new_dict);
+        if (!value_bytes)
+          return value_bytes.error();
+        value = MDB_val{value_bytes->size(), const_cast<void*>(static_cast<const void*>(value_bytes->data()))};
+        MONERO_LMDB_CHECK(mdb_cursor_put(ranges_cur.get(), &key, &value, 0));
+      }
+
+      return {std::move(out)};
+    });
+  }
+
   expect<void> storage::add_webhook(const webhook_type type, const boost::optional<account_address>& address, const webhook_value& event)
   {
     if (event.second.url != "zmq")
@@ -2253,8 +2572,10 @@ namespace db
         return {error::bad_webhook};
 
       lmkey = lmdb::to_val(key);
-      const epee::byte_slice value = webhooks.make_value(event.first, event.second);
-      lmvalue = MDB_val{value.size(), const_cast<void*>(static_cast<const void*>(value.data()))};
+      const expect<epee::byte_slice> value = webhooks.make_value(event.first, event.second);
+      if (!value)
+        return value.error();
+      lmvalue = MDB_val{value->size(), const_cast<void*>(static_cast<const void*>(value->data()))};
       MONERO_LMDB_CHECK(mdb_cursor_put(webhooks_cur.get(), &lmkey, &lmvalue, 0));
       return success();
     });

src/db/storage.h

@@ -52,6 +52,8 @@ namespace db
     MONERO_CURSOR(spends);
     MONERO_CURSOR(images);
     MONERO_CURSOR(requests);
+    MONERO_CURSOR(subaddress_ranges);
+    MONERO_CURSOR(subaddress_indexes);
 
     MONERO_CURSOR(blocks);
     MONERO_CURSOR(accounts_by_address);
@@ -133,6 +135,13 @@ namespace db
     expect<request_info>
       get_request(request type, account_address const& address, cursor::requests cur = nullptr) noexcept;
 
+    //! \return All subaddresses activated for account `id`.
+    expect<std::vector<subaddress_dict>> get_subaddresses(account_id id, cursor::subaddress_ranges cur = nullptr) noexcept;
+
+    //! \return A specific subaddress index
+    expect<address_index>
+      find_subaddress(account_id id, crypto::public_key const& spend_public, cursor::subaddress_indexes& cur) noexcept;
+
     //! \return All webhook values associated with user `key` and `payment_id`.
     expect<std::vector<webhook_value>>
       find_webhook(webhook_key const& key, crypto::hash8 const& payment_id, cursor::webhooks cur = nullptr);
@@ -243,6 +252,24 @@ namespace db
     expect<std::pair<std::size_t, std::vector<webhook_tx_confirmation>>>
       update(block_id height, epee::span<const crypto::hash> chain, epee::span<const lws::account> accts);
 
+    /*!
+      Adds subaddresses to an account. Upon success, an account will
+      immediately begin tracking them in the scanner.
+      
+      \param id of the account to associate new indexes
+      \param addresss of the account (needed to generate subaddress publc key)
+      \param view_key of the account (needed to generate subaddress public key)
+      \param subaddrs Range of subaddress indexes that need to be added to the
+        database. Indexes _may_ overlap with existing indexes.
+      \param max_subaddresses The maximum number of subaddresses allowed per
+        account.
+
+      \return The new ranges of subaddress indexes added to the database
+        (whereas `subaddrs` may overlap with existing indexes).
+      */
+    expect<std::vector<subaddress_dict>>
+      upsert_subaddresses(account_id id, const account_address& address, const crypto::secret_key& view_key, std::vector<subaddress_dict> subaddrs, std::uint32_t max_subaddresses);
+
     /*!
       Add webhook to be tracked in the database. The webhook will "call"
       the specified URL with JSON/msgpack information when the event occurs.