mirror of
https://github.com/Such-Software/grin-lws.git
synced 2026-10-17 07:31:34 +02:00
Phase 4+5: genuine dual Postgres/SQLite + end-to-end scanner proof
Building an end-to-end integration test (mock grin node → real tick() → SQLite store) surfaced the real dual-DB blockers in sqlx::Any — NOT the placeholder issue the review flagged: - Any truncates i64 -> i32 for SQLite (12345678900 -> -539222988), which would corrupt grin amounts. Store outputs.value as a decimal string, sum in Rust. (Probe-documented; other int columns are < 2^31 for decades.) - Any can't decode SQLite BOOLEAN; is_coinbase/spent are now INTEGER 0/1. - DEFAULT now() -> DEFAULT CURRENT_TIMESTAMP (portable). The integration test now proves the whole pipeline end-to-end on SQLite: node UTXO listing → view-only rewind (golden output) → store with value + key_id → correct balance → spend-by-absence reconcile → zero. One schema + one query set now serve both backends.
This commit is contained in:
jwinterm
@@ -32,17 +32,21 @@ CREATE TABLE IF NOT EXISTS accounts (
|
||||
CREATE TABLE IF NOT EXISTS outputs (
|
||||
"commit" TEXT PRIMARY KEY,
|
||||
rewind_hash TEXT NOT NULL REFERENCES accounts(rewind_hash) ON DELETE CASCADE,
|
||||
value BIGINT NOT NULL,
|
||||
-- nanogrin as a DECIMAL STRING. sqlx's `Any` driver truncates i64 -> i32
|
||||
-- for SQLite, which would corrupt amounts > ~2.1e9 (grin values reach
|
||||
-- ~1e16), so the value is stored as text and summed in Rust. Other integer
|
||||
-- columns stay well under 2^31 for decades.
|
||||
value TEXT NOT NULL,
|
||||
height BIGINT NOT NULL,
|
||||
mmr_index BIGINT NOT NULL,
|
||||
is_coinbase BOOLEAN NOT NULL DEFAULT false,
|
||||
is_coinbase INTEGER NOT NULL DEFAULT 0, -- 0/1 (Any can't decode SQLite BOOLEAN)
|
||||
lock_height BIGINT NOT NULL DEFAULT 0,
|
||||
-- Recovered from the proof message during rewind (grin_recover_output
|
||||
-- equivalent, server-side). Enables direct spend by the client.
|
||||
key_id TEXT,
|
||||
n_child INTEGER,
|
||||
-- Spend tracking: set when this output's commitment appears as a tx input.
|
||||
spent BOOLEAN NOT NULL DEFAULT false,
|
||||
spent INTEGER NOT NULL DEFAULT 0, -- 0/1 (Any can't decode SQLite BOOLEAN)
|
||||
spent_height BIGINT
|
||||
);
|
||||
|
||||
|
||||
@@ -4,12 +4,17 @@
|
||||
//! Uses `sqlx::Any` so a single build serves both Postgres (scale) and SQLite
|
||||
//! (single operator) — the driver is chosen at runtime from `DATABASE_URL`.
|
||||
//!
|
||||
//! Dialect: the SQL below uses Postgres `$N` placeholders and runs unchanged on
|
||||
//! BOTH Postgres and SQLite through sqlx's `Any` driver (SQLite accepts `$N`
|
||||
//! numbered parameters and binds them positionally). Verified end-to-end by the
|
||||
//! `sqlite_probe` test, which applies the real migration + reused-placeholder
|
||||
//! upserts against SQLite. `DEFAULT CURRENT_TIMESTAMP` (not `now()`) keeps the
|
||||
//! schema portable.
|
||||
//! Dialect: one schema + one set of queries serve BOTH Postgres and SQLite via
|
||||
//! sqlx's `Any` driver, verified end-to-end by the `sqlite_probe` test (real
|
||||
//! migration + reused-placeholder upserts) and the scanner integration test.
|
||||
//! Portability required working around `Any`'s quirks, NOT `$N` placeholders
|
||||
//! (SQLite accepts those and binds positionally):
|
||||
//! - `Any` truncates i64 -> i32 for SQLite, so `outputs.value` (nanogrin, up
|
||||
//! to ~1e16) is stored as a DECIMAL STRING and summed in Rust. Other integer
|
||||
//! columns stay well under 2^31 for decades.
|
||||
//! - `Any` can't decode SQLite `BOOLEAN`, so boolean flags (`is_coinbase`,
|
||||
//! `spent`) are `INTEGER` 0/1.
|
||||
//! - `DEFAULT CURRENT_TIMESTAMP` (not `now()`) is portable.
|
||||
//!
|
||||
//! SECURITY: only view-only data is ever stored. No spend key, no private key.
|
||||
|
||||
@@ -181,10 +186,10 @@ pub async fn insert_output(pool: &AnyPool, out: &StoredOutput, rewind_hash: &str
|
||||
)
|
||||
.bind(&out.commit)
|
||||
.bind(rewind_hash)
|
||||
.bind(out.value as i64)
|
||||
.bind(out.value.to_string())
|
||||
.bind(out.height as i64)
|
||||
.bind(out.mmr_index as i64)
|
||||
.bind(out.is_coinbase)
|
||||
.bind(out.is_coinbase as i64)
|
||||
.bind(out.lock_height as i64)
|
||||
.bind(&out.key_id)
|
||||
.bind(out.n_child.map(|n| n as i32))
|
||||
@@ -195,7 +200,7 @@ pub async fn insert_output(pool: &AnyPool, out: &StoredOutput, rewind_hash: &str
|
||||
|
||||
/// Mark an output spent (its commitment left the node's unspent set).
|
||||
pub async fn mark_spent(pool: &AnyPool, commit: &str, spent_height: i64) -> Result<()> {
|
||||
sqlx::query("UPDATE outputs SET spent = true, spent_height = $2 WHERE \"commit\" = $1")
|
||||
sqlx::query("UPDATE outputs SET spent = 1, spent_height = $2 WHERE \"commit\" = $1")
|
||||
.bind(commit)
|
||||
.bind(spent_height)
|
||||
.execute(pool)
|
||||
@@ -206,7 +211,7 @@ pub async fn mark_spent(pool: &AnyPool, commit: &str, spent_height: i64) -> Resu
|
||||
/// Un-mark a spend (a reorg brought a previously-spent output back into the
|
||||
/// unspent set). Self-heals an over-eager spend reconcile.
|
||||
pub async fn mark_unspent(pool: &AnyPool, commit: &str) -> Result<()> {
|
||||
sqlx::query("UPDATE outputs SET spent = false, spent_height = NULL WHERE \"commit\" = $1")
|
||||
sqlx::query("UPDATE outputs SET spent = 0, spent_height = NULL WHERE \"commit\" = $1")
|
||||
.bind(commit)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
@@ -216,7 +221,7 @@ pub async fn mark_unspent(pool: &AnyPool, commit: &str) -> Result<()> {
|
||||
/// Every currently-unspent commitment across all accounts — the set the scanner
|
||||
/// re-checks against the node to detect spends by absence.
|
||||
pub async fn all_unspent_commits(pool: &AnyPool) -> Result<Vec<String>> {
|
||||
let rows = sqlx::query("SELECT \"commit\" FROM outputs WHERE spent = false")
|
||||
let rows = sqlx::query("SELECT \"commit\" FROM outputs WHERE spent = 0")
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows.iter().map(|r| r.get::<String, _>("commit")).collect())
|
||||
@@ -226,7 +231,7 @@ pub async fn all_unspent_commits(pool: &AnyPool) -> Result<Vec<String>> {
|
||||
pub async fn unspent_outputs(pool: &AnyPool, rewind_hash: &str) -> Result<Vec<StoredOutput>> {
|
||||
let rows = sqlx::query(
|
||||
"SELECT \"commit\", value, height, mmr_index, is_coinbase, lock_height, key_id, n_child \
|
||||
FROM outputs WHERE rewind_hash = $1 AND spent = false ORDER BY height ASC",
|
||||
FROM outputs WHERE rewind_hash = $1 AND spent = 0 ORDER BY height ASC",
|
||||
)
|
||||
.bind(rewind_hash)
|
||||
.fetch_all(pool)
|
||||
@@ -235,18 +240,25 @@ pub async fn unspent_outputs(pool: &AnyPool, rewind_hash: &str) -> Result<Vec<St
|
||||
Ok(rows.iter().map(row_to_output).collect())
|
||||
}
|
||||
|
||||
/// Sum a set of decimal-string `value` rows into `(total, count)`. Summed in
|
||||
/// Rust because `value` is stored as text (see schema) and `Any` truncates
|
||||
/// large integers for SQLite. `u128` accumulator avoids overflow across many
|
||||
/// outputs; the total is clamped to `u64` (grin's supply fits comfortably).
|
||||
fn sum_value_rows(rows: &[sqlx::any::AnyRow]) -> (u64, u64) {
|
||||
let total: u128 = rows
|
||||
.iter()
|
||||
.map(|r| r.get::<String, _>("value").parse::<u64>().unwrap_or(0) as u128)
|
||||
.sum();
|
||||
(total.min(u64::MAX as u128) as u64, rows.len() as u64)
|
||||
}
|
||||
|
||||
/// (total unspent, count) for an account — the balance read.
|
||||
pub async fn account_totals(pool: &AnyPool, rewind_hash: &str) -> Result<(u64, u64)> {
|
||||
let row = sqlx::query(
|
||||
"SELECT COALESCE(SUM(value),0) AS total, COUNT(*) AS n \
|
||||
FROM outputs WHERE rewind_hash = $1 AND spent = false",
|
||||
)
|
||||
.bind(rewind_hash)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
let total: i64 = row.try_get("total").unwrap_or(0);
|
||||
let n: i64 = row.try_get("n").unwrap_or(0);
|
||||
Ok((total.max(0) as u64, n.max(0) as u64))
|
||||
let rows = sqlx::query("SELECT value FROM outputs WHERE rewind_hash = $1 AND spent = 0")
|
||||
.bind(rewind_hash)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(sum_value_rows(&rows))
|
||||
}
|
||||
|
||||
/// (unlocked total, count) for an account: unspent AND spendable at `tip_height`
|
||||
@@ -258,17 +270,15 @@ pub async fn account_unlocked_totals(
|
||||
rewind_hash: &str,
|
||||
tip_height: i64,
|
||||
) -> Result<(u64, u64)> {
|
||||
let row = sqlx::query(
|
||||
"SELECT COALESCE(SUM(value),0) AS total, COUNT(*) AS n \
|
||||
FROM outputs WHERE rewind_hash = $1 AND spent = false AND lock_height <= $2",
|
||||
let rows = sqlx::query(
|
||||
"SELECT value FROM outputs \
|
||||
WHERE rewind_hash = $1 AND spent = 0 AND lock_height <= $2",
|
||||
)
|
||||
.bind(rewind_hash)
|
||||
.bind(tip_height)
|
||||
.fetch_one(pool)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
let total: i64 = row.try_get("total").unwrap_or(0);
|
||||
let n: i64 = row.try_get("n").unwrap_or(0);
|
||||
Ok((total.max(0) as u64, n.max(0) as u64))
|
||||
Ok(sum_value_rows(&rows))
|
||||
}
|
||||
|
||||
/// The height at which the scanner last processed this account.
|
||||
@@ -283,10 +293,11 @@ pub async fn account_scan_height(pool: &AnyPool, rewind_hash: &str) -> Result<Op
|
||||
fn row_to_output(r: &sqlx::any::AnyRow) -> StoredOutput {
|
||||
StoredOutput {
|
||||
commit: r.get::<String, _>("commit"),
|
||||
value: r.get::<i64, _>("value").max(0) as u64,
|
||||
// value is stored as a decimal string (see schema note on Any/i64).
|
||||
value: r.get::<String, _>("value").parse::<u64>().unwrap_or(0),
|
||||
height: r.get::<i64, _>("height").max(0) as u64,
|
||||
mmr_index: r.get::<i64, _>("mmr_index").max(0) as u64,
|
||||
is_coinbase: r.get::<bool, _>("is_coinbase"),
|
||||
is_coinbase: r.get::<i64, _>("is_coinbase") != 0,
|
||||
lock_height: r.get::<i64, _>("lock_height").max(0) as u64,
|
||||
key_id: r.try_get::<Option<String>, _>("key_id").ok().flatten(),
|
||||
n_child: r
|
||||
@@ -417,7 +428,7 @@ pub async fn rollback_to(pool: &AnyPool, fork_height: i64) -> Result<()> {
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
// Spends recorded above the fork are undone (the input may be unspent again).
|
||||
sqlx::query("UPDATE outputs SET spent = false, spent_height = NULL WHERE spent_height > $1")
|
||||
sqlx::query("UPDATE outputs SET spent = 0, spent_height = NULL WHERE spent_height > $1")
|
||||
.bind(fork_height)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
+164
@@ -237,3 +237,167 @@ async fn handle_reorg(
|
||||
db::set_cursor(pool, fork_index.saturating_sub(1) as i64, fork as i64, &fork_hash).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod integration {
|
||||
//! End-to-end: drive the real `tick()` against a MOCK grin node serving the
|
||||
//! golden-vector output, over a real SQLite store. Proves the whole pipeline
|
||||
//! (node listing → view-only rewind → store → balance → spend reconcile).
|
||||
use std::collections::HashSet;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use axum::{extract::State, routing::post, Json, Router};
|
||||
use serde_json::{json, Value};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
use super::*;
|
||||
use crate::config::Config;
|
||||
use crate::secret::Secret;
|
||||
|
||||
// Golden depth-3 (Grim) output — its rewind_hash/commit/proof are the ones
|
||||
// grin's own ProofBuilder produced (see src/rewind.rs golden vectors).
|
||||
const RH: &str = "c2f675f2b05b5bed7d7f01480c6f2aaf368fe4da5e169c3b6d18bb354ba784a2";
|
||||
const COMMIT: &str = "099c6701a2c0cf2c02ada5a051a87e554524bcfa7d77339caf35b607fc1a1b2226";
|
||||
const PROOF: &str = "69d7d34b41a9cff366101cd1473fd28dde406de95fa29d403a30684f93a6c857409eb08742b6fd8704c73ea65e982f3f332dbb6f2c61458b6acb901b8d98be9b01cf44cbcf9a8e087638ceabd7666c2bc9728d46ce21a5560da9710e69d4ae6099ad49666851f4181b2fea9f10ff400a5fc789ef132b96668bedcc8697349798747fc68dcb32c081f39810d7d05846f955cf398abf358ec4c013c6cf35e7abb4552a1bf8a2eaff0a2c4bd5a3f232fd323f1cd59588623dfdbbd3276e1e4dbc5c00038cb18ba9ee83d70a803bcbb5f32972153454a093bbee48b14c2653ed34c5bc94b0220d10f3b97f4914d62c0288e9146ad34527634736ff28db7dc6171cee1bed2bbe3c85788417778556ceb259fe8fd7571ff884b6dacc375d56d7a528cc037993c0402c6f7b2ca13940501e798b6476ae24c4b65cca0805e98de96819b246997de4f3680bf81b3c7dbf4c456c29677e3a3be281993e06b8c8aae14559672f5103b80f5aaba187ea0c74903af9896433aec723063b695efd2254b831143c7335058496aa4a8d8a90c0bfb4260da8790f32073f4f809a59f93ef33c0379d20a48212c1ac36181eeef59ababfb1b6f9a0ce4c9baaa83299c21ccad09a0493ad79f35083b00a149d72745feb8e6776fb16f06128a37f1ffd406476deeb5e7f35208b3cc7544197be7def4509fdcbd92d8bd72e3fd0437b3f249c247981ba70bc2ebf985c3efa546ceaf953e7061b33b4f1a5da78a32406b6258483efb1c64e49b77e3ca960e1f9f1be3779d657fd879ad8d282a3bf0a19d2c6f3a0b161e4fb0e2c2dff2efbb7fbf8be5b355b22f0fe2a2166ebe0627b9006cb740a190a50dcde3c068734f9554efaa9a196c7a985239ddce0b246c34c8627297fa3b3050cac82d7af9ab7063e8873f000f533f9a250cdc777c4980cf569f6c4011fcb3dfb5b1eb995c";
|
||||
const VALUE: u64 = 12_345_678_900;
|
||||
const MMR: u64 = 3;
|
||||
const HEIGHT: u64 = 100;
|
||||
|
||||
struct MockNode {
|
||||
highest_index: u64,
|
||||
tip_height: u64,
|
||||
tip_hash: String,
|
||||
/// Currently-unspent outputs (mmr_index, commit, proof).
|
||||
utxos: Vec<(u64, String, String)>,
|
||||
/// Commits the node still lists as unspent (for get_outputs).
|
||||
present: HashSet<String>,
|
||||
}
|
||||
|
||||
async fn rpc(State(m): State<Arc<Mutex<MockNode>>>, Json(req): Json<Value>) -> Json<Value> {
|
||||
let m = m.lock().unwrap();
|
||||
let method = req["method"].as_str().unwrap_or("");
|
||||
let params = &req["params"];
|
||||
let ok = |v: Value| Json(json!({ "result": { "Ok": v } }));
|
||||
match method {
|
||||
"get_tip" => ok(json!({
|
||||
"height": m.tip_height, "last_block_pushed": m.tip_hash,
|
||||
"prev_block_to_last": "", "total_difficulty": 0
|
||||
})),
|
||||
"get_header" => ok(json!({ "hash": m.tip_hash, "height": m.tip_height })),
|
||||
"get_unspent_outputs" => {
|
||||
let start = params[0].as_u64().unwrap_or(0);
|
||||
let end = params[1].as_u64().unwrap_or(u64::MAX);
|
||||
let outs: Vec<Value> = m
|
||||
.utxos
|
||||
.iter()
|
||||
.filter(|(idx, _, _)| *idx >= start && *idx <= end)
|
||||
.map(|(idx, commit, proof)| {
|
||||
json!({
|
||||
"output_type": "Transaction", "commit": commit, "spent": false,
|
||||
"proof": proof, "block_height": HEIGHT, "mmr_index": idx
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
let last = outs
|
||||
.iter()
|
||||
.filter_map(|o| o["mmr_index"].as_u64())
|
||||
.max()
|
||||
.unwrap_or(m.highest_index);
|
||||
ok(json!({
|
||||
"highest_index": m.highest_index,
|
||||
"last_retrieved_index": last,
|
||||
"outputs": outs
|
||||
}))
|
||||
}
|
||||
"get_outputs" => {
|
||||
let commits = params[0].as_array().cloned().unwrap_or_default();
|
||||
let present: Vec<Value> = commits
|
||||
.iter()
|
||||
.filter_map(|c| c.as_str())
|
||||
.filter(|c| m.present.contains(*c))
|
||||
.map(|c| json!({ "commit": c }))
|
||||
.collect();
|
||||
ok(Value::Array(present))
|
||||
}
|
||||
"get_pmmr_indices" => {
|
||||
ok(json!({ "highest_index": m.highest_index, "last_retrieved_index": 1, "outputs": [] }))
|
||||
}
|
||||
_ => Json(json!({ "result": { "Err": "unknown method" } })),
|
||||
}
|
||||
}
|
||||
|
||||
fn cfg_for(url: String) -> Config {
|
||||
Config {
|
||||
bind_addr: "127.0.0.1:0".parse().unwrap(),
|
||||
database_url: Secret::new(String::new()),
|
||||
node_foreign_api_url: url,
|
||||
node_foreign_api_secret: Secret::new(String::new()),
|
||||
scan_poll_secs: 1,
|
||||
scan_batch_blocks: 100,
|
||||
restore_max_depth_days: 0,
|
||||
admin_key: Secret::new(String::new()),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scan_discovers_then_spends_the_golden_output() {
|
||||
// ── mock node serving the golden output as an unspent UTXO ──
|
||||
let state = Arc::new(Mutex::new(MockNode {
|
||||
highest_index: MMR,
|
||||
tip_height: 500,
|
||||
tip_hash: "tiphash".into(),
|
||||
utxos: vec![(MMR, COMMIT.into(), PROOF.into())],
|
||||
present: HashSet::from([COMMIT.to_string()]),
|
||||
}));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let app = Router::new().route("/v2/foreign", post(rpc)).with_state(state.clone());
|
||||
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||
let node = GrinNode::new(&cfg_for(format!("http://{addr}/v2/foreign"))).unwrap();
|
||||
|
||||
// ── real SQLite store + migration ──
|
||||
sqlx::any::install_default_drivers();
|
||||
let path = std::env::temp_dir().join("grinlws_scan_it.db");
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let pool = sqlx::any::AnyPoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect(&format!("sqlite://{}?mode=rwc", path.display()))
|
||||
.await
|
||||
.unwrap();
|
||||
let schema = include_str!("../migrations/0001_init.sql");
|
||||
let no_comments: String = schema
|
||||
.lines()
|
||||
.map(|l| l.find("--").map(|i| &l[..i]).unwrap_or(l))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
for s in no_comments.split(';') {
|
||||
let s = s.trim();
|
||||
if !s.is_empty() {
|
||||
sqlx::query(s).execute(&pool).await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
db::register_account(&pool, RH, HEIGHT as i64).await.unwrap();
|
||||
|
||||
// ── tick 1: discover ──
|
||||
tick(&pool, &node).await.expect("tick 1");
|
||||
let (total, count) = db::account_totals(&pool, RH).await.unwrap();
|
||||
assert_eq!((total, count), (VALUE, 1), "golden output discovered + stored");
|
||||
let (unlocked, _) = db::account_unlocked_totals(&pool, RH, 500).await.unwrap();
|
||||
assert_eq!(unlocked, VALUE, "regular output is spendable now");
|
||||
let outs = db::unspent_outputs(&pool, RH).await.unwrap();
|
||||
assert_eq!(outs[0].key_id.as_deref(), Some("0300000000000000000000000700000000"));
|
||||
|
||||
// ── spend it: remove from the node's unspent set ──
|
||||
{
|
||||
let mut m = state.lock().unwrap();
|
||||
m.utxos.clear();
|
||||
m.present.clear();
|
||||
}
|
||||
// ── tick 2: reconcile by absence ──
|
||||
tick(&pool, &node).await.expect("tick 2");
|
||||
assert_eq!(db::account_totals(&pool, RH).await.unwrap().0, 0, "spent → zero balance");
|
||||
|
||||
let _ = std::fs::remove_file(&path);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user