From db5ed2e8aa14122f5714f5a36b6e8bfd00dd8f51 Mon Sep 17 00:00:00 2001 From: jwinterm Date: Mon, 1 Jun 2026 08:17:05 -0400 Subject: [PATCH] payouts: defer when Monero outputs are locked instead of failing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spent outputs (including change) lock for ~10 blocks on Monero. When a payout batch has no unlocked balance to cover it, attempting transfer_split fails noisily and, for batches, gets parked in needs_review for an operator. Add a pre-flight unlocked-balance check that reverts the batch to 'pending' (re-checked every new-block tick and batch interval) so it sends as soon as outputs unlock — no manual intervention. Also classify insufficient-funds RPC errors as pre-broadcast/safe-to-retry: single -> pending (not failed), batch -> pending without an operator alert (not needs_review), since no tx went on-chain and there's no double-pay risk. --- src/game/gameModeManager.js | 66 ++++++++++++++++++++++++++++++------- test/batchPayout.test.js | 54 ++++++++++++++++++++++++++++-- 2 files changed, 106 insertions(+), 14 deletions(-) diff --git a/src/game/gameModeManager.js b/src/game/gameModeManager.js index ea14f60..fa4387d 100644 --- a/src/game/gameModeManager.js +++ b/src/game/gameModeManager.js @@ -17,6 +17,14 @@ const { ValidationError, normalizeError } = require('../utils/errors'); const paymentConfig = require('../config/paymentConfig'); const money = require('../money/atomic'); +// A wallet "not enough (unlocked) money" error is raised BEFORE the tx is broadcast, so it +// is SAFE to retry (no double-pay risk) — unlike an ambiguous post-broadcast failure. Monero +// locks spent outputs (incl. change) for ~10 blocks, so this is the expected error when all +// outputs are temporarily locked. +function isInsufficientFundsError(message) { + return /not enough (unlocked )?(money|balance|outputs)|insufficient|no unlocked|unlocked balance/i.test(String(message || '')); +} + const DEFAULT_SINGLE_GAME_PRICE = 5000000000; // 0.005 XMR or 0.05 WOW depending on currency decimals const DEFAULT_CREDITS_PACKAGE_PRICE = 50000000000; @@ -1399,6 +1407,31 @@ class GameModeManager { if (pending.rows.length === 0) return; + // MONERO OUTPUT LOCKING: a spent output (incl. the change output) is locked for + // ~10 blocks. If we don't have enough UNLOCKED balance to cover this batch right + // now, DEFER it (revert to pending) instead of attempting a transfer that would + // fail — the next batch interval and every new-block tick re-check, so it sends as + // soon as outputs unlock. Batching already minimises how many change outputs we + // create; this handles the "all outputs currently locked" window. + try { + const totalNeeded = money.sum(pending.rows.map(p => p.amount)); // BigInt atomic + const bal = await this.walletService.getBalance(); + // unlocked_balance arrives as a JS number from the RPC; coerce to integer BigInt. + const unlocked = (bal && !bal.error) ? BigInt(Math.round(Number(bal.unlocked_balance) || 0)) : null; + if (unlocked !== null && unlocked < totalNeeded) { + const ids = pending.rows.map(p => p.id); + await this.db.query( + `UPDATE payouts SET status = 'pending', last_error = $2, last_retry_at = NOW() WHERE id = ANY($1)`, + [ids, 'Deferred: insufficient unlocked balance (Monero outputs locked ~10 blocks)'] + ).catch(() => {}); + console.warn(`⏳ Deferring ${pending.rows.length} payout(s): unlocked ${unlocked} < needed ${totalNeeded}. Outputs likely locked; will retry next block/interval.`); + return; + } + } catch (balErr) { + // If the balance check itself fails, fall through and let the transfer try. + console.error('Payout balance pre-check failed, proceeding:', balErr.message); + } + console.log(`📦 Processing batch of ${pending.rows.length} pending payout(s)`); // Build destinations for transfer_split @@ -1435,10 +1468,14 @@ class GameModeManager { }); console.log(`💸 Single payout ${p.id} completed: ${result.txHash}`); } catch (err) { - console.error(`❌ Single payout ${p.id} failed:`, err.message); + // Insufficient-unlocked-funds is a pre-broadcast error -> retry on the next + // batch run (revert to pending). Other failures -> 'failed' (retry service, + // which re-checks the chain before re-sending). + const status = isInsufficientFundsError(err.message) ? 'pending' : 'failed'; + console.error(`❌ Single payout ${p.id} failed -> ${status}:`, err.message); await this.db.query( - `UPDATE payouts SET status = 'failed', last_error = $1, last_retry_at = NOW() WHERE id = $2`, - [err.message, p.id] + `UPDATE payouts SET status = $1, last_error = $2, last_retry_at = NOW() WHERE id = $3`, + [status, err.message, p.id] ).catch(() => {}); } } else { @@ -1473,17 +1510,22 @@ class GameModeManager { }); console.log(`💸 Batch payout completed: ${pending.rows.length} payouts in tx ${txHash}`); } catch (err) { - console.error(`❌ Batch payout failed (${pending.rows.length} payouts):`, err.message); - // CRITICAL: a batch failure is ambiguous — transfer_split may have already - // broadcast on-chain even though the RPC response errored. These rows have no - // tx_hash, so the retry service's blockchain guard can't protect them and an - // auto-retry could DOUBLE-PAY. Mark 'needs_review' (which the retry service - // does NOT pick up) and alert an operator instead. + // An insufficient-unlocked-funds error is raised BEFORE broadcast (no tx + // went out), so it's safe to retry: revert to 'pending' and let the next + // run send it once outputs unlock — no operator alert needed (transient). + // + // ANY OTHER error is ambiguous: transfer_split may have broadcast on-chain + // even though the RPC response errored. Those rows have no tx_hash, so the + // retry service's blockchain guard can't protect them and an auto-retry + // could DOUBLE-PAY. Mark 'needs_review' (retry service skips it) + alert. + const fundsIssue = isInsufficientFundsError(err.message); + const status = fundsIssue ? 'pending' : 'needs_review'; + console.error(`❌ Batch payout failed (${pending.rows.length} payouts) -> ${status}:`, err.message); await this.db.query( - `UPDATE payouts SET status = 'needs_review', last_error = $1, last_retry_at = NOW() WHERE id = ANY($2)`, - [String(err.message).slice(0, 500), ids] + `UPDATE payouts SET status = $1, last_error = $2, last_retry_at = NOW() WHERE id = ANY($3)`, + [status, String(err.message).slice(0, 500), ids] ).catch(() => {}); - if (this.alertService && typeof this.alertService.sendAlert === 'function') { + if (!fundsIssue && this.alertService && typeof this.alertService.sendAlert === 'function') { this.alertService.sendAlert('batch_payout_failed', { subject: '⚠️ Batch payout failed — manual review required', html: `

A batch of ${pending.rows.length} payout(s) failed and was marked needs_review to avoid a possible double-payout.

` diff --git a/test/batchPayout.test.js b/test/batchPayout.test.js index 7a06efc..65ddf57 100644 --- a/test/batchPayout.test.js +++ b/test/batchPayout.test.js @@ -89,6 +89,8 @@ describe('Batch payout dispatch', () => { test('an ambiguous batch failure is marked needs_review, not failed', async () => { const walletService = { + // Enough unlocked balance so the pre-flight check passes and we reach the transfer. + getBalance: jest.fn().mockResolvedValue({ balance: '10000000000000', unlocked_balance: '10000000000000' }), processPayout: jest.fn(), processBatchPayout: jest.fn().mockRejectedValue(new Error('RPC timeout')) }; @@ -101,11 +103,59 @@ describe('Batch payout dispatch', () => { await gmm._processPendingPayouts(); // Failure handling uses db.query (outside a transaction) to mark the batch needs_review. - const failCall = db.query.mock.calls.find(c => /needs_review/i.test(c[0])); + // Status is passed as a bound param ($1), so check the params array, not the SQL text. + const failCall = db.query.mock.calls.find(c => Array.isArray(c[1]) && c[1][0] === 'needs_review'); expect(failCall).toBeDefined(); expect(failCall[0]).toMatch(/id = ANY/i); // Must NOT mark them 'failed' (which the retry service would auto-resend). - const failedCall = db.query.mock.calls.find(c => /status = 'failed'/i.test(c[0])); + const failedCall = db.query.mock.calls.find(c => Array.isArray(c[1]) && c[1][0] === 'failed'); expect(failedCall).toBeUndefined(); }); + + test('MONERO LOCKING: defers the batch to pending when unlocked balance is insufficient (no transfer attempted)', async () => { + // TWO_PENDING needs 200000000000 + 300000000000 = 500000000000 atomic. + // Unlocked balance is far below that (outputs locked ~10 blocks) -> defer, do not send. + const walletService = { + getBalance: jest.fn().mockResolvedValue({ balance: '500000000000', unlocked_balance: '1000000000' }), + processPayout: jest.fn(), + processBatchPayout: jest.fn() + }; + const { gmm, db } = buildGmm(walletService); + + db._mockClient.query + .mockResolvedValueOnce({ rows: TWO_PENDING }) + .mockResolvedValue({ rows: [] }); + + await gmm._processPendingPayouts(); + + // No on-chain transfer was attempted. + expect(walletService.processBatchPayout).not.toHaveBeenCalled(); + expect(walletService.processPayout).not.toHaveBeenCalled(); + + // The defer UPDATE sets status = 'pending' literally in SQL and passes ids as $1. + const deferSql = db.query.mock.calls.find(c => /SET status = 'pending'/i.test(c[0]) && /id = ANY/i.test(c[0])); + expect(deferSql).toBeDefined(); + expect(deferSql[1][0]).toEqual([10, 11]); // ids array param + }); + + test('a pre-broadcast insufficient-funds batch error is retried (pending), not needs_review', async () => { + const walletService = { + getBalance: jest.fn().mockResolvedValue({ balance: '10000000000000', unlocked_balance: '10000000000000' }), + processPayout: jest.fn(), + processBatchPayout: jest.fn().mockRejectedValue(new Error('not enough unlocked money to transfer')) + }; + const { gmm, db } = buildGmm(walletService); + + db._mockClient.query + .mockResolvedValueOnce({ rows: TWO_PENDING }) + .mockResolvedValue({ rows: [] }); + + await gmm._processPendingPayouts(); + + // A funds error means nothing broadcast -> safe to retry: status='pending', NOT needs_review. + const pendingCall = db.query.mock.calls.find(c => Array.isArray(c[1]) && c[1][0] === 'pending' && /id = ANY/i.test(c[0])); + expect(pendingCall).toBeDefined(); + const reviewCall = db.query.mock.calls.find(c => Array.isArray(c[1]) && c[1][0] === 'needs_review'); + expect(reviewCall).toBeUndefined(); + }); });