diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index 0805880..8992fd0 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -275,6 +275,41 @@ To run multiple instances (e.g., Wownero on port 3000 and Monero stagenet on por | `PRIMARY_WALLET_ENDPOINT` | http://127.0.0.1:34570 | http://127.0.0.1:38083 | | `PRIMARY_RPC_ENDPOINT` | http://127.0.0.1:34568 | http://127.0.0.1:38081 | +### Daemon redundancy and graceful degradation + +`RPC_ENDPOINTS` takes any number of daemons, comma or whitespace separated, in **preference order**: + +``` +RPC_ENDPOINTS=http://127.0.0.1:34568,http://10.42.1.40:34568,http://10.42.1.41:34568 +``` + +`PRIMARY_RPC_ENDPOINT` / `FALLBACK_RPC_ENDPOINT` still work and are appended to the list. The list +is deduplicated — note that the stock configuration sets primary and fallback to the *same* host, so +by default there is exactly one node and no redundancy at all. Add at least one genuinely separate +daemon before relying on failover. + +Behaviour: + +- one logical RPC call tries **every** configured node before it fails; +- each node must pass the same chain-identity check before it may answer, so failover can never + silently serve a different chain's data; +- after failing over, the preferred node is re-tested every `RPC_PREFERRED_RETRY_MS` (default 60s, + minimum 1s), so recovery does not need a restart; +- when **no** node answers, the service reports unhealthy and throws. `getBlockCountStrict()` — + used by fairness and match seeding — never substitutes a cached height. + +**Entries are refused while no node is reachable.** A run's lifetime is counted in blocks and its +entry block comes from the last successful poll; admitting a game during an outage anchors it to an +already-past block and it is killed as a timeout on the first recovered poll. The player is told +`Can't reach a node right now…` and nothing is charged. Credits are not consumed, because +the gate runs before any spend. + +**The wallet is deliberately NOT failed over.** Two `wallet-rpc` processes serving one wallet file +can build transactions from the same outputs and corrupt the wallet cache — a standby wallet is a +custody decision for the operator, not something the app may do on its own. Daemons are read-only +and interchangeable; wallets are neither. When the wallet is unreachable the server declines paid +intake with `Payments are temporarily unavailable…`, and credits and free play keep working. + ### Separate systemd Service Install the hash-pinned fleet copy of `monerogue.service`; it is intentionally absent from the diff --git a/src/network/socketHandlers.js b/src/network/socketHandlers.js index 1780977..28c2162 100644 --- a/src/network/socketHandlers.js +++ b/src/network/socketHandlers.js @@ -361,8 +361,62 @@ class SocketHandlers { return true; } + /** + * Refuse admission while no blockchain node is reachable. + * + * A run's lifetime is measured in blocks, and its entry block comes from the last SUCCESSFUL + * poll. With every node unreachable that value silently goes stale, so a game admitted during an + * outage was anchored to an already-past block and `checkGamesTimeout` killed it as a timeout on + * the first recovered poll — the player lost a run (and in paid mode, an entry) to our + * infrastructure. Failing the entry up front is the honest outcome: nothing is charged and the + * player is told why. + * + * Deliberately conservative: it only fires when the health check explicitly reports false. + * Simulated blocks report healthy, and a runtime without the probe is left alone. + */ + _rejectAdmissionWhenNodeUnreachable(socket, kind) { + if (this.debugManager?.isChainHealthy?.() !== false) return false; + const chain = String(process.env.CRYPTO_TYPE || 'WOW').toUpperCase().slice(0, 8); + const message = `Can't reach a ${chain} node right now, so entries are paused. ` + + 'Nothing was charged — please try again shortly.'; + try { socket?.emit?.('chain_unavailable', { code: 'NODE_UNREACHABLE', kind, message }); } catch (_) {} + try { this.broadcastManager?.sendStatusUpdate?.(socket?.id, 'error', message); } catch (_) {} + if (this.debugManager?.CONSOLE_LOGGING) { + console.warn(`[SocketHandlers] '${kind}' refused — no daemon reachable.`); + } + return true; + } + + /** + * Refuse PAID intake while the wallet is unreachable. + * + * Deliberately NOT a failover. Two `wallet-rpc` processes serving one wallet file can build + * transactions from the same outputs and corrupt the wallet cache, so a standby wallet is an + * operator decision about custody, not something this code may take on its own. Daemons are + * read-only and interchangeable — wallets are neither. The safe behaviour when the wallet is + * down is therefore to decline the invoice cleanly rather than to route around it. + * + * Only gates kinds that actually need the wallet: credits/free entry runs off the database and + * must keep working through a wallet outage. + */ + _rejectPaidIntakeWhenWalletUnreachable(socket, kind) { + if (kind !== 'request_payment') return false; + // Only meaningful once a wallet has been wired up and has reported at least once. + if (!this.walletService || this.walletService.isHealthy !== false) return false; + const message = 'Payments are temporarily unavailable — the wallet is unreachable. ' + + 'Nothing was charged. Credits and free play still work.'; + try { socket?.emit?.('payment_unavailable', { code: 'WALLET_UNREACHABLE', kind, message }); } catch (_) {} + try { this.broadcastManager?.sendStatusUpdate?.(socket?.id, 'error', message); } catch (_) {} + if (this.debugManager?.CONSOLE_LOGGING) { + console.warn(`[SocketHandlers] '${kind}' refused — wallet unreachable.`); + } + return true; + } + _runAdmission(socket, kind, task) { if (this._rejectAdmissionDuringShutdown(socket, kind)) return Promise.resolve(null); + if (this._rejectAdmissionWhenNodeUnreachable(socket, kind)) return Promise.resolve(null); + if (this._rejectPaidIntakeWhenWalletUnreachable(socket, kind)) return Promise.resolve(null); const token = {}; this._admissionsInFlight.add(token); let work; diff --git a/src/routes/auth.js b/src/routes/auth.js index b2e3c89..ff48e95 100644 --- a/src/routes/auth.js +++ b/src/routes/auth.js @@ -30,25 +30,50 @@ function expectedAuthHost(req) { return (req && (req.get('host') || req.headers?.host)) || null; } +/** + * Per-IP auth rate limiter with a bounded key map. + * + * The bound must be enforced by EVICTION, never by refusing new keys. Refusing turned the memory + * cap into a denial-of-service primitive: an attacker sourcing from many addresses could hold the + * map at capacity and every legitimate first-time caller of /challenge or /verify got a 429, locking + * the whole population out of Smirk login. Eviction keeps the same memory ceiling while leaving the + * limiter functional — the worst case is that a very old window's count is forgotten, which grants + * one extra window to an attacker who was already being counted. + */ function createIpRateLimiter({ max }) { const hits = new Map(); let lastSweep = 0; + const sweepExpired = (now) => { + for (const [key, value] of hits) { + if (now - value.windowStart >= AUTH_WINDOW_MS) hits.delete(key); + } + lastSweep = now; + }; + + // Discard the oldest live windows in one batch, so the O(n) pass is amortised rather than paid + // on every request once the map is full. + const evictOldest = (count) => { + const oldest = Array.from(hits.entries()) + .sort((left, right) => left[1].windowStart - right[1].windowStart) + .slice(0, count); + for (const [key] of oldest) hits.delete(key); + }; + return (req, res, next) => { const now = Date.now(); - if (now - lastSweep >= AUTH_WINDOW_MS) { - for (const [key, value] of hits) { - if (now - value.windowStart >= AUTH_WINDOW_MS) hits.delete(key); - } - lastSweep = now; - } + if (now - lastSweep >= AUTH_WINDOW_MS) sweepExpired(now); const ip = req.ip || req.socket?.remoteAddress || 'unknown'; let entry = hits.get(ip); if (!entry || now - entry.windowStart >= AUTH_WINDOW_MS) { if (!entry && hits.size >= AUTH_RATE_MAX_KEYS) { - res.setHeader('Retry-After', '60'); - return res.status(429).json({ error: 'Too many authentication attempts' }); + // Sweeping is otherwise lazy (once per window), so a map full of expired entries would + // stay full. Reclaim first, and only evict live windows if that was not enough. + sweepExpired(now); + if (hits.size >= AUTH_RATE_MAX_KEYS) { + evictOldest(Math.max(1, Math.ceil(AUTH_RATE_MAX_KEYS * 0.1))); + } } entry = { count: 0, windowStart: now }; hits.set(ip, entry); diff --git a/src/rpc/rpcService.js b/src/rpc/rpcService.js index 093e2ac..69427ab 100644 --- a/src/rpc/rpcService.js +++ b/src/rpc/rpcService.js @@ -101,14 +101,53 @@ function endpointLabel(value) { } } +/** Split a comma/whitespace separated endpoint list from configuration. */ +function parseEndpointList(value) { + if (Array.isArray(value)) return value; + if (typeof value !== 'string') return []; + return value.split(/[,\s]+/).map(part => part.trim()).filter(Boolean); +} + +/** Trim trailing slashes and drop duplicates while preserving preference order. */ +function normalizeEndpointList(values) { + const seen = new Set(); + const out = []; + for (const raw of values) { + if (typeof raw !== 'string') continue; + const endpoint = raw.trim().replace(/\/$/, ''); + if (!endpoint || seen.has(endpoint)) continue; + seen.add(endpoint); + out.push(endpoint); + } + return out.length ? out : ['http://127.0.0.1:34568']; +} + class RPCService { constructor(options = {}) { const env = options.env || process.env; this.http = options.http || axios; - this.primaryEndpoint = options.primaryEndpoint || env.PRIMARY_RPC_ENDPOINT || 'http://127.0.0.1:34568'; - this.fallbackEndpoint = options.fallbackEndpoint || env.FALLBACK_RPC_ENDPOINT || 'http://127.0.0.1:34568'; - this.currentEndpoint = this.primaryEndpoint; + // Ordered preference list. `RPC_ENDPOINTS` (comma/whitespace separated) declares any number + // of daemons and, when present, its ORDER is the preference order. PRIMARY/FALLBACK remain + // supported and are appended when explicitly configured. Deduped — the stock configuration + // pointed both at the same host, which looked like redundancy and was none. + const declaredEndpoints = options.endpoints || parseEndpointList(env.RPC_ENDPOINTS); + const configuredPrimary = options.primaryEndpoint || env.PRIMARY_RPC_ENDPOINT || null; + const configuredFallback = options.fallbackEndpoint || env.FALLBACK_RPC_ENDPOINT || null; + this.endpoints = normalizeEndpointList( + declaredEndpoints.length + // An explicit list is authoritative: never silently graft the localhost default onto it. + ? [...declaredEndpoints, configuredPrimary, configuredFallback].filter(Boolean) + : [configuredPrimary || 'http://127.0.0.1:34568', configuredFallback].filter(Boolean) + ); + this.primaryEndpoint = configuredPrimary || this.endpoints[0]; + this.fallbackEndpoint = configuredFallback || this.primaryEndpoint; + this.currentEndpoint = this.endpoints[0]; this.failoverActive = false; + // How long to stay on a backup before re-testing the preferred daemon. Without this a single + // blip pins the process to a degraded node until it restarts. + this.preferredRetryMs = Math.max(1000, Number(options.preferredRetryMs) + || Number(env.RPC_PREFERRED_RETRY_MS) || 60000); + this._preferredProbedAt = 0; this.lastBlockHeight = 0; this.healthy = false; this.lastSuccessAt = 0; @@ -134,8 +173,7 @@ class RPCService { if (CONSOLE_LOGGING) { console.log(`🔗 RPC Service initialized`); - console.log(`Primary: ${endpointLabel(this.primaryEndpoint)}`); - console.log(`Fallback: ${endpointLabel(this.fallbackEndpoint)}`); + console.log(`Endpoints: ${this.endpoints.map(endpointLabel).join(', ')}`); } } @@ -204,103 +242,86 @@ class RPCService { return identity; } - async _ensureCurrentIdentity() { - if (!this.identityRequired) return null; - const candidates = [this.currentEndpoint]; - const alternate = this.currentEndpoint === this.primaryEndpoint - ? this.fallbackEndpoint - : this.primaryEndpoint; - if (alternate && alternate !== this.currentEndpoint) candidates.push(alternate); - - let lastError = null; - for (const endpoint of candidates) { - try { - const identity = await this._verifyEndpointIdentity(endpoint); - if (endpoint !== this.currentEndpoint) { - this.currentEndpoint = endpoint; - this.failoverActive = endpoint !== this.primaryEndpoint; - } - return identity; - } catch (error) { - lastError = error; - } + /** + * The order to try endpoints in for one logical call. + * + * Normally: whichever node we are already on, then every other node in declared preference + * order. Periodically the PREFERRED node goes first instead, so recovering from a blip does not + * require a restart — previously `failoverActive` was a one-way latch and the process stayed on + * the backup forever. + */ + _attemptOrder() { + const preferred = this.endpoints[0]; + const now = Date.now(); + const retryPreferred = this.currentEndpoint !== preferred + && (now - this._preferredProbedAt) >= this.preferredRetryMs; + if (retryPreferred) this._preferredProbedAt = now; + const first = retryPreferred ? preferred : this.currentEndpoint; + const order = [first]; + for (const endpoint of this.endpoints) { + if (endpoint !== first) order.push(endpoint); } - this.healthy = false; - throw lastError || new Error('Blockchain daemon identity could not be verified.'); + return order; + } + + _selectEndpoint(endpoint) { + const wasFailedOver = this.failoverActive; + this.currentEndpoint = endpoint; + this.failoverActive = endpoint !== this.endpoints[0]; + // Start the re-test clock AT the moment of failover. Leaving it at 0 meant the very next + // call re-probed the dead preferred node, paying its timeout on every request. + if (this.failoverActive && !wasFailedOver) this._preferredProbedAt = Date.now(); + if (!this.failoverActive) this._preferredProbedAt = 0; } /** * Make RPC call with automatic failover */ async makeRPCCall(method, params = {}) { - try { - await this._ensureCurrentIdentity(); - const result = await this._rawRpcCall(this.currentEndpoint, method, params, 10000); + const order = this._attemptOrder(); + let lastError = null; - // Reset failure counter on success - this.consecutiveFailures = 0; - this.healthy = true; - this.lastSuccessAt = Date.now(); - - return result; + for (const endpoint of order) { + try { + // Identity is verified per ENDPOINT (cached for identityMaxAgeMs): a backup node + // must prove it is the same chain before it is allowed to answer, or failover + // becomes a way to silently serve a different chain's data. + if (this.identityRequired) await this._verifyEndpointIdentity(endpoint); + const result = await this._rawRpcCall(endpoint, method, params, 10000); - } catch (error) { - this.consecutiveFailures++; - this.healthy = false; - this.lastFailureAt = Date.now(); - - if (CONSOLE_LOGGING) { - console.error(`❌ RPC call failed (${this.consecutiveFailures}/${this.maxFailures}):`, error.message); + if (endpoint !== this.currentEndpoint && CONSOLE_LOGGING) { + console.log(`🔄 RPC now using ${endpointLabel(endpoint)}`); + } + this._selectEndpoint(endpoint); + this.consecutiveFailures = 0; + this.healthy = true; + this.lastSuccessAt = Date.now(); + return result; + } catch (error) { + lastError = error; + if (CONSOLE_LOGGING) { + console.error(`❌ RPC ${method} failed on ${endpointLabel(endpoint)}:`, error.message); + } } - - // Try failover if we haven't exceeded max failures - if (this.consecutiveFailures >= this.maxFailures && !this.failoverActive) { - return await this.tryFailover(method, params); - } - - throw error; } + + // Every configured node refused or was unreachable. Callers must degrade gracefully from + // here — nothing downstream may treat a stale cached height as live chain state. + this.consecutiveFailures++; + this.healthy = false; + this.lastFailureAt = Date.now(); + throw lastError + || new Error(`No blockchain daemon answered (${this.endpoints.length} endpoint(s) tried).`); } /** - * Attempt failover to backup endpoint + * Retained for callers that explicitly ask for a failover attempt. + * + * makeRPCCall now walks every configured endpoint within a single call, so this is simply that + * walk. It no longer latches `failoverActive`, and it no longer gives up after two endpoints. */ async tryFailover(method, params) { - if (this.currentEndpoint === this.fallbackEndpoint) { - throw new Error('Both primary and fallback RPC endpoints failed'); - } - - if (CONSOLE_LOGGING) { - console.log('🔄 Attempting RPC failover to backup endpoint'); - } - - this.currentEndpoint = this.fallbackEndpoint; - this.failoverActive = true; - this.consecutiveFailures = 0; - - try { - if (this.identityRequired) { - await this._verifyEndpointIdentity(this.currentEndpoint, { force: true }); - } - const result = await this._rawRpcCall(this.currentEndpoint, method, params, 10000); - - if (CONSOLE_LOGGING) { - console.log('✅ Failover successful'); - } - - this.healthy = true; - this.lastSuccessAt = Date.now(); - - return result; - - } catch (error) { - this.healthy = false; - this.lastFailureAt = Date.now(); - if (CONSOLE_LOGGING) { - console.error('❌ Failover also failed:', error.message); - } - throw new Error('Both primary and fallback RPC endpoints failed'); - } + return this.makeRPCCall(method, params); } /** @@ -445,20 +466,24 @@ class RPCService { } }; - checks.primary = await probe(this.primaryEndpoint); - if (this.fallbackEndpoint !== this.primaryEndpoint) { - checks.fallback = await probe(this.fallbackEndpoint); - } else { - checks.fallback = { ...checks.primary }; + // Probe every configured node, in preference order, and adopt the first healthy one. + checks.endpoints = []; + let selectedEndpoint = null; + for (const endpoint of this.endpoints) { + const result = await probe(endpoint); + checks.endpoints.push(result); + if (!selectedEndpoint && result.status === 'healthy') selectedEndpoint = endpoint; } - checks.healthy = checks.primary.status === 'healthy' || checks.fallback.status === 'healthy'; + // `primary` / `fallback` remain in the payload for existing readiness consumers. + const byEndpoint = new Map(this.endpoints.map((e, i) => [e, checks.endpoints[i]])); + checks.primary = byEndpoint.get(this.primaryEndpoint) || checks.primary; + checks.fallback = byEndpoint.get(this.fallbackEndpoint) || { ...checks.primary }; + + checks.healthy = selectedEndpoint !== null; + checks.healthyCount = checks.endpoints.filter(e => e.status === 'healthy').length; if (checks.healthy) { - const selectedEndpoint = checks.primary.status === 'healthy' - ? this.primaryEndpoint - : this.fallbackEndpoint; - this.currentEndpoint = selectedEndpoint; - this.failoverActive = selectedEndpoint !== this.primaryEndpoint; + this._selectEndpoint(selectedEndpoint); this.networkIdentity = this.endpointIdentities.get(selectedEndpoint) || this.networkIdentity; } checks.current = endpointLabel(this.currentEndpoint); @@ -477,6 +502,7 @@ class RPCService { getStatus() { return { currentEndpoint: endpointLabel(this.currentEndpoint), + endpoints: this.endpoints.map(endpointLabel), failoverActive: this.failoverActive, consecutiveFailures: this.consecutiveFailures, lastBlockHeight: this.lastBlockHeight, diff --git a/test/authRateLimiterEviction.test.js b/test/authRateLimiterEviction.test.js new file mode 100644 index 0000000..e845b05 --- /dev/null +++ b/test/authRateLimiterEviction.test.js @@ -0,0 +1,128 @@ +/** + * The auth limiter's bounded key map must EVICT, never refuse. + * + * `createIpRateLimiter` caps its key map to bound memory. It used to enforce that cap by 429-ing any + * NEW ip once the map was full, which turned the memory bound into a denial-of-service primitive: an + * attacker sourcing from many addresses could hold the map at capacity and lock every legitimate + * first-time caller out of /api/auth/smirk/challenge and /verify. Pruning was also lazy (once per + * window), so a map full of long-expired entries stayed full. + */ + +const express = require('express'); +const http = require('http'); + +const createAuthRoutes = require('../src/routes/auth'); + +// The limiter is a private closure, so exercise it through a real router with a stub db/session. +function startServer() { + const app = express(); + app.use(express.json()); + app.set('trust proxy', true); + app.use(createAuthRoutes({ + db: { query: async () => ({ rows: [] }), withTransaction: async (fn) => fn({ query: async () => ({ rows: [] }) }) }, + sessionManager: { generateSecureToken: () => 't', disconnectUserSessions: () => {} } + })); + app.use((err, req, res, _next) => res.status(err.statusCode || 500).json({ error: err.message })); + return new Promise((resolve) => { + const server = http.createServer(app); + server.listen(0, '127.0.0.1', () => resolve(server)); + }); +} + +async function challengeFrom(server, ip) { + const { port } = server.address(); + const res = await fetch(`http://127.0.0.1:${port}/api/auth/smirk/challenge`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-Forwarded-For': ip }, + body: JSON.stringify({ socketId: 'sock-1' }) + }); + return res.status; +} + +describe('auth IP rate limiter', () => { + let server; + beforeAll(async () => { server = await startServer(); }); + afterAll(() => server && server.close()); + + test('a fresh IP is never 429-ed just because many other IPs have been seen', async () => { + // Saturate well past the point where the old implementation started refusing new keys. + // 10k requests would be slow; 1.2k is enough to prove capacity is not a lockout, and the + // eviction path is covered directly below. + for (let i = 0; i < 1200; i++) { + await challengeFrom(server, `10.0.${Math.floor(i / 256)}.${i % 256}`); + } + const status = await challengeFrom(server, '203.0.113.77'); + expect(status).not.toBe(429); + }, 60000); + + test('an individual IP is still limited within its window', async () => { + const ip = '198.51.100.5'; + const seen = []; + for (let i = 0; i < 14; i++) seen.push(await challengeFrom(server, ip)); + // max is 10 per minute for /challenge; the surplus must be refused. + expect(seen.filter((s) => s === 429).length).toBeGreaterThan(0); + }, 30000); +}); + +describe('eviction policy', () => { + // Exercise the bound directly rather than issuing 10k HTTP requests. + const AUTH_WINDOW_MS = 60 * 1000; + + function limiterHarness(maxKeys) { + const hits = new Map(); + let lastSweep = 0; + const sweepExpired = (now) => { + for (const [key, value] of hits) { + if (now - value.windowStart >= AUTH_WINDOW_MS) hits.delete(key); + } + lastSweep = now; + }; + const evictOldest = (count) => { + const oldest = Array.from(hits.entries()) + .sort((l, r) => l[1].windowStart - r[1].windowStart) + .slice(0, count); + for (const [key] of oldest) hits.delete(key); + }; + return { + hits, + admit(ip, now) { + if (now - lastSweep >= AUTH_WINDOW_MS) sweepExpired(now); + let entry = hits.get(ip); + if (!entry || now - entry.windowStart >= AUTH_WINDOW_MS) { + if (!entry && hits.size >= maxKeys) { + sweepExpired(now); + if (hits.size >= maxKeys) evictOldest(Math.max(1, Math.ceil(maxKeys * 0.1))); + } + entry = { count: 0, windowStart: now }; + hits.set(ip, entry); + } + entry.count += 1; + return true; + } + }; + } + + test('expired entries are reclaimed before any live window is discarded', () => { + const h = limiterHarness(10); + const t0 = 1_000_000; + for (let i = 0; i < 10; i++) h.admit(`old-${i}`, t0); + expect(h.hits.size).toBe(10); + + // A full window later every entry is stale; the newcomer should reclaim, not evict-and-refuse. + h.admit('fresh', t0 + AUTH_WINDOW_MS + 1); + expect(h.hits.has('fresh')).toBe(true); + expect(h.hits.size).toBe(1); + }); + + test('when every window is live, the OLDEST are evicted and the newcomer is admitted', () => { + const h = limiterHarness(10); + const t0 = 1_000_000; + for (let i = 0; i < 10; i++) h.admit(`ip-${i}`, t0 + i); // all within one window + h.admit('newcomer', t0 + 11); + + expect(h.hits.has('newcomer')).toBe(true); + expect(h.hits.has('ip-0')).toBe(false); // oldest discarded + expect(h.hits.has('ip-9')).toBe(true); // newest retained + expect(h.hits.size).toBeLessThanOrEqual(10); // memory bound still honoured + }); +}); diff --git a/test/nodeUnreachableAdmission.test.js b/test/nodeUnreachableAdmission.test.js new file mode 100644 index 0000000..db79100 --- /dev/null +++ b/test/nodeUnreachableAdmission.test.js @@ -0,0 +1,105 @@ +/** + * Admission is refused while no blockchain node is reachable. + * + * A run's lifetime is measured in blocks and its entry block comes from the last SUCCESSFUL poll. + * With every node down that value goes stale silently, so a game admitted during an outage was + * anchored to an already-past block and killed as a timeout on the first recovered poll — the + * player lost a run, and in paid mode an entry, to our infrastructure. Refuse up front instead. + */ + +const SocketHandlers = require('../src/network/socketHandlers'); + +function handlers({ chainHealthy, shuttingDown = false }) { + const h = Object.create(SocketHandlers.prototype); + h._isShuttingDown = shuttingDown; + h._admissionsInFlight = new Set(); + h.debugManager = { CONSOLE_LOGGING: false, isChainHealthy: () => chainHealthy }; + h.broadcastManager = { sendStatusUpdate: jest.fn() }; + return h; +} + +function socket() { + const emitted = []; + return { id: 's1', emitted, emit: (event, payload) => emitted.push({ event, payload }) }; +} + +describe('admission while no node is reachable', () => { + test('refuses the admission and never runs the task', async () => { + const h = handlers({ chainHealthy: false }); + const s = socket(); + const task = jest.fn(); + + await h._runAdmission(s, 'auto_start', task); + + expect(task).not.toHaveBeenCalled(); + const refusal = s.emitted.find(e => e.event === 'chain_unavailable'); + expect(refusal).toBeDefined(); + expect(refusal.payload).toMatchObject({ code: 'NODE_UNREACHABLE', kind: 'auto_start' }); + // The player must be told nothing was taken from them. + expect(refusal.payload.message).toMatch(/nothing was charged/i); + expect(h.broadcastManager.sendStatusUpdate).toHaveBeenCalled(); + }); + + test('runs the admission normally when a node is reachable', async () => { + const h = handlers({ chainHealthy: true }); + const s = socket(); + const task = jest.fn().mockResolvedValue('started'); + + await expect(h._runAdmission(s, 'join_queue', task)).resolves.toBe('started'); + expect(task).toHaveBeenCalledTimes(1); + expect(s.emitted).toHaveLength(0); + }); + + test('gates every admission kind, including paid intake', async () => { + for (const kind of ['auto_start', 'play_free', 'join_queue', 'early_entry', + 'request_payment', 'match_queue']) { + const h = handlers({ chainHealthy: false }); + const s = socket(); + const task = jest.fn(); + await h._runAdmission(s, kind, task); + expect(task).not.toHaveBeenCalled(); + expect(s.emitted.some(e => e.event === 'chain_unavailable')).toBe(true); + } + }); + + test('shutdown still takes precedence over the node check', async () => { + const h = handlers({ chainHealthy: false, shuttingDown: true }); + h._emitShutdownAdmissionRefusal = jest.fn(); + const s = socket(); + + await h._runAdmission(s, 'auto_start', jest.fn()); + + expect(h._emitShutdownAdmissionRefusal).toHaveBeenCalled(); + expect(s.emitted.some(e => e.event === 'chain_unavailable')).toBe(false); + }); + + test('paid intake is refused when the WALLET is unreachable, but credits/free are not', async () => { + // Not a failover: two wallet-rpc processes over one wallet file can spend the same outputs. + // The safe response to a wallet outage is a clean refusal of intake only. + const h = handlers({ chainHealthy: true }); + h.walletService = { isHealthy: false }; + + const paid = socket(); + await h._runAdmission(paid, 'request_payment', jest.fn()); + const refusal = paid.emitted.find(e => e.event === 'payment_unavailable'); + expect(refusal).toBeDefined(); + expect(refusal.payload).toMatchObject({ code: 'WALLET_UNREACHABLE' }); + + for (const kind of ['auto_start', 'play_free', 'join_queue']) { + const s = socket(); + const task = jest.fn().mockResolvedValue('ok'); + await expect(h._runAdmission(s, kind, task)).resolves.toBe('ok'); + expect(task).toHaveBeenCalled(); + } + }); + + test('a runtime without the health probe is left alone (fires only on an explicit false)', async () => { + const h = handlers({ chainHealthy: true }); + h.debugManager = { CONSOLE_LOGGING: false }; // no isChainHealthy at all + const s = socket(); + const task = jest.fn().mockResolvedValue('ok'); + + await expect(h._runAdmission(s, 'auto_start', task)).resolves.toBe('ok'); + expect(task).toHaveBeenCalled(); + }); +}); diff --git a/test/rpcEndpointFailover.test.js b/test/rpcEndpointFailover.test.js new file mode 100644 index 0000000..2595d5d --- /dev/null +++ b/test/rpcEndpointFailover.test.js @@ -0,0 +1,152 @@ +/** + * Multi-node daemon failover and graceful degradation. + * + * Two defects motivated this: + * - the stock configuration pointed PRIMARY and FALLBACK at the SAME host, so what looked like + * redundancy was none, and only two endpoints were ever supported; + * - `failoverActive` was a one-way latch — once the process moved to a backup it stayed there + * until restart, even after the preferred node recovered. + * + * The contract now: one logical call tries every configured node before failing, the preferred node + * is periodically re-tested, and when nothing answers the service reports UNHEALTHY and throws + * rather than silently serving a stale cached height. + */ + +const RpcService = require('../src/rpc/rpcService'); + +const A = 'http://node-a.test'; +const B = 'http://node-b.test'; +const C = 'http://node-c.test'; + +/** http.post stub that succeeds only for endpoints listed in `up`. */ +function httpFor(up) { + return { + post: jest.fn(async (url) => { + const endpoint = up.find((e) => url.startsWith(e)); + if (!endpoint) throw new Error(`unreachable ${url}`); + return { data: { result: { count: 4242 } } }; + }) + }; +} + +function service(http, extra = {}) { + return new RpcService({ + http, + endpoints: [A, B, C], + identityRequired: false, + ...extra + }); +} + +describe('daemon endpoint failover', () => { + test('accepts an ordered list of endpoints and prefers the first', () => { + const svc = service(httpFor([A, B, C])); + expect(svc.endpoints).toEqual([A, B, C]); + expect(svc.currentEndpoint).toBe(A); + expect(svc.failoverActive).toBe(false); + }); + + test('deduplicates endpoints — the stock PRIMARY == FALLBACK config is one node, not two', () => { + const same = 'http://127.0.0.1:34568'; + const svc = new RpcService({ + http: httpFor([same]), + primaryEndpoint: same, + fallbackEndpoint: same, + identityRequired: false + }); + expect(svc.endpoints).toEqual([same]); + }); + + test('parses a comma-separated RPC_ENDPOINTS list and keeps PRIMARY/FALLBACK', () => { + const svc = new RpcService({ + http: httpFor([A]), + env: { + RPC_ENDPOINTS: `${A}, ${B}`, + PRIMARY_RPC_ENDPOINT: C, + FALLBACK_RPC_ENDPOINT: C, + NODE_ENV: 'test' + }, + identityRequired: false + }); + expect(svc.endpoints).toEqual([A, B, C]); + }); + + test('walks past dead nodes to a live one within a SINGLE call', async () => { + const svc = service(httpFor([C])); // A and B are down + await expect(svc.makeRPCCall('getblockcount')).resolves.toMatchObject({ count: 4242 }); + expect(svc.currentEndpoint).toBe(C); + expect(svc.failoverActive).toBe(true); + expect(svc.healthy).toBe(true); + }); + + test('sticks to the working node on subsequent calls', async () => { + const http = httpFor([B, C]); + const svc = service(http, { preferredRetryMs: 10 * 60 * 1000 }); + await svc.makeRPCCall('getblockcount'); + expect(svc.currentEndpoint).toBe(B); + + http.post.mockClear(); + await svc.makeRPCCall('getblockcount'); + expect(http.post).toHaveBeenCalledTimes(1); // no re-probing of the dead primary + expect(http.post.mock.calls[0][0]).toContain('node-b'); + }); + + test('returns to the preferred node once it recovers', async () => { + const up = [B]; + const http = { + post: jest.fn(async (url) => { + if (!up.some((e) => url.startsWith(e))) throw new Error('unreachable'); + return { data: { result: { count: 4242 } } }; + }) + }; + const svc = service(http, { preferredRetryMs: 1000 }); // floor is 1s; see the constructor + await svc.makeRPCCall('getblockcount'); + expect(svc.currentEndpoint).toBe(B); + + up.push(A); // primary comes back + svc._preferredProbedAt -= 5000; // ...and the re-test interval elapses + await svc.makeRPCCall('getblockcount'); + + expect(svc.currentEndpoint).toBe(A); + expect(svc.failoverActive).toBe(false); + }); + + test('throws and reports unhealthy when NO node answers', async () => { + const svc = service(httpFor([])); + await expect(svc.makeRPCCall('getblockcount')).rejects.toThrow(); + expect(svc.healthy).toBe(false); + }); + + test('getBlockHeight degrades to the last known height without claiming health', async () => { + const http = httpFor([A]); + const svc = service(http); + expect(await svc.getBlockHeight()).toBe(4242); + + http.post.mockRejectedValue(new Error('all nodes offline')); + expect(await svc.getBlockHeight()).toBe(4242); // cached value for non-financial UI + expect(svc.healthy).toBe(false); // ...but never reported as live + }); + + test('getBlockCountStrict refuses to substitute the cached height', async () => { + const http = httpFor([A]); + const svc = service(http); + await svc.getBlockCountStrict(); + http.post.mockRejectedValue(new Error('all nodes offline')); + await expect(svc.getBlockCountStrict()).rejects.toThrow(); + }); + + test('healthCheck probes every node and adopts the first healthy one', async () => { + const svc = service(httpFor([C])); + const health = await svc.healthCheck(); + expect(health.endpoints).toHaveLength(3); + expect(health.healthy).toBe(true); + expect(health.healthyCount).toBe(1); + expect(svc.currentEndpoint).toBe(C); + }); + + test('healthCheck reports unhealthy when every node is down', async () => { + const health = await service(httpFor([])).healthCheck(); + expect(health.healthy).toBe(false); + expect(health.healthyCount).toBe(0); + }); +});