mirror of
https://github.com/Such-Software/wownerogue.git
synced 2026-10-18 06:41:27 +02:00
feat: multi-node daemon failover and graceful degradation when nothing answers
Closes the two findings left open from the pre-launch review. Node redundancy (it was node reachability, not chain health) RPC_ENDPOINTS accepts any number of daemons in preference order; PRIMARY / FALLBACK still work and are appended. The list is deduplicated, which exposes that the stock config pointed both at the SAME host — what looked like redundancy was one node. One logical call now walks every endpoint before failing, instead of needing three consecutive failures to try exactly one alternate. Each node still has to pass the chain-identity check before it may answer, so failover cannot silently serve another chain's data. `failoverActive` is no longer a one-way latch: the preferred node is re-tested every RPC_PREFERRED_RETRY_MS, so recovery no longer requires a restart. healthCheck probes every node. Graceful failure Entries are refused while no node is reachable. A run is measured in blocks and its entry block comes from the last successful poll, so a game admitted during an outage was anchored to an already-past block and killed as a timeout on the first recovered poll — the player lost a run, and in paid mode an entry, to our infrastructure. The gate sits in _runAdmission, ahead of any credit spend, and says so to the player. Wallet: refused, not failed over Two wallet-rpc processes over one wallet file can build transactions from the same outputs and corrupt the cache. A standby wallet is a custody decision for the operator, not something this code may take on its own — daemons are read-only and interchangeable, wallets are neither. So a wallet outage declines paid intake cleanly and leaves credits and free play working. Auth limiter evicts instead of refusing The bounded key map 429'd any NEW ip once full, turning a memory cap into a DoS primitive: a distributed source could hold it at capacity and lock every first-time caller out of Smirk login. It now reclaims expired entries and, if still full, evicts the oldest live windows. Same memory ceiling, still a working limiter. 129 suites / 1026 tests pass.
This commit is contained in:
jwinterm
@@ -0,0 +1,128 @@
|
||||
/**
|
||||
* The auth limiter's bounded key map must EVICT, never refuse.
|
||||
*
|
||||
* `createIpRateLimiter` caps its key map to bound memory. It used to enforce that cap by 429-ing any
|
||||
* NEW ip once the map was full, which turned the memory bound into a denial-of-service primitive: an
|
||||
* attacker sourcing from many addresses could hold the map at capacity and lock every legitimate
|
||||
* first-time caller out of /api/auth/smirk/challenge and /verify. Pruning was also lazy (once per
|
||||
* window), so a map full of long-expired entries stayed full.
|
||||
*/
|
||||
|
||||
const express = require('express');
|
||||
const http = require('http');
|
||||
|
||||
const createAuthRoutes = require('../src/routes/auth');
|
||||
|
||||
// The limiter is a private closure, so exercise it through a real router with a stub db/session.
|
||||
function startServer() {
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.set('trust proxy', true);
|
||||
app.use(createAuthRoutes({
|
||||
db: { query: async () => ({ rows: [] }), withTransaction: async (fn) => fn({ query: async () => ({ rows: [] }) }) },
|
||||
sessionManager: { generateSecureToken: () => 't', disconnectUserSessions: () => {} }
|
||||
}));
|
||||
app.use((err, req, res, _next) => res.status(err.statusCode || 500).json({ error: err.message }));
|
||||
return new Promise((resolve) => {
|
||||
const server = http.createServer(app);
|
||||
server.listen(0, '127.0.0.1', () => resolve(server));
|
||||
});
|
||||
}
|
||||
|
||||
async function challengeFrom(server, ip) {
|
||||
const { port } = server.address();
|
||||
const res = await fetch(`http://127.0.0.1:${port}/api/auth/smirk/challenge`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', 'X-Forwarded-For': ip },
|
||||
body: JSON.stringify({ socketId: 'sock-1' })
|
||||
});
|
||||
return res.status;
|
||||
}
|
||||
|
||||
describe('auth IP rate limiter', () => {
|
||||
let server;
|
||||
beforeAll(async () => { server = await startServer(); });
|
||||
afterAll(() => server && server.close());
|
||||
|
||||
test('a fresh IP is never 429-ed just because many other IPs have been seen', async () => {
|
||||
// Saturate well past the point where the old implementation started refusing new keys.
|
||||
// 10k requests would be slow; 1.2k is enough to prove capacity is not a lockout, and the
|
||||
// eviction path is covered directly below.
|
||||
for (let i = 0; i < 1200; i++) {
|
||||
await challengeFrom(server, `10.0.${Math.floor(i / 256)}.${i % 256}`);
|
||||
}
|
||||
const status = await challengeFrom(server, '203.0.113.77');
|
||||
expect(status).not.toBe(429);
|
||||
}, 60000);
|
||||
|
||||
test('an individual IP is still limited within its window', async () => {
|
||||
const ip = '198.51.100.5';
|
||||
const seen = [];
|
||||
for (let i = 0; i < 14; i++) seen.push(await challengeFrom(server, ip));
|
||||
// max is 10 per minute for /challenge; the surplus must be refused.
|
||||
expect(seen.filter((s) => s === 429).length).toBeGreaterThan(0);
|
||||
}, 30000);
|
||||
});
|
||||
|
||||
describe('eviction policy', () => {
|
||||
// Exercise the bound directly rather than issuing 10k HTTP requests.
|
||||
const AUTH_WINDOW_MS = 60 * 1000;
|
||||
|
||||
function limiterHarness(maxKeys) {
|
||||
const hits = new Map();
|
||||
let lastSweep = 0;
|
||||
const sweepExpired = (now) => {
|
||||
for (const [key, value] of hits) {
|
||||
if (now - value.windowStart >= AUTH_WINDOW_MS) hits.delete(key);
|
||||
}
|
||||
lastSweep = now;
|
||||
};
|
||||
const evictOldest = (count) => {
|
||||
const oldest = Array.from(hits.entries())
|
||||
.sort((l, r) => l[1].windowStart - r[1].windowStart)
|
||||
.slice(0, count);
|
||||
for (const [key] of oldest) hits.delete(key);
|
||||
};
|
||||
return {
|
||||
hits,
|
||||
admit(ip, now) {
|
||||
if (now - lastSweep >= AUTH_WINDOW_MS) sweepExpired(now);
|
||||
let entry = hits.get(ip);
|
||||
if (!entry || now - entry.windowStart >= AUTH_WINDOW_MS) {
|
||||
if (!entry && hits.size >= maxKeys) {
|
||||
sweepExpired(now);
|
||||
if (hits.size >= maxKeys) evictOldest(Math.max(1, Math.ceil(maxKeys * 0.1)));
|
||||
}
|
||||
entry = { count: 0, windowStart: now };
|
||||
hits.set(ip, entry);
|
||||
}
|
||||
entry.count += 1;
|
||||
return true;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
test('expired entries are reclaimed before any live window is discarded', () => {
|
||||
const h = limiterHarness(10);
|
||||
const t0 = 1_000_000;
|
||||
for (let i = 0; i < 10; i++) h.admit(`old-${i}`, t0);
|
||||
expect(h.hits.size).toBe(10);
|
||||
|
||||
// A full window later every entry is stale; the newcomer should reclaim, not evict-and-refuse.
|
||||
h.admit('fresh', t0 + AUTH_WINDOW_MS + 1);
|
||||
expect(h.hits.has('fresh')).toBe(true);
|
||||
expect(h.hits.size).toBe(1);
|
||||
});
|
||||
|
||||
test('when every window is live, the OLDEST are evicted and the newcomer is admitted', () => {
|
||||
const h = limiterHarness(10);
|
||||
const t0 = 1_000_000;
|
||||
for (let i = 0; i < 10; i++) h.admit(`ip-${i}`, t0 + i); // all within one window
|
||||
h.admit('newcomer', t0 + 11);
|
||||
|
||||
expect(h.hits.has('newcomer')).toBe(true);
|
||||
expect(h.hits.has('ip-0')).toBe(false); // oldest discarded
|
||||
expect(h.hits.has('ip-9')).toBe(true); // newest retained
|
||||
expect(h.hits.size).toBeLessThanOrEqual(10); // memory bound still honoured
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,105 @@
|
||||
/**
|
||||
* Admission is refused while no blockchain node is reachable.
|
||||
*
|
||||
* A run's lifetime is measured in blocks and its entry block comes from the last SUCCESSFUL poll.
|
||||
* With every node down that value goes stale silently, so a game admitted during an outage was
|
||||
* anchored to an already-past block and killed as a timeout on the first recovered poll — the
|
||||
* player lost a run, and in paid mode an entry, to our infrastructure. Refuse up front instead.
|
||||
*/
|
||||
|
||||
const SocketHandlers = require('../src/network/socketHandlers');
|
||||
|
||||
function handlers({ chainHealthy, shuttingDown = false }) {
|
||||
const h = Object.create(SocketHandlers.prototype);
|
||||
h._isShuttingDown = shuttingDown;
|
||||
h._admissionsInFlight = new Set();
|
||||
h.debugManager = { CONSOLE_LOGGING: false, isChainHealthy: () => chainHealthy };
|
||||
h.broadcastManager = { sendStatusUpdate: jest.fn() };
|
||||
return h;
|
||||
}
|
||||
|
||||
function socket() {
|
||||
const emitted = [];
|
||||
return { id: 's1', emitted, emit: (event, payload) => emitted.push({ event, payload }) };
|
||||
}
|
||||
|
||||
describe('admission while no node is reachable', () => {
|
||||
test('refuses the admission and never runs the task', async () => {
|
||||
const h = handlers({ chainHealthy: false });
|
||||
const s = socket();
|
||||
const task = jest.fn();
|
||||
|
||||
await h._runAdmission(s, 'auto_start', task);
|
||||
|
||||
expect(task).not.toHaveBeenCalled();
|
||||
const refusal = s.emitted.find(e => e.event === 'chain_unavailable');
|
||||
expect(refusal).toBeDefined();
|
||||
expect(refusal.payload).toMatchObject({ code: 'NODE_UNREACHABLE', kind: 'auto_start' });
|
||||
// The player must be told nothing was taken from them.
|
||||
expect(refusal.payload.message).toMatch(/nothing was charged/i);
|
||||
expect(h.broadcastManager.sendStatusUpdate).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test('runs the admission normally when a node is reachable', async () => {
|
||||
const h = handlers({ chainHealthy: true });
|
||||
const s = socket();
|
||||
const task = jest.fn().mockResolvedValue('started');
|
||||
|
||||
await expect(h._runAdmission(s, 'join_queue', task)).resolves.toBe('started');
|
||||
expect(task).toHaveBeenCalledTimes(1);
|
||||
expect(s.emitted).toHaveLength(0);
|
||||
});
|
||||
|
||||
test('gates every admission kind, including paid intake', async () => {
|
||||
for (const kind of ['auto_start', 'play_free', 'join_queue', 'early_entry',
|
||||
'request_payment', 'match_queue']) {
|
||||
const h = handlers({ chainHealthy: false });
|
||||
const s = socket();
|
||||
const task = jest.fn();
|
||||
await h._runAdmission(s, kind, task);
|
||||
expect(task).not.toHaveBeenCalled();
|
||||
expect(s.emitted.some(e => e.event === 'chain_unavailable')).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('shutdown still takes precedence over the node check', async () => {
|
||||
const h = handlers({ chainHealthy: false, shuttingDown: true });
|
||||
h._emitShutdownAdmissionRefusal = jest.fn();
|
||||
const s = socket();
|
||||
|
||||
await h._runAdmission(s, 'auto_start', jest.fn());
|
||||
|
||||
expect(h._emitShutdownAdmissionRefusal).toHaveBeenCalled();
|
||||
expect(s.emitted.some(e => e.event === 'chain_unavailable')).toBe(false);
|
||||
});
|
||||
|
||||
test('paid intake is refused when the WALLET is unreachable, but credits/free are not', async () => {
|
||||
// Not a failover: two wallet-rpc processes over one wallet file can spend the same outputs.
|
||||
// The safe response to a wallet outage is a clean refusal of intake only.
|
||||
const h = handlers({ chainHealthy: true });
|
||||
h.walletService = { isHealthy: false };
|
||||
|
||||
const paid = socket();
|
||||
await h._runAdmission(paid, 'request_payment', jest.fn());
|
||||
const refusal = paid.emitted.find(e => e.event === 'payment_unavailable');
|
||||
expect(refusal).toBeDefined();
|
||||
expect(refusal.payload).toMatchObject({ code: 'WALLET_UNREACHABLE' });
|
||||
|
||||
for (const kind of ['auto_start', 'play_free', 'join_queue']) {
|
||||
const s = socket();
|
||||
const task = jest.fn().mockResolvedValue('ok');
|
||||
await expect(h._runAdmission(s, kind, task)).resolves.toBe('ok');
|
||||
expect(task).toHaveBeenCalled();
|
||||
}
|
||||
});
|
||||
|
||||
test('a runtime without the health probe is left alone (fires only on an explicit false)', async () => {
|
||||
const h = handlers({ chainHealthy: true });
|
||||
h.debugManager = { CONSOLE_LOGGING: false }; // no isChainHealthy at all
|
||||
const s = socket();
|
||||
const task = jest.fn().mockResolvedValue('ok');
|
||||
|
||||
await expect(h._runAdmission(s, 'auto_start', task)).resolves.toBe('ok');
|
||||
expect(task).toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,152 @@
|
||||
/**
|
||||
* Multi-node daemon failover and graceful degradation.
|
||||
*
|
||||
* Two defects motivated this:
|
||||
* - the stock configuration pointed PRIMARY and FALLBACK at the SAME host, so what looked like
|
||||
* redundancy was none, and only two endpoints were ever supported;
|
||||
* - `failoverActive` was a one-way latch — once the process moved to a backup it stayed there
|
||||
* until restart, even after the preferred node recovered.
|
||||
*
|
||||
* The contract now: one logical call tries every configured node before failing, the preferred node
|
||||
* is periodically re-tested, and when nothing answers the service reports UNHEALTHY and throws
|
||||
* rather than silently serving a stale cached height.
|
||||
*/
|
||||
|
||||
const RpcService = require('../src/rpc/rpcService');
|
||||
|
||||
const A = 'http://node-a.test';
|
||||
const B = 'http://node-b.test';
|
||||
const C = 'http://node-c.test';
|
||||
|
||||
/** http.post stub that succeeds only for endpoints listed in `up`. */
|
||||
function httpFor(up) {
|
||||
return {
|
||||
post: jest.fn(async (url) => {
|
||||
const endpoint = up.find((e) => url.startsWith(e));
|
||||
if (!endpoint) throw new Error(`unreachable ${url}`);
|
||||
return { data: { result: { count: 4242 } } };
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
function service(http, extra = {}) {
|
||||
return new RpcService({
|
||||
http,
|
||||
endpoints: [A, B, C],
|
||||
identityRequired: false,
|
||||
...extra
|
||||
});
|
||||
}
|
||||
|
||||
describe('daemon endpoint failover', () => {
|
||||
test('accepts an ordered list of endpoints and prefers the first', () => {
|
||||
const svc = service(httpFor([A, B, C]));
|
||||
expect(svc.endpoints).toEqual([A, B, C]);
|
||||
expect(svc.currentEndpoint).toBe(A);
|
||||
expect(svc.failoverActive).toBe(false);
|
||||
});
|
||||
|
||||
test('deduplicates endpoints — the stock PRIMARY == FALLBACK config is one node, not two', () => {
|
||||
const same = 'http://127.0.0.1:34568';
|
||||
const svc = new RpcService({
|
||||
http: httpFor([same]),
|
||||
primaryEndpoint: same,
|
||||
fallbackEndpoint: same,
|
||||
identityRequired: false
|
||||
});
|
||||
expect(svc.endpoints).toEqual([same]);
|
||||
});
|
||||
|
||||
test('parses a comma-separated RPC_ENDPOINTS list and keeps PRIMARY/FALLBACK', () => {
|
||||
const svc = new RpcService({
|
||||
http: httpFor([A]),
|
||||
env: {
|
||||
RPC_ENDPOINTS: `${A}, ${B}`,
|
||||
PRIMARY_RPC_ENDPOINT: C,
|
||||
FALLBACK_RPC_ENDPOINT: C,
|
||||
NODE_ENV: 'test'
|
||||
},
|
||||
identityRequired: false
|
||||
});
|
||||
expect(svc.endpoints).toEqual([A, B, C]);
|
||||
});
|
||||
|
||||
test('walks past dead nodes to a live one within a SINGLE call', async () => {
|
||||
const svc = service(httpFor([C])); // A and B are down
|
||||
await expect(svc.makeRPCCall('getblockcount')).resolves.toMatchObject({ count: 4242 });
|
||||
expect(svc.currentEndpoint).toBe(C);
|
||||
expect(svc.failoverActive).toBe(true);
|
||||
expect(svc.healthy).toBe(true);
|
||||
});
|
||||
|
||||
test('sticks to the working node on subsequent calls', async () => {
|
||||
const http = httpFor([B, C]);
|
||||
const svc = service(http, { preferredRetryMs: 10 * 60 * 1000 });
|
||||
await svc.makeRPCCall('getblockcount');
|
||||
expect(svc.currentEndpoint).toBe(B);
|
||||
|
||||
http.post.mockClear();
|
||||
await svc.makeRPCCall('getblockcount');
|
||||
expect(http.post).toHaveBeenCalledTimes(1); // no re-probing of the dead primary
|
||||
expect(http.post.mock.calls[0][0]).toContain('node-b');
|
||||
});
|
||||
|
||||
test('returns to the preferred node once it recovers', async () => {
|
||||
const up = [B];
|
||||
const http = {
|
||||
post: jest.fn(async (url) => {
|
||||
if (!up.some((e) => url.startsWith(e))) throw new Error('unreachable');
|
||||
return { data: { result: { count: 4242 } } };
|
||||
})
|
||||
};
|
||||
const svc = service(http, { preferredRetryMs: 1000 }); // floor is 1s; see the constructor
|
||||
await svc.makeRPCCall('getblockcount');
|
||||
expect(svc.currentEndpoint).toBe(B);
|
||||
|
||||
up.push(A); // primary comes back
|
||||
svc._preferredProbedAt -= 5000; // ...and the re-test interval elapses
|
||||
await svc.makeRPCCall('getblockcount');
|
||||
|
||||
expect(svc.currentEndpoint).toBe(A);
|
||||
expect(svc.failoverActive).toBe(false);
|
||||
});
|
||||
|
||||
test('throws and reports unhealthy when NO node answers', async () => {
|
||||
const svc = service(httpFor([]));
|
||||
await expect(svc.makeRPCCall('getblockcount')).rejects.toThrow();
|
||||
expect(svc.healthy).toBe(false);
|
||||
});
|
||||
|
||||
test('getBlockHeight degrades to the last known height without claiming health', async () => {
|
||||
const http = httpFor([A]);
|
||||
const svc = service(http);
|
||||
expect(await svc.getBlockHeight()).toBe(4242);
|
||||
|
||||
http.post.mockRejectedValue(new Error('all nodes offline'));
|
||||
expect(await svc.getBlockHeight()).toBe(4242); // cached value for non-financial UI
|
||||
expect(svc.healthy).toBe(false); // ...but never reported as live
|
||||
});
|
||||
|
||||
test('getBlockCountStrict refuses to substitute the cached height', async () => {
|
||||
const http = httpFor([A]);
|
||||
const svc = service(http);
|
||||
await svc.getBlockCountStrict();
|
||||
http.post.mockRejectedValue(new Error('all nodes offline'));
|
||||
await expect(svc.getBlockCountStrict()).rejects.toThrow();
|
||||
});
|
||||
|
||||
test('healthCheck probes every node and adopts the first healthy one', async () => {
|
||||
const svc = service(httpFor([C]));
|
||||
const health = await svc.healthCheck();
|
||||
expect(health.endpoints).toHaveLength(3);
|
||||
expect(health.healthy).toBe(true);
|
||||
expect(health.healthyCount).toBe(1);
|
||||
expect(svc.currentEndpoint).toBe(C);
|
||||
});
|
||||
|
||||
test('healthCheck reports unhealthy when every node is down', async () => {
|
||||
const health = await service(httpFor([])).healthCheck();
|
||||
expect(health.healthy).toBe(false);
|
||||
expect(health.healthyCount).toBe(0);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user